File manager - Edit - /home/u478019808/domains/bestandroidphones.store/public_html/static/img/logo/gslib.tar
Back
impersonation_credentials.py 0000644 00000005006 15025013616 0012362 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of credentials that refreshes using the iamcredentials API.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import datetime from oauth2client import client from gslib.iamcredentials_api import IamcredentailsApi class ImpersonationCredentials(client.OAuth2Credentials): _EXPIRY_FORMAT = '%Y-%m-%dT%H:%M:%SZ' def __init__(self, service_account_id, scopes, credentials, logger): self._service_account_id = service_account_id self.api = IamcredentailsApi(logger, credentials) response = self.api.GenerateAccessToken(service_account_id, scopes) self.access_token = response.accessToken self.token_expiry = self._ConvertExpiryTime(response.expireTime) super(ImpersonationCredentials, self).__init__(self.access_token, None, None, None, self.token_expiry, None, None, scopes=scopes) @property def service_account_id(self): return self._service_account_id def _refresh(self, http): # client.Oauth2Credentials converts scopes into a set, so we need to convert # back to a list before making the API request. response = self.api.GenerateAccessToken(self._service_account_id, list(self.scopes)) self.access_token = response.accessToken self.token_expiry = self._ConvertExpiryTime(response.expireTime) def _ConvertExpiryTime(self, value): return datetime.datetime.strptime(value, ImpersonationCredentials._EXPIRY_FORMAT) __pycache__/resumable_streaming_upload.cpython-39.pyc 0000644 00000012357 15025013616 0017010 0 ustar 00 a +(Wg" � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd l mZ G d d � d e�ZdS ) z-Helper class for streaming resumable uploads.� )�absolute_import)�print_function)�division)�unicode_literalsN)�CommandException)�GetJsonResumableChunkSize)�UTF8c @ sV e Zd ZdZddd�Zedd� �Zddd �Zd d � Zd d � Z e j fdd�Z dd� Z dS )�#ResumableStreamingJsonUploadWrappera# Wraps an input stream in a buffer for resumable uploads. This class takes a non-seekable input stream, buffers it, and exposes it as a stream with limited seek capabilities such that it can be used in a resumable JSON API upload. max_buffer_size bytes of buffering is supported. Fc C sL || _ |s&|t� k r&td|t� f ��|| _t�� | _d| _d| _d| _ dS )av Initializes the wrapper. Args: stream: Input stream. max_buffer_size: Maximum size of internal buffer; should be >= the chunk size of the resumable upload API to ensure that at least one full chunk write can be replayed in the event of a server error. test_small_buffer: Skip check for buffer size vs. chunk size, for testing. z�Resumable streaming upload created with buffer size %s, JSON resumable upload chunk size %s. Buffer size must be >= JSON resumable upload chunk size to ensure that uploads can be resumed.r N) �_orig_fpr r �_max_buffer_size� collections�deque�_buffer� _buffer_start� _buffer_end� _position)�self�stream�max_buffer_sizeZtest_small_buffer� r �//opt/gsutil/gslib/resumable_streaming_upload.py�__init__) s � z,ResumableStreamingJsonUploadWrapper.__init__c C s t | jdd�S )z<Returns the mode of the underlying file descriptor, or None.�modeN)�getattrr �r r r r r C s z(ResumableStreamingJsonUploadWrapper.mode���c C sV |du p|dk }|r | j }n|}d}g }| j| jk r�| j}d}|t| j| � | jk rr|t| j| �7 }|d7 }q>|| jk r�|dkr�t| j| �}| j| } || } t| |�} |�| j| | | | � � || 8 }||7 }|d7 }| j| 7 _qr|�r>| j� |�} t| �} |�s| }n|�| � d� |�}| j| 7 _�n|�r"| j� |�} |�s\| }n|�| � d� |�}t| �} | �rR| j| 7 _| j�| � | j| 7 _d}| j| j | j k�r�| j� � }| jt|�7 _�q�|�rR| j | j| j }|�rR| j� || d� � | j|8 _n0t j�r>|�r>dd� |D �}|�rNd� |�nd}|S )z�"Reads from the wrapped stream. Args: size: The amount of bytes to read. If omitted or negative, the entire contents of the stream will be read and returned. Returns: Bytes from the wrapped stream. Nr � � c S s$ g | ] }t |t�r |�t�n|�qS r )� isinstance�str�encoder )�.0Zbdr r r � <listcomp>� s �z<ResumableStreamingJsonUploadWrapper.read.<locals>.<listcomp>)r r r r �lenr �min�appendr �read�join�popleft� appendleft�six�PY3)r �sizeZread_all_bytes�bytes_remaining�dataZ buffered_dataZ pos_in_bufferZ buffer_indexZ buffer_lenZoffset_from_positionZbytes_available_this_buffer� read_size�new_data�data_lenZ oldest_dataZ refill_amountr r r r& H s~ �� ��z(ResumableStreamingJsonUploadWrapper.readc C s | j S )z$Returns the current stream position.)r r r r r �tell� s z(ResumableStreamingJsonUploadWrapper.tellc C s dS )z/Returns true since limited seek support exists.Tr r r r r �seekable� s z,ResumableStreamingJsonUploadWrapper.seekablec C s� |t jkr<|| jk s || jkr4td|| j| jf ��|| _nT|t jkr�|| jkrbtd|| jf ��| �| j�rpqb| j|8 _ntd||f ��dS )z�Seeks on the buffered stream. Args: offset: The offset to seek to; must be within the buffer bounds. whence: Must be os.SEEK_SET. Raises: CommandException if an unsupported seek mode or position is used. z�Unable to resume upload because of limited buffering available for streaming uploads. Offset %s was requested, but only data from %s to %s is buffered.zHInvalid SEEK_END offset %s on streaming upload. Only %s can be buffered.z;Invalid seek mode on streaming upload. (mode %s, offset %s)N) �os�SEEK_SETr r r r �SEEK_ENDr r& )r �offset�whencer r r �seek� s" � � �z(ResumableStreamingJsonUploadWrapper.seekc C s | j �� S )N)r �closer r r r r: � s z)ResumableStreamingJsonUploadWrapper.closeN)F)r )�__name__� __module__� __qualname__�__doc__r �propertyr r& r2 r3 r4 r5 r9 r: r r r r r s e!r )r> � __future__r r r r r r4 r* �gslib.exceptionr �gslib.utils.boto_utilr �gslib.utils.constantsr �objectr r r r r �<module> s __pycache__/command_argument.cpython-39.pyc 0000644 00000010402 15025013616 0014721 0 ustar 00 a +(Wgc � @ sT d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ G dd � d e�Z d S ) z<Contains classes related to argparse-based argument parsing.� )�absolute_import)�print_function)�division)�unicode_literals� )� CompleterTypec @ s� e Zd ZdZdd� Zedd� �Zedd� �Zedd � �Zed d � �Z ed d � �Z edd� �Z edd� �Z edd� �Z edd� �Zedd� �Zedd� �ZdS )�CommandArgumentzArgparse style argument.c O s0 d}d|v r|d }|d= || _ || _|| _dS )a� Constructs an argparse argument with the given data. See add_argument in argparse for description of the options. The only deviation from the argparse arguments is the 'completer' parameter. If 'completer' is present, it's used as the argcomplete completer for the argument. Args: *args: Position args to pass to argparse add_argument **kwargs: Named args to pass to argparse add_argument N� completer)�args�kwargsr )�selfr r r � r �%/opt/gsutil/gslib/command_argument.py�__init__ s zCommandArgument.__init__c C s t ddtjd�S )zEConstructs an argument that takes 0 or more Cloud URLs as parameters.�file�*��nargsr �r r � CLOUD_OBJECTr r r r � MakeZeroOrMoreCloudURLsArgument/ s �z/CommandArgument.MakeZeroOrMoreCloudURLsArgumentc C s t ddtjd�S )zEConstructs an argument that takes 0+ Cloud bucket URLs as parameters.r r r �r r � CLOUD_BUCKETr r r r �%MakeZeroOrMoreCloudBucketURLsArgument6 s �z5CommandArgument.MakeZeroOrMoreCloudBucketURLsArgumentc C s t d| tjd�S )zDConstructs an argument that takes N Cloud bucket URLs as parameters.r r r ��nr r r � MakeNCloudBucketURLsArgument= s �z,CommandArgument.MakeNCloudBucketURLsArgumentc C s t d| tjd�S )z=Constructs an argument that takes N Cloud URLs as parameters.r r r r r r r �MakeNCloudURLsArgumentD s �z&CommandArgument.MakeNCloudURLsArgumentc C s t ddtjd�S )z?Constructs an argument that takes 0 or more Cloud or File URLs.r r r �r r �CLOUD_OR_LOCAL_OBJECTr r r r �%MakeZeroOrMoreCloudOrFileURLsArgumentK s �z5CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgumentc C s t d| tjd�S )zEConstructs an argument that takes N Cloud or File URLs as parameters.r r r r r r r � MakeNCloudOrFileURLsArgumentR s �z,CommandArgument.MakeNCloudOrFileURLsArgumentc C s t ddtjd�S )zDConstructs an argument that takes 0 or more File URLs as parameters.r r r �r r � LOCAL_OBJECTr r r r � MakeZeroOrMoreFileURLsArgumentY s �z.CommandArgument.MakeZeroOrMoreFileURLsArgumentc C s t d| tjd�S )z<Constructs an argument that takes N File URLs as parameters.r r r" r r r r �MakeNFileURLsArgument` s �z%CommandArgument.MakeNFileURLsArgumentc C s t ddtjd�S )z=Constructs an argument that takes a File URL or a canned ACL.r r r )r r �LOCAL_OBJECT_OR_CANNED_ACLr r r r � MakeFileURLOrCannedACLArgumentg s �z.CommandArgument.MakeFileURLOrCannedACLArgumentc C s t dtjd�S )z1Constructs an argument that takes arbitrary text.�text)r �r r �NO_OPr r r r �MakeFreeTextArgumentn s z$CommandArgument.MakeFreeTextArgumentc C s t ddtjd�S )z4Constructs an argument that takes multiple bindings.�binding�+r r) r r r r � MakeOneOrMoreBindingsArguments s z-CommandArgument.MakeOneOrMoreBindingsArgumentN)�__name__� __module__� __qualname__�__doc__r � staticmethodr r r r r r! r$ r% r' r+ r. r r r r r s0 r N) r2 � __future__r r r r � tab_completer �objectr r r r r �<module> s __pycache__/ui_controller.cpython-39.pyc 0000644 00000074543 15025013616 0014301 0 ustar 00 a +(Wg�� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZddl Z ddl Z ddl m Z dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z ddl m Z ddl m Z ddl m!Z! e j"Z#G dd� de$�Z%d*d d �Z&G d d � d e$�Z'G d d!� d!e'�Z(G d"d#� d#e'�Z)G d$d%� d%e$�Z*G d&d'� d'e$�Z+G d(d)� d)e j,�Z-dS )+zFBase classes for gsutil UI controller, UIThread and MainThreadUIQueue.� )�absolute_import)�print_function)�division)�unicode_literals)�dequeN)�queue)�LogPerformanceSummaryParams)�LogRetryableError)� FileMessage)� FinalMessage)�MetadataMessage)�PerformanceSummaryMessage)�ProducerThreadMessage)�ProgressMessage)�RetryableErrorMessage)�SeekAheadMessage)� StatusMessage)�parallelism_framework_util)� DecimalShort)� HumanReadableWithDecimalPlaces)�MakeHumanReadable)� PrettyTimec @ s e Zd ZdZdZdZdZdZdS )�EstimationSourcezenum for total size source.� � � � N)�__name__� __module__� __qualname__�__doc__�PRODUCER_THREAD_FINAL�SEEK_AHEAD_THREAD�PRODUCER_THREAD_ESTIMATE�INDIVIDUAL_MESSAGES� r% r% �"/opt/gsutil/gslib/ui_controller.pyr 2 s r r c C sb t | |d�}|�� }ttt|d ���dkrTt|d �| d }|d d|� |d< dd�|� S )z�Adjusts proper width for printing num_bytes in readable format. Args: num_bytes: The number of bytes we must display. decimal_places: The standard number of decimal places. Returns: String of fixed width representing num_bytes. )�decimal_placesr i� r Nz%9s� )r �split�int�round�float�len�join)� num_bytesr' Zhuman_readable� number_formatZlast_characterr% r% r& �BytesToFixedWidthStringF s �r1 c @ s� e Zd ZdZG dd� de�Zd#d d �Zd d� Zdd� Zdd� Z dd� Z dd� Z dd� Z dd� Z dd � Zejfd d �Zd d � Zejfd!d"�Zd S )$�StatusMessageManagerz�General manager for common functions shared by data and metadata managers. This subclass has the responsibility of having a common constructor and the same handler for SeekAheadMessages and ProducerThreadMessages. c @ s e Zd ZdZdd� ZdS )z+StatusMessageManager._ThroughputInformationz�Class that contains all information needed for throughput calculation. This _ThroughputInformation is used to track progress and time at several points of our operation. c C s || _ || _dS )z�Constructor of _ThroughputInformation. Args: progress: The current progress, in bytes/second or objects/second. report_time: Float representing when progress was reported (seconds since Epoch). N)�progress�time)�selfr3 Z report_timer% r% r& �__init__h s z4StatusMessageManager._ThroughputInformation.__init__N�r r r r r6 r% r% r% r& �_ThroughputInformationa s r8 r �333333�?� � FN�P c C s� || _ || _|| _|| _|| _|| _|| _|| _tj | _ tj | _ d| _ d| _ | jrX| jnt�� | _| j| _| j| _d| _t� | _d| _g d�| _d| _d| _d| _ d| _d| _dS )aj Instantiates a StatusMessageManager. Args: update_message_period: Minimum period for refreshing and displaying new information. A non-positive value will ignore any time restrictions imposed by this field, but it will affect throughput and time remaining estimations. update_spinner_period: Minimum period for refreshing and displaying the spinner. A non-positive value will ignore any time restrictions imposed by this field. sliding_throughput_period: Sliding period for throughput calculation. A non-positive value will make it impossible to calculate the throughput. first_throughput_latency: Minimum waiting time before actually displaying throughput info. A non-positive value will ignore any time restrictions imposed by this field. quiet_mode: If True, do not print status messages (but still process them for analytics reporting as necessary). custom_time: If a custom start_time is desired. Used for testing. verbose: Tells whether or not the operation is on verbose mode. console_width: Width to display on console. This should not adjust the visual output, just the space padding. For proper visualization, we recommend setting this field to at least 80. r g )�/�-�\�|FN)�update_message_period�update_spinner_period�sliding_throughput_period�first_throughput_latency� quiet_mode� custom_time�verbose� console_widthr r$ �num_objects_source�total_size_source� num_objects� total_sizer4 �refresh_message_time� start_time�refresh_spinner_time� throughputr � old_progress�last_progress_time�spinner_char_list�current_spinner_index�objects_finished�object_report_change� final_message� r5 rA rB rC rD rE rF rG rH r% r% r& r6 s s6 $�� z StatusMessageManager.__init__c C s | j | j S )zpReturns the current spinner character. Returns: char_to_print: Char to be printed as the spinner )rS rT �r5 r% r% r& � GetSpinner� s z StatusMessageManager.GetSpinnerc C s | j d t| j� | _ dS )z&Updates the current spinner character.r N)rT r- rS rY r% r% r& � UpdateSpinner� s �z"StatusMessageManager.UpdateSpinnerc C s� |j rH| jtjkr"tj| _|j| _| jtjkrD|jrDtj| _|j| _dS | jtjkrdtj| _|j| _| jtjkr�|jr�tj| _|j| _dS )zqHandles a ProducerThreadMessage. Args: status_message: The ProducerThreadMessage to be processed. N) �finishedrI r r! rK rJ �sizerL r# �r5 �status_messager% r% r& � _HandleProducerThreadMessage� s$ � �z1StatusMessageManager._HandleProducerThreadMessagec C sv d|j }|jr>|dt|j� 7 }| jtjkr>tj| _|j| _| jtjkrZtj| _|j | _ |d7 }| jsr|� |� dS )z�Handles a SeekAheadMessage. Args: status_message: The SeekAheadMessage to be processed. stream: Stream to print messages. z,Estimated work for this command: objects: %sz, total size: %s� N) rK r] r rJ r r" rL rI rE �write)r5 r_ �streamZestimate_messager% r% r& �_HandleSeekAheadMessage� s �� z,StatusMessageManager._HandleSeekAheadMessagec C s t |jd� dS )zyHandles a PerformanceSummaryMessage. Args: status_message: The PerformanceSummaryMessage to be processed. )� uses_sliceN)r re r^ r% r% r& � _HandlePerformanceSummaryMessage� s z5StatusMessageManager._HandlePerformanceSummaryMessagec C s || j | jkS )z�Decides whether enough time has passed to start tracking throughput. Args: cur_time: current time. Returns: Whether or not we should track the throughput. )rN rD �r5 �cur_timer% r% r& �ShouldTrackThroughput s z*StatusMessageManager.ShouldTrackThroughputc C s$ || j | jk}| j}|s | jo"|S )z�Decides whether or not it is time for printing a new progress. Args: cur_time: current time. Returns: Whether or not we should print the progress. )rM rA rK rV )r5 rh Zsufficient_time_elapsedZnonzero_reportr% r% r& �ShouldPrintProgress s ���z(StatusMessageManager.ShouldPrintProgressc C s || j | jko| jS )z�Decides whether or not it is time for updating the spinner character. Args: cur_time: Current time. Returns: Whether or not we should update and print the spinner. )rO rB rL rg r% r% r& �ShouldPrintSpinner s �z'StatusMessageManager.ShouldPrintSpinnerc C s$ | � � | js |�| �� d � dS )z�Prints a spinner character. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. � N)r[ rE rb rZ )r5 rc r% r% r& � PrintSpinner+ s z!StatusMessageManager.PrintSpinnerc C s� t | j�dkr0|| jd j | jkr0| j�� q | js:dS | jd }| j|jkrZd| _dS ||j | j|j | _td| j�| _dS )a� Updates throughput if the required period for calculation has passed. The throughput is calculated by taking all the progress (objects or bytes) processed within the last sliding_throughput_period seconds, and dividing that by the time period between the oldest progress time within that range and the last progress measurement, which are defined by oldest_progress[1] and last_progress_time, respectively. Among the pros of this approach, a connection break or a sudden change in throughput is quickly noticeable. Furthermore, using the last throughput measurement rather than the current time allows us to have a better estimation of the actual throughput. Args: cur_time: Current time to check whether or not it is time for a new throughput measurement. cur_progress: The current progress, in number of objects finished or in bytes. r r N) r- rQ r4 rC �popleftrR rP r3 �max)r5 rh Z cur_progressZoldest_progressr% r% r& �UpdateThroughput6 s � �z%StatusMessageManager.UpdateThroughputc C sb dt | j� }| jr&|dt| j� 7 }| jt|� }| js^|�d| d t|d�d d � dS )z�Prints a final message to indicate operation succeeded. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. z#Operation completed over %s objectsz/%sra �.r r( N) r rK rL r rH r- rE rb ro )r5 rc �string_to_print�remaining_widthr% r% r& �PrintFinalSummaryMessageY s �� ��z-StatusMessageManager.PrintFinalSummaryMessage)r r9 r: r; FNFr< )r r r r �objectr8 r6 rZ r[ r` rd rf ri rj rk �sys�stderrrm rp rt r% r% r% r& r2 Z s, � N #r2 c sP e Zd ZdZd� fd d � Zd d � Zd d� Zdd� Zej fdd�Z dd� Z � Z S )�MetadataManagera� Manages shared state for metadata operations. This manager is specific for metadata operations. Among its main functions, it receives incoming StatusMessages, storing all necessary data about the current and past states of the system necessary to display to the UI. It also provides methods for calculating metrics such as throughput and estimated time remaining. Finally, it provides methods for displaying messages to the UI. r r9 r: r; FNr< c s$ t t| �j||||||||d� dS )zhInstantiates a MetadataManager. See argument documentation in StatusMessageManager base class. �rA rB rC rD rE rF rG rH N)�superrx r6 rX �� __class__r% r& r6 v s ��zMetadataManager.__init__c C s | j S )zaGets the progress for a MetadataManager. Returns: The number of finished objects. )rU rY r% r% r& � GetProgress� s zMetadataManager.GetProgressc C s` | j d7 _ | jtjkr0tj| _| jd7 _d| _|j| _| j | jkr\| jtjkr\d| _ dS )zeHandles a MetadataMessage. Args: status_message: The MetadataMessage to be processed. r TN) rU rI r r$ rK rV r4 rR r! rW r^ r% r% r& �_HandleMetadataMessage� s �z&MetadataManager._HandleMetadataMessagec C s� d| _ t|t�r | �||� nTt|t�r4| �|� n>t|t�rJ| �|� n(t|t�r^t |� nt|t �rr| � |� | j � | �| j|j�� dS )z�Processes a message from _MainThreadUIQueue or _UIThread. Args: status_message: The StatusMessage item to be processed. stream: Stream to print messages. FN)rV � isinstancer rd r r` r r~ r r r rf rQ �appendr8 rU r4 �r5 r_ rc r% r% r& �ProcessMessage� s �z MetadataManager.ProcessMessagec C sB | j | j }| jr || j }nd}| �� }| jtjkr�dt| j� d t| j � d }| j | jkrhd}n"dtdt dt | j� | j �� }|d }ndt| j� d }d }| j | j | j kr�d | j }| jtjkr�| jr�d t|� } q�d } nd }d } d } | j||||| d�} | jt| � } | j�s>|�| t| d�d d � dS )ak Prints progress and throughput/time estimation. Prints total number of objects and number of finished objects with the percentage of work done, potentially including the throughput (in objects/second) and estimated time remaining. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. N�[r= z objects]�100�%3d�c �d �% Done� z%.2f objects/s�ETA z\{char_to_print} {objects_completed} {percentage_completed} {throughput} {time_remaining_str})� char_to_print�objects_completed�percentage_completedrP �time_remaining_strr r( rl )rK rU rP rZ rI r r" r �minr* r, rM rN rD r# r �formatrH r- rE rb ro ) r5 rc �total_remaining�time_remainingr� r� � percentager� rP r� � format_strrr rs r% r% r& � PrintProgress� sX �� �� � � ��z MetadataManager.PrintProgressc C s t |ttttttf�rdS dS �z�Determines whether this manager is suitable for handling status_message. Args: status_message: The StatusMessage object to be analyzed. Returns: True if this message can be properly handled by this manager, False otherwise. TF)r r r r r r r r^ r% r% r& �CanHandleMessage� s ��z MetadataManager.CanHandleMessage)r r9 r: r; FNFr< ) r r r r r6 r} r~ r� rv rw r� r� � __classcell__r% r% r{ r& rx k s �@rx c sx e Zd ZdZG dd� de�Zd � fd d � Zd d � Zdd� Zdd� Z dd� Z dd� Z dd� Z e jfdd�Zdd� Z� ZS ) � DataManagera� Manages shared state for data operations. This manager is specific for data operations. Among its main functions, it receives incoming StatusMessages, storing all necessary data about the current and past states of the system necessary to display to the UI. It also provides methods for calculating metrics such as throughput and estimated time remaining. Finally, it provides methods for displaying messages to the UI. c @ s e Zd ZdZdd� ZdS )z DataManager._ProgressInformationz�Class that contains all progress information needed for a given file. This _ProgressInformation is used as the value associated with a file_name in the dict that stores the information about all processed files. c C s d| _ d| _i | _|| _dS )zbConstructor of _ProgressInformation. Args: size: The total size of the file. r N)�new_progress_sum�existing_progress_sum�dictr] )r5 r] r% r% r& r6 s z)DataManager._ProgressInformation.__init__Nr7 r% r% r% r& �_ProgressInformation s r� r r9 r: r; FNc sT t t| �j||||||||d� d| _d| _d| _d| _i | _d| _d| _ d| _ dS )zdInstantiates a DataManager. See argument documentation in StatusMessageManager base class. ry Tr N) rz r� r6 � first_item�total_progress� new_progress�existing_progress�individual_file_progress�component_total�finished_components�existing_componentsrX r{ r% r& r6 3 s( �� zDataManager.__init__c C s | j S )znGets the progress for a DataManager. Returns: The number of processed bytes in this operation. )r� rY r% r% r& r} Z s zDataManager.GetProgressc C s: |j s�| jr0| js0|j| _| j| _| j| _d| _|jj}|j rD|j nd|_ | � |j �| j |< | j t jkr~t j| _ | jd7 _| jt jkr�t j| _| j|j 7 _d| _n�| jd7 _|jj}| j | }|j|j }| j|j | 7 _| j|j | 7 _|j| _| j |= d| _| j| jk�r6| j t jk�r6d| _dS )zsHandles a FileMessage that describes a file. Args: status_message: the FileMessage to be processed. Fr r TN)r\ r� rF r4 rM rN Zlast_throughput_time�src_url� url_stringr] r� r� rI r r$ rK rJ rL rV rU r� r� r� r� rR r! rW )r5 r_ � file_name� file_progress�total_bytes_transferredr% r% r& �_HandleFileDescriptionb sB � � �z"DataManager._HandleFileDescriptionc C sL |j }|tjkpJ|tjkpJ|tjkpJ|tjkpJ|tjkpJ|tjkpJ|tjkS )a Tells whether or not this FileMessage represent a file. This is needed because FileMessage is used by both files and components. Args: file_message: The FileMessage to be analyzed. Returns: Whether or not this represents a file. ) � message_typer � FILE_DOWNLOAD� FILE_UPLOAD�FILE_CLOUD_COPY�FILE_DAISY_COPY�FILE_LOCAL_COPY� FILE_REWRITE� FILE_HASH)r5 � file_messager� r% r% r& �_IsFile� s ������zDataManager._IsFilec C s� |j tjkr�|js�| jd7 _|jj}| j| }|j|j f}d|j f|j |< | j |j 7 _ | j |j 7 _ | j|j 7 _�nP|j tjks�|j tjk�r�|j�s | jd7 _|j tjk�r�|jj}| j| }| j |j7 _ |j|j f}d|jf|j |< | j |j7 _ | j|j7 _n�| jd7 _|jj}| j| }|j|j f}||j v �rb|j | nd}| j |j t|� 7 _ | j|j t|� 7 _|j| _| j|j t|� 7 _|j |d |d f|j |< dS )zxHandles a FileMessage that describes a component. Args: status_message: The FileMessage to be processed. r r �r r N)r� r �EXISTING_COMPONENTr\ r� r� r� r� � component_num�dst_urlr] r� r� r� r� �COMPONENT_TO_UPLOAD�COMPONENT_TO_DOWNLOADr� �bytes_already_downloadedr� �sumr� r4 rR r� �r5 r_ r� r� �key� last_updater% r% r& �_HandleComponentDescription� sT � � � � � �z'DataManager._HandleComponentDescriptionc C s� |j j}| j| }|j|jf}||jv r2|j| nd}| j|d 8 _| j|j|d 7 _| j|j|d 7 _| j |j|d 7 _ |j|d f|j|< |j | _ dS )z�Handles a ProgressMessage that tracks progress of a file or component. Args: status_message: The ProgressMessage to be processed. r� r r N) r� r� r� r� r� r� �processed_bytesr� r� r� r4 rR r� r% r% r& �_HandleProgressMessage� s � �z"DataManager._HandleProgressMessagec C s� d| _ t|t�r | �|� n�t|t�r4| �||� ntt|t�rj| �|�rT| �|� n | � |� t |d� n>t|t �r�| � |� n(t|t �r�t|� nt|t�r�| �|� | j�| �| j|j�� dS )z�Processes a message from _MainThreadUIQueue or _UIThread. Args: status_message: The StatusMessage item to be processed. stream: Stream to print messages. Here only for SeekAheadThread F)r� N)rV r r r` r rd r r� r� r� r r r� r r r rf rQ r� r8 r� r4 r� r% r% r& r� � s& �zDataManager.ProcessMessagec C st | j | j }| jr || j }nd}| �� }| jtjkrXdt| j� d t| j � d }ndt| j� d }dt | j�t | j �f }| j tjkr�| j | jkr�d}n"dt dt d t| j� | j �� }|d }nd }| j| j | jk�r t | j�d } | j tjk�r| j�rd t|� } nd } nd } d } d} | j||||| | d�} | jt| � } | j�sp|�| t| d�d d � dS )a� Prints progress and throughput/time estimation. If a ProducerThreadMessage or SeekAheadMessage has been provided, it outputs the number of files completed, number of total files, the current progress, the total size, and the percentage it represents. If none of those have been provided, it only includes the number of files completed, the current progress and total size (which might be updated), with no percentage as we do not know if more files are coming. It may also include time estimation (available only given ProducerThreadMessage or SeekAheadMessage provided) and throughput. For that to happen, there is an extra condition of at least first_throughput_latency seconds having been passed since the UIController started, and that either the ProducerThread or the SeekAheadThread have estimated total number of files and total size. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. Nr� r= z files]z[%s/%s]r� r� r� r� r� r� z/sr� zl{char_to_print} {objects_completed}{bytes_progress} {percentage_completed} {throughput} {time_remaining_str})r� r� �bytes_progressr� rP r� r r( rl )rL r� rP rZ rI r r" r rU rK r1 rJ r� r* r, rM rN rD r# r r� rH r- rE rb ro )r5 rc r� r� r� r� r� r� r� rP r� r� rr rs r% r% r& r� " sb ���� �� ���zDataManager.PrintProgressc C s t |tttttttf�r dS dS r� )r r r r r r r r r^ r% r% r& r� v s � z DataManager.CanHandleMessage)r r9 r: r; FNFN)r r r r ru r� r6 r} r� r� r� r� r� rv rw r� r� r� r% r% r{ r& r� s& �'58&Tr� c @ s. e Zd ZdZddd �Zdd d �Zdd d �ZdS )� UIControllerz�Controller UI class to integrate _MainThreadUIQueue and _UIThread. This class receives messages from _MainThreadUIQueue and _UIThread and send them to an appropriate manager, which will then processes and store data about them. r r9 r: r; FNc C s\ || _ || _|| _|| _|| _d| _|| _|| _d| _g | _ d| _ d| _ |rXt |d�| _ dS )a� Instantiates a UIController. Args: update_message_period: Minimum period for refreshing and displaying new information. A non-positive value will ignore any time restrictions imposed by this field. update_spinner_period: Minimum period for refreshing and displaying the spinner. A non-positive value will ignore any time restrictions imposed by this field. sliding_throughput_period: Sliding period for throughput calculation. A non-positive value will make it impossible to calculate the throughput. first_throughput_latency: Minimum waiting time before actually displaying throughput info. A non-positive value will ignore any time restrictions imposed by this field. quiet_mode: If True, do not print status messages (but still process them for analytics reporting as necessary). custom_time: If a custom start_time is desired. Used for testing. verbose: Tells whether or not the operation is on verbose mode. dump_status_messages_file: File path for logging all received status messages, for debugging purposes. Nr< F�ab) rG rA rB rC rD �managerrE rF rH �early_estimation_messages�printed_final_message�dump_status_message_fp�open) r5 rA rB rC rD rE rF rG �dump_status_messages_filer% r% r& r6 � s zUIController.__init__c C s� | j �||� | j �|�rN| j �|�r:| j �|| j �� � | j �|� || j _| j �|�rn| j � |� || j _ t |t �s�| j j r�| j jr�| js�d| _t| j jd� | j �|� dS )a� Processes a message, updates throughput and prints progress. Args: status_message: Message to be processed. Could be None if UIThread cannot retrieve message from status_queue. stream: stream to print messages. Usually sys.stderr, but customizable for testing. cur_time: Message time. Used to determine if it is time to refresh output, or calculate throughput. T)�num_objects_transferredN)r� r� rj ri rp r} r� rM rk rm rO r r rW rK r� r rt )r5 r_ rc rh r% r% r& �_HandleMessage� s& ���zUIController._HandleMessagec C s� t |t�sd|tkr`| js`t| j| j| j| j| j | j | j | j d�| _| j D ]}| j|||jd� qHdS | jr�| j�t|�� | j�d� |s�|j}| j�sTt |t�s�t |t�r�| j �|� dS t |t��rt| j| j| j| j| j | j | j | j d�| _| j D ]}| �|||� q�nFt| j| j| j| j| j | j | j | j d�| _| j D ]}| �|||� �q>| j�|��s�t |t��szt |t��r�t| j| j| j| j| j | j | j d�| _| j D ]}| �|||� �q�ndS | �|||� dS )a� Coordinates UI manager and calls appropriate function to handle message. Args: status_message: Message to be processed. Could be None if UIThread cannot retrieve message from status_queue. stream: Stream to print messages. Usually sys.stderr, but customizable for testing. cur_time: Message time. Used to determine if it is time to refresh output, or calculate throughput. ry )rh Nra )rA rB rC rD rF rG rH )r r �_ZERO_TASKS_TO_DO_ARGUMENTr� r� rA rB rC rD rE rF rG rH r� r� r4 r� rb �strr r r� r rx r� r r )r5 r_ rc rh Zestimation_messager% r% r& �Call� s� � � � � � � � zUIController.Call)r r9 r: r; FNFN)N)N)r r r r r6 r� r� r% r% r% r& r� � s � 1 r� c s* e Zd ZdZ� fdd�Zddd�Z� ZS )�MainThreadUIQueuea� Handles status display and processing in the main thread / master process. This class emulates a queue to cover main-thread activity before or after Apply, as well as for the single-threaded, single-process case, i.e., _SequentialApply. When multiple threads or processes are used during calls to Apply, the main thread is waiting for work to complete, and this queue must remain unused until Apply returns. Code producing arguments for Apply (such as the NameExpansionIterator) must not post messages to this queue to avoid race conditions with the UIThread. This class sends the messages it receives to UIController, which decides the correct course of action. c s t t| ��� || _|| _dS )z�Instantiates a _MainThreadUIQueue. Args: stream: Stream for printing messages. ui_controller: UIController to manage messages. N)rz r� r6 � ui_controllerrc )r5 rc r� r{ r% r& r6 L s zMainThreadUIQueue.__init__Nc C s | j �|| j� d S )N)r� r� rc )r5 r_ �timeoutr% r% r& �putY s zMainThreadUIQueue.put)N)r r r r r6 r� r� r% r% r{ r& r� = s r� c s* e Zd ZdZd� fdd� Zdd� Z� ZS )�UIThreada� Responsible for centralized printing across multiple processes/threads. This class pulls status messages that are posted to the centralized status queue and coordinates displaying status and progress to the user. It is used only during calls to _ParallelApply, which in turn is called only when multiple threads and/or processes are used. This class sends the messages it receives to UIController, which decides the correct course of action. r c s2 t t| ��� || _|| _|| _|| _| �� dS )z�Instantiates a _UIThread. Args: status_queue: Queue for reporting status updates. stream: Stream for printing messages. ui_controller: UI controller to manage messages. timeout: Timeout for getting a message. N)rz r� r6 � status_queuerc r� r� �start)r5 r� rc r� r� r{ r% r& r6 k s zUIThread.__init__c C s� zRz| j j| jd�}W n tjy2 d }Y qY n0 | j�|| j� |tkrqPqW n4 t y� } z | j� d| � W Y d }~n d }~0 0 d S )N)r� zException in UIThread: %s ) r� �getr� �Queue�Emptyr� r� rc r� � Exceptionrb )r5 r_ �er% r% r& �run| s z UIThread.run)r )r r r r r6 r� r� r% r% r{ r& r� _ s r� )r ).r � __future__r r r r � collectionsr rv � threadingr4 � six.movesr r� � gslib.metricsr r �gslib.thread_messager r r r r r r r r � gslib.utilsr �gslib.utils.unit_utilr r r r �ZERO_TASKS_TO_DO_ARGUMENTr� ru r r1 r2 rx r� r� r� �Threadr� r% r% r% r& �<module> sT � $ 2" __pycache__/impersonation_credentials.cpython-39.pyc 0000644 00000003446 15025013616 0016657 0 ustar 00 a +(Wg � @ sj d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z G d d � d ej �Z dS ) zJImplementation of credentials that refreshes using the iamcredentials API.� )�absolute_import)�print_function)�division)�unicode_literalsN)�client)�IamcredentailsApic s< e Zd ZdZ� fdd�Zedd� �Zdd� Zdd � Z� Z S ) �ImpersonationCredentialsz%Y-%m-%dT%H:%M:%SZc s^ || _ t||�| _| j�||�}|j| _| �|j�| _t t | �j | jd d d | jd d |d� d S )N)�scopes) �_service_account_idr �api�GenerateAccessToken� accessToken� access_token�_ConvertExpiryTime� expireTime� token_expiry�superr �__init__)�self�service_account_idr � credentials�logger�response�� __class__� �./opt/gsutil/gslib/impersonation_credentials.pyr s �z!ImpersonationCredentials.__init__c C s | j S �N)r )r r r r r 1 s z+ImpersonationCredentials.service_account_idc C s0 | j �| jt| j��}|j| _| �|j�| _ d S r ) r r r �listr r r r r r )r �httpr r r r �_refresh5 s �z!ImpersonationCredentials._refreshc C s t j �|tj�S r )�datetime�strptimer �_EXPIRY_FORMAT)r �valuer r r r = s �z+ImpersonationCredentials._ConvertExpiryTime) �__name__� __module__� __qualname__r# r �propertyr r r � __classcell__r r r r r s r ) �__doc__� __future__r r r r r! � oauth2clientr Zgslib.iamcredentials_apir �OAuth2Credentialsr r r r r �<module> s __pycache__/__main__.cpython-39.pyc 0000644 00000051441 15025013616 0013131 0 ustar 00 a +(WgY� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlmZ ddlmZ dd lmZ ddlZdd lmZ dd lmZ dd l m Z dd l m Z ddl m!Z! ddl"Z"ddlZddl#m$Z$m%Z% ddlm&Z& ze � \Z'Z(W n� e�y� Z) z~e)j*�rVe)j+nde)j+ Z+de+ Z,ze%j-e,dej.d� W n e/�y� ej.�0e,� Y n0 e)�r�e&�1e)� e�2d� W Y dZ)[)n dZ)[)0 0 e!e(e&j3�4� �e_5e" j6ej57 _6ddl7Z7ddl8Z8ddl9m:Z: ddl9m;Z< ddlm=Z= ddlm>Z> ddl?m@Z@ ddl?mAZA dd l?mBZB dd l?mCZC dd l?mDZD dd lEmFZF ddlGmH mI mZJ dd l#mKZK dd!l#mLZL dd"l#m$Z$ dd#lMmNZN dd$lMmOZO dd%lMmPZP g d&�ZQz ddlRZRW n eS�y Y n0 d'�T� ZUd(�T� ZVd)�T� ZWdaXd*aYd+d,� ZZd-d.� Z[dLd/d0�Z\d1d2� Z]G d3d4� d4e j^�Z_e j`fd5d6�Zad7d8� Zbd9d:� Zcd;d<� Zdd=d>� Zed?d@� ZfdAdB� ZgdCdD� ZhdEdF� ZidMdGdH�ZjdIdJ� ZkeldKk�r�e�2eb� � dS )Nz7Main module for Google Cloud Storage command line tool.� )�absolute_import)�print_function)�division)�unicode_literalsN)� configparser)�range)� exceptions)�CommandException)�ControlCException)� check_python_version_support)�GetArgumentsAndOptions)� GetUserAgent)� system_util� text_util)�metrics�CommandException: %s�%s � ��end�file� )� reauth_creds)�errors)�context_config)�wildcard_iterator)�AccessDeniedException)�ArgumentException)�BadRequestException)�ProjectIdException)�ServiceException)� CommandRunner)� boto_util)� constants)r )�GetCaughtSignals)�InitializeSignalHandling)�RegisterSignalHandler)�proxy� proxy_port� proxy_user� proxy_passa� ***************************** WARNING ***************************** *** You are running gsutil with debug output enabled. *** Be aware that debug output includes authentication credentials. *** Make sure to remove the value of the Authorization header for *** each HTTP request printed to the console prior to posting to *** a public medium such as a forum post or Stack Overflow. ***************************** WARNING ***************************** aK ***************************** WARNING ***************************** *** You are running gsutil with trace output enabled. *** Be aware that trace output includes authentication credentials *** and may include the contents of any files accessed during the trace. ***************************** WARNING ***************************** a� ***************************** WARNING ***************************** *** You are running gsutil with the "https_validate_certificates" config *** variable set to False. This option should always be set to True in *** production environments to protect against man-in-the-middle attacks, *** and leaking of user data. ***************************** WARNING ***************************** Fc C s" t � tjj�� jr tj�� dS )aT Cleans up if process is killed with SIGINT, SIGQUIT or SIGTERM. Note that this method is called after main() has been called, so it has access to all the modules imported at the start of main(). Args: signal_num: Unused, but required in the method signature. cur_stack_frame: Unused, but required in the method signature. N)�_Cleanup�gslib�utils�parallelism_framework_util�$CheckMultiprocessingAvailableAndInit� is_available�commandZ TeardownMultiprocessingProcesses)� signal_num�cur_stack_frame� r4 � /opt/gsutil/gslib/__main__.py�_CleanupSignalHandler� s r6 c C s. t �� D ] } zt�| � W q Y q0 qd S )N)r" ZGetCleanupFiles�os�unlink)Zfnamer4 r4 r5 r+ � s r+ c C s� t tjkstr.t�� }dt�dd|�| f }nd| }ztj |dt j d� W n t yj t j � |� Y n0 |rzt�|� t �d� dS ) z�Outputs message to stderr and exits gsutil with code 1. This function should only be called in single-process, single-threaded mode. Args: message: Message to print to stderr. exception: The exception that caused gsutil to fail. z(DEBUG: Exception stack trace: %s %s �\n� r r r r N)� debug_levelr# �DEBUGLEVEL_DUMP_REQUESTS�test_exception_traces� tracebackZ format_exc�re�subr � print_to_fd�sys�stderr�UnicodeDecodeError�writer � LogFatalError�exit)�message� exception� stack_trace�errr4 r4 r5 �_OutputAndExit� s � rL c C s | � d� t�d� d S )N�helpr )�RunNamedCommandrB rG )�command_runnerr4 r4 r5 �_OutputUsageAndExit� s rP c s" e Zd ZdZd� fdd� Z� ZS )�GsutilFormatterz<A logging.Formatter that supports logging microseconds (%f).Nc s, |rt j �|j��|�S tt| �j||d�S )N)�datefmt)�datetimeZ fromtimestampZcreated�strftime�superrQ � formatTime)�self�recordrR �� __class__r4 r5 rV � s zGsutilFormatter.formatTime)N)�__name__� __module__� __qualname__�__doc__rV � __classcell__r4 r4 rY r5 rQ � s rQ c C sF d}d}t ||d�}t�� }|�|� t�� }|�|� |�| � dS )zASimilar to logging.basicConfig() except it always adds a handler.z3%(levelname)s %(asctime)s %(filename)s] %(message)sz%m%d %H:%M:%S.%f)ZfmtrR N)rQ �loggingZ StreamHandlerZ setFormatter� getLoggerZ addHandler�setLevel)�levelZ log_formatZ date_formatZ formatterZhandlerZ root_loggerr4 r4 r5 �_ConfigureRootLogger� s rd c # C sx t � dd l} dd l} dd l} ddlm} ddlm} | jj � � j rZ| j � � | j� � n | j �� zLdd l}t�� \}}|j�||� |j�| jj �� � |�| jj �� � W n ty� Y n0 t� \}}|s�t|��t�d� t � � t� � t � }t j �std�!t"�#d����i } d} d} d} da$d } d }da%d }t&tj'�dk�rVtj'�(d� t)j*�+d d ��s�t)j*�,d ��s�t)j*�-d � t)j*�.d d d � t/� D ]}t0|t1� �q��z�t2D �]�\}}|d v �r�t3j4a$�q�|d v �r�t$t3j4k�r�t3j5a$nt3j4a$�q�|dv �r t6|� �q�|dv �rB|�7d�\}}}|�s4t6|� || |�8� < n�|dv �rRd } n�|dv �rbd } n�|dk�rr|}n�|dv �r�d } n�|dv �r�|t3_9n�|dk�r�|}n�|dk�r�|} n�|dk�r�d a%dt3_:nv|dv �r�|�7d�\}}}|�s�t6|� |�7d�\}}}|�r|�st6|� t)j*�,|��s0t)j*�-|� t)j*�;|||� �q�t �<� t _=t>j?t2d � t$t@_A| �rrtjB�CtD� t$t3j4k�rZtjB�CtE� tFtGjHd � |�Id d g� g }d D ]4}z|�Jt)j*�K|�� W n tLjM�y� Y n0 �q�tNt&|��D ](} || d } | tOv �r�| d!f|| < �q�tjB�Cd"d#�!tj'� � tjB�Cd$t �P� � tjB�Cd%tQ|� � n`| �rftGjRntGjS} tGjR} tGjT} tF| d � tUjVjW�X| � tUjYjZjW�X| � tUj[j\�X| � t]j\�X| � dd l^}!|!j_j`|!_`t ja�s�tjB�Ctb� | �r�d&}"n,tc�s�d}"n tcd }"|"d'k�rtd�etG�f� � tg� thji�jd(d)�d*v �rBtk|�W tl� S tm||"tcdd � | t$| | ||d+� W tl� S tl� 0 d S ),Nr )� oauth2_client)�credentials_libr � zcYour boto configuration has is_secure = False. Gsutil cannot be run this way, for security reasons.FrM �BotoZhttps_validate_certificatesT)z-dz--debug)z-Dz--detailedDebug)z-?z--help)z-hz--header�:)z-mz--multithreaded)z-qz--quietz-u)z-vz --version)z-iz --impersonate-service-accountz--perf-trace-tokenz --trace-tokenz--testexceptiontracesi )z-oz--option�=)Z global_opts)rc �verz-l)rh �GSUtilZREDACTEDzCommand being run: %s � zconfig_file_list: %s z config: %s �version�testZ _ARGCOMPLETE�0)�1�2)�args�headersr; � trace_token�parallel_operations�perf_trace_token� user_project)nr% Zgslib.boto_translationZ gslib.commandZ&gslib.utils.parallelism_framework_util�gcs_oauth2_boto_pluginre Zapitools.base.pyrf r- r. r/ r0 r1 Z"InitializeMultiprocessingVariablesZboto_translationZ InitializeThreadingVariablesr ZGetGsutilClientIdAndSecretZ oauth2_helperZ SetFallbackClientIdAndSecretZSetLockZ CreateLockZSetCredentialsCacheFileLock� ImportErrorr r rB rG r" ZMonkeyPatchBotoZMonkeyPatchHttpr! ZBOTO_IS_SECURE�join�textwrap�wrapr; r= �len�argv�append�boto�configZ has_optionZ has_sectionZ add_sectionZsetboolr$ r& r6 �optsr# r< Z%DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADSrP � partition�lowerZIMPERSONATE_SERVICE_ACCOUNTZLONG_RETRY_WARN_SEC�setZConfigureCertsFileZconfigured_certs_filer ZLogCommandParams�httplib2Z debuglevelrC rE � TRACE_WARNING� DEBUG_WARNINGrd r` �DEBUGrN �extend�itemsr ZNoSectionErrorr �CONFIG_KEYS_TO_REDACT�GetFriendlyConfigFilePaths�strZWARNING�INFOZERROR� oauth2client�client�loggerrb ZcontribZmultiprocess_file_storageZ transportZ_LOGGERr �six�moves�inputZ CERTIFICATE_VALIDATION_ENABLED� HTTP_WARNINGrs r Zcreate_context_configra � _CheckAndWarnForProxyDifferencesr7 �environ�get�_PerformTabCompletionr+ �#_RunNamedCommandAndHandleExceptions)#r, re rf ry Zgsutil_client_idZgsutil_client_secret� supportedrK rO rt rv �quietrn ru rw rx r2 �o�aZhdr_name�_Zhdr_valZopt_section_nameZ opt_valueZ opt_sectionZopt_nameZ config_itemsZconfig_section�iZconfig_item_keyZroot_logger_levelZoa2c_logger_levelZ+oa2c_multiprocess_file_storage_logger_levelr� � command_namer4 r4 r5 �main� sB �� � � �� � � � � � r� c C sZ t j�ddd�} t j�ddd �s&| �rVdD �](}|tjv r*tj| r*g }t�|�}|jt j�ddd �kr�|� dt j�ddd �||jf � |j t j�ddd �kr�|� dt j�ddd �||j f � |j t j�dd d �kr�|� d | � |j s�| �r|j | k�r|� d | ||j f � |�rLt j�d �t�d | ��� t j�dd �|� � tj|= q*d S )Nrh r( r r' )Z http_proxyZ https_proxyZ HTTPS_PROXYz6Boto proxy host: "%s" differs from %s proxy host: "%s"r) z6Boto proxy user: "%s" differs from %s proxy user: "%s"r* z2Boto proxy password differs from %s proxy passwordz6Boto proxy port: "%s" differs from %s proxy port: "%s"rg z�WARNING: Proxy configuration is present in both the %s environment variable and boto configuration, but configuration differs. boto configuration proxy values will be used. Differences detected:z %s )r� r� Zgetintr� r7 r� r" ZProxyInfoFromEnvironmentVarZ proxy_hostr� r) r* r( rB rC rE r{ r| r} )Z boto_portZ proxy_env_varZdiffering_valuesZ proxy_infor4 r4 r5 r� � sl ����� ������� � �� ���r� c C s � t d| | d� d S )Nz Failure: %s.�rH rI )rL ��er4 r4 r5 �_HandleUnknownFailure� s r� c C s, | j rt| j| d� ntd| j | d� d S )Nr� r )� informationalrL �reasonr� r4 r4 r5 �_HandleCommandException� s r� c C sT t dkr>d�t�t�� ��}td| t�dd|�f t� d� ntd| t� d� dS ) z�Called when user hits ^C. This function prints a brief message instead of the normal Python stack trace (unless -D option is used). Args: signal_num: Signal that was caught. cur_stack_frame: Unused. � r z@DEBUG: Caught CTRL-C (signal %d) - Exception stack trace: %sr9 r: �rI z#Caught CTRL-C (signal %d) - exitingN) r; r{ r> � format_listZ extract_stackrL r? r@ r )r2 r3 rJ r4 r4 r5 �_HandleControlC� s ���r� c C s ddl }|�� dS )zFCalled when user hits ^\, so we can force breakpoint a running gsutil.r N)�pdbZ set_trace)r2 r3 r� r4 r4 r5 �_HandleSigQuit s r� c C s. t j�dd�}d}|r"|d| 7 }|d7 }|S )z�Constructs a help string for an access control error. Args: reason: e.reason string from caught exception. Returns: Contructed help text. rl �default_project_idaX Your request resulted in an AccountProblem (403) error. Usually this happens if you attempt to create a bucket without first having enabled billing for the project you are using. Please ensure billing is enabled for your project by following the instructions at `Google Cloud Platform Console<https://support.google.com/cloud/answer/6158867>`. z�In the project overview, ensure that the Project Number listed for your project matches the project ID (%s) from your boto config file. a� If the above doesn't resolve your AccountProblem, please send mail to buganizer-system+187143@google.com requesting assistance, noting the exact command you ran, the fact that you received a 403 AccountProblem error, and your project ID. Please do not post your project ID on StackOverflow. Note: It's possible to use Google Cloud Storage without enabling billing if you're only listing or reading objects for which you're authorized, or if you're uploading objects to a bucket billed to a project that has billing enabled. But if you're attempting to create buckets or upload objects to a bucket owned by your own project, you must first enable billing for that project.)r� r� � get_value)r� r� Z acct_helpr4 r4 r5 � _ConstructAccountProblemHelp s ���� r� c C s� t �� sPtj�ddd�sPt�� r2d�t� d��}nd�t� d��}t || d� nV| j r�| j dkst| j d kstd | j v r�d �|�� d �d kr�t d�t� t | j ���| d� d S )NZTestsZ bypass_anonymous_access_warningFrg z�You are attempting to access protected data with no configured credentials. Please visit https://cloud.google.com/console#/project and sign up for an account, and then run the "gcloud auth login" command to configure gsutil to use these credentials.z�You are attempting to access protected data with no configured credentials. Please visit https://cloud.google.com/console#/project and sign up for an account, and then run the "gsutil config" command to configure gsutil to use these credentials.r� ZAccountProblemzAccount disabled.z3account for the specified project has been disabled�,zgs://���r� )r" ZHasConfiguredCredentialsr� r� r� r �InvokedViaCloudSdkr{ r| r} rL r� �findr� )r� rs rH r4 r4 r5 �"_CheckAndHandleCredentialException> s: �����������r� c C s* z@t tjtdd� tjs$t tjt� | j|||||||d|d� W S t y� } z>t | �� d�dkrpt d| d� nt t | �| d� W Y d } ~ �n�d } ~ 0 t y� } zt| � W Y d } ~ �nld } ~ 0 tjy� } z tt | j�� W Y d } ~ �n8d } ~ 0 tjj�y2 } z t d | j | d� W Y d } ~ �n�d } ~ 0 tjj�yn } z t d | j | d� W Y d } ~ �n�d } ~ 0 tjj�y� } zt d | d� W Y d } ~ �n�d } ~ 0 tjj�y� } z t t | �| d� W Y d } ~ �nPd } ~ 0 t�yP } zZ| j t j k�s tj�r(| j t j k�r(t� � �s(t �!d � nt d| j" | d� W Y d } ~ �n�d } ~ 0 t#�y� } zJ| j t j k�s�tj�r�| j t j k�r�t� � �s�t �!d � n� W Y d } ~ �n|d } ~ 0 t$j%�y� } z t | j&| d� W Y d } ~ �nFd } ~ 0 t'�y } zt d| d� W Y d } ~ �nd } ~ 0 t(�yb } z0| j&dk�r@t)| |� t | | d� W Y d } ~ �n�d } ~ 0 t*�y� } z$t)| |� t | | d� W Y d } ~ �n�d } ~ 0 t+�y� } zt | | d� W Y d } ~ �n^d } ~ 0 t,�y } zt | | d� W Y d } ~ �n,d } ~ 0 t-j.j/t0j1t0j2f�yZ } z2t�3� �r8t d| d� n t d| d� W Y d } ~ �n�d } ~ 0 t4j5�y� } z4t dt6| j7dd�| j8�p�df | d� W Y d } ~ �n�d } ~ 0 t9j:�y4 } zr| j;d t j k�r�t d| d� nF| j;d t j<k�rd�=|��>d��rt d�=t?�@d��| d� ntA| � W Y d } ~ n�d } ~ 0 t-j.jB�y| } z,t dd�=t?�@d | �� | d� W Y d } ~ n�d } ~ 0 tCjD�y� t�3� �r�t d � nt d � Y nz tE�y$ } z`d �=tF�G� �} d t | �v �r�t d!| | d� n d"t | �v �rt d#| | d� tA| � W Y d } ~ n d } ~ 0 0 d S )$z/Runs the command and handles common exceptions.T)Zis_final_handler)rw Zcollect_analyticsrx Zsecret_access_keyr� zdMissing credentials for the given URI(s). Does your boto config file contain all needed credentials?r� r� NzInvalidUriError: %s.zInvalidUrlError: %s.�NotReadyToAuthenticater z OSError: %s.z�You are attempting to perform an operation that requires a project id, with none configured. Please re-run gsutil config and make sure to follow the instructions for finding and entering your default project id.ZMissingSecurityHeaderz<Your credentials are invalid. Please run $ gcloud auth loginz�Your credentials are invalid. For more help, see "gsutil help creds", or re-run the gsutil config command (see "gsutil help config").zHttpError: %s, %s�statusr a< Got a "Broken pipe" error. This can happen to clients using Python 2.x, when the server sends an error response and then closes the socket (see http://bugs.python.org/issue5542). If you are trying to upload a large object you might retry with a small (say 200k) object, and see if you get a more specific error code.rm zs3://rg ac Got a "Connection reset by peer" error. One way this can happen is when copying data to/from an S3 regional bucket. If you are using a regional S3 bucket you could try re-running this command using the regional S3 endpoint, for example s3://s3-<region>.amazonaws.com/your-bucket. For details about this problem see https://github.com/boto/boto/issues/2207z %s z�Failed to retrieve valid credentials (%s). Make sure you selected and pasted the ENTIRE authorization code (including any numeric prefix e.g. '4/').zKYou must re-authenticate with your SAML IdP. Please run $ gcloud auth loginzGYou must re-authenticate with your SAML IdP. Please run $ gsutil configz, zmac verify failurez�Encountered an error while refreshing access token. If you are using a service account, please verify that the gs_service_key_file_password field in your config file(s), %s, is correct.zasn1 encoding routinesz�Encountered an error while refreshing access token. If you are using a service account, please verify that the gs_service_key_file field in your config file(s), %s, is correct.)Hr& �signal�SIGINTr� r Z IS_WINDOWS�SIGQUITr� rN �AttributeErrorr� r� rL r r� �getoptZ GetoptError�msgr� rI ZInvalidUriErrorrH r, �InvalidUrlErrorZ auth_handlerr� �ExternalBinaryError�OSError�errnoZEPIPE�EINVALZIsRunningInteractivelyrB rG �strerror�IOErrorr ZWildcardExceptionr� r r r� r r r r� r� ZHttpAccessTokenRefreshError�google_auth_exceptionsZ OAuthErrorZ RefreshErrorr� �apitools_exceptionsZ HttpError�getattr�response�content�socket�errorrs Z ECONNRESETr{ �containsr| r} r� ZFlowExchangeError� reauth_errorsZ ReauthSamlLoginRequiredError� Exceptionr" r� ) rO r� rs rt r; ru rv rw rx r� Z config_pathsr4 r4 r5 r� c s � �& $(("&� �� (� �� $� " """� �� �� �$��� ���� ����r� c C st zddl }ddl}W n4 tyH } z td| |d� W Y d}~n d}~0 0 |jdd�}| �|� |j|tjd� dS )z6Performs gsutil-specific tab completion for the shell.r NzIA library required for performing tab completion was not found. Cause: %sr� F)Zadd_help)� exit_method) � argcomplete�argparserz rL ZArgumentParserZ ConfigureCommandArgumentParsersZ autocompleterB rG )rO r� r� r� Zparserr4 r4 r5 r� s �� r� �__main__)N)NNr NFNN)mr^ � __future__r r r r rS r� r� r` r7 r? r� r� rB r| r> r� Z six.movesr r Z google.authr r� �gslib.exceptionr, r r �gslib.utils.version_checkr Zgslib.utils.arg_helperr Z gslib.utils.user_agent_helperr r� Z gslib.utilsr r r r� rs r� r� r� rK rA rC rD rE rF rG ZMetricsCollectorZ IsDisabled� USER_AGENTZ UserAgentr� r� Z google_reauthr r r� r r Zgslib.cloud_apir r r r r Zgslib.command_runnerr! Zapitools.base.py.exceptions�base�pyr� r" r# Zgslib.sig_handlingr$ r% r& r� ry rz �lstripr� r� r� r; r= r6 r+ rL rP Z FormatterrQ r� rd r� r� r� r� r� r� r� r� r� r� r[ r4 r4 r4 r5 �<module> s� c-&' � 3 __pycache__/tz_utc.cpython-39.pyc 0000644 00000004066 15025013616 0012722 0 ustar 00 a +(Wg� � @ s` d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ G dd � d e�Z d S ) z9Provides a tzinfo subclass to represent the UTC timezone.� )�absolute_import)�print_function)�division)�unicode_literals�� timedelta)�tzinfoc @ s( e Zd ZdZdd� Zdd� Zdd� ZdS ) �UTCa� Timezone information class used to convert datetime timestamps to UTC. This class is necessary to convert timestamps to UTC. By default Python datetime objects are timezone unaware. This created problems when interacting with cloud object timestamps which are timezone-aware. This issue appeared when handling the timeCreated metadata attribute; the values returned by the service were placed in RFC 3339 format in the storage_v1_messages module. RFC 3339 requires a timezone in any timestamp. This caused problems as the datetime object elsewhere in the code was timezone unaware and was different by exactly one hour. The main problem was that the local system used daylight savings time which adjusted the timestamp ahead by one hour. c C s t d�S )z�An offset of the number of minutes away from UTC this tzinfo object is. Returns: A time duration of zero. UTC is zero minutes away from UTC. r r ��self�_� r �/opt/gsutil/gslib/tz_utc.py� utcoffset( s z UTC.utcoffsetc C s dS )zvA method to retrieve the name of this timezone object. Returns: The name of the timezone (i.e. 'UTC'). r r r r r r �tzname0 s z UTC.tznamec C s t d�S )zA fixed offset to handle daylight savings time (DST). Returns: A time duration of zero as UTC does not use DST. r r r r r r �dst8 s zUTC.dstN)�__name__� __module__� __qualname__�__doc__r r r r r r r r s r N) r � __future__r r r r �datetimer r r r r r r �<module> s __pycache__/command.cpython-39.pyc 0000644 00000177162 15025013616 0013040 0 ustar 00 a +(WgN� � @ sj d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlmZ ddlZdd lmZ dd lmZ dd lmZ dd l m Z dd l m Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl%m'Z' ddl(m)Z) ddl(m*Z* ddl(m+Z+ ddl(m,Z, ddl(m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl2m4Z4 dd l2m5Z5 dd l2m6Z6 dd l2m7Z7 dd l8m9Z9 dd l8m:Z: dd!l8m;Z; dd"l8m<Z< dd#l8m=Z= dd$l>m?Z@ dd%lAmBZB dd&lAmCZC dd'lAmDZD dd(lAmEZE dd)lFmGZG dd*lFmHZH dd+lFmIZI dd,lJmKZK dd-lJmLZL dd.lMmNZN dd/lMmOZO ddlPZdd0lPmQZQ dd1lPmRZR dd2lPmSZS dd3lPmTZT dd4lPmUZU dd5lPmVZV dd6lPmWZW dd7lPmXZX dd8lPmYZY dd9lZm[Z[ dd:l\m]Z] dd;l^m_Z_ dd<l^m`Z` dd=lambZb dd>lamcZc dd?lamdZd dd@lemfZf ddAlgmhZi zddBljmkZl W n em�y� dZlY n0 dCZndDZodEdF� ZpdGdH� ZqdIdJ� ZrdKdL� Zsd�dMdN�ZtdOdP� Zug ZvdQdR� ZwdSdT� ZxdUdV� ZydWZzdXZ{dYdZ� Z|dCZ}d[d\gd[d]gd[d^gd_d]gd_d^gd`d\gd`d]gd`d^gg da�g db�dcd]gddd]gddd^gded]gded^gdf�Z~dgdh� Zdidj� Z�dkdl� Z�edmg dn��Z�G dodp� dpe$e]�Z�G dqdr� dre��Z�G dsdt� dtedtdu��Z�dvdw� Z�G dxdy� dyej��Z�G dzd{� d{e��Z�G d|d}� d}ej��Z�G d~d� de��Z�d�d�� Z�G d�d�� d�e��Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� Z�dS )�ax Base class for gsutil commands. In addition to base class code, this file contains helpers that depend on base class state (such as GetAndPrintAcl) In general, functions that depend on class state and that are used by multiple commands belong in this file. Functions that don't depend on class state belong in util.py, and non-shared helpers belong in individual subclasses. � )�absolute_import)�print_function)�division)�unicode_literalsN)� namedtuple)� StorageUri)�AccessDeniedException)�ArgumentException)�ServiceException)�CloudApiDelegator)� ApiSelector)�GsutilApiMapFactory)�CommandException)� HelpProvider)�CaptureThreadStatException)�LogPerformanceSummaryParams)�CopyObjectInfo)�CopyObjectsIterator)�NameExpansionIterator)�NameExpansionResult)� SeekAheadNameExpansionIterator)�PluralityCheckableIterator)�SeekAheadThread)�ChildProcessSignalHandler)�GetCaughtSignals)� KillProcess)� MultithreadedMainSignalHandler)�RegisterSignalHandler)� HaveFileUrls)�HaveProviderUrls)�StorageUrlFromString)�UrlsAreForSingleProvider)� UrlsAreMixOfBucketsAndObjects)�storage_v1_messages)� FinalMessage)�MetadataMessage)�PerformanceSummaryMessage)�ProducerThreadMessage)�MainThreadUIQueue)� UIController)�UIThread)�GetFriendlyConfigFilePaths)�!GetMaxConcurrentCompressedUploads)�NO_MAX)�UTF8)� AtomicDict)�$CheckMultiprocessingAvailableAndInit)�multiprocessing_context)�ProcessAndThreadSafeInt)�PutToQueueWithTimeout)�SEEK_AHEAD_JOIN_TIMEOUT)� ShouldProhibitMultiprocessing)�UI_THREAD_JOIN_TIMEOUT)�ZERO_TASKS_TO_DO_ARGUMENT)�RsyncDiffToApply)�GcloudStorageCommandMixin)� GetTermLines)� IS_WINDOWS)�AclTranslation)�GetNonMetadataHeaders)�PRIVATE_DEFAULT_OBJ_ACL)�CreateWildcardIterator)�queue)�Random� i� c C sJ t �| �}|jsFd|_|�t jj� t �� }|�t � d�� |� |� |S )a� Fetches a logger with the given name that resembles 'print' output. Initial Logger Configuration: The logger abides by gsutil -d/-D/-DD/-q options. If none of those options were specified at invocation, the returned logger will display all messages logged with level INFO or above. Log propagation is disabled. If a logger with the specified name has already been created and configured, it is not reconfigured, e.g.: foo = CreateOrGetGsutilLogger('foo') # Creates and configures Logger "foo". foo.setLevel(logging.DEBUG) # Change level from INFO to DEBUG foo = CreateOrGetGsutilLogger('foo') # Does not reset level to INFO. Args: command_name: (str) Command name to create logger for. Returns: A logging.Logger object. Fz %(message)s) �logging� getLogger�handlers� propagate�setLevel�root�level� StreamHandler� setFormatter� Formatter� addHandler)� command_name�logZ log_handler� rP � /opt/gsutil/gslib/command.py�CreateOrGetGsutilLoggert s rR c C s | j �|� d S �N)�logger� exception��cls�erP rP rQ �_DefaultExceptionHandler� s rY c C s< | j s dS |j}|�� r8tj�|j�r8| j�d|� dS dS )NTz Skipping symbolic link %s...F) �exclude_symlinks�expanded_storage_url� IsFileUrl�os�path�islink� object_namerT �info)Zcommand_instance�urlZ exp_src_urlrP rP rQ �_UrlArgChecker� s rc c G s dS )NTrP )� unused_argsrP rP rQ �DummyArgChecker� s re c C s | j ||d�S )N�� thread_state)� SetAclFunc)rW �name_expansion_resultrg rP rP rQ �SetAclFuncWrapper� s rj c C s | j �t|�� d| _dS )zDException handler that maintains state about post-completion status.FN)rT �error�str�everything_set_okayrV rP rP rQ �SetAclExceptionHandler� s rn c C s t rtt dd �rt �� d S )N�atfork)� CryptoRandom�getattrro rP rP rP rQ �_CryptoRandomAtFork� s rr c C s t �t�} t�| � | S rS )r1 �Queue�MAX_QUEUE_SIZE�queues�append�Z new_queuerP rP rQ �_NewMultiprocessingQueue� s rx c C s t � t�} t�| � | S rS )rs rt ru rv rw rP rP rQ �_NewThreadsafeQueue� s ry i�~ i0u c C s t j�ddt�S )N�GSUtil�task_estimation_threshold)�boto�config�getint�!DEFAULT_TASK_ESTIMATION_THRESHOLDrP rP rP rQ �_GetTaskEstimationThreshold� s �r� �acl�ch�get�set�cors�defacl)rC r� �off)rC r� �onrC � versioning�web)ZchaclZgetaclZsetaclZgetcorsZsetcorsZchdefaclZ getdefaclZ setdefaclZdisableloggingZ enableloggingZ getloggingZ getversioningZ setversioningZ getwebcfgZ setwebcfgc C s� t �� ag ag at�� atd�at td�a t td�a t td�a t� � at�� at td�atd�atd�at td�at td�at td�at�� atd�at�t�at� t � �a dS )a� Initializes module-level variables that will be inherited by subprocesses. On Windows, a multiprocessing.Manager object should only be created within an "if __name__ == '__main__':" block. This function must be called, otherwise every command that calls Command.Apply will fail. While multiprocessing variables are initialized at the beginning of gsutil execution, new processes and threads are created only by calls to Command.Apply. When multiple processes and threads are used, the flow of startup/teardown looks like this: 1. __main__: initializes multiprocessing variables, including any necessary Manager processes (here and in gslib.utils.parallelism_framework_util). 2. __main__: Registers signal handlers for terminating signals responsible for cleaning up multiprocessing variables and manager processes upon exit. 3. Command.Apply registers signal handlers for the main process to kill itself after the cleanup handlers registered by __main__ have executed. 4. If worker processes have not been created for the current level of recursive calls, Command.Apply creates those processes. ---- Parallel operations start here, so steps are no longer numbered. ---- - Command.Apply in the main thread starts the ProducerThread. - The Producer thread adds task arguments to the global task queue. - It optionally starts the SeekAheadThread which estimates total work for the Apply call. - Command.Apply in the main thread starts the UIThread, which will consume messages from the global status queue, process them, and display them to the user. - Each worker process creates a thread pool to perform work. - The worker process registers signal handlers to kill itself in response to a terminating signal. - The main thread of the worker process moves items from the global task queue to the process-local task queue. - Worker threads retrieve items from the process-local task queue, perform the work, and post messages to the global status queue. - Worker threads may themselves call Command.Apply. - This creates a new pool of worker subprocesses with the same size as the main pool. This pool is shared amongst all Command.Apply calls at the given recursion depth. - This reuses the global UIThread, global status queue, and global task queue. - This starts a new ProducerThread. - A SeekAheadThread is not started at this level; only one such thread exists at the top level, and it provides estimates for top-level work only. - The ProducerThread runs out of tasks, or the user signals cancellation. - The ProducerThread cancels the SeekAheadThread (if it is running) via an event. - The ProducerThread enqueues special terminating messages on the global task queue and global status queue, signaling the UI Thread to shut down and the main thread to continue operation. - In the termination case, existing processes exit in response to terminating signals from the main process. ---- Parallel operations end here. ---- 5. Further top-level calls to Command.Apply can be made, which will repeat all of the steps made in #4, except that worker processes will be reused. T)�managerN) r1 �Managerr� �consumer_pools� task_queues�Lock�caller_id_lockr2 �caller_id_counterr/ � total_tasks�call_completed_map�global_return_values_map� Condition�need_pool_or_done_cond�worker_checking_level_lock�caller_id_finished_count�new_pool_needed�current_max_recursive_level�shared_vars_map�shared_vars_list_map� thread_stats�dict� class_map� failure_countrs rt �glob_status_queue�BoundedSemaphorer, �!concurrent_compressed_upload_lockrP rP rP rQ �"InitializeMultiprocessingVariables s, I �r� c C s t � t�� tjjj�� dS )z7Should be called by signal handlers prior to shut down.N)�ShutDownGsutilr� �shutdown�gslib�utils�parallelism_framework_util�top_level_managerrP rP rP rQ � TeardownMultiprocessingProcesses� s r� c C s~ t d�at� at�� at� at� at d�a t d�a t � t �a t� at�� at� at� at� ag at� at�� at�t� �adS )aD Initializes module-level variables used when running multi-threaded. When multiprocessing is not available (or on Windows where only 1 process is used), thread-safe analogs to the multiprocessing global variables must be initialized. This function is the thread-safe analog to InitializeMultiprocessingVariables. FN)r2 r� r/ r� � threadingr� r� r� r� r� r� rs rt r� r� r� r� r� r� r� r� r� r� r� r, r� rP rP rP rQ � InitializeThreadingVariables� s& �r� � CommandSpec� rN �usage_synopsis�command_name_aliases�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�supported_private_args�argparse_argumentsc s� e Zd ZdZdZdZdZdZeddde ddddddddf dd ��Z d d � Z e e �Z d d � Zdd� ZdM� fdd� Zdd� Zdd� ZdNdd�Zdd� ZdOdd �Zd d � Zd d � ZdPd!d"�ZdQd#d$�Zd%d&� Zd'd(� Zd)d*� Z d+d,� Z d-d.� Z d/d0� Z d1d2� Z dRd3d4�Z!dSd5d6�Z"d7d8� Z#dTd9d:�Z$d;d<� Z%d=d>� Z&G d?d@� d@e'�Z(de)ddddddfdAdB�Z*dCdD� Z+dEdF� Z,dUdGdH�Z-dIdJ� Z.dKdL� Z/� Z0S )V�Commandz#Base class for all gsutil commands.N) r� r� �iamZkms�labelrC � notificationZ retentionr� r ���� Fc C s8 t | ||p g ||||||| p"tjg| p*tj| | p2g d� S )z2Creates an instance of CommandSpec, with defaults.r� )r� r �XMLr� rP rP rQ �CreateCommandSpec s �zCommand.CreateCommandSpecc C s | j jS rS )� command_specrN ��selfrP rP rQ �_GetDefaultCommandName s z Command._GetDefaultCommandNamec C s | j jS )z�Calculate the index in args of the first URL arg. Returns: Index of the first URL arg (according to the command spec). )r� r� r� rP rP rQ �_CalculateUrlsStartArg$ s z Command._CalculateUrlsStartArgc C sL t �| jd�}|rH|dd� | }| j�d�t�d| j| jd� ��� |S )zBMap deprecated aliases to the corresponding new command, and warn.N� � z�You are using a deprecated alias, "%(used_alias)s", for the "%(command_name)s" command. This will stop working on 9/9/2014. Please use "%(command_name)s" with the appropriate sub-command in the future. See "gsutil help %(command_name)s" for details.)Z used_aliasrN ) � OLD_ALIAS_MAPr� �command_alias_usedrT �warn�join�textwrap�wraprN )r� �argsZnew_command_argsrP rP rQ �_TranslateDeprecatedAliases, s ���� z#Command._TranslateDeprecatedAliasesc sB t � �� || _|| _|| _|| _|| _| | _|| _| | _ || _ || _ d| _ d| _ d| _| | _d| _t| j�| _| r�| D ]} | j�| � qx| jdur�t| j�| _nd| _| jdu r�td| j ��| j�tj� | _ t | j t j � ddd�d�a!| �"|�| _#| �$� | jj%| �&� d�| _t'| j#�| jj(k �s<t'| j#�| jj)k�rD| �*� | j| j+v�rZ| �,� | jj-t.j/gd�}| jj0t.j/d�}t1�2| j ||�| _3d| _4t5| j | j3| jt6t7j8t!�| j| j| j| j| j d � | _9d | _:t;�r�| j:d 7 _:t<j=| _>| j:| j>7 _:| j?�r4| j?D ]*\}}|d k�s$|d k�rd| _ �q4�qt@� jA| _BdS )ag Instantiates a Command. Args: command_runner: CommandRunner (for commands built atop other commands). args: Command-line args (arg0 = actual arg, not command name ala bash). headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). trace_token: Trace token to pass to the API implementation. parallel_operations: Should command operations be executed in parallel? bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. gsutil_api_class_map_factory: Creates map of cloud storage interfaces. Settable for testing/mocking. logging_filters: Optional list of logging. Filters to apply to this command's logger. command_alias_used: The alias that was actually used when running this command (as opposed to the "official" command name, which will always correspond to the file name). perf_trace_token: Performance measurement trace token to use when making API calls. user_project: Project to be billed for this request. Implementation note: subclasses shouldn't need to define an __init__ method, and instead depend on the shared initialization that happens here. If you do define an __init__ method in a subclass you'll need to explicitly call super().__init__(). But you're encouraged not to do this, because it will make changing the __init__ interface more painful. FNzA"%s" command implementation is missing a command_spec definition.rz �dump_status_messages_file)� quiet_moder� )r� )�gs�s3��debug� http_headers� trace_token�perf_trace_token� user_projectr� zpython z-rz-RT)C�super�__init__�command_runner� unparsed_args�headersr� r� r� �parallel_operationsr� �bucket_storage_uri_class� gsutil_api_class_map_factoryrZ �recursion_requested� all_versionsr� �seek_ahead_gsutil_apirR rN rT � addFilterr= �non_metadata_headersr� r � isEnabledForrC �INFOr� r) r| r} r� � ui_controllerr� r� � ParseSubOpts�_replacer� �lenr� r� �$RaiseWrongNumberOfArgumentsException�&_commands_with_subcommands_and_subopts�CheckArgumentsr� r r� r� r � GetApiMap�gsutil_api_map� project_idr r( �sys�stderr� gsutil_apiZ gsutil_cmdr; r� � GSUTIL_PATHZ gsutil_path�sub_optsr0 � is_available� multiprocessing_is_available)r� r� r� r� r� r� r� r� r� �logging_filtersr� r� r� Z log_filter� support_map� default_map�oZunused_a�� __class__rP rQ r� ? s� * ��� ������� �zCommand.__init__c C sp t | j�| jjk r<| jjdkr"dnd}d| j| jj|f }nd| j| jjf }|d| jj| jf 7 }t|��dS )zCRaises exception for wrong number of arguments supplied to command.r� �sr� z/The %s command requires at least %d argument%s.z,The %s command accepts at most %d arguments.z4 Usage: %s For additional help run: gsutil help %sN)r� r� r� r� rN r� r� r )r� Ztail_str�messagerP rP rQ r� � s � � �z,Command.RaiseWrongNumberOfArgumentsExceptionc C s d| j j| jf }t|��dS )z?Raises exception for specifying an invalid argument to command.zRIncorrect option(s) specified. Usage: %s For additional help run: gsutil help %sN)r� r� rN r )r� r rP rP rQ � RaiseInvalidArgumentException� s �z%Command.RaiseInvalidArgumentExceptionTc C s� |du r| j }n|}z"t�|| jj| jjp,g �\}}W n tjyR | �� Y n0 |rt|| | _| _ |r�| �� n|r�t d��||fS dS )ae Parses sub-opt args. Args: check_args: True to have CheckArguments() called after parsing. args: List of args. If None, self.args will be used. should_update_sub_opts_and_args: True if self.sub_opts and self.args should be updated with the values returned after parsing. Else return a tuple of sub_opts, args returned by getopt.getopt. This is done to allow this method to be called from get_gcloud_storage_args in which case we do not want to update self.sub_opts and self.args. Raises: RaiseInvalidArgumentException: Invalid args specified. NzIRequested to check arguments but sub_opts and args have not been updated.) r� �getoptr� r� r� � GetoptErrorr r� r� � TypeError)r� Z check_argsr� Z should_update_sub_opts_and_argsr� Zparsed_sub_optsZ parsed_argsrP rP rQ r� � s" � zCommand.ParseSubOptsc C s\ | j js,t| j| j jd� �r,td| j ��| j jsXt| j| j jd� �rXtd| j ��dS )a Checks that command line arguments match the command_spec. Any commands in self._commands_with_subcommands_and_subopts are responsible for calling this method after handling initial parsing of their arguments. This prevents commands with sub-commands as well as options from breaking the parsing of getopt. TODO: Provide a function to parse commands and sub-commands more intelligently once we stop allowing the deprecated command versions. Raises: CommandException if the arguments don't match. NzN"%s" command does not support "file://" URLs. Did you mean to use a gs:// URL?z1"%s" command does not support provider-only URLs.) r� r� r r� r� r rN r� r r� rP rP rQ r� s ����zCommand.CheckArgumentsc C s t || j|| j| jd�S )a� Helper to instantiate gslib.WildcardIterator. Args are same as gslib.WildcardIterator interface, but this method fills in most of the values from instance state. Args: url_string: URL string naming wildcard objects to iterate. all_versions: If true, the iterator yields all versions of objects matching the wildcard. If false, yields just the live object version. Returns: WildcardIterator for use by caller. )r� r� rT )r? r� r� rT )r� � url_stringr� rP rP rQ �WildcardIterator$ s �zCommand.WildcardIteratorc C s< | j s6t| j| jt�d�t| j| j| j | j | j d� | _ | j S )aW Helper to instantiate a Cloud API instance for a seek-ahead iterator. This must be separate from the core command.gsutil_api instance for thread-safety, since other iterators typically use that instance and the SeekAheadIterator operates in parallel. Returns: Cloud API instance for use by the seek-ahead iterator. �dummyr� ) r� r r� r� rC rD r� r� r� r� r� r� r� rP rP rQ �GetSeekAheadGsutilApi9 s � z Command.GetSeekAheadGsutilApic C s t d| j ��dS )a Abstract function in base class. Subclasses must implement this. The return value of this function will be used as the exit status of the process, so subclass commands should return an integer exit code (0 for success, a value in [1,255] for failure). z5Command %s is missing its RunCommand() implementationN)r rN r� rP rP rQ � RunCommandV s �zCommand.RunCommandc C s6 g }t tt|��}t|�r(| js(td��|D ]x}|�� r�|�� r�| jrZd|_|� |j � q�| � |j �j dgd�D ]$}t |ddd|jdd�} || | � qpq,|� |j � q,t|�dk�rt| j| j| j| j|| j| j| jp�| j|d � } t| j| j| �� || j| jd �} | j|| || j | d � | j�s2| j�s2td ��dS ) a� Sets the standard or default object ACL depending on self.command_name. Args: acl_func: ACL function to be passed to Apply. acl_excep_handler: ACL exception handler to be passed to Apply. url_strs: URL strings on which to set ACL. object_fields: If present, list of object metadata fields to retrieve; if None, default name expansion iterator fields will be used. Raises: CommandException if an ACL could not be set. z/Cannot operate on a mix of buckets and objects.�*�id�� bucket_fieldsFN)Zsource_storage_urlZis_multi_source_requestZ!is_multi_top_level_source_request�names_containerr[ �expanded_resultr� )r� �continue_on_error�bucket_listing_fields)r� )� fail_on_error�seek_ahead_iterator�'ACLs for some objects could not be set.) �list�mapr r" r� r � IsCloudUrl�IsBucketr` rv r r � IterBucketsr � storage_urlr� r rN r� rT r� r� r r� r r �Applyrm ) r� �acl_func�acl_excep_handler�url_strsZ object_fieldsZmulti_threaded_url_args�urlsrb �blrZname_expansion_for_urlZname_expansion_iteratorr rP rP rQ � ApplyAclFuncg sh �� � � � �zCommand.ApplyAclFuncc C s� |r| j rJ �|}n| j}| j r$dnd}|j}| j�d||� |�|j�tjkrh|jdkrh| � ||� n | � ||� t |j t t�� d�� dS )z�Sets the object ACL for the name_expansion_result provided. Args: name_expansion_result: NameExpansionResult describing the target object. thread_state: If present, use this gsutil Cloud API instance for the set. zdefault object ACL�ACLzSetting %s on %s...r� )� message_timeN)�def_aclr� r[ rT ra �GetApiSelector�schemer r� �_SetAclXmlPassthrough�_SetAclGsutilApir3 � status_queuer% �time)r� ri rg r� Z op_stringrb rP rP rQ rh � s � �zCommand.SetAclFuncc C s� |j }ztz(tj|_ |j| j|| j| j|jd� W n@ typ } z(| j rZd| _ | j � |� n� W Y d}~n d}~0 0 W ||_ n||_ 0 dS )a� Sets the ACL for the URL provided using the XML passthrough functions. This function assumes that self.def_acl, self.canned, and self.continue_on_error are initialized, and that self.acl_arg is either an XML string or a canned ACL string. Args: url: CloudURL to set the ACL on. gsutil_api: gsutil Cloud API to use for the ACL set. Must support XML passthrough functions. )�canned� def_obj_acl�providerFN) � prefer_apir r� �XmlPassThroughSetAcl�acl_argr/ r( r* r r rm rT rk )r� rb r� Zorig_prefer_apirX rP rP rQ r+ � s � z Command._SetAclXmlPassthroughc C s� �zB|� � r�| jr~| jr:|j|jt�� | j|jdgd� q�t � | jtj �}|sX|� t � tj|d�}|j|j||jdgd� n\| jr�|j|jt�� | j|jdgd� n4t � | jtj�}tj|d�}|j|j||jdgd� nf| j�r |j|j|jt�� |j|j| jd� n8t � | jtj �}tj|d�}|j|j|j||j|jd� W nj t�yl } z� W Y d }~nLd }~0 t�y� } z*| j�r�d | _| j�|� n� W Y d }~n d }~0 0 d S ) ad Sets the ACL for the URL provided using the gsutil Cloud API. This function assumes that self.def_acl, self.canned, and self.continue_on_error are initialized, and that self.acl_arg is either a JSON string or a canned ACL string. Args: url: CloudURL to set the ACL on. gsutil_api: gsutil Cloud API to use for the ACL set. r )�canned_def_aclr1 �fields)�defaultObjectAcl)r1 r6 )� canned_aclr1 r6 )r� )r1 � generationr8 )r1 r9 NF)r r( r/ � PatchBucket� bucket_name�apitools_messages�Bucketr4 r* r< � JsonToMessage�ObjectAccessControlrv r> �BucketAccessControl�PatchObjectMetadatar` �Objectr9 r r r rm rT rk ) r� rb r� r0 �bucket_metadataZ bucket_aclZ object_acl�object_metadatarX rP rP rQ r, � s� �� ���� ��� � zCommand._SetAclGsutilApic C s� | j d }| j dd� }t|�s.td| j ��tj�|�rxt�|dt ��}|� � }W d � n1 sf0 Y d| _ n<t j |d | jd| jd�}|�� }||vr�td| ��d | _ d | _|| _| �|||� | js�| js�td ��dS ) z�Sets ACLs on the self.args using the passed-in acl function. Args: acl_func: ACL function to be passed to Apply. acl_excep_handler: ACL exception handler to be passed to Apply. r r� Nz,"%s" command spanning providers not allowed.�rF)r� �validater� zInvalid canned ACL "%s".Tr )r� r! r rN r] r^ �isfile�codecs�openr. �readr/ r| � storage_urir� r� � canned_aclsrm r4 r% r )r� r r! r4 �url_args�frK rL rP rP rQ �SetAclCommandHelper< s2 � &� zCommand.SetAclCommandHelperc C s, ddl m} |r(| j�d�t�d��� dS )z�Warns service account users who have received an AccessDenied error. When one of the metadata-related commands fails due to AccessDenied, user must ensure that they are listed as an Owner in the API console. r )�IS_SERVICE_ACCOUNTr� a� It appears that your service account has been denied access while attempting to perform a metadata operation. If you believe that you should have access to this metadata (i.e., if it is associated with your account), please make sure that your service accounts email address is listed as an Owner in the Permissions tab of the API console. See "gsutil help creds" for further information. N)Z$gcs_oauth2_boto_plugin.oauth2_pluginrP rT �warningr� r� r� )r� rP rP rP rQ �_WarnServiceAccountsg s ��z Command._WarnServiceAccountsc C s� | � |�}t|�}| j�|j�tjkr�|jdkr�z&| jj|| j|jd�}t |� � � W q� t y� } z| � � � W Y d}~q�d}~0 0 nN| j dkr�|jj}|s�| j�d|||� n|jj}|s�| �|� t t�|�� dS )z�Prints the standard or default object ACL depending on self.command_name. Args: url_str: URL string to get ACL for. r� )r0 r1 Nr� a� No default object ACL present for %s. This could occur if the default object ACL is private, in which case objects created in this bucket will be readable only by their creators. It could also mean you do not have OWNER permission on %s and therefore do not have permission to read the default object ACL. It could also mean that %s has Bucket Policy Only enabled and therefore object ACLs and default object ACLs are disabled (see https://cloud.google.com/storage/docs/bucket-policy-only).)�#GetAclCommandBucketListingReferencer r� r) r* r r� �XmlPassThroughGetAclr( �print�to_xmlr rR rN � root_objectr7 rT r� r� �_ListAccessControlsAclr< �JsonFromMessage)r� �url_strr$ rb r� �_rP rP rQ �GetAndPrintAcl s2 �� � zCommand.GetAndPrintAclc C sD |� � r | jj|j|jd�S |�� r<| jj|j|j|jd�S dS dS )at Returns either bucket or object access controls for a storage url. Args: storage_url: StorageUrl object representing the bucket or object. Returns: BucketAccessControls, ObjectAccessControls, or None if storage_url does not represent a cloud bucket or cloud object. Raises: ServiceException if there was an error in the request. )r1 N)r r� �ListBucketAccessControlsr; r* �IsObject�ListObjectAccessControlsr` )r� r rP rP rQ rX � s ��z Command._ListAccessControlsAclc C s� t |�}|�� r*t| �|�jdgd��}n.| jdkr<dg}ndg}t| �|�j|d��}|�� rhtd��|� � r�td|| jf ��t |�d S ) a' Gets a single bucket listing reference for an acl get command. Args: url_str: URL string to get the bucket listing reference for. Returns: BucketListingReference for the URL string. Raises: CommandException if string did not result in exactly one reference. r� )r r� r7 r �No URLs matchedzD%s matched more than one URL, which is not allowed by the %s commandr ) r r^ r r � IterObjectsrN r �IsEmptyr � HasPluralityr )r� rZ � wildcard_urlZplurality_iterr rP rP rQ rS � s0 �� ����z+Command.GetAclCommandBucketListingReferencec C sD | j ||d�}|�� r(td|| jf ��t|�d }t|j�|jfS )aP Gets a single bucket URL based on the command arguments. Args: arg: String argument to get bucket URL for. bucket_fields: Fields to populate for the bucket. Returns: (StorageUrl referring to a single bucket, Bucket metadata). Raises: CommandException if args did not match exactly one bucket. r zD%s matched more than one URL, which is not allowed by the %s commandr )�GetBucketUrlIterFromArgrc r rN r r r rW )r� �argr � plurality_checkable_iteratorr$ rP rP rQ �GetSingleBucketUrlFromArg� s �� z!Command.GetSingleBucketUrlFromArgc C sP t |�}|�� r|�� r&td| j ��t| �|�j|d��}|�� rLtd��|S )a- Gets a single bucket URL based on the command arguments. Args: arg: String argument to iterate over. bucket_fields: Fields to populate for the bucket. Returns: PluralityCheckableIterator over buckets. Raises: CommandException if iterator matched no buckets. z""%s" command must specify a bucketr r` ) r r r^ r rN r r r rb )r� rf r Zarg_urlrg rP rP rQ re � s ��z Command.GetBucketUrlIterFromArgc C s2 t j}|r"|D ]}|| j�� qi t _d t _d S rS )r � provider_pool� connection�close)r� Zconnection_pool�irP rP rQ �_ResetConnectionPool s z Command._ResetConnectionPoolc C s� | j s |rh|s$tj�ddtjjj�}|dk r8td| ��|sRtj�ddtjjj�}|dk rptd| ��nd}d}t � \}}|r�|dkr�td� t � d|d � t � �f ����| jd k}|r�|d kr�|dkr�|r�| j�d � | j�d |� | j�d|� ||fS )a� Determines the values of process_count and thread_count. These values are used for parallel operations. If we're not performing operations in parallel, then ignore existing values and use process_count = thread_count = 1. Args: process_count: A positive integer or None. In the latter case, we read the value from the .boto config file. thread_count: A positive integer or None. In the latter case, we read the value from the .boto config file. parallel_operations_override: Used to override self.parallel_operations. This allows the caller to safely override the top-level flag for a single call. print_macos_warning: Print a warning about parallel processing on MacOS if true. Returns: (process_count, thread_count): The number of processes and threads to use, respectively. rz �parallel_process_countr� z$Invalid parallel_process_count "%d".�parallel_thread_countz#Invalid parallel_thread_count "%d".r� z�It is not possible to set process_count > 1 on %s. Please update your config file(s) (located at %s) and set "parallel_process_count = 1".z, r �macOSa^ If you experience problems with multiprocessing on MacOS, they might be related to https://bugs.python.org/issue33725. You can disable multiprocessing by editing your .boto config or by adding the following flag to your command: `-o "GSUtil:parallel_process_count=1"`. Note that multithreading is still available even if you disable multiprocessing. zprocess count: %dzthread count: %d)r� r| r} r~ r� �commandsZ DEFAULT_PARALLEL_PROCESS_COUNTr Z DEFAULT_PARALLEL_THREAD_COUNTr5 r� r� r� r+ �recursive_apply_levelrT ra r� )r� � process_count� thread_count� parallel_operations_overrideZprint_macos_warning� should_prohibit_multiprocessing�os_name�is_main_threadrP rP rQ �_GetProcessAndThreadCount( sL ���� ��� �z!Command._GetProcessAndThreadCountc C s� t � t�� t�� }W d � n1 s*0 Y t�| �}| jd7 _d|_d|_d|_|t |< dt |< dt |< dt |< g t |< |S )zBSet up the state for a caller id, corresponding to one Apply call.Nr� r� Fr )r� r� � Increment�GetValue�copyrr rT r� r� r� r� r� r� r� )r� � caller_idrW rP rP rQ �_SetUpPerCallerStaten s & z Command._SetUpPerCallerStatec C s� g }t � }t�|� t�� t�� tkr0td��t|�D ]B}t t �}t j | j ||||fd�}d|_|�|� t� |�� q8t||�} t �| � dS )z7Create a new pool of processes that call _ApplyThreads.z,Recursion depth of Apply calls is too great.)�targetr� TN)rx r� rv r� rz r{ �MAX_RECURSIVE_DEPTHr �ranger� r� r1 �Process� _ApplyThreads�daemonrr �start� _ConsumerPool) r� � num_processes� num_threadsr- � processes� task_queuer[ rr �p� consumer_poolrP rP rQ �_CreateNewConsumerPool� s( �� z Command._CreateNewConsumerPoolc @ s e Zd ZdZdZdZdZdS )z Command.ParallelOverrideReasonzAEnum class to describe purpose of overriding parallel operations.�sliceZspeedZperfdiagN)�__name__� __module__� __qualname__�__doc__�SLICEZSPEEDZPERFDIAGrP rP rP rQ �ParallelOverrideReason� s r� c C sb |r,i } |D ] } t | | �| | < t| | d� q | �|||�\}}| jdkoP| jdk}|rbt||d� | op|| dk} |r�| js�|dkr�t| jd� | � � }|r�|t |< |D ]} dt || f< q�| jr�|nd}|| dk�r| j |||||||| | | |d� |�rt � n| �|||||| | � |�rN|D ](} | | t �|| f� }t| | |� �q$| �r^t�|�S dS )a_ Calls _Parallel/SequentialApply based on multiprocessing availability. Args: func: Function to call to process each argument. args_iterator: Iterable collection of arguments to be put into the work queue. exception_handler: Exception handler for WorkerThread class. shared_attrs: List of attributes to manage across sub-processes. arg_checker: Used to determine whether we should process the current argument or simply skip it. Also handles any logging that is specific to a particular type of argument. parallel_operations_override: A string (see ParallelOverrideReason) describing the reason to override self.parallel_operations. This allows the caller to safely override the top-level flag for a single call. process_count: The number of processes to use. If not specified, then the configured default will be used. thread_count: The number of threads per process. If not specified, then the configured default will be used.. should_return_results: If true, then return the results of all successful calls to func in a list. fail_on_error: If true, then raise any exceptions encountered when executing func. This is only applicable in the case of process_count == thread_count == 1. seek_ahead_iterator: If present, a seek-ahead iterator that will provide an approximation of the total number of tasks and bytes that will be iterated by the ProducerThread. Used only if multiple processes and/or threads are used. Returns: Results from spawned threads. r r� )r� r� r� )rT )r ru N)rq �setattrry rr �sequential_caller_idr r� r0 rT r~ r� r� �_ParallelApply�_AggregateThreadStats�_SequentialApplyr� r� )r� �func� args_iterator�exception_handlerZ shared_attrs� arg_checkerru rs rt �should_return_resultsr r Zoriginal_shared_vars_values�namerx r} Zusable_processes_countZ final_valuerP rP rQ r � sp 3� �� � ��� � �z Command.Applyc C sN t j�ddd�dkr(t j�ddd�dksJ| j�dt�d| jj � d � dS ) z2Outputs a suggestion to the user to use gsutil -m.rz rn r r� ro r� z�==> NOTE: You are performing a sequence of gsutil operations that may run significantly faster if you instead use gsutil -m %s ... Please see the -m section under "gsutil help options" for further information about when gsutil -m can be advantageous.N) r| r} r~ rT ra r� �fillr� rN r� rP rP rQ �_MaybeSuggestGsutilDashM6 s � ���z Command._MaybeSuggestGsutilDashMc C s� t dd| j| j| j| jd�}t|�}d} z t|�} W n� tyf } zW Y d} ~ �q@W Y d} ~ n�d} ~ 0 ty� } znt � |r�� nTz|| | � W n8 ty� } z | j � d|t � � � W Y d} ~ n d} ~ 0 0 W Y d} ~ q(W Y d} ~ n d} ~ 0 0 | d7 } | tk�s | t dk�r| �� || | �r(t|| |||||�} |�| | � q(| t }|t� k�r\| �� t| jjtt�� �� |j�|| � | �|� dS )a Performs all function calls sequentially in the current thread. No other threads or processes will be spawned. This degraded functionality is used when the multiprocessing module is not available or the user requests only one thread and one process. NF)r� r� r� r� r �4Caught exception while handling exception for %s: %sr� ) � WorkerThreadr� r� r� r� �iter�next� StopIteration� Exception�_IncrementFailureCountrT r� � traceback� format_exc�#OFFER_GSUTIL_M_SUGGESTION_THRESHOLD�#OFFER_GSUTIL_M_SUGGESTION_FREQUENCYr� �Task� PerformTaskr: r3 r� r- r$ r. �shared_vars_updater�Update�_ProcessSourceUrlTypes)r� r� r� r� r} r� r� r � worker_threadZsequential_call_countr� rX r[ �taskZ#lines_since_suggestion_last_printedrP rP rQ r� B sZ � "�" � �� �zCommand._SequentialApplyc C s | j dk} | | jjkr4| j dkr4t�tt�� d�� ts\| r\tj tj fD ]} t | t dd� qHt s�|dkrvt �t� � n>t� }t �|� t|| j|| j| j| jt| j| j| j| jd� |dk�rVz�| s�t�� | j t � � k�r2t �@ | r�| � ||t� n t j!dd� t �"� t �#� W d � n1 �s(0 Y W | �sTt�$� n| �sRt�$� 0 nx| �s�zft�� | j t%� k�r�t&� t� }t �|� t|| j|| j| j| jt| j| j| j| jd� W t�$� n t�$� 0 t | j }| �r�| �s�d} t't(�(| �|||||||| | | �rtndd� }d}| �r.t)tt*j+t,�}t �l t-| �rPW d � �q�n8|dk�r�| �r�t � � �r�t �!� | � ||t� t �"� t �#� W d � n1 �s�0 Y �q.| �r�t.tt/� |j0t1d � | �2|j3� |j4�r�|j4�|j5�r�| �r�|j5�| �r | �s t.tt6t�� �� dS ) a� Dispatches input arguments across a thread/process pool. Pools are composed of parallel OS processes and/or Python threads, based on options (-m or not) and settings in the user's config file. If only one OS process is requested/available, dispatch requests across threads in the current OS process. In the multi-process case, we will create one pool of worker processes for each level of the tree of recursive calls to Apply. E.g., if A calls Apply(B), and B ultimately calls Apply(C) followed by Apply(D), then we will only create two sets of worker processes - B will execute in the first, and C and D will execute in the second. If C is then changed to call Apply(E) and D is changed to call Apply(F), then we will automatically create a third set of processes (lazily, when needed) that will be used to execute calls to E and F. This might look something like: Pool1 Executes: B / \ Pool2 Executes: C D / \ Pool3 Executes: E F Apply's parallelism is generally broken up into 4 cases: - If process_count == thread_count == 1, then all tasks will be executed by _SequentialApply. - If process_count > 1 and thread_count == 1, then the main thread will create a new pool of processes (if they don't already exist) and each of those processes will execute the tasks in a single thread. - If process_count == 1 and thread_count > 1, then this process will create a new pool of threads to execute the tasks. - If process_count > 1 and thread_count > 1, then the main thread will create a new pool of processes (if they don't already exist) and each of those processes will, upon creation, create a pool of threads to execute the tasks. Args: caller_id: The caller ID unique to this call to command.Apply. See command.Apply for description of other arguments. r r� T)�is_final_handler) r� r� r� r� r- r� r� r� r� )� reset_valueN)r r- ��timeout)7rr r� r� r� �putr& r. r; �signal�SIGINT�SIGTERMr r r� rv rx ry � WorkerPoolrT r� r� r� r� r� r� r� r� �acquirer� r{ r� r� r� �Reset� notify_all�wait�release� _GetCurrentMaxRecursiveLevel�"_IncrementCurrentMaxRecursiveLevel�ProducerThreadr| r* r� r� r� r� r3 r7 r� r6 r� r� �unknown_exception�iterator_exceptionr$ )r� r� r� r� r} r� rs rt r� r r ru rx � signal_numr� Zproducer_threadZ ui_threadrP rP rQ r� � s� : � � � � * � � � ��, zCommand._ParallelApplyc C s* t |t�sdS t|j|j|j|jd� dS )z6Logs the URL type information to analytics collection.N)�is_daisy_chain� has_file_src� has_cloud_src�provider_types)� isinstancer r r� r� r� r� )r� r� rP rP rQ r� t s �z Command._ProcessSourceUrlTypesc C s� |dksJ d��t � t� D ]}t�|t� q | �� || _t| }t�|�}t || j || j | j | j || j| j| j| jd� }d} |jdd�s�t�d� q�|�� } | jtkr�|�| � | d7 } q�|�� q�dS ) a� Assigns the work from the multi-process global task queue. Work is assigned to an individual process for later consumption either by the WorkerThreads or (if thread_count == 1) this thread. Args: thread_count: The number of threads used to perform the work. If 1, then perform all work in this thread. process_count: The number of processes used to perform the work. recursive_apply_level: The depth in the tree of recursive calls to Apply of this thread. status_queue: Multiprocessing/threading queue for progress reporting and performance aggregation. r� zCInvalid state, calling command._ApplyThreads with only one process.� �worker_semaphorer� r� r� r- r� r� r� r� r F)�blockingg{�G�z�?N)rr r r� r rm rr r� r� r� r� rT r� r� r� r� r� r� r� r� r. �sleepr� r� r7 �AddTaskr� ) r� rt rs rr r- Z catch_signalr� r� Z worker_poolZ num_enqueuedr� rP rP rQ r� } s> � � zCommand._ApplyThreads)NNNN)FNT)F)N)N)N)N)T)NN)1r� r� r� r� r� r� rr r� � staticmethodr- r� r� �propertyrN r� r� r� r� r r� r� r r r r% rh r+ r, rO rR r\ rX rS rh re rm ry r~ r� �objectr� rc r r� r� r� r� r� � __classcell__rP rP r rQ r� � s� � � � ) � T J++& � F%� R � k r� c @ s e Zd Zdd� Zdd� ZdS )r� c C s || _ || _d S rS )r� r� )r� r� r� rP rP rQ r� � s z_ConsumerPool.__init__c C s | j D ]}t|j� qd S rS )r� r �pid)r� �processrP rP rQ �ShutDown� s z_ConsumerPool.ShutDownN)r� r� r� r� r� rP rP rP rQ r� � s r� c @ s e Zd ZdZdS )r� a= Task class representing work to be completed. Args: func: The function to be executed. args: The arguments to func. caller_id: The globally-unique caller ID corresponding to the Apply call. exception_handler: The exception handler to use if the call to func fails. should_return_results: True iff the results of this function should be returned from the Apply call. arg_checker: Used to determine whether we should process the current argument or simply skip it. Also handles any logging that is specific to a particular type of argument. fail_on_error: If true, then raise any exceptions encountered when executing func. This is only applicable in the case of process_count == thread_count == 1. N)r� r� r� r� rP rP rP rQ r� � s r� zUfunc args caller_id exception_handler should_return_results arg_checker fail_on_errorc C s t | |t�S )a� Initializes and runs the seek-ahead thread. We defer starting this thread until it is needed, since it is only useful when the ProducerThread iterates more results than it can store on the global task queue. Args: seek_ahead_iterator: Iterator that yields SeekAheadResults. seek_ahead_thread_cancel_event: threading.Event for signaling the seek-ahead thread to terminate. Returns: The thread object for the initialized thread. )r r� )r � seek_ahead_thread_cancel_eventrP rP rQ �_StartSeekAheadThread� s �r� c s* e Zd ZdZd� fdd� Zdd� Z� ZS )r� z<Thread used to enqueue work for other processes and threads.Nc sv t t| ��� || _|| _|| _|| _|| _|| _|| _ || _ | | _ t � | _ d| _d| _d| _| | _| | _| �� dS )a Initializes the producer thread. Args: cls: Instance of Command for which this ProducerThread was created. args_iterator: Iterable collection of arguments to be put into the work queue. caller_id: Globally-unique caller ID corresponding to this call to Apply. func: The function to be called on each element of args_iterator. task_queue: The queue into which tasks will be put, to later be consumed by Command._ApplyThreads. should_return_results: True iff the results for this call to command.Apply were requested. exception_handler: The exception handler to use when errors are encountered during calls to func. arg_checker: Used to determine whether we should process the current argument or simply skip it. Also handles any logging that is specific to a particular type of argument. fail_on_error: If true, then raise any exceptions encountered when executing func. This is only applicable in the case of process_count == thread_count == 1. seek_ahead_iterator: If present, a seek-ahead iterator that will provide an approximation of the total number of tasks and bytes that will be iterated by the ProducerThread. status_queue: status_queue to inform task_queue estimation. Only valid when calling from the main thread, else None. Even if this is the main thread, the status_queue will only properly work if args is a collection of NameExpansionResults, which is the type that gives us initial information about files to be processed. Otherwise, nothing will be added to the queue. TN)r� r� r� r� rW r� r} r� r� r� r� r �_SharedVariablesUpdater�shared_variables_updaterr� r� r� r r- r� ) r� rW r� r} r� r� r� r� r� r r r- r rP rQ r� s" *zProducerThread.__init__c C s. d}d }d }d }d }d }d}d }�zX�zpd} t | j�| _zt| j�}W n� tyx } zW Y d } ~ �q�W Y d } ~ n�d } ~ 0 t�y } z�t� | jr�| | _� nnz| �| j | � W n< ty� } z$| j j � d| j t �� � W Y d } ~ n d } ~ 0 0 | j�| j| j � W Y d } ~ q8W Y d } ~ n d } ~ 0 0 | �| j |�r8|d7 }| j�r�|d �s�t|t��snt|t��snt|t��r�t| jt|| t�� �� t|t��s�t|t��r�|j�r�t � |j�} d| v �r�| t | d �7 } n"t|t��r�|j �r�| t |j �7 } |�s^|d u �rt � }|dk�rd}nH||k�r^| j!�rZt"�#� }t$| j!|�}t%j&�'dd d ��rZ|j(t)d � d}|}t*| j || j| j| j+| j| j�}|r8| j,�-|� q8W n4 t�y� } z| j�s�| | _.W Y d } ~ n d } ~ 0 0 W |t/| j< |�s�t*d t0| jd d d d �}| j,�-|� |d u�r |�1� |j(t)d � | j�rft|t��sJt|t��sJt|t��rft| jt|| t�� dd �� t2| jt3�'| j�� n�|t/| j< |�s�t*d t0| jd d d d �}| j,�-|� |d u�r�|�1� |j(t)d � | j�rt|t��s�t|t��s�t|t��rt| jt|| t�� dd �� t2| jt3�'| j�� 0 d S ) Nr Fr� r� �d �sizeTrz Ztask_estimation_forcer� )�finished)4r� r� r� r� r� r� r r� r� rW rT r� r� r� r� r� r� r} r� r- r� r r r8 r3 r' r. r �json�loads�intZ copy_sizer� r r� �Eventr� r| r} r� r� r4 r� r� r� r� r� r� r7 r� � _NotifyIfDoner� ) r� Z num_tasksZcur_taskZ last_taskr{ Zseek_ahead_threadr� Z seek_ahead_thread_consideredr� � total_sizerX r[ Zjson_expanded_resultrP rP rQ �runI s " �" ��� � � � � ������ �� � ������ �zProducerThread.run)NN)r� r� r� r� r� r� r� rP rP r rQ r� s �<r� c @ s" e Zd ZdZddd�Zdd� ZdS ) r� z3Pool of worker threads to which tasks can be added.Nr c C s� |d u |d u ksJ �| | _ | | _| | _| | _|p4t� | _g | _t|�D ]@} t| j||||||| j | j| j| jd� }| j� |� |� � qFd S )Nr� ) r� r� r� r� ry r� �threadsr� r� rv r� )r� rt rT r� r� r� r� r� r- r� r� r� r� r[ r� rP rP rQ r� � s. � zWorkerPool.__init__c C s | j �|� dS )zCAdds a task to the task queue; used only in the multi-process case.N)r� r� )r� r� rP rP rQ r� � s zWorkerPool.AddTask) NNNNr NNNNN)r� r� r� r� r� r� rP rP rP rQ r� � s � .r� c sJ e Zd ZdZd� fdd� Zedd� �Zedd � �Zd d � Zd d � Z � Z S )r� a8 Thread where all the work will be performed. This makes the function calls for Apply and takes care of all error handling, return value propagation, and shared_vars. Note that this thread is NOT started upon instantiation because the function- calling logic is also used in the single-threaded case. Nr c s� t t| ��� t�� | _t�� | _|| _|| _ d| _ i | _ t � | _ || _| | _| | _| | _d| _|r�|r�t|||||| j| j| j| jd� | _dS )ar Initializes the worker thread. Args: task_queue: The thread-safe queue from which this thread should obtain its work. logger: Logger to use for this thread. worker_semaphore: threading.BoundedSemaphore to be released each time a task is completed, or None for single-threaded execution. bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. gsutil_api_map: Map of providers and API selector tuples to api classes which can be used to communicate with those providers. Used for the instantiating CloudApiDelegator class. debug: debug level for the CloudApiDelegator class. status_queue: Queue for reporting status updates. user_project: Project to be billed for this request. TN)r� r� r� r� r� )r� r� r� r] �getpidr� r. � init_timer� r� r� �cached_classesr� r� r� r� r� r� �thread_gsutil_apir ) r� r� rT r� r� r� r� r- r� r� r� r� r rP rQ r� s2 �zWorkerThread.__init__c C sX t �| j| jf�du r,t| j�t | j| jf< t | j| jf }|�� |t | j| jf< dS )zEUpdate the thread_stats AtomicDict before task_queue.get() is called.N)r� r� r� �ident� _ThreadStatr� �StartBlockedTime�r� � thread_statrP rP rQ �_StartBlockedTimeG s z WorkerThread._StartBlockedTimec C s, t | j| jf }|�� |t | j| jf< dS )zDUpdate the thread_stats AtomicDict after task_queue.get() is called.N)r� r� r� �EndBlockedTimer� rP rP rQ �_EndBlockedTimeS s z WorkerThread._EndBlockedTimec C s, |j }z�z0|j||j| jd�}|jr6tj||gg d� W n~ ty� } zft� |j rX� nJz|� ||� W n8 ty� } z |j � d|t �� � W Y d}~n d}~0 0 W Y d}~n d}~0 0 W | jr�| j�� | j�||� t�|d�}t||� n8| j�r| j�� | j�||� t�|d�}t||� 0 dS )z�Makes the function call for a task. Args: task: The Task to perform. cls: The instance of a class which gives context to the functions called by the Task's function. E.g., see SetAclFuncWrapper. rf )� default_valuer� Nr� )r} r� r� r� r� r� rz r� r� r r� rT r� r� r� r� r� r� r� r� r� )r� r� rW r} �resultsrX r[ �num_donerP rP rQ r� Z s8 � �2 � zWorkerThread.PerformTaskc C st | � � | j�� }| �� |jtkr&q |j}| j�|d �}|sbt�t | �}t |j �|_ || j|< | � ||� q d S rS )r� r� r� r� r� r7 r} r� r| r� rR rN rT r� )r� r� r} rW rP rP rQ r� � s zWorkerThread.run) NNNr NNNNN) r� r� r� r� r� r r� r� r� r� r� rP rP r rQ r� � s" �; &r� c @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd S ) r� z1Stores thread idle and execution time statistics.c C s d| _ || _t�� | _d| _d S )Nr )�total_idle_time�end_block_timer. �start_block_time�total_execution_time)r� r� rP rP rQ r� � s z_ThreadStat.__init__c C s( t � � | _| j| j }| j|7 _d S rS )r. r� r� r� )r� Z exec_timerP rP rQ r� � s z _ThreadStat.StartBlockedTimec C s( t � � | _| j| j }| j|7 _d S rS )r. r� r� r� )r� Z idle_timerP rP rQ r� � s z_ThreadStat.EndBlockedTimec C s: | j | jkr"| j|| j 7 _n| j|| j 7 _dS )z)Decide final stats upon Apply completion.N)r� r� r� r� )r� �end_timerP rP rQ � AggregateStat� s z_ThreadStat.AggregateStatN)r� r� r� r� r� r� r� r� rP rP rP rQ r� � s r� c C sL t � � } d }}t�� D ]"}|�| � ||j7 }||j7 }qt||d� dS )z�At the end of the top-level Apply call, aggregate the thread stats dict. This should only be called in the main process and thread because it logs to the MetricsCollector. r )�thread_idle_time�thread_execution_timeN)r. r� �valuesr� r� r� r )�cur_timer� r� r� rP rP rQ r� � s �r� c @ s e Zd ZdZdd� Zdd� ZdS )r� a� Used to update shared variable for a class in the global map. Note that each thread will have its own instance of the calling class for context, and it will also have its own instance of a _SharedVariablesUpdater. This is used in the following way: 1. Before any tasks are performed, each thread will get a copy of the calling class, and the globally-consistent value of this shared variable will be initialized to whatever it was before the call to Apply began. 2. After each time a thread performs a task, it will look at the current values of the shared variables in its instance of the calling class. 2.A. For each such variable, it computes the delta of this variable between the last known value for this class (which is stored in a dict local to this class) and the current value of the variable in the class. 2.B. Using this delta, we update the last known value locally as well as the globally-consistent value shared across all classes (the globally consistent value is simply increased by the computed delta). c C s i | _ d S rS )�last_shared_var_valuesr� rP rP rQ r� � s z _SharedVariablesUpdater.__init__c C s\ t �|d�}|rX|D ]B}||f}| j�|d�}t||�| }|| | j|< t�||� qdS )z.Update any shared variables with their deltas.Nr )r� r� r rq r� rz )r� r} rW Z shared_varsr� �key� last_value�deltarP rP rQ r� � s z _SharedVariablesUpdater.UpdateN)r� r� r� r� r� r� rP rP rP rQ r� � s r� c C sP t | }||krL|dkrLt� dt| < t�� W d � n1 sB0 Y dS )a� Notify any threads waiting for results that something has finished. Each waiting thread will then need to check the call_completed_map to see if its work is done. Note that num_done could be calculated here, but it is passed in as an optimization so that we have one less call to a globally-locked data structure. Args: caller_id: The caller_id of the function whose progress we're checking. num_done: The number of tasks currently completed for that caller_id. r TN)r� r� r� r� )r} r� Z num_to_dorP rP rQ r� � s r� c C sT t D ] } z | �� W q Y q0 qtD ] }|�� q(z t�� W n Y n0 dS )zEShut down all processes in consumer pools in preparation for exiting.N)ru Zcancel_join_threadr� r� r� )�qr� rP rP rQ r� s r� c C s t �� S rS )r� r{ rP rP rP rQ r� ! s r� c C s t �� d S rS )r� rz rP rP rP rQ r� & s r� c C s t �� d S rS )r� rz rP rP rP rQ r� + s r� c C s t �� d S rS )r� � DecrementrP rP rP rQ �DecrementFailureCount0 s r c C s t �� S )zAReturns the number of failures processed during calls to Apply().)r� r{ rP rP rP rQ �GetFailureCount5 s r c C s t �� dS )zEResets the failure_count variable to 0 - useful if error is expected.N)r� r� rP rP rP rQ �ResetFailureCount; s r )N)�r� � __future__r r r r rH � collectionsr r| r r� rC r] r� r� r� r� r. r� r| �boto.storage_urir r� �gslib.cloud_apir r r �gslib.cloud_api_delegatorr �gslib.cs_api_mapr r �gslib.exceptionr Zgslib.help_providerr Z gslib.metricsr r Zgslib.name_expansionr r r r r Z"gslib.plurality_checkable_iteratorr Zgslib.seek_ahead_threadr �gslib.sig_handlingr r r r r �gslib.storage_urlr r r r! r" �"gslib.third_party.storage_apitoolsr# r< �gslib.thread_messager$ r% r& r' Zgslib.ui_controllerr( r) r* Zgslib.utils.boto_utilr+ r, �gslib.utils.constantsr- r. �&gslib.utils.parallelism_framework_utilr/ r0 r1 r2 r3 r4 r5 r6 r7 Zgslib.utils.rsync_utilr8 Zgslib.utils.shim_utilr9 Zgslib.utils.system_utilr: r; � gslib.utils.translation_helperr< r= r> Zgslib.wildcard_iteratorr? � six.movesr@ rs ZCryptorA rp � ImportErrorr� r� rR rY rc re rj rn ru rr rx ry rt r r� r� r� r� r� r� r� r� r� r� r� r� �Threadr� r� r� r� r� r� r� r� r� r� r� r r r rP rP rP rQ �<module> s: % � %� g �� <6 '. __pycache__/tracker_file.cpython-39.pyc 0000644 00000042017 15025013616 0014042 0 ustar 00 a +(Wg�` � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl mZ ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ dZdZdZ dZ G dd� de �Z dd� Z dd� Z!dd� Z"d8dd�Z#d d � Z$d9d d �Z%d d!� Z&d"d#� Z'd:d$d%�Z(d&d'� Z)d(d)� Z*d;d*d+�Z+d<d,d-�Z,d.d/� Z-d0d1� Z.d2d3� Z/d=d4d5�Z0d6d7� Z1dS )>z0Helper functions for tracker file functionality.� )�absolute_import)�print_function)�division)�unicode_literalsN)�config)�CommandException)�GetGsutilStateDir)�ResumableThreshold)�UTF8)�GetMd5)�CreateDirIfNeeded�d zCouldn't write tracker file (%s): %s. This can happen if gsutil is configured to save tracker files to an unwritable directory)�encryption_key_sha256�serialization_datac @ s$ e Zd ZdZdZdZdZdZdZdS )�TrackerFileType�upload�downloadZdownload_componentZparallel_uploadZsliced_download�rewriteN) �__name__� __module__� __qualname__�UPLOAD�DOWNLOAD�DOWNLOAD_COMPONENT�PARALLEL_UPLOAD�SLICED_DOWNLOAD�REWRITE� r r �!/opt/gsutil/gslib/tracker_file.pyr 4 s r c C sP t | tj�r | �t�}| }n| }| �t�}t�|�}d|�� d |dd� S )a" Apply a hash function (SHA1) to shorten the passed file name. The spec for the hashed file name is as follows: TRACKER_<hash>_<trailing> where hash is a SHA1 hash on the original file name and trailing is the last 16 chars from the original file name. Max file name lengths vary by operating system so the goal of this function is to ensure the hashed version takes fewer than 100 characters. Args: filename: file name to be hashed. May be unicode or bytes. Returns: shorter, hashed version of passed file name ZTRACKER_�.i�N) � isinstance�six� text_type�encoder �decode�hashlib�sha1� hexdigest)�filename�filename_bytesZ filename_str�mr r r � _HashFilename= s r+ c C s& t �ddtj�t� d��} t| � | S )a Looks up or creates the gsutil tracker file directory. This is the configured directory where gsutil keeps its resumable transfer tracker files. This function creates it if it doesn't already exist. Returns: The pathname to the tracker directory. �GSUtil�resumable_tracker_dirz tracker-files)r �get�os�path�joinr r )Z tracker_dirr r r �CreateTrackerDirIfNeededY s �r2 c C s( t �ddd| ||||f �}t|tj�S )aU Gets the tracker file name described by the arguments. Args: src_bucket_name: Source bucket (string). src_obj_name: Source object (string). dst_bucket_name: Destination bucket (string). dst_obj_name: Destination object (string) api_selector: API to use for this operation. Returns: File path to tracker file. �[/\\]�_z!rewrite__%s__%s__%s__%s__%s.token)�re�sub�_HashAndReturnPathr r )�src_bucket_nameZ src_obj_name�dst_bucket_name� dst_obj_name� api_selector�res_tracker_file_namer r r �GetRewriteTrackerFilePathh s ���r= c C s� |t jkr(t�ddd| j| j|f �}n�|t jkrTt�dddtj� | j�|f �}n�|t j kr�t�dddtj� | j�||f �}nf|t j kr�t�ddd| j| j||f �}n<|t j kr�t�dddtj� | j�|f �}n|t j kr�t� �t||�S )a� Gets the tracker file name described by the arguments. Args: dst_url: Destination URL for tracker file. tracker_file_type: TrackerFileType for this operation. api_selector: API to use for this operation. src_url: Source URL for the source file name for parallel uploads. component_num: Component number if this is a download component, else None. Returns: File path to tracker file. r3 r4 z resumable_upload__%s__%s__%s.urlz resumable_download__%s__%s.etagz#resumable_download__%s__%s__%d.etagz#parallel_upload__%s__%s__%s__%s.urlz sliced_download__%s__%s.etag)r r r5 r6 � bucket_name� object_namer r/ r0 �realpathr r r r �NotImplementedErrorr7 )�dst_url�tracker_file_typer; �src_url� component_numr< r r r �GetTrackerFilePath� sB �� �� �� �� �� rF c C s2 t t| tj|�� t| |�}|D ] }t |� q dS )z�Deletes all tracker files corresponding to an object download. Args: dst_url: StorageUrl describing the destination file. api_selector: The Cloud API implementation used. N)�DeleteTrackerFilerF r r �!GetSlicedDownloadTrackerFilePaths)rB r; Z tracker_files� tracker_filer r r �DeleteDownloadTrackerFiles� s � rJ c C s� t | tj|�}|g}|du r�d}zVz t|d�}t�|�d }W n( ttfyf | Y W |rd|�� S 0 W |r�|�� n|r�|�� 0 t |�D ] }|� t | tj ||d�� q�|S )a Gets a list of sliced download tracker file paths. The list consists of the parent tracker file path in index 0, and then any existing component tracker files in [1:]. Args: dst_url: Destination URL for tracker file. api_selector: API to use for this operation. num_components: The number of component tracker files, if already known. If not known, the number will be retrieved from the parent tracker file on disk. Returns: File path to tracker file. N�r�num_components�rE ) rF r r �open�json�load�IOError� ValueError�close�range�appendr )rB r; rL Zparallel_tracker_file_pathZtracker_file_pathsrI �ir r r rH � s6 � � � ��rH c C sF t � }t| �}dt|��� |f }d|tj|f }t|�tk sBJ �|S )z�Hashes and returns a tracker file path. Args: res_tracker_file_name: The tracker file name prior to it being hashed. tracker_file_type: The TrackerFileType of res_tracker_file_name. Returns: Final (hashed) tracker file path. z%s_%sz%s%s%s)r2 r+ �str�lowerr/ �sep�len� MAX_TRACKER_FILE_NAME_LENGTH)r< rC r- Zhashed_tracker_file_name�tracker_file_nameZtracker_file_pathr r r r7 � s ��r7 c C s | rt j�| �rt �| � d S )N)r/ r0 �exists�unlink)r\ r r r rG s rG c C sz | r*| j r*| jr*| jr*|r*|j r*|jr*|s.dS t� } | ||||||| ||| f D ]"} | durN| �t�| ��d�� qN| �� S )a` Creates an MD5 hex digest of the parameters for a rewrite call. Resuming rewrites requires that the input parameters are identical. Thus, the rewrite tracker file needs to represent the input parameters. For easy comparison, hash the input values. If a user does a performs a same-source/same-destination rewrite via a different command (for example, with a changed ACL), the hashes will not match and we will restart the rewrite from the beginning. Args: src_obj_metadata: apitools Object describing source object. Must include bucket, name, and etag. dst_obj_metadata: apitools Object describing destination object. Must include bucket and object name projection: Projection used for the API call. src_generation: Optional source generation. gen_match: Optional generation precondition. meta_gen_match: Optional metageneration precondition. canned_acl: Optional canned ACL string. max_bytes_per_call: Optional maximum bytes rewritten per call. src_dec_key_sha256: Optional SHA256 hash string of decryption key for source object. dst_enc_key_sha256: Optional SHA256 hash string of encryption key for destination object. fields: Optional fields to include in response to call. Returns: MD5 hex digest Hash of the input parameters, or None if required parameters are missing. Nr ) �bucket�name�etagr �updater! r" r# r' ) �src_obj_metadata�dst_obj_metadata� projection�src_generation� gen_match�meta_gen_match� canned_acl�max_bytes_per_call�src_dec_key_sha256�dst_enc_key_sha256�fields�md5_hashZ input_paramr r r �HashRewriteParameters s< ) �������ro c C s� d}|s dS z�zBt | d�}|�� �d�}||krN|�� �d�W W |rL|�� S W nF ty� } z.|jtjkr�tj� d| |j f � W Y d}~n d}~0 0 W |r�|�� n|r�|�� 0 dS )a~ Attempts to read a rewrite tracker file. Args: tracker_file_name: Tracker file path string. rewrite_params_hash: MD5 hex digest of rewrite call parameters constructed by HashRewriteParameters. Returns: String rewrite_token for resuming rewrite requests if a matching tracker file exists, None otherwise (which will result in starting a new rewrite). NrK � zGCouldn't read Copy tracker file (%s): %s. Restarting copy from scratch.) rN �readline�rstriprS rQ �errno�ENOENT�sys�stderr�write�strerror)r\ �rewrite_params_hashrI Z existing_hash�er r r �ReadRewriteTrackerFileK s, � �� �r{ c C s t | d||f � dS )a Writes a rewrite tracker file. Args: tracker_file_name: Tracker file path string. rewrite_params_hash: MD5 hex digest of rewrite call parameters constructed by HashRewriteParameters. rewrite_token: Rewrite token string returned by the service. z%s %s N)�_WriteTrackerFile)r\ ry Z rewrite_tokenr r r �WriteRewriteTrackerFilel s �r} c C s� | j s J �d}| jt� k r"||fS |j}|du r8tj} ntj} |d| 7 }t|| ||d�}d} �zz�t|d�} | tju r�| � � � d�} | | j kr�||fW W | r�| � � S nP| tju r�t � | �� �} | d | j kr�| d | jkr�|| d fW W | r�| � � S |�d | � W nZ ttf�yd } z<t| t��s:| jtjk�rP|�d |t| �f � W Y d} ~ n d} ~ 0 0 W | �r�| � � n| �r�| � � 0 | tju �r�t|d | j � n| tju �r�t|| |� ||fS ) a? Checks for a download tracker file and creates one if it does not exist. The methodology for determining the download start point differs between normal and sliced downloads. For normal downloads, the existing bytes in the file are presumed to be correct and have been previously downloaded from the server (if a tracker file exists). In this case, the existing file size is used to determine the download start point. For sliced downloads, the number of bytes previously retrieved from the server cannot be determined from the existing file size, and so the number of bytes known to have been previously downloaded is retrieved from the tracker file. Args: src_obj_metadata: Metadata for the source object. Must include etag and generation. dst_url: Destination URL for tracker file. logger: For outputting log messages. api_selector: API to use for this operation. start_byte: The start byte of the byte range for this download. existing_file_size: Size of existing file for this download on disk. component_num: The component number, if this is a component of a parallel download, else None. Returns: tracker_file_name: The name of the tracker file, if one was used. download_start_byte: The first byte that still needs to be downloaded. Nz component %drM rK rp ra � generation�download_start_bytezPTracker file doesn't match for download of %s. Restarting download from scratch.zOCouldn't read download tracker file (%s): %s. Restarting download from scratch.z%s )ra �sizer r? r r r rF rN rq rr rS rO �loads�readr~ �warnrQ rR r rs rt rW r| �!WriteDownloadComponentTrackerFile)rc rB �loggerr; � start_byte�existing_file_sizerE r\ Z download_namerC rI � etag_value�component_datarz r r r � ReadOrCreateDownloadTrackerFilez sh ! � � � �� � � �r� c C s | j s J �d}| jt� k r |S |du r.tj}ntj}t||||d�}d}z�z�t|d�}|tju r�|�� � d�} | | j kr�|W W |r�|� � S nL|tju r�t � |� � �} | d | j kr�| d | jkr�| d W W |r�|� � S W n ttfy� Y n0 W |�r|� � n|�r|� � 0 |S )af Returns the download starting point. The methodology of this function is the same as in ReadOrCreateDownloadTrackerFile, with the difference that we are not interested here in possibly creating a tracker file. In case there is no tracker file, this means the download starting point is start_byte. Args: src_obj_metadata: Metadata for the source object. Must include etag and generation. dst_url: Destination URL for tracker file. api_selector: API to use for this operation. start_byte: The start byte of the byte range for this download. existing_file_size: Size of existing file for this download on disk. component_num: The component number, if this is a component of a parallel download, else None. Returns: download_start_byte: The first byte that still needs to be downloaded. NrM rK rp ra r~ r )ra r� r r r r rF rN rq rr rS rO r� r� r~ rQ rR ) rc rB r; r� r� rE r\ rC rI r� r� r r r �GetDownloadStartByte� sL � � � � � r� c C s$ |j |j|d�}t| t�|�� dS )z�Updates or creates a download component tracker file on disk. Args: tracker_file_name: The name of the tracker file. src_obj_metadata: Metadata for the source object. Must include etag. current_file_pos: The current position in the file. )ra r~ r N)ra r~ r| rO �dumps)r\ rc Zcurrent_file_posr� r r r r� s �r� c C s� zXt �| t jt jB t jB d�}t �|d��}|�|� W d � n1 sJ0 Y W dS ttfy� } zt | |j ��W Y d}~n d}~0 0 dS )z/Creates a tracker file, storing the input data.i� �wNF) r/ rN �O_WRONLY�O_CREAT�O_TRUNC�fdopenrw rQ �OSError�#RaiseUnwritableTrackerFileExceptionrx )r\ �data�fdZtfrz r r r r| + s �(r| c C sN zt �|�}W n0 ty> } zt| |j��W Y d}~n d}~0 0 t| |� dS )zrCreate a tracker file and write json data to it. Raises: TypeError: If the data is not JSON serializable N)rO r� � TypeErrorr� rx r| )r\ r� �json_str�errr r r �WriteJsonDataToTrackerFile7 s "r� c C s� d}d}d}�zVzjt | d�}|�� }t�|�}|t |krDd}d}n4|t W W |rZ|�� |rj|�d| � |rvt| � S W n� t y� } z(|j t j kr�|�d| |j � W Y d}~n�d}~0 t tf�y2 } z\d}|dur�d}n>|W Y d}~W |r�|�� |�r|�d| � |�r t| � S W Y d}~n d}~0 0 W |�rD|�� |�rV|�d| � |�r�t| � n0|�rt|�� |�r�|�d| � |�r�t| � 0 dS )a� Reads tracker data from an upload tracker file if it exists. Deletes the tracker file if it uses an old format or the desired encryption key has changed. Args: tracker_file_name: Tracker file name for this upload. logger: logging.Logger for outputting log messages. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. Returns: Serialization data if the tracker file already exists (resume existing upload), None otherwise. NFrK Tz�Upload tracker file (%s) does not match current encryption key. Restarting upload from scratch with a new tracker file that uses the current encryption key.zKCouldn't read upload tracker file (%s): %s. Restarting upload from scratch.)rN r� rO r� � ENCRYPTION_UPLOAD_TRACKER_ENTRY�"SERIALIZATION_UPLOAD_TRACKER_ENTRYrS r� rG rQ rs rt rx �KeyErrorrR ) r\ r� r rI Zremove_tracker_fileZencryption_restartZ tracker_dataZ tracker_jsonrz r r r �GetUploadTrackerDataD sx �� ���� ��r� c C s t t| |f ��dS )z:Raises an exception when unable to write the tracker file.N)r �&TRACKER_FILE_UNWRITABLE_EXCEPTION_TEXT)r\ Z error_strr r r r� s �r� )NN)N)NNNNNNNN)N)N)N)2�__doc__� __future__r r r r rs r% rO r/ r5 ru r! �botor �gslib.exceptionr �gslib.utils.boto_utilr r �gslib.utils.constantsr �gslib.utils.hashing_helperr Zgslib.utils.system_utilr r[ r� r� r� �objectr r+ r2 r= rF rJ rH r7 rG ro r{ r} r� r� r� r| r� r� r� r r r r �<module> sn � � 3 � + � C! � ^ � F ; __pycache__/parallel_tracker_file.cpython-39.pyc 0000644 00000021623 15025013616 0015716 0 ustar 00 a +(Wg�0 � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZddl Z ddl Z ddl Z ddl Z ddl mZ dd lmZmZ dd lmZ ed d �ZG d d� de�Zdd� Zdd� Zdd� Zddd�Zd dd�Zd dd�Z dS ) zAHelper functions for composite upload tracker file functionality.� )�absolute_import)�print_function)�division)�unicode_literals)� namedtupleN)�CommandException)�WriteJsonDataToTrackerFile�#RaiseUnwritableTrackerFileException)�UTF8�ObjectFromTrackerzobject_name generationc @ s$ e Zd ZdZdZdZdZdZdZdS )� _CompositeUploadTrackerEntryz7Enum class for composite upload tracker file JSON keys.� components�component_nameZcomponent_generation�encryption_key_sha256�prefixN) �__name__� __module__� __qualname__�__doc__�COMPONENTS_LIST�COMPONENT_NAME�COMPONENT_GENERATION� ENC_SHA256�PREFIX� r r �*/opt/gsutil/gslib/parallel_tracker_file.pyr & s r c C s d}d}g }d}z�z`t | d�}|�� }t�|�}|tj }|tj }|tj D ] }|�t |tj |tj �� qNW nv t y� } z(| j t jkr�|�d| | j� W Y d} ~ n@d} ~ 0 ttfy� } z d}t|�\}}W Y d} ~ n d} ~ 0 0 W |r�|�� n|�r|�� 0 |||fS )a� Read the tracker file from the last parallel composite upload attempt. If it exists, the tracker file is of the format described in WriteParallelUploadTrackerFile or a legacy format. If the file doesn't exist or is formatted incorrectly, then the upload will start from the beginning. This function is not thread-safe and must be protected by a lock if called within Command.Apply. Args: tracker_file_name: The name of the tracker file to read parse. logger: logging.Logger for outputting log messages. Returns: enc_key_sha256: Encryption key SHA256 used to encrypt the existing components, or None if an encryption key was not used. component_prefix: String prefix used in naming the existing components, or None if no prefix was found. existing_components: A list of ObjectFromTracker objects representing the set of files that have already been uploaded. N�rz^Couldn't read upload tracker file (%s): %s. Restarting parallel composite upload from scratch.)�open�read�json�loadsr r r r �appendr r r �IOError�errno�ENOENT�warn�strerror�KeyError� ValueError�_ParseLegacyTrackerData�close) �tracker_file_name�logger�enc_key_sha256r �existing_components� tracker_file� tracker_data� tracker_json� component�er r r � ReadParallelUploadTrackerFile/ s@ �� �$ � r4 c C sz | � d�dd� }d}g }|rr|d }d}|t|�d k rr|| ||d }}|sXd}|�t||�� |d7 }q*||fS )at Parses a legacy parallel composite upload tracker file. Args: tracker_data: Legacy tracker file contents. Returns: component_prefix: The prefix used in naming the existing components, or None if no prefix was found. existing_components: A list of ObjectFromTracker objects representing the set of files that have already been uploaded. � N���r � � )�split�lenr! r )r0 Zold_tracker_datar r. �i�name� generationr r r r) k s r) c C s� t jr.t|t�r|�t�}t|t�r.|�t�}|r�||kr�zR|�d| � g } |D ] } |�� } | j| _| j | _ qP|j || | t j j |jjd� W n, dd� |D �} |�dd�| �� Y n0 dg fS ||fS )a� Validates that tracker data matches the current encryption key. If the data does not match, makes a best-effort attempt to delete existing temporary component objects encrypted with the old key. Args: tracker_file_name: String file name of tracker file. existing_enc_sha256: Encryption key SHA256 used to encrypt the existing components, or None if an encryption key was not used. existing_prefix: String prefix used in naming the existing components, or None if no prefix was found. existing_components: A list of ObjectFromTracker objects representing the set of files that have already been uploaded. current_enc_key_sha256: Current Encryption key SHA256 that should be used to encrypt objects. bucket_url: Bucket URL in which the components exist. command_obj: Command class for calls to Apply. logger: logging.Logger for outputting log messages. delete_func: command.Apply-callable function for deleting objects. delete_exc_handler: Exception handler for delete_func. Returns: prefix: existing_prefix, or None if the encryption key did not match. existing_components: existing_components, or empty list if the encryption key did not match. z�Upload tracker file (%s) does not match current encryption key. Deleting old components and restarting upload from scratch with a new tracker file that uses the current encryption key.)� arg_checker� parallel_operations_overridec S s g | ] }|j �qS r )� object_name)�.0r2 r r r � <listcomp>� s z8ValidateParallelCompositeTrackerData.<locals>.<listcomp>zrFailed to delete some of the following temporary objects: %s (Continuing on to re-upload components from scratch.)r5 N)�six�PY3� isinstance�str�encoder r% �Cloner@ r= �Apply�gslib�command�DummyArgChecker�ParallelOverrideReason�SPEED�join)r+ Zexisting_enc_sha256�existing_prefixr. Zcurrent_enc_key_sha256� bucket_url� command_objr, Z delete_funcZdelete_exc_handlerZcomponents_to_deleter2 �urlZcomponent_namesr r r �$ValidateParallelCompositeTrackerData� sB � � �� rT c C s t t�dd�t| � d �S )aN Generates a random prefix for component objects. Args: encryption_key_sha256: Encryption key SHA256 that will be used to encrypt the components. This is hashed into the prefix to avoid collision during resumption with a different encryption key. Returns: String prefix for use in the composite upload. r7 l �c( l d( )rF �random�randint�hash�r r r r � GenerateComponentObjectPrefix� s �rY c C sr |�Z t | |�\}}}||kr2td|||jf ��|g}|| } t| || |d� W d � n1 sd0 Y dS )a� Rewrites an existing tracker file with info about the uploaded component. Follows the format described in _CreateParallelUploadTrackerFile. Args: tracker_file_name: Tracker file to append to. tracker_file_lock: Thread and process-safe Lock protecting the tracker file. component: ObjectFromTracker describing the object that was uploaded. logger: logging.Logger for outputting log messages. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. zygsutil client error: encryption key SHA256 (%s) in tracker file does not match encryption key SHA256 (%s) of component %srX N)r4 r r@ � WriteParallelUploadTrackerFile) r+ �tracker_file_lockr2 r, r �existing_enc_key_sha256r r. Znewly_completed_componentsZcompleted_componentsr r r �)WriteComponentToParallelUploadTrackerFile� s* ������r] c C sf t jrt|t�r|�d�}g }|D ] }|�tj|jtj |j i� q"tj |tj |tj |i}t| |� dS )a� Writes information about components that were successfully uploaded. The tracker file is serialized JSON of the form: { "encryption_key_sha256": sha256 hash of encryption key (or null), "prefix": Prefix used for the component objects, "components": [ { "component_name": Component object name, "component_generation": Component object generation (or null), }, ... ] } where N is the number of components that have been successfully uploaded. This function is not thread-safe and must be protected by a lock if called within Command.Apply. Args: tracker_file_name: The name of the parallel upload tracker file. prefix: The generated prefix that used for uploading any existing components. components: A list of ObjectFromTracker objects that were uploaded. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. �asciiN)rC rD rE �bytes�decoder! r r r@ r r= r r r r )r+ r r r Ztracker_componentsr2 �tracker_file_datar r r rZ s ��rZ )N)N)N) r � __future__r r r r � collectionsr r# r rU rC rJ �gslib.exceptionr �gslib.tracker_filer r �gslib.utils.constantsr r �objectr r4 r) rT rY r] rZ r r r r �<module> s. < I � % � __pycache__/boto_translation.cpython-39.pyc 0000644 00000136643 15025013616 0015002 0 ustar 00 a +(WgV � @ sV d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlmZ ddlmZ ddlZdd l m Z ddl Z dd l m Z dd l m Z dd l!m"Z" dd l#m$Z$ ddl%m&Z' ddl(m)Z) ddl*m+Z, ddl-m.Z. ddl/m0Z0 ddl1Z ddl2Z ddl3m4Z4 ddl5m6Z6 ddl5m7Z7 ddl5m8Z8 ddl5m9Z9 ddl5m:Z: ddl5m;Z; ddl5m<Z< ddl5m=Z= dd l5m>Z> dd l5m?Z? dd l5m@Z@ dd l5mAZA ddlBZCdd lDmEZE dd!lDmFZF dd"lGmHZH dd#lGmIZI dd$lJmKZK dd%lJmLZL dd&lMmNZO dd'lPmQZQ dd(lRmSZS dd)lRmTZT dd*lRmUZU dd+lVmWZW dd,lVmXZX dd-lYmZZZ dd.lYm[Z[ dd/lYm\Z\ dd0lYm]Z] dd1lYm^Z^ dd2lYm_Z_ dd3l`maZa dd4l`mbZb dd5lcmdZd dd6lemfZf dd7lgmhZh dd8limjZj dd9limkZk dd:limlZl dd;limmZm dd<limnZn dd=limoZo dd>limpZp dd?limqZq dd@limrZr ddAlimsZs ddBlimtZt ddClimuZu ddDlimvZv ddElwmxZx ejy�r�ezZ{e j|j}e j|j~e j|j=e j|j?e j|je j|j�fZ�e��� a�dFa�ej�dGej�dH�Z�e��dI�Z�dJdK� Z�dLdM� Z�G dNdO� dOe��Z�G dPdQ� dQe9�Z�dS )Rz?XML/boto gsutil Cloud API implementation for GCS and Amazon S3.� )�absolute_import)�print_function)�division)�unicode_literalsN)� parseString)� _exceptions)� http_client)�config)�handler)�Cors)�LifecycleConfig)�CORSConfiguration)� DeleteMarker)� Lifecycle)�Prefix)�Tags)�BotoResumableUpload)�AccessDeniedException)�ArgumentException)�BadRequestException)�CloudApi)�NotEmptyException)�NotFoundException)�PreconditionException)�ResumableDownloadException)� ResumableUploadAbortException)�ResumableUploadException)�!ResumableUploadStartOverException)�ServiceException)�CommandException)�InvalidUrlError)�GOOG_PROJ_ID_HDR)�PopulateProjectId)�GenerationFromUrlAndString)�StorageUrlFromString)�storage_v1_messages)�parallelism_framework_util)�ConfigureNoOpAuthIfNeeded)�GetMaxRetryDelay)� GetNumRetries)�ListToGetFields)�ValidateDstObjectMetadata)�DEFAULT_FILE_BUFFER_SIZE)�REQUEST_REASON_ENV_VAR)�REQUEST_REASON_HEADER_KEY)�S3_DELETE_MARKER_GUID)�UTF8)�XML_PROGRESS_CALLBACKS)�Base64EncodeHash)�Base64ToHexHash)� AddAcceptEncodingGzipIfNeeded)�multiprocessing_context)�EncodeStringAsLong)�AclTranslation)� AddS3MarkerAclToObjectMetadata)�CorsTranslation)� CreateBucketNotFoundException)�%CreateNotFoundExceptionForObjectWrite)� CreateObjectNotFoundException)�DEFAULT_CONTENT_TYPE)�HeadersFromObjectMetadata)�LabelTranslation)�LifecycleTranslation)�REMOVE_CORS_CONFIG)� S3MarkerAclFromObjectMetadata)�UnaryDictToXml)�TWO_MIBFz.*non-\s*existent\s*object)�flagsz^"*[a-fA-F0-9]{32}"*$c C sH dddd�}|� � D ].\}}t�d|d�}|durt| jj||� qdS )z:Update Boto Key object with user config's custom endpoint.�host�port� host_header)�gs_hostZgs_portZgs_host_header� CredentialsN)�itemsr �get�setattr�bucket� connection)�keyZ user_setting_to_key_attributeZ user_settingZ key_attributeZuser_setting_value� rQ �%/opt/gsutil/gslib/boto_translation.py�_AddCustomEndpointToKey� s �rS c C s t �� at�dd�adS )z�Perform necessary initialization for multiprocessing. See gslib.command.InitializeMultiprocessingVariables for an explanation of why this is necessary. �ir N)r& � CreateLock�boto_auth_initialized_lockr5 �Value�boto_auth_initializedrQ rQ rQ rR �"InitializeMultiprocessingVariables� s rY c @ s e Zd ZdZdd� Zdd� ZdS )� DownloadProxyCallbackHandlerzFIntermediary callback to keep track of the number of bytes downloaded.c C s || _ || _d S �N)� _start_byte� _callback)�self� start_byte�callbackrQ rQ rR �__init__� s z%DownloadProxyCallbackHandler.__init__c C s | j r| � | j| |� dS )z�Saves necessary data and then calls the given Cloud API callback. Args: bytes_downloaded: Number of bytes processed so far. total_size: Total size of the ongoing operation. N)r] r\ )r^ Zbytes_downloaded� total_sizerQ rQ rR �call� s z!DownloadProxyCallbackHandler.callN)�__name__� __module__� __qualname__�__doc__ra rc rQ rQ rQ rR rZ � s rZ c s" e Zd ZdZdn� fdd� Zdd� Zdodd �Zdpd d �Zdqd d �Zdrdd�Z dsdd�Z dtdd�Z dudd�Z dd� Z ddddejjddddddf dd�Zdeddfdd �Zddedfd d �Zdvd d �Zdwd!d"�Zdxd#d$�Zdyd%d&�Zdzd'd(�Zd{d)d*�Zd+d,� Zd|d-d.�Z d}d/d0�Z d~d1d2�Z dd3d4�Z d�d5d6�Z d�d7d8�Z!d9d:� Z"d;d<� Z#d=d>� Z$d?d@� Z%d�dAdB�Z&d�dCdD�Z'd�dEdF�Z(d�dGdH�Z)dIdJ� Z*dKdL� Z+dMdN� Z,dOdP� Z-dQdR� Z.d�dSdT�Z/d�dUdV�Z0d�dWdX�Z1d�dYdZ�Z2d�d\d]�Z3d^d_� Z4d`da� Z5dbdc� Z6ddde� Z7dfdg� Z8dhdi� Z9djdk� Z:dldm� Z;� Z<S )��BotoTranslationaA Boto-based XML translation implementation of gsutil Cloud API. This class takes gsutil Cloud API objects, translates them to XML service calls, and translates the results back into gsutil Cloud API objects for use by the caller. This class does not support encryption and ignores encryption and decryption parameters. Behavior when encountering encrypted objects is undefined. TODO: Implement support. This class does not support handling a Requester Pays user project for billing, and any given user project will be ignored. TODO: Support user_project. Nr c sz t t| �j|||||||| d� |} t�, t� ttt�r@dandt_W d � n1 sZ0 Y t j � ddd�| _ dS )a� Performs necessary setup for interacting with the cloud storage provider. Args: bucket_storage_uri_class: boto storage_uri class, used by APIs that provide boto translation or mocking. logger: logging.logger for outputting log messages. status_queue: Queue for relaying status to UI. provider: Provider prefix describing cloud storage provider to connect to. 'gs' and 's3' are supported. Function implementations ignore the provider argument and use this one instead. credentials: Unused. debug: Debug level for the API implementation (0..3). http_headers (dict|None): Arbitrary headers to be included in every request. trace_token: Unused in this subclass. perf_trace_token: Performance trace token to use when making API calls ('gs' provider only). user_project: Unused in this subclass )�provider�debug� http_headers� trace_token�perf_trace_tokenT� N�GSUtil�default_api_version�1) �superrh ra rV r' � isinstancerX �bool�value�botor � get_value� api_version) r^ �bucket_storage_uri_class�logger� status_queueri � credentialsrj rk rl rm � user_project�_�� __class__rQ rR ra � s$ � $ �zBotoTranslation.__init__c C s dS )z*Service account credentials unused for S3.NrQ �r^ rQ rQ rR �GetServiceAccountId� s z#BotoTranslation.GetServiceAccountIdc C sh |}| � |�}| �� }z| j|jd|d�|d�W S tyb } z| j||d� W Y d}~n d}~0 0 dS )�,See CloudApi class for function doc strings.T��validate�headers��fields�� bucket_nameN)�_StorageUriForBucket�_CreateBaseHeaders�_BotoBucketToBucket� get_bucket� TRANSLATABLE_BOTO_EXCEPTIONS�_TranslateExceptionAndRaise)r^ r� ri r� r~ � bucket_urir� �erQ rQ rR � GetBucket� s ��zBotoTranslation.GetBucketc c s� |}t |d�}| �� }| jdkr,t|�|t< zbtjd| j d| j| jd�}|j |d�}|D ]0} | jdkrz| j � � | j krzqZ| j | |d�V qZW n. t y� } z| �| � W Y d } ~ n d } ~ 0 0 d S ) r� �� list_fields�gsz%s://F��suppress_consec_slashesry rj �r� �s3r� N)r* r� ri r" r! rv � storage_uriry rj �get_all_buckets�name�lowerr� r� r� ) r^ � project_idri r� r~ � get_fieldsr� Z provider_uriZ buckets_iterrN r� rQ rQ rR � ListBuckets� s& � zBotoTranslation.ListBucketsc C sv |}| � |�} | �� } | �|| � �z|jrJt�|j�} | j| �� | d� |r|| �� } || vrjt d| ��| j || j | d� |r�| �� } || vr�t d| ��| j || j | d� |j r�|j tkr�g |_ t�|j �} | j| d| d� |j�rt�|j�} | j| �� | d� |j�rjt�|j�}z| �� j|| d� W n8 tjj�yh } z|j dk�rT� W Y d}~n d}~0 0 |j �r�t � |j �}| j |d| d� |j!�r�|j!j"�r�|j!j#�r�| j$|j!j"|j!j#d| d� n| j%d| d� |j&�r�| j'|j&| d� |j(�r| j)|j(j*| d� |j+�r.|j+j,}|j+j-}| j.||| d� | j/||d�W S t0�yp } z| j1||d� W Y d}~n d}~0 0 dS )r� r� �Invalid canned ACL "%s".F�� Nr� r� )2r� r� �_AddPreconditionsToHeaders�aclr7 �BotoAclFromMessage� set_xml_acl�to_xml� canned_aclsr �set_acl� object_name� set_def_acl�corsrA r9 ZBotoCorsFromMessage�set_cors�defaultObjectAcl�set_def_xml_acl�labelsr? ZBotoTagsFromMessager� �set_tagsrv � exception�GSResponseError�status� lifecycler@ ZBotoLifecycleFromMessage�configure_lifecycle�logging� logBucket�logObjectPrefix�enable_logging�disable_logging� storageClass�set_storage_class� versioning�configure_versioning�enabled�website�mainPageSuffix� notFoundPage�set_website_configr� r� r� )r^ r� �metadata� canned_acl�canned_def_acl� preconditionsri r� r~ r� r� �boto_aclr� � boto_cors� boto_tagsr� �boto_lifecycle�main_page_suffixZ error_pagerQ rQ rR � PatchBucket s� � � � ��zBotoTranslation.PatchBucketc C s |}| � |�}d}|r&|jr&|j�� }| �� } |jdkr�t|�| t< d} |rX|jrX|j} |rz|jrz|jj rzt |jj �| d< z|j | || d� W n2 t y� } z| j | |d� W Y d} ~ n d} ~ 0 0 nHz|j | |d� W n4 t �y } z| j | |d� W Y d} ~ n d} ~ 0 0 | j||d�S ) r� � r� z x-goog-bucket-retention-period)r� �location� storage_classr� N)r� r� r� )r� r� r� r� �schemer" r! r� �retentionPolicy�retentionPeriod�str� create_bucketr� r� r� ) r^ r� r� r� ri r� r~ r� r� r� r� r� rQ rQ rR � CreateBucketd s: ��� &$z BotoTranslation.CreateBucketc C s ||f}| � |�}| �� }z|j|d� W n� t�y } z�| j||d�}|r�d|jv r�zN|�� r�| jdkr~td| |j d��td| |j d��ntd| |j d��W n2 ty� } z| j | |d� W Y d } ~ n d } ~ 0 0 n*|r�|j d kr�t d | ��n| j ||d� W Y d }~n d }~0 0 d S ) r� r� r� �BucketNotEmptyr� z�VersionedBucketNotEmpty (%s). Currently, gsutil does not support listing or removing S3 DeleteMarkers, so you may need to delete these using another tool to successfully delete this bucket.�r� z VersionedBucketNotEmpty (%s)�BucketNotEmpty (%s)N� zBucket %s does not exist.) r� r� � delete_bucketr� �_TranslateBotoException�reason�get_versioning_configri r r� r� r ) r^ r� r� ri r~ r� r� r� �translated_exception�e2rQ rQ rR � DeleteBucket� sD �� ���� &z BotoTranslation.DeleteBucketc c s� |}t |d�}| �|�} | �� } |du p.d|v } |du pHtdd� |D ��} z | j|pVd|p\d|| d�} W n2 ty� } z| j||d� W Y d}~n d}~0 0 z�| D ]�}| r�t|t�r�t � |j t j j �V q�| r�|}tg d ��}|r�|�r2|�|��s2d}t|d d��r |j}t|d d��r |j}| j||j |d �}| j||d �}t � |t j j�V q�W n4 t�y� } z| j||d� W Y d}~n d}~0 0 dS )r� r� N�prefixesc s s | ]}|� d �V qdS )zitems/N�� startswith��.0�fieldrQ rQ rR � <genexpr>� s z.BotoTranslation.ListObjects.<locals>.<genexpr>r� )�prefix� delimiter� all_versionsr� r� )rN �etagr� �updated� generation�metageneration�sizer� � version_id�r� r� )r* r� r� �any� list_bucketr� r� rs r r �CsObjectOrPrefixr� �CsObjectOrPrefixType�PREFIX�set�issubset�getattrr� r� � _GetBotoKey�_BotoKeyToObject�OBJECT)r^ r� r� r� r� ri r� r~ r� r� r� Zyield_prefixesZ yield_objectsZ objects_iterr� rP Zkey_to_convertZkey_http_fieldsr� Z return_objectrQ rQ rR � ListObjects� sZ � � $� ���� zBotoTranslation.ListObjectsc C s\ |}z | j | j|||d�|d�W S tyV } z | j||||d� W Y d}~n d}~0 0 dS )r� r� r� �r� r� r� N)r� r� r� r� )r^ r� r� r� ri r� r~ r� rQ rQ rR �GetObjectMetadata� s ���z!BotoTranslation.GetObjectMetadatac s � fdd�S )aA Curries a digester object into a form consumable by boto. Key instantiates its own digesters by calling hash_algs[alg]() [note there are no arguments to this function]. So in order to pass in our caught-up digesters during a resumable download, we need to pass the digester object but don't get to look it up based on the algorithm name. Here we use a lambda to make lookup implicit. Args: digester_object: Input object to be returned by the created function. Returns: A function which when called will return the input object. c s � S r[ rQ rQ ��digester_objectrQ rR �<lambda> � z0BotoTranslation._CurryDigester.<locals>.<lambda>rQ )r^ r rQ r rR �_CurryDigester s z BotoTranslation._CurryDigesterFc C s$ | � � }t||d� | dur.d| | f |d< n*| dkrDd| |d< n| dk rXd| |d< | r�t�| �}t�t�|d ��}| jd kr�t|� n| j |||d �}| r�| jd kr�i }| D ]}| � | | �||< q�ni }|p�d}| r�t�| �d }|�rt t |�t t�}nt}z`|tjju �r6| j|| | ||| ||d � n2|tjju �r\| j||| |||d � n td| ��W n8 t�y� } z | j||||d� W Y d}~n d}~0 0 | jdk�r | �r G dd� dt�}| D ]T}|tjjk�r�| dk�s t|dd��r ||jv �s d| |< n||j| �| |< �q�dS )r� )�compressed_encodingNz bytes=%s-%s�ranger z bytes=%s-zbytes=%s�urlr� r� rb )r� r` � num_callbacks� hash_algs)�progress_callback�num_progress_callbacksr� r z Unsupported DownloadStrategy: %sr r� c @ s e Zd ZdZdd� Zdd� ZdS )z6BotoTranslation.GetObjectMedia.<locals>.HashToDigestera Wrapper class to expose hash digests. boto creates its own digesters in s3's get_file, returning on-the-fly hashes only by way of key.local_hashes. To propagate the digest back to the caller, this stub class implements the digest() function. c S s || _ d S r[ ��hash_val)r^ r rQ rQ rR ra r s z?BotoTranslation.GetObjectMedia.<locals>.HashToDigester.__init__c S s | j S r[ r r� rQ rQ rR �digestu s z=BotoTranslation.GetObjectMedia.<locals>.HashToDigester.digestN)rd re rf rg ra r rQ rQ rQ rR �HashToDigesterj s r � local_hashes)r� r4 �json�loads�pickle�binascii� a2b_base64ri rS r� r �max�intrD r1 r �DownloadStrategy� RESUMABLE�_PerformResumableDownload�ONE_SHOT�_PerformSimpleDownloadr r� r� �objectr� r )r^ r� r� �download_streamri r� � object_sizer �download_strategyr_ �end_byter �serialization_data� digesters�decryption_tupler� Zserialization_dictrP r �algrb r r� r Zalg_namerQ rQ rR �GetObjectMedia s� � ���� � � �� z BotoTranslation.GetObjectMediac C sB z|j |||||d� W n$ ty< |j ||||d� Y n0 d S )N)�cb�num_cbr� r )r* r+ r� )�get_contents_to_file� TypeError)r^ r! rP r r r� r rQ rQ rR r � s � �z&BotoTranslation._PerformSimpleDownloadc C sF t jttjtjf} |jjj} t � } d} |} z�t ||�}|� � }d|v rP|� d� d||f |d< z |j |||j|d|d� W n( ty� |j |||j|dd� Y n0 |�� W dS | �yH } z|| dkr�| j�d t|�� t|t��r4|jtjk�r4| jd k�r |j |||j|dd� n|j |||j|d|d� W Y d}~njd}~0 tjj�y� } zD|jtjj j k�rzt|j ��n| dk�r�| j�d |j � W Y d}~n d}~0 0 |� � }|| k�r�|} d} n| d7 } | | k�r�td ��z |� � W n t j!�y Y n0 t"t#�#� d | t$� �}| dk�r6| j�d| |� t%�&|� q*dS )a\ Downloads bytes from key to fp, resuming as needed. Args: fp: File pointer into which data should be downloaded. start_byte: Start byte of the download. end_byte: End byte of the download. key: Key object from which data is to be downloaded headers: Headers to send when retrieving the file callback: (optional) a callback function that will be called to report progress on the download. The callback should accept two integer parameters. The first integer represents the number of bytes that have been successfully transmitted from the service. The second represents the total number of bytes that need to be transmitted. num_callbacks: (optional) If a callback is specified with the callback parameter, this determines the granularity of the callback by defining the maximum number of times the callback will be called during the file transfer. hash_algs: Dict of hash algorithms to apply to downloaded bytes. Raises: ResumableDownloadException on error. r r z bytes=%d-%d�Range)�override_num_retriesr )r/ Nrn zCaught exception (%s)r� zBCaught boto.exception.ResumableDownloadException (%s) - will retryzeToo many resumable download attempts failed without progress. You might try this download again later� zWGot retryable failure (%d progress-less in a row). Sleeping %d seconds before re-trying)'r � HTTPException�IOError�socket�error�gaierrorrN rO rj r) rZ �copy�pop�get_filerc r- �flushrz �info�reprrs �errno�EPIPEri rv r� r � disposition� ResumableTransferDisposition�ABORT_CUR_PROCESS�message�tell�close�IncompleteRead�min�randomr( �time�sleep)r^ �fpr_ r$ rP r� r` r r Zretryable_exceptionsrj � num_retriesZprogress_less_iterationsZlast_progress_byteZ cb_handlerr� Zsleep_time_secsrQ rQ rR r � s� � � � ��� � � � �z)BotoTranslation._PerformResumableDownloadc C sv |} | j |||d�} | �� } t|| j�} i } t� }d}t�| �D ](\}}d}|du r`|�|� q@|| |< q@| �|| � |r�z| j | |d| d� W n6 t y� } z | j ||||d� W Y d}~n d}~0 0 |j �r.t �|j �}z| j|�� || d� W n8 t �y, } z | j ||||d� W Y d}~n d}~0 0 |�rd| �� }||v�rRtd| ��| j|| j| d� | j||||d �S ) r� r� FTNr� r )�key_namer� r� )r� r� )�_StorageUriForObjectr� r> ri r� �six� iteritems�addr� � set_metadatar� r� r� r7 r� r� r� r� r r� r� r )r^ r� r� r� r� r� r� ri r� r~ � object_urir� Z meta_headers� metadata_plus�metadata_minusZmetadata_changed�k�vr� r� r� rQ rQ rR �PatchObjectMetadata sh � � � � � �z#BotoTranslation.PatchObjectMetadatac C s |j |||||dd� d S )NT)�md5�policyr* r� �rewind)�set_contents_from_file)r^ �dst_uri� upload_streamrW r� r r� rQ rQ rR �_PerformSimpleUpload^ s �z$BotoTranslation._PerformSimpleUploadc C s� |� � �� r |j||||d� n�t�� \}}z�t|d��4}|�t�} | r^|�| � |�t�} qDW d � n1 sr0 Y t|d�� } |j | ||d� W d � n1 s�0 Y W t � |� t � |� nt � |� t � |� 0 d S )N)rX r* r� �wb�rb)rX r� ) � get_provider�supports_chunked_transfer�set_contents_from_stream�tempfile�mkstemp�open�readr, �writerZ �osrC �unlink) r^ r[ r\ r� r r� Ztemp_fhZ temp_pathZout_fpZ stream_bytesZin_fprQ rQ rR �_PerformStreamingUploadl s, � * �& � z'BotoTranslation._PerformStreamingUploadc C s* t || j|d�} | j||||||d� d S )N)Z resume_url)r� r* r� )r rz ZSendFile) r^ rP r\ Z upload_size�tracker_callbackr� r% r r� Zresumable_uploadrQ rQ rR �_PerformResumableUpload� s ��z'BotoTranslation._PerformResumableUploadc C s� t |� | �� }|�t|| j�� |jrbd|v rN|d d|j�d� 7 < nd|j�d� |d< |jr�d|v r�|d d|j�d� 7 < nd|j�d� |d< d|v r�|d s�d|d< | �||� | � |j |j �}||fS ) a Shared upload implementation. Args: object_metadata: Object metadata describing destination object. preconditions: Optional gsutil Cloud API preconditions. Returns: Headers dictionary, StorageUri for upload (based on inputs) z x-goog-hashz ,crc32c=%s� z crc32c=%sz,md5=%szmd5=%s� content-typezapplication/octet-stream) r+ r� �updater> ri �crc32c�rstrip�md5Hashr� rL rN r� )r^ �object_metadatar� r� r[ rQ rQ rR � _UploadSetup� s4 � � � � �z BotoTranslation._UploadSetupc C s� z,| � ||� | �� }| j|j|d�|d�W S tjjy� } zB|jrtt� |j� t ��rtt d� t�d|j ����n� W Y d}~n d}~0 0 dS )a� Set ACLs on an uploaded object and return its metadata. Args: dst_uri: Generation-specific StorageUri describing the object. object_metadata: Metadata for the object, including an ACL if applicable. fields: If present, return only these Object metadata fields. Returns: gsutil Cloud API Object metadata. Raises: CommandException if the object was overwritten / deleted concurrently. r� r� rm z�Uploaded object (%s) was deleted or overwritten immediately after it was uploaded. This can happen if you attempt to upload to the same object multiple times concurrently.N)� _SetObjectAclr� r� �get_keyrv r� �InvalidUriErrorrA �NON_EXISTENT_OBJECT_REGEX�match�encoder0 r �join�textwrap�wrap�uri)r^ r[ rs r� r� r� rQ rQ rR �_HandleSuccessfulUpload� s ����z'BotoTranslation._HandleSuccessfulUploadc C sV | � � }|jr.t�|j�}|j|�� |d� n$| jdkrRt|�}|rR|j||d� dS )zCSets the ACL (if present in object_metadata) on an uploaded object.r� r� N)r� r� r7 r� r� r� ri rB )r^ rs r[ r� r� �s3_aclrQ rQ rR ru � s z BotoTranslation._SetObjectAclc C s� | r t d��| jdkr,| j||||| |d�S | j||d�\} }|sNtd| ��z4| j|j| d�||||||| d� | j||| d�W S ty� } z0t | j|j �}| j ||j |j |d � W Y d }~n d }~0 0 d S ) r� z.XML API does not support gzip-encoded uploads.r� )r� r� r� r� �r� z;No tracker callback function set for resumable upload of %sr� )r� r% r r� r� �r� r� �not_found_exceptionN) �NotImplementedErrorri � UploadObjectrt r rl �new_keyr r� r; rN r� r� )r^ r\ rs r� r� r� r% rk r �encryption_tupleri r� � gzip_encodedr� r[ r� r� rQ rQ rR �UploadObjectResumable� sT � �� �����z%BotoTranslation.UploadObjectResumablec C s� | r t d��| j||d�\} } z&| j| |||| d� | j| ||d�W S ty� } z0t| j|j�} | j| |j|j | d� W Y d} ~ n d} ~ 0 0 dS )r� �-XML API does not suport gzip-encoded uploads.r� )r� r r� r� r� N) r� rt rj r r� r; ri rN r� r� )r^ r\ rs r� r r� r� ri r� r� r� r[ r� r� rQ rQ rR �UploadObjectStreaming! s2 � ����z%BotoTranslation.UploadObjectStreamingc C s� | r t d��| j||d�\} } zHd} |jr@t|j�|j�d�g} | j| || ||| d� | j| || d�W S ty� } z0t| j |j �}| j ||j |j |d� W Y d}~n d}~0 0 dS )r� r� r� Nz "')rW r� r r� r� r� ) r� rt rr r3 �stripr] r r� r; ri rN r� r� )r^ r\ rs r� r� r� r r� ri r� r� r� r[ rW r� r� rQ rQ rR r� B s> � �����z BotoTranslation.UploadObjectc C sv |}| � � }| �||� | j|||d�}z|jd|d� W n6 typ } z | j| |||d� W Y d} ~ n d} ~ 0 0 dS )r� r� Fr� r N)r� r� rL � delete_keyr� r� ) r^ r� r� r� r� ri r~ r� r~ r� rQ rQ rR � DeleteObjectm s ��z BotoTranslation.DeleteObjectc C s0 | } |durt d��| �|j|j�} d}| jdkr:|}d}| �� }|�t|| j�� | �||� |rr||| � � j < |j r|dnd}| jdkr�t |�}|r�d}|j p�d}z,| j|j|j|||||d�}| j|| d�W S t�y* } z@t| j|j| j|j|j|d�}| j||j|j|d � W Y d}~n d}~0 0 dS ) r� Nz*XML API does not suport max_bytes_per_callr� TF)� preserve_aclr� r� �src_version_id�src_generationr� )Z src_provider�src_bucket_nameZsrc_object_namer� r� )r� rL rN r� ri r� ro r> r� r` � acl_headerr� rB r� �copy_keyr� r� r; r� )r^ �src_obj_metadata�dst_obj_metadatar� r� r� r �max_bytes_per_callr� r' ri r� r~ r[ r� r� r� r� Zdst_storage_classr� r� r� rQ rQ rR � CopyObject� sZ � ���zBotoTranslation.CopyObjectc C s� |}t |� |j}d|_|j} d|_| �� } | �t|| j�� | �|| � |js`t |_|j| d< | � | |�} g } |D ]"} | j | | j| j d�}| � |� qtz | j | | d� | j| ||d�W S ty� } z | �||j|j� W Y d}~n d}~0 0 dS )r� Nrn r� r� r� )r+ r� rN r� ro r> ri r� � contentTyper= rL r� �append�composer r� r� )r^ �src_objs_metadatar� r� r� ri r� r~ Z dst_obj_nameZdst_bucket_namer� r[ Zsrc_componentsZsrc_objZsrc_urir� rQ rQ rR � ComposeObject� s< � � �z BotoTranslation.ComposeObjectc C sB |r>| j dkr>|jdur&t|j�|d< |jdur>t|j�|d< dS )z'Adds preconditions (if any) to headers.r� Nzx-goog-if-generation-matchz x-goog-if-metageneration-match)ri � gen_matchr� �meta_gen_match)r^ r� r� rQ rQ rR r� � s �z*BotoTranslation._AddPreconditionsToHeadersc C s` i }| j r|�| j � | jdkr*| j|d< | jdkrD| jrD| j|d< tj�t�}|r\||t < |S )z:Creates base headers used for all API calls in this class.r� zx-goog-api-version�cookie) rk ro ri rx rm rh �environrL r- r. )r^ Z base_headersZrequest_reasonrQ rQ rR r� � s z"BotoTranslation._CreateBaseHeadersc C s |r t �|�r |�d��� S dS )z�Returns an MD5 from the etag iff the etag is a valid MD5 hash. Args: src_etag: Object etag for which to return the MD5. Returns: MD5 in hex string format, or None. �"'N)� MD5_REGEX�searchr� r� )r^ Zsrc_etagrQ rQ rR �_GetMD5FromETag s z BotoTranslation._GetMD5FromETagc C s" t jd| j|f d| j| jdd�S )z�Returns a boto storage_uri for the given bucket name. Args: bucket: Bucket name (string). Returns: Boto storage_uri for the bucket. z%s://%sF)r� ry rj r� )rv r� ri ry rj )r^ rN rQ rQ rR r� s �z$BotoTranslation._StorageUriForBucketc C s6 d| j ||f }|r |d| 7 }tj|d| j| jd�S )a8 Returns a boto storage_uri for the given object. Args: bucket: Bucket name (string). object_name: Object name (string). generation: Generation or version_id of object. If None, live version of the object is used. Returns: Boto storage_uri for the object. z %s://%s/%sz#%sFr� )ri rv r� ry rj )r^ rN r� r� Z uri_stringrQ rQ rR rL ) s �z$BotoTranslation._StorageUriForObjectc C s~ | j |||d�}z2| �� }|j|d�}|s>td| j|||d��|W S tyx } z | j||||d� W Y d}~n d}~0 0 dS )z�Gets the boto key for an object. Args: bucket_name: Bucket containing the object. object_name: Object name. generation: Generation or version of the object to retrieve. Returns: Boto key for the object. r� r� Z404r N)rL r� rv r< ri r� r� )r^ r� r� r� rQ r� rP r� rQ rQ rR r� > s* � ��zBotoTranslation._GetBotoKeyc C sh | � |j�}tj|j|jd�}| �� }| jdk�rB|r>d|v rRt|d�rR|�� |_|r^d|v r�z(t � |j |d��D ]}|j � |� qrW nT ty� } z<| j||jd�}|r�t|t�r�n| j||jd� W Y d}~n d}~0 0 |r�d |v �r>z |j|d�} t�| �|_W n6 t�y< } z | j||jd� W Y d}~n d}~0 0 |�rNd |v �r�z*t �|j|d��D ]}|j� |� �qbW nZ t�y� } z@| j||jd�}|�r�t|t��r�n| j||jd� W Y d}~n d}~0 0 |�r�d |v �rHz,|j|d�j} | �rtj� � |_ | |j _ W n6 t�yF } z | j||jd� W Y d}~n d}~0 0 |�rXd |v �r�z |j |d�} t �!| �|_"W n6 t�y� } z | j||jd� W Y d}~n d}~0 0 |�r�d |v �rhzt|j#|d�} | �r.d| v �r.| d } d| v }d| v }|�s�|�r.tj�$� |_%|�r | d |j%_&|�r.| d |j%_'W n6 t�yf } z | j||jd� W Y d}~n d}~0 0 |�rxd|v �r$zt|j(|d�}|�r�d|v �r�|d }d|v }d|v }|�s�|�r�tj�)� |_*|�r�|d |j*_+|�r�|d |j*_,W n6 t�y" } z | j||jd� W Y d}~n d}~0 0 |�r4d|v �rB|j-|d�|_.|�rRd|v �r�zfz |�/� �0� }t1�2|�|_3W nD t4j5j6�y� } z&| jdk�r�|j7dk�s�� W Y d}~n d}~0 0 W n6 t�y� } z | j||jd� W Y d}~n d}~0 0 |�r d|v �rd|j8|d�}|�rd| jdk�rHd|v �rH|d dk�rHtjj9d d �|_:n | jdk�rdtjj9d d �|_:|S ) z�Constructs an apitools Bucket from a boto bucket. Args: bucket: Boto bucket. fields: If present, construct the apitools Bucket with only this set of metadata fields. Returns: apitools Bucket. )r� �idr� r� �get_storage_classr� r� r� Nr� r� � encryptionr� r� ZLoggingZLogObjectPrefixZ LogBucketr� �WebsiteConfigurationZMainPageSuffixZ NotFoundPager� r� r� r� r� � Versioning�EnabledT)r� );r� r� �apitools_messages�Bucketr� ri �hasattrr� r� r7 ZBotoBucketAclToMessage�get_aclr� r� r� r� rs r r� �get_corsr9 ZBotoCorsToMessager� �BotoObjectAclToMessage� get_def_aclr� �get_encryption_config�default_kms_key_name�EncryptionValuer� �defaultKmsKeyName�get_lifecycle_configr@ ZBotoLifecycleToMessager� �get_logging_config� LoggingValuer� r� r� �get_website_config� WebsiteValuer� r� r� � get_locationr� r� �get_tagsr? ZBotoTagsToMessager� rv r� �StorageResponseErrorr� r� �VersioningValuer� )r^ rN r� r� Zcloud_api_bucketr� r� r� r� r� Zkeynamer� Z boto_loggingZlogging_configZlog_object_prefix_presentZlog_bucket_presentZ boto_websiteZwebsite_configZmain_page_suffix_presentZnot_found_page_presentr� r� rQ rQ rR r� ] s� � ���& & ���&�� & �& �& �� & � & � � �z#BotoTranslation._BotoBucketToBucketc C s� d}|r&d|v s&t dd� |D ��dkr0| �|�}d}|r@d|v rLt|dd�}d}|r\d|v rht|d d�}d}|rxd |v r�t|d d�}| �|�}d}|r�d |v r�| �|�}d} |r�d |v r�| �|�} d} |r�d|v r�t|dd�} | r�| �d�} d} |r�d|v �r,t|d��r,d|jv �r,t � |jd �� d�} d} |�r@d|v �r�t|d��rpd|jv �rpt � |jd �� d�} nD| � t|dd���r�t | � |j��} n | jdk�r�| j�d|j|� d} |�r�d|v �r�t�t�|tj��} d}|�r�d|v �r�|j�p�d}d}|�rd|v �r t|dd�}tj�r\| �r@t| t��r@| � d �} | �r\t| t��r\| � d �} t j |j j |j ||j!|j"|j#||| | | |||| || |d �}| �$||� |�r�d |v �r�t%t&| j�|�}| j'|||d � |S ) a Constructs an apitools Object from a boto key. Args: key: Boto key to construct Object from. fields: If present, construct the apitools Object with only this set of metadata fields. Returns: apitools Object corresponding to key. Nr� c S s g | ]}|� d �r|�qS )z metadata/r� r� rQ rQ rR � <listcomp>� r z4BotoTranslation._BotoKeyToObject.<locals>.<listcomp>rn � cacheControl� cache_control�componentCount�component_count�contentDisposition�content_dispositionr� � timeCreatedr� r� rp � cloud_hashes� rr rW r� zMNon-MD5 etag (%s) present for key %s, data integrity checks are not possible.� mediaLinkr� r r� �_storage_class�ascii)rN r� r� �contentEncoding�contentLanguager� r� r� r� rp rr r� r� r� r� r� r� r� r� r� )(�len� _TranslateBotoKeyCustomMetadatar� �_TranslateBotoKeyGeneration� _TranslateBotoKeyMetageneration�_TranslateBotoKeyTimestampr� r� r� �base64� b64encoderq r� r2 r� ri rz �warnr � b2a_base64r �dumps�HIGHEST_PROTOCOLr� rM �PY3rs �bytes�decoder� �ObjectrN r� �content_encoding�content_language� content_type�_TranslateDeleteMarkerr# r$ �_TranslateBotoKeyAcl)r^ rP r� �custom_metadatar� r� r� r� r� Z time_createdr� rp Zmd5_hashZ media_linkr� r� �cloud_api_object�generation_strrQ rQ rR r� � s� �� � � � ��z BotoTranslation._BotoKeyToObjectc C s` d}t |dd�r\tjjg d�}t�|j�D ]0\}}|�� dkr@q*|j� tjjj ||d�� q*|S )zCPopulates an apitools message from custom metadata in the boto key.Nr� )�additionalPropertieszcontent-language�rP ru ) r� r� r� � MetadataValuerM rN r� r� r� r� �AdditionalProperty)r^ rP r� rT rU rQ rQ rR r� [ s � ��z/BotoTranslation._TranslateBotoKeyCustomMetadatac C sJ d}| j dkr&t|dd�rFt|j�}n | j dkrFt|dd�rFt|j�}|S )zFReturns the generation/version_id number from the boto key if present.Nr� r� r� r� )ri r� �longr� r6 r� )r^ rP r� rQ rQ rR r� j s z+BotoTranslation._TranslateBotoKeyGenerationc C s( d}| j dkr$t|dd�r$t|j�}|S )z?Returns the metageneration number from the boto key if present.Nr� r� )ri r� r� r� )r^ rP r� rQ rQ rR r� u s z/BotoTranslation._TranslateBotoKeyMetagenerationc C s� |j r�d|j v r"|j �d�d }n|j �d�d }d}ztj�||�W S ty� zd}tj�|j |�W Y S ty� Y Y dS 0 Y n0 dS )z�Parses the timestamp from the boto key into an datetime object. This avoids a dependency on dateutil. Args: key: Boto key to get timestamp from. Returns: datetime object if string is parsed successfully, None otherwise. �.�ZZ000Zz.000000Zz%Y-%m-%dT%H:%M:%S.%fZz%a, %d %b %Y %H:%M:%S %ZN)� last_modifiedrq �datetime�strptime� ValueError)r^ rP Zkey_us_timestamp�fmtrQ rQ rR r� } s z*BotoTranslation._TranslateBotoKeyTimestampc C sD t |t�r@|js$tj�� |_g |j_|jj�tjjjt dd�� dS )zCMarks deleted objects with a metadata value (for S3 compatibility).Tr� N) rs r r� r� r� r� r� r� r� r/ )r^ rP r� rQ rQ rR r� � s ��z&BotoTranslation._TranslateDeleteMarkerc C s� | j |jj|j|d�}| �� }zV| jdkrR|j|d�}t�|�D ]}|j� |� q@| jdkrr|j |d�}t ||� W n6 t j jy� } z|jdkr�n� W Y d}~n d}~0 0 dS )z8Updates cloud_api_object with the ACL from the boto key.r� r� r� r� � N)rL rN r� r� ri r� r7 r� r� r� � get_xml_aclr8 rv r� r� r� ) r^ rP r� r� �storage_uri_for_keyr� Zkey_aclr� r� rQ rQ rR r� � s" � z$BotoTranslation._TranslateBotoKeyAclc C s$ | j |||||d�}|r |�n� dS )a Translates a Boto exception and raises the translated or original value. Args: e: Any Exception. bucket_name: Optional bucket name in request that caused the exception. object_name: Optional object name in request that caused the exception. generation: Optional generation in request that caused the exception. not_found_exception: Optional exception to raise in the not-found case. Raises: Translated CloudApi exception, or the original exception if it was not translatable. )r� r� r� r� N)r� )r^ r� r� r� r� r� r� rQ rQ rR r� � s �z+BotoTranslation._TranslateExceptionAndRaisec C s t |tjj��r |jdkr.t|j|j|jd�S |jdksB|jdkrVt|j|j|jd�S |jdkr�|rvt |d|j� |S |r�|r�t |j| j |||d�S t |j| j |�S t |j|j|jd�S |jdkr�|jr�d |jv r�td | |j|jd�S |jd k� r�t|j�S |jd k�r t|j|j|jd�S t |tjj��r@td |j|jd�S t |tjj��rdt|j|j|jd�S t |tjj��r�|j�r�t�|j��r�t |jdd�S t|j�S t |tjj��r�|jtjjj k�r�t |j�S |jtjjj k�r�t|j�S t|j�S t |tjj ��rt |j�S dS )aX Translates boto exceptions into their gsutil Cloud API equivalents. Args: e: Any exception in TRANSLATABLE_BOTO_EXCEPTIONS. bucket_name: Optional bucket name in request that caused the exception. object_name: Optional object name in request that caused the exception. generation: Optional generation in request that caused the exception. not_found_exception: Optional exception to raise in the not-found case. Returns: ServiceException for translatable exceptions, None otherwise. Because we're using isinstance, check for subtypes first. i� )r� �bodyi� r� r� r� r� i� r� r� i� i� zBucket already exists.r� N) rs rv r� r� r� r �coder� r rM r< ri r: r rA r r r �StorageCreateErrorr �BotoServerErrorrw rx ry r r r> r? �ABORTr � START_OVERr )r^ r� r� r� r� r� rQ rQ rR r� � sh � �� � � z'BotoTranslation._TranslateBotoExceptionc C sh z4t j|jd| j| jd�}|r(|�� W S |�� W S W n. tyb } z| �|� W Y d}~n d}~0 0 dS ��5See CloudApiDelegator class for function doc strings.Fr� N) rv r� � url_stringry rj r� r� r� r� )r^ � storage_url� def_obj_aclr~ r� rQ rQ rR �XmlPassThroughGetAcl/ s � z$BotoTranslation.XmlPassThroughGetAclTc C s� | � � }z�tj|jd| j| jd�}|r�|rZ|�� }||vrFtd| ��|j||j |d� q�|�� }||vrvtd| ��|j ||j |d� n*|r�|j ||j |d� n|j ||j |d� W n. t y� } z| �|� W Y d}~n d}~0 0 dS )r� Fr� r� r� N)r� rv r� r� ry rj r� r r� r� r� r� r� r� r� ) r^ �acl_textr� �cannedr� r� r~ r� r� rQ rQ rR �XmlPassThroughSetAcl> s. � z$BotoTranslation.XmlPassThroughSetAclc C s� | � � }|jdkrt� }nt� }t�|d�}ztj�||� W nD t j y� } z*t d|� � |� � |�� f ��W Y d}~n d}~0 0 z,tj|jd| j| jd�}|j|d|d� W n. ty� } z| �|� W Y d}~n d}~0 0 dS )r� r� Nz3Requested CORS is invalid: %s at line %s, column %sFr� r� )r� r� �S3Corsr r � XmlHandler�xml�saxr � SaxExceptions�SAXParseExceptionr � getMessage� getLineNumber�getColumnNumberrv r� r� ry rj r� r� r� )r^ � cors_textr� r� Zcors_obj�hr� r~ rQ rQ rR �XmlPassThroughSetCors_ s. ���z%BotoTranslation.XmlPassThroughSetCorsc C s� | � � }tj|jd| j| jd�}z|jd|d�}W n. ty` } z| �|� W Y d}~n d}~0 0 t j j � |� � �t��}|jdd�S �r� Fr� r� Nz )�indent)r� rv r� r� ry rj r� r� r� r �dom�minidomr r� rz r0 � toprettyxml)r^ r� r� r~ r� r� � parsed_xmlrQ rQ rR �XmlPassThroughGetCorsz s � z%BotoTranslation.XmlPassThroughGetCorsc C s� | � � }z*tj|jd| j| jd�}|jd|d�}W n. ty` } z| �|� W Y d}~n d}~0 0 t j j � |� � �t��}|jdd�S r )r� rv r� r� ry rj r� r� r� r r r r r� rz r0 r )r^ r� r� r~ r� r� r rQ rQ rR �XmlPassThroughGetLifecycle� s � z*BotoTranslation.XmlPassThroughGetLifecyclec C s� | � � }|jdkrt� }nt� }t�|d�}ztj�||� W nD t j y� } z*t d|� � |� � |�� f ��W Y d}~n d}~0 0 z,tj|jd| j| jd�}|j|d|d� W n. ty� } z| �|� W Y d}~n d}~0 0 dS )r� r� Nz?Requested lifecycle config is invalid: %s at line %s, column %sFr� r� )r� r� � S3Lifecycler r r r r r r r r r r r rv r� r� ry rj r� r� r� )r^ �lifecycle_textr� r� Z lifecycle_objr r� r~ rQ rQ rR �XmlPassThroughSetLifecycle� s. ���z*BotoTranslation.XmlPassThroughSetLifecyclec C sp | � � }z,tj|jd| j| jd�}t|j|d��}W n. tyb } z| � |� W Y d}~n d}~0 0 t |�� � S �r� Fr� r� N) r� rv r� r� ry rj rC r� r� r� �XmlParseStringr )r^ r� r� r~ Zlogging_config_xmlr� rQ rQ rR �XmlPassThroughGetLogging� s � � z(BotoTranslation.XmlPassThroughGetLoggingc C sl | � � }z(tj|jd| j| jd�}|�� �� }W n. ty^ } z| � |� W Y d}~n d}~0 0 t |�� � S r� ) r� rv r� r� ry rj r� � get_xml_tagsr� r� r r )r^ r� r� r~ Ztagging_config_xmlr� rQ rQ rR �XmlPassThroughGetTagging� s � z(BotoTranslation.XmlPassThroughGetTaggingc C s� | � � }t� }t�|d�}ztj�||� W nD tjyp } z*t d|� � |� � |� � f ��W Y d}~n d}~0 0 z.t j|jd| j| jd�}|�� j||d� W n. ty� } z| �|� W Y d}~n d}~0 0 dS )r� NzDRequested labels/tagging config is invalid: %s at line %s, column %sFr� r� )r� r r r r r r r r r r r r rv r� r� ry rj r� r� r� r� )r^ Z tagging_textr� r� Ztags_objr r� r~ rQ rQ rR �XmlPassThroughSetTagging� s* ���z(BotoTranslation.XmlPassThroughSetTaggingc C sp | � � }z,tj|jd| j| jd�}t|j|d��}W n. tyb } z| � |� W Y d}~n d}~0 0 t |�� � S r ) r� rv r� r� ry rj rC r� r� r� r r )r^ r� r� r~ Zweb_config_xmlr� rQ rQ rR �XmlPassThroughGetWebsite� s � z(BotoTranslation.XmlPassThroughGetWebsite)NNr NNNN)NN)NNN)NNNNN)NNNN)NN)NNNNN)NNN)NNNNN)NNNN)NNN)NNNN)N)N) NNNNNNNNNF)NNNNNNF)NNNNNNNF)NNN) NNNNNNNNN)NNNN)N)N)N)N)N)NNNN)NNNN)F)TF)=rd re rf rg ra r� r� r� r� r� r� r r r r r r r) r1 r r rV r] rj rl rt r ru r� r� r� r� r� r� r� r� r� r� rL r� r� r� r� r� r� r� r� r� r� r� r r r r r r r r r r! � __classcell__rQ rQ r rR rh � sJ �1 � O � % ( � D � � v� � � C � � " � ' ! � 5 � $ � . � � I � + p � � S � !rh )�rg � __future__r r r r r� r r� r< r rh r rF �rer3 rc r| � threadingrG r Zxml.dom.minidomr r �xml.saxr r rM � six.movesr rv r r � boto.gs.corsr �boto.gs.lifecycler � boto.s3.corsr r �boto.s3.deletemarkerr �boto.s3.lifecycler r �boto.s3.prefixr �boto.s3.taggingr �boto.exception� boto.utilsZgslib.boto_resumable_uploadr �gslib.cloud_apir r r r r r r r r r r r Zgslib.devshell_auth_plugin�gslib�gslib.exceptionr r Zgslib.project_idr! r" �gslib.storage_urlr# r$ �"gslib.third_party.storage_apitoolsr% r� � gslib.utilsr& Zgslib.utils.boto_utilr' r( r) Z gslib.utils.cloud_api_helperr* r+ �gslib.utils.constantsr, r- r. r/ r0 r1 Zgslib.utils.hashing_helperr2 r3 �gslib.utils.metadata_utilr4 �&gslib.utils.parallelism_framework_utilr5 �gslib.utils.text_utilr6 Z gslib.utils.translation_helperr7 r8 r9 r: r; r< r= r> r? r@ rA rB rC �gslib.utils.unit_utilrD r� r r� r� r� rw r� r� r� �LockrV rX �compile�DOTALLrx r� rS rY r rZ rh rQ rQ rQ rR �<module> s� � � __pycache__/bucket_listing_ref.cpython-39.pyc 0000644 00000010354 15025013616 0015251 0 ustar 00 a +(Wg� � @ sx d Z ddlmZ ddlmZ ddlmZ ddlmZ G dd� de�ZG dd � d e�ZG d d � d e�Z G d d � d e�Z dS )z>Classes for cloud/file references yielded by gsutil iterators.� )�absolute_import)�print_function)�division)�unicode_literalsc @ sX e Zd ZdZG dd� de�Zedd� �Zedd� �Zdd � Z d d � Z d d � Z dd� Z dS )�BucketListingRefa Base class for a reference to one fully expanded iterator result. This allows polymorphic iteration over wildcard-iterated URLs. The reference contains a fully expanded URL string containing no wildcards and referring to exactly one entity (if a wildcard is contained, it is assumed this is part of the raw string and should never be treated as a wildcard). Each reference represents a Bucket, Object, or Prefix. For filesystem URLs, Objects represent files and Prefixes represent directories. The root_object member contains the underlying object as it was retrieved. It is populated by the calling iterator, which may only request certain fields to reduce the number of server requests. For filesystem URLs, root_object is not populated. c @ s e Zd ZdZdZdZdZdS )z&BucketListingRef._BucketListingRefTypez,Enum class for describing BucketListingRefs.�bucket�object�prefixN)�__name__� __module__� __qualname__�__doc__�BUCKET�OBJECT�PREFIX� r r �'/opt/gsutil/gslib/bucket_listing_ref.py�_BucketListingRefType) s r c C s | j S �N�� _url_string��selfr r r � url_string/ s zBucketListingRef.url_stringc C s | j S r )� _ref_typer r r r � type_name3 s zBucketListingRef.type_namec C s | j | jjkS r )r r r r r r r �IsBucket7 s zBucketListingRef.IsBucketc C s | j | jjkS r )r r r r r r r �IsObject: s zBucketListingRef.IsObjectc C s | j | jjkS r )r r r r r r r �IsPrefix= s zBucketListingRef.IsPrefixc C s | j S r r r r r r �__str__@ s zBucketListingRef.__str__N) r r r r r r �propertyr r r r r r r r r r r s r c s" e Zd ZdZd� fdd� Z� ZS )�BucketListingBucketz&BucketListingRef subclass for buckets.Nc s0 t t| ��� | jj| _|j| _|| _|| _ dS )z�Creates a BucketListingRef of type bucket. Args: storage_url: StorageUrl containing a bucket. root_object: Underlying object metadata, if available. N) �superr! �__init__r r r r r � storage_url� root_object�r r$ r% �� __class__r r r# G s z BucketListingBucket.__init__)N�r r r r r# � __classcell__r r r' r r! D s r! c s" e Zd ZdZd� fdd� Z� ZS )�BucketListingPrefixz'BucketListingRef subclass for prefixes.Nc s0 t t| ��� | jj| _|j| _|| _|| _ dS )z�Creates a BucketListingRef of type prefix. Args: storage_url: StorageUrl containing a prefix. root_object: Underlying object metadata, if available. N) r" r+ r# r r r r r r$ r% r& r' r r r# X s z BucketListingPrefix.__init__)Nr) r r r' r r+ U s r+ c s" e Zd ZdZd� fdd� Z� ZS )�BucketListingObjectz&BucketListingRef subclass for objects.Nc s0 t t| ��� | jj| _|j| _|| _|| _ dS )z�Creates a BucketListingRef of type object. Args: storage_url: StorageUrl containing an object. root_object: Underlying object metadata, if available. N) r" r, r# r r r r r r$ r% r& r' r r r# i s z BucketListingObject.__init__)Nr) r r r' r r, f s r, N) r � __future__r r r r r r r! r+ r, r r r r �<module> s - __pycache__/tab_complete.cpython-39.pyc 0000644 00000024547 15025013616 0014056 0 ustar 00 a +(Wg+ � @ sf d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z ddl mZ dd l mZ dd l mZ dd lmZ dd lmZ dd lmZ dZdZd�� ZG dd� de�ZG dd� de�Z G dd� de�Z G dd� de�Z G dd� dej �Z G dd � d e!�Z"G d d � d e�Z#G d d � d e�Z$G d!d"� d"e�Z%d#d$� Z&d%d&� Z'dS )'zShell tab completion.� )�absolute_import)�print_function)�division)�unicode_literalsN)�CannedACLStrings)�IsFileUrlString)�StorageUrlFromString)� StripOneSlash)� GetTabCompletionCacheFilename)�GetTabCompletionLogFilename)�CreateWildcardIterator� i� z� Tab completion aborted (took >%ss), you may complete the command manually. The timeout can be adjusted in the gsutil configuration file. c @ s$ e Zd ZdZdZdZdZdZdZdS )� CompleterTypeZ cloud_bucketZ cloud_objectZcloud_or_local_objectZ local_objectZlocal_object_or_canned_aclZno_opN) �__name__� __module__� __qualname__� CLOUD_BUCKET� CLOUD_OBJECT�CLOUD_OR_LOCAL_OBJECT� LOCAL_OBJECT�LOCAL_OBJECT_OR_CANNED_ACL�NO_OP� r r �!/opt/gsutil/gslib/tab_complete.pyr / s r c @ s e Zd ZdZdd� Zdd� ZdS )�LocalObjectCompleterz!Completer object for local files.c C s ddl m} |� | _d S )Nr )�FilesCompleter)Zargcomplete.completersr �files_completer)�selfr r r r �__init__; s z LocalObjectCompleter.__init__c K s | j |fi |��S �N)r )r �prefix�kwargsr r r �__call__B s z LocalObjectCompleter.__call__N�r r r �__doc__r r" r r r r r 8 s r c @ s e Zd ZdZdd� Zdd� ZdS )� LocalObjectOrCannedACLCompleterz|Completer object for local files and canned ACLs. Currently, only Google Cloud Storage canned ACL names are supported. c C s t � | _d S r )r �local_object_completer)r r r r r L s z(LocalObjectOrCannedACLCompleter.__init__c s, | j � fi |��}� fdd�tD �}|| S )Nc s g | ]}|� � �r|�qS r �� startswith)�.0�acl�r r r � <listcomp>Q � z<LocalObjectOrCannedACLCompleter.__call__.<locals>.<listcomp>)r& r )r r r! Z local_objects� canned_aclsr r+ r r" O s z(LocalObjectOrCannedACLCompleter.__call__Nr# r r r r r% F s r% c @ s< e Zd ZdZdd� Zedd� �Zdd� Zdd � Zd d � Z d S ) �TabCompletionCachez!Cache for tab completion results.c C s || _ || _|| _|| _d S r )r �results� timestamp�partial_results)r r r0 r1 r2 r r r r X s zTabCompletionCache.__init__c C s� z\t | d��>}t�|�� �}|d }|d }|d }|d }W d � n1 sP0 Y W n" ty~ d}g }d}d}Y n0 t||||�S ) z�Instantiates the cache from a file. Args: filename: The file to load. Returns: TabCompletionCache instance with loaded data or an empty cache if the file cannot be loaded �rr r0 r1 �partial-resultsNr F)�open�json�loads�read� Exceptionr/ )�filename�fpZ cache_dictr r0 r1 r2 r r r � LoadFromFile^ s * z TabCompletionCache.LoadFromFilec s� t � � }|| j tkrdS d}� | jkr0| j}n<| jsl� �| j�rl� �d�| j�d�krl� fdd�| jD �}|dur�t � � | _|S dS )z>Returns the cached results for prefix or None if not in cache.N�/c s g | ]}|� � �r|�qS r r' )r) �xr+ r r r, � r- z7TabCompletionCache.GetCachedResults.<locals>.<listcomp>)�timer1 �TAB_COMPLETE_CACHE_TTLr r0 r2 r( �count)r r � current_timer0 r r+ r �GetCachedResultsz s � z#TabCompletionCache.GetCachedResultsc C s || _ || _|| _t�� | _dS )zBUpdates the in-memory cache with the results for the given prefix.N)r r0 r2 r? r1 )r r r0 r2 r r r � UpdateCache� s z TabCompletionCache.UpdateCachec C sl t �| j| j| j| jd��}z8t|d��}|�|� W d � n1 sH0 Y W n tyf Y n0 dS )z'Writes out the cache to the given file.)r r0 r4 r1 �wN) r6 �dumpsr r0 r2 r1 r5 �write�IOError)r r: �json_strr; r r r � WriteToFile� s � , z TabCompletionCache.WriteToFileN) r r r r$ r � staticmethodr< rC rD rJ r r r r r/ U s r/ c s( e Zd ZdZ� fdd�Zdd� Z� ZS )�CloudListingRequestThreadz@Thread that performs a listing request for the given URL string.c s* t t| ��� d| _|| _|| _d| _dS )z�Instantiates Cloud listing request thread. Args: wildcard_url_str: The URL to list. gsutil_api: gsutil Cloud API instance to use. TN)�superrL r �daemon�_wildcard_url_str� _gsutil_apir0 )r Zwildcard_url_str� gsutil_api�� __class__r r r � s z"CloudListingRequestThread.__init__c C s4 t | j| j�jdgd�}dd� t�|t�D �| _d S )N�name)�bucket_listing_fieldsc S s g | ] }t |��qS r )�str)r) �cr r r r, � s z1CloudListingRequestThread.run.<locals>.<listcomp>)r rO rP �IterAll� itertools�islice�_TAB_COMPLETE_MAX_RESULTSr0 )r �itr r r �run� s �� �z CloudListingRequestThread.run)r r r r$ r r] � __classcell__r r rR r rL � s rL c @ s e Zd ZdS )� TimeoutErrorN)r r r r r r r r_ � s r_ c @ s* e Zd ZdZd dd�Zdd� Zdd� Zd S ) �CloudObjectCompleterz Completer object for Cloud URLs.Fc C s || _ || _dS )z�Instantiates completer for Cloud URLs. Args: gsutil_api: gsutil Cloud API instance to use. bucket_only: Whether the completer should only match buckets. N)rP � _bucket_only)r rQ � bucket_onlyr r r r � s z CloudObjectCompleter.__init__c C sL t || j�}|�� |�|� |�� rBddl}|�t| � t� �|j }|S )a< Perform a remote listing request for the given wildcard URL. Args: wildcard_url: The wildcard URL to list. timeout: Time limit for the request. Returns: Cloud resources matching the given wildcard URL. Raises: TimeoutError: If the listing does not finish within the timeout. r N) rL rP �start�join�is_alive� argcomplete�warn�_TIMEOUT_WARNINGr_ r0 )r � wildcard_url�timeoutZrequest_threadrf r0 r r r �_PerformCloudListing� s z)CloudObjectCompleter._PerformCloudListingc K s0 |s d}n t |�rg S |d }t|�}| jr8|�� s8g S tj�ddd�}|dkrTg S t�� }t� t � �}|� |�}d} |d ur�|} d} nfzJ| � ||�} | jr�t | �d kr�t| d �g} t | �tk} |�|| | � W n ty� d } g } Y n0 |�t � � t�� } t | �} | | }td | | || | |f � | S ) Nzgs://�*�GSUtil�tab_completion_timeout� r � z (from cache)� z (request timeout)z:%s results%s in %.2fs, %.2f results/second for prefix: %s )r r ra �IsBucket�boto�config�getintr? r/ r< r rC rk �lenr r[ rD r_ rJ �_WriteTimingLog)r r r! ri �urlrj � start_time�cacheZcached_resultsZtiming_log_entry_typer0 r2 �end_timeZ num_resultsZelapsed_secondsr r r r" � sN ���z CloudObjectCompleter.__call__N)F)r r r r$ r rk r" r r r r r` � s r` c @ s e Zd ZdZdd� Zdd� ZdS )�CloudOrLocalObjectCompleterz�Completer object for Cloud URLs or local files. Invokes the Cloud object completer if the input looks like a Cloud URL and falls back to local file completer otherwise. c C s t |�| _t� | _d S r )r` �cloud_object_completerr r& )r rQ r r r r # s z$CloudOrLocalObjectCompleter.__init__c K s& t |�r| j}n| j}||fi |��S r )r r& r} )r r r! � completerr r r r" ' s z$CloudOrLocalObjectCompleter.__call__Nr# r r r r r| s r| c @ s e Zd ZdZdd� ZdS )� NoOpCompleterz(Completer that always returns 0 results.c K s g S r r )r Z unused_prefix� unused_kwargsr r r r" 2 s zNoOpCompleter.__call__N)r r r r$ r" r r r r r / s r c C sz | t jkrt|�S | t jkr"t� S | t jkr2t� S | t jkrHt|dd�S | t j krZt|�S | t j krjt � S t d| ��dS )a Create a completer instance of the given type. Args: completer_type: The type of completer to create. gsutil_api: gsutil Cloud API instance to use. Returns: A completer instance. Raises: RuntimeError: if completer type is not supported. T)rb zUnknown completer "%s"N) r r r| r r r r% r r` r r r � RuntimeError)Zcompleter_typerQ r r r � MakeCompleter6 s r� c C sJ t j�ddd�rFtt� d��}|�| � W d � n1 s<0 Y dS )zAWrite an entry to the tab completion timing log, if it's enabled.rm �tab_completion_time_logsF�abN)rs rt �getboolr5 r rG )�messager; r r r rw Q s rw )(r$ � __future__r r r r rY r6 � threadingr? rs � boto.gs.aclr �gslib.storage_urlr r r Zgslib.utils.boto_utilr r Zgslib.wildcard_iteratorr r@ r[ �rstriprh �objectr r r% r/ �ThreadrL r9 r_ r` r| r r� rw r r r r �<module> s< RX __pycache__/plurality_checkable_iterator.cpython-39.pyc 0000644 00000006316 15025013616 0017331 0 ustar 00 a +(Wg� � @ sZ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZG dd� dej�Z dS ) zHIterator wrapper for checking wrapped iterator's emptiness or plurality.� )�absolute_import)�print_function)�division)�unicode_literalsNc @ sJ e Zd ZdZdd� Zddd�Zdd� Zd d � Zd d � Zd d� Z dd� Z dS )�PluralityCheckableIteratora� Iterator wrapper class. Allows you to check whether the wrapped iterator is empty and whether it has more than 1 element. This iterator accepts three types of values from the iterator it wraps: 1. A yielded element (this is the normal case). 2. A raised exception, which will be buffered and re-raised when it is reached in this iterator. 3. A yielded tuple of (exception, stack trace), which will be buffered and raised with it is reached in this iterator. c C s || _ d | _g | _d| _d S )NF)� orig_iterator� base_iterator�head�underlying_iter_empty)�self�it� r �1/opt/gsutil/gslib/plurality_checkable_iterator.py�__init__0 s z#PluralityCheckableIterator.__init__� c C s� | j s�t| j�|k r�zj| js(t| j�| _t| j�}d| _ t|t�rlt|d t �rl| j� d|d |d f� n| j� d|f� W q t y� d| _ Y q t y� } z&| j� d|t � � d f� W Y d}~q d}~0 0 q t| j�S ) a Populates self.head from the underlying iterator. Args: num_elements: Populate until self.head contains this many elements (or until the underlying iterator runs out). Returns: Number of elements at self.head after execution complete. Fr � exceptionr �elementT� N)r �lenr r �iterr �next� isinstance�tuple� Exception�append� StopIteration�sys�exc_info)r � num_elements�er r r � _PopulateHead8 s 2z(PluralityCheckableIterator._PopulateHeadc C s | S �Nr �r r r r �__iter__W s z#PluralityCheckableIterator.__iter__c C sN | � � rD| j�d�}|d dkr(|d S t�|d j|d |d ��t� �d S )Nr r r r )r r �pop�six�reraise� __class__r )r Z item_tupler r r �__next__Z s z#PluralityCheckableIterator.__next__c C s | � � S r! �r r" r r r �IsEmptyc s z"PluralityCheckableIterator.IsEmptyc C s | j dd�dkS )Nr )r r r) r" r r r � HasPluralityf s z'PluralityCheckableIterator.HasPluralityc C sD | � � r@| jd d dkr@| jd }t�|d j|d |d ��dS )z9Raises an exception if the first iterated element raised.r r r r N)r r r% r& r' )r Zexception_tupler r r � PeekExceptionk s �z(PluralityCheckableIterator.PeekExceptionN)r ) �__name__� __module__� __qualname__�__doc__r r r# r( r* r+ r, r r r r r # s r ) r0 � __future__r r r r r r% �Iteratorr r r r r �<module> s __pycache__/no_op_credentials.cpython-39.pyc 0000644 00000001650 15025013616 0015075 0 ustar 00 a +(Wg# � @ sH d Z ddlmZ ddlmZ ddlmZ ddlmZ G dd� de�ZdS ) z;No-op implementation of credentials for JSON HTTP requests.� )�absolute_import)�print_function)�division)�unicode_literalsc @ s$ e Zd Zdd� Zdd� Zdd� ZdS )�NoOpCredentialsc C s d S �N� )�selfr r �&/opt/gsutil/gslib/no_op_credentials.py�__init__ s zNoOpCredentials.__init__c C s |S r r )r Zhttp_objr r r � authorize s zNoOpCredentials.authorizec C s d S r r )r �storer r r � set_store s zNoOpCredentials.set_storeN)�__name__� __module__� __qualname__r r r r r r r r s r N)�__doc__� __future__r r r r �objectr r r r r �<module> s __pycache__/name_expansion.cpython-39.pyc 0000644 00000050033 15025013616 0014411 0 ustar 00 a +(Wg�w � @ s| d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z ddl Z ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlZ ddlmZ G dd� de�Z G dd� de�Z G dd� de�Z d)dd�Z G dd� de�Z G dd � d e�Z!G d d � d e�Z"G d d � d e�Z#e�$d!d"d#g�Z%e�$d$d%d&g�Z&G d'd(� d(e j'�Z(dS )*aB Name expansion iterator and result classes. Name expansion support for the various ways gsutil lets users refer to collections of data (via explicit wildcarding as well as directory, bucket, and bucket subdir implicit wildcarding). This class encapsulates the various rules for determining how these expansions are done. � )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�CommandException)�NO_URLS_MATCHED_GENERIC)�NO_URLS_MATCHED_TARGET)�PluralityCheckableIterator)�SeekAheadResult)�storage_v1_messages)�StorageUrlFromStringc @ s e Zd ZdZdd� Zdd� ZdS )�NameExpansionResulta* Holds one fully expanded result from iterating over NameExpansionIterator. The member data in this class need to be pickleable because NameExpansionResult instances are passed through Multiprocessing.Queue. In particular, don't include any boto state like StorageUri, since that pulls in a big tree of objects, some of which aren't pickleable (and even if they were, pickling/unpickling such a large object tree would result in significant overhead). The state held in this object is needed for handling the various naming cases (e.g., copying from a single source URL to a directory generates different dest URL names than copying multiple URLs to a directory, to be consistent with naming rules used by the Unix cp command). For more details see comments in _NameExpansionIterator. c C s6 || _ || _|| _|| _|| _|r,t�|�nd| _dS )a� Instantiates a result from name expansion. Args: source_storage_url: StorageUrl that was being expanded. is_multi_source_request: bool indicator whether multiple input URLs or src_url_str expanded to more than one BucketListingRef. is_multi_top_level_source_request: same as is_multi_source_request but measured before recursion. names_container: Bool indicator whether src_url names a container. expanded_storage_url: StorageUrl that was expanded. expanded_result: cloud object metadata in MessageToJson form (for pickleability), if any was iterated; None otherwise. Consumers must call JsonToMessage to get an apitools Object. N)�source_storage_url�is_multi_source_request�!is_multi_top_level_source_request�names_container�expanded_storage_urlr � MessageToJson�expanded_result)�selfr r r r r r � r �#/opt/gsutil/gslib/name_expansion.py�__init__@ s ���z NameExpansionResult.__init__c C s d| j S )Nz%s)r �r r r r �__repr__Y s z NameExpansionResult.__repr__N)�__name__� __module__� __qualname__�__doc__r r r r r r r / s r c @ s* e Zd ZdZd dd�Zdd� Zd d � ZdS ) �_NameExpansionIteratorzlClass that iterates over all source URLs passed to the iterator. See details in __iter__ function doc. FTNc C st || _ || _|| _|| _|| _|| _|| _| j�� | j_|| _ | | _ | | _ | | _ | s^t dg�n| | _ddd�| _dS )a� Creates a NameExpansionIterator. Args: command_name: name of command being run. debug: Debug level to pass to underlying iterators (range 0..3). logger: logging.Logger object. gsutil_api: Cloud storage interface. Settable for testing/mocking. url_strs: PluralityCheckableIterator of URL strings needing expansion. recursion_requested: True if -r specified on command-line. If so, listings will be flattened so mapped-to results contain objects spanning subdirectories. all_versions: Bool indicating whether to iterate over all object versions. cmd_supports_recursion: Bool indicating whether this command supports a '-r' flag. Useful for printing helpful error messages. project_id: Project id to use for bucket retrieval. ignore_symlinks: If True, ignore symlinks during iteration. continue_on_error: If true, yield no-match exceptions encountered during iteration instead of raising them. bucket_listing_fields: Iterable fields to include in expanded results. Ex. ['name', 'acl']. Underyling iterator is responsible for converting these to list-style format ['items/name', 'items/acl']. If this is None, only the object name is included in the result. Examples of _NameExpansionIterator with recursion_requested=True: - Calling with one of the url_strs being 'gs://bucket' will enumerate all top-level objects, as will 'gs://bucket/' and 'gs://bucket/*'. - 'gs://bucket/**' will enumerate all objects in the bucket. - 'gs://bucket/abc' will enumerate either the single object abc or, if abc is a subdirectory, all objects under abc and any of its subdirectories. - 'gs://bucket/abc/**' will enumerate all objects under abc or any of its subdirectories. - 'file:///tmp' will enumerate all files under /tmp, as will 'file:///tmp/*' - 'file:///tmp/**' will enumerate all files under /tmp or any of its subdirectories. Example if recursion_requested=False: calling with gs://bucket/abc/* lists matching objects or subdirs, but not sub-subdirs or objects beneath subdirs. Note: In step-by-step comments below we give examples assuming there's a gs://bucket with object paths: abcd/o1.txt abcd/o2.txt xyz/o1.txt xyz/o2.txt and a directory file://dir with file paths: dir/a.txt dir/b.txt dir/c/ �namez**�*)TFN)� command_name�debug�logger� gsutil_api�url_strs�recursion_requested� all_versions� HasPlurality� has_plurality�cmd_supports_recursion� project_id�ignore_symlinks�continue_on_error�set�bucket_listing_fields�_flatness_wildcard) r r# r$ r% r& r' r( r) r, r- r. r/ r1 r r r r c s A�z _NameExpansionIterator.__init__c c sl | j D �]^}t|�}|�� rT|�� s,|�� rT| j jr<td��t|ddd|dd�V qd}|�� r�|� � r�| j s�t | � |�j dgd��}n.t | � |�j| jdd��}|�� r�|� � r�d}|�� }| j jp�|}| j| j }| j r�t| ||| j�}nt|�}t |�}|�� �rh| j�r\ztt| ��W n8 t�yX } z | t�� d fV W Y d} ~ n d} ~ 0 0 n tt| ��t t|| j | j| j| j ��} | �� }| j j�p�|} | D ]�\} } |�p�| }| � � �r�t|| ||| j | j d�V n�t| j �}|�� �r�d | t!j"|f }n |j#|d �}t | � |�j$| jd ��}|�p,|�� }| j j�p:|} |D ] } t|| |d| j | j d�V �q@�q�qdS ) a9 Iterates over all source URLs passed to the iterator. For each src url, expands wildcards, object-less bucket names, subdir bucket names, and directory names, and generates a flat listing of all the matching objects/files. You should instantiate this object using the static factory function NameExpansionIterator, because consumers of this iterator need the PluralityCheckableIterator wrapper built by that function. Yields: gslib.name_expansion.NameExpansionResult. Raises: CommandException: if errors encountered. zPMultiple URL strings are not supported with streaming ("-") URLs or named pipes.FN)r r r r r r �id)� bucket_fieldsT)r1 �expand_top_level_buckets� z%s%s%s��wildcard_suffix�r1 )%r' r � IsFileUrl�IsStream�IsFifor+ r r � IsCloudUrl�IsBucketr( r �WildcardIterator� IterBuckets�IterAllr1 r* r2 � _ImplicitBucketSubdirIterator� _NonContainerTuplifyIterator�IsEmptyr/ r �sys�exc_info�_OmitNonRecursiveIteratorr# r, r% �IsObject� storage_url� root_object� url_string�os�sep�CreatePrefixUrl� IterObjects)r �url_strrI Zsrc_names_bucketZpost_step1_iterZsrc_url_expands_to_multir �subdir_exp_wildcardZpost_step2_iter�eZpost_step3_iterr r �blrZsrc_names_container� expanded_urlZurl_to_iterateZwc_iterr r r �__iter__� s� ����� ��� � * �� � � � ��� ��z _NameExpansionIterator.__iter__c C s" t jj|| j| j| j| j| jd�S )a2 Helper to instantiate gslib.WildcardIterator. Args are same as gslib.WildcardIterator interface, but this method fills in most of the values from instance state. Args: url_string: URL string naming wildcard objects to iterate. Returns: Wildcard iterator over URL string. )r) r- r. r% )�gslib�wildcard_iterator�CreateWildcardIteratorr& r) r- r. r% )r rK r r r r? Q s �z'_NameExpansionIterator.WildcardIterator)FTNFFN)r r r r r rU r? r r r r r ] s � V r c @ s" e Zd ZdZd dd�Zdd� ZdS ) � SeekAheadNameExpansionIteratorz�Creates and wraps a _NameExpansionIterator and yields SeekAheadResults. Unlike the NameExpansionIterator, which can make API calls upon __init__ to check for plurality, this iterator does no work until the first iteration occurs. FTNc C sP |dv o | | _ | j r dgnd} t||t�d�|t|�||||| d| d� | _dS )z5Initializes a _NameExpansionIterator with the inputs.)�cp�mv�rewrite�sizeN�dummyT�r) r, r- r. r/ r1 )�count_data_bytesr �logging� getLoggerr �name_expansion_iterator) r r# r$ r& r' r( r) r, r- r. Zfile_size_will_changer1 r r r r n s$ ��z'SeekAheadNameExpansionIterator.__init__c c sL | j D ]@}| jr>|jr>t�tj|j�}|jp.d}t|d�V qt� V qd S )Nr )Z data_bytes) rc r` r r � JsonToMessage�apitools_messages�Objectr] r )r �name_expansion_resultZiterated_metadataZ iterated_sizer r r rU � s � z'SeekAheadNameExpansionIterator.__iter__)FTNFF�r r r r r rU r r r r rY f s � %rY FTc C sD t |�}t| ||||||||| | | d� } t | �} | �� r@tt��| S )ar Static factory function for instantiating _NameExpansionIterator. This wraps the resulting iterator in a PluralityCheckableIterator and checks that it is non-empty. Also, allows url_strs to be either an array or an iterator. Args: command_name: name of command being run. debug: Debug level to pass to underlying iterators (range 0..3). logger: logging.Logger object. gsutil_api: Cloud storage interface. Settable for testing/mocking. url_strs: Iterable URL strings needing expansion. recursion_requested: True if -r specified on command-line. If so, listings will be flattened so mapped-to results contain objects spanning subdirectories. all_versions: Bool indicating whether to iterate over all object versions. cmd_supports_recursion: Bool indicating whether this command supports a '-r' flag. Useful for printing helpful error messages. project_id: Project id to use for the current command. ignore_symlinks: If True, ignore symlinks during iteration. continue_on_error: If true, yield no-match exceptions encountered during iteration instead of raising them. bucket_listing_fields: Iterable fields to include in expanded results. Ex. ['name', 'acl']. Underyling iterator is responsible for converting these to list-style format ['items/name', 'items/acl']. If this is None, only the object name is included in the result. Raises: CommandException if underlying iterator is empty. Returns: Name expansion iterator instance. For example semantics, see comments in NameExpansionIterator.__init__. r_ )r r rD r r ) r# r$ r% r& r' r( r) r, r- r. r/ r1 rc r r r �NameExpansionIterator� s& /� ri c @ s e Zd ZdZdd� Zdd� ZdS )rC z�Iterator that produces the tuple (False, blr) for each iterated value. Used for cases where blr_iter iterates over a set of BucketListingRefs known not to name containers. c C s || _ dS )zTInstantiates iterator. Args: blr_iter: iterator of BucketListingRef. N��blr_iter)r rk r r r r � s z%_NonContainerTuplifyIterator.__init__c c s | j D ]}d|fV qd S )NFrj )r rS r r r rU � s z%_NonContainerTuplifyIterator.__iter__Nrh r r r r rC � s rC c @ s e Zd ZdZdd� Zdd� ZdS )rG a� Iterator wrapper for that omits certain values for non-recursive requests. This iterates over tuples of (names_container, BucketListingReference) and omits directories, prefixes, and buckets from non-recurisve requests so that we can properly calculate whether the source URL expands to multiple URLs. For example, if we have a bucket containing two objects: bucket/foo and bucket/foo/bar and we do a non-recursive iteration, only bucket/foo will be yielded. c C s" || _ || _|| _|| _|| _dS )a� Instanties the iterator. Args: tuple_iter: Iterator over names_container, BucketListingReference from step 2 in the NameExpansionIterator recursion_requested: If false, omit buckets, dirs, and subdirs command_name: Command name for user messages cmd_supports_recursion: Command recursion support for user messages logger: Log object for user messages N)� tuple_iterr( r# r, r% )r rl r( r# r, r% r r r r s z"_OmitNonRecursiveIterator.__init__c c s| | j D ]p\}}| jsl|�� slt|j�}|�� r4d}n|j}| jrX| j� d||j| j � qv| j� d||j� q||fV qd S )N� directoryz-Omitting %s "%s". (Did you mean to do %s -r?)zOmitting %s "%s".) rl r( rH r rK r: � type_namer, r% �infor# )r r rS rT �descr r r rU s �z"_OmitNonRecursiveIterator.__iter__Nrh r r r r rG � s rG c @ s e Zd ZdZdd� Zdd� ZdS )rB a� Iterator wrapper that performs implicit bucket subdir expansion. Each iteration yields tuple (names_container, expanded BucketListingRefs) where names_container is true if URL names a directory, bucket, or bucket subdir. For example, iterating over [BucketListingRef("gs://abc")] would expand to: [BucketListingRef("gs://abc/o1"), BucketListingRef("gs://abc/o2")] if those subdir objects exist, and [BucketListingRef("gs://abc") otherwise. c C s || _ || _|| _|| _dS )a� Instantiates the iterator. Args: name_exp_instance: calling instance of NameExpansion class. blr_iter: iterator over BucketListingRef prefixes and objects. subdir_exp_wildcard: wildcard for expanding subdirectories; expected values are ** if the mapped-to results should contain objects spanning subdirectories, or * if only one level should be listed. bucket_listing_fields: Fields requested in enumerated results. N)rk �name_exp_instancerQ r1 )r rq rk rQ r1 r r r r 3 s z&_ImplicitBucketSubdirIterator.__init__c c s� | j D ]�}|�� rjt|j�j| jd�}t| j�|�j | j d��}|� � s^|D ]}d|fV qLq�d|fV q|� � r~d|fV qt d| ��qd S )Nr7 r9 TFz7_ImplicitBucketSubdirIterator got a bucket reference %s)rk �IsPrefixr rK rN rQ r rq r? rA r1 rD rH r )r rS � prefix_urlZimplicit_subdir_iteratorZexp_blrr r r rU E s& � �� �z&_ImplicitBucketSubdirIterator.__iter__Nrh r r r r rB ' s rB c @ s e Zd ZdZdd� ZdS )�CopyObjectInfozARepresents the information needed for copying a single object. c C s@ |j | _ |j| _|j| _|j| _|j| _|j| _|| _|| _dS )aE Instantiates the object info from name expansion result and destination. Args: name_expansion_result: StorageUrl that was being expanded. exp_dst_url: StorageUrl of the destination. have_existing_dst_container: Whether exp_url names an existing directory, bucket, or bucket subdirectory. N)r r r r r r � exp_dst_url�have_existing_dst_container)r rg ru rv r r r r ` s �zCopyObjectInfo.__init__N)r r r r r r r r r rt \ s rt �DestinationInforu rv �%NameExpansionIteratorDestinationTuple�name_expansion_iter� destinationc @ s( e Zd ZdZdd� Zdd� Zdd� ZdS ) �CopyObjectsIteratora� Iterator wrapper for copying objects and keeping track of source URL types. This is used in the cp command for copying from multiple source to multiple destinations. It takes a list of NameExpansionIteratorDestinationTuple. It wraps them and return CopyObjectInfo objects that wraps NameExpansionResult with the destination. It's used also for collecting analytics PerformanceSummary info, because there may be multiple source URLs and we want to know if any of them are file URLs, if any of them are cloud URLs, if any of them require daisy chain operations, and if any use different providers. The source URL type information will be aggregated at the end of _SequentialApply or _ParallelApply. c C s< || _ d| _d| _g | _|| _t| j�}|j| _|j| _ dS )a Instantiates the iterator. Args: name_expansion_dest_iter: NameExpansionIteratorDestinationTuple iterator. is_daisy_chain: The -D option in cp might have already been specified, in which case we do not need to check again for daisy chain operations. FN) �is_daisy_chain� has_file_src� has_cloud_src�provider_types�name_expansion_dest_iter�nextry �current_expansion_iterrz �current_destination)r r� r| �name_expansion_dest_tupler r r r � s z CopyObjectsIterator.__init__c C s | S )Nr r r r r rU � s z CopyObjectsIterator.__iter__c C s� zt | j�}W n4 tyB t | j�}|j| _|j| _| �� Y S 0 t|| jj | jj �}| j sn|j � � rnd| _ | js�|j �� r�d| _| jj �� r�| jj j}nd}| js�|dur�|j �� r�|j j|kr�d| _|j j| jvr�| j�|j j� |S )z@Keeps track of URL types as the command iterates over arguments.TN)r� r� � StopIterationr� ry rz r� �__next__rt ru rv r} r r: r~ r= �schemer| r �append)r rg r� �eltZdst_url_schemer r r r� � s8 �� � �z CopyObjectsIterator.__next__N)r r r r r rU r� r r r r r{ � s r{ )FTNFFN))r � __future__r r r r � collectionsra rL rE �six�apitools.base.pyr rV �gslib.exceptionr r r �"gslib.plurality_checkable_iteratorr �gslib.seek_ahead_threadr �"gslib.third_party.storage_apitoolsr re �gslib.wildcard_iteratorr �objectr r rY ri rC rG rB rt � namedtuplerw rx �Iteratorr{ r r r r �<module> s` . > � C25�� �� __pycache__/storage_url.cpython-39.pyc 0000644 00000040756 15025013616 0013746 0 ustar 00 a +(Wg�@ � @ s| d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl m Z ddl m Z dd l mZ dd l mZ e�d �Ze�d �Ze�d �Ze�d�Ze�d�Ze�d�Ze�d�Zeddg�ZG dd� de�ZG dd� de�Z G dd� de�Z dd� Z d d � Z d d � Z d d!� Z!d"d#� Z"d$d%� Z#d&d'� Z$d6d(d)�Z%d*d+� Z&d,d-� Z'd.d/� Z(d0d1� Z)d2d3� Z*d4d5� Z+dS )7z*File and Cloud URL representation classes.� )�absolute_import)�print_function)�division)�unicode_literalsN)�CommandException)�InvalidUrlError)� system_util)� text_utilz(?P<provider>[^:]*)://$z.(?P<provider>[^:]*)://(?P<bucket>[^/]*)/{0,1}$z6(?P<provider>[^:]*)://(?P<bucket>[^/]*)/(?P<object>.*)z&(?P<object>.+)#(?P<generation>[0-9]+)$z"(?P<object>.+)#(?P<version_id>.+)$z([^:]*://)(?P<filepath>.*)z[*?\[\]]�.�..c @ sr e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zdd d�Z dd� Z e dd� �Z e dd� �Z dd� Zdd� Zd S )� StorageUrlz4Abstract base class for file and Cloud Storage URLs.c C s t d��d S )NzClone not overridden��NotImplementedError��self� r � /opt/gsutil/gslib/storage_url.py�Clone6 s zStorageUrl.Clonec C s t d��d S )NzIsFileUrl not overriddenr r r r r � IsFileUrl9 s zStorageUrl.IsFileUrlc C s t d��d S )NzIsCloudUrl not overriddenr r r r r � IsCloudUrl< s zStorageUrl.IsCloudUrlc C s t d��d S )NzIsStream not overriddenr r r r r �IsStream? s zStorageUrl.IsStreamc C s t d��d S )NzIsFifo not overriddenr r r r r �IsFifoB s zStorageUrl.IsFifoNc C s t d��dS )ai Returns a prefix of this URL that can be used for iterating. Args: wildcard_suffix: If supplied, this wildcard suffix will be appended to the prefix with a trailing slash before being returned. Returns: A prefix of this URL that can be used for iterating. If this URL contains a trailing slash, it will be stripped to create the prefix. This helps avoid infinite looping when prefixes are iterated, but preserves other slashes so that objects with '/' in the name are handled properly. For example, when recursively listing a bucket with the following contents: gs://bucket// <-- object named slash gs://bucket//one-dir-deep a top-level expansion with '/' as a delimiter will result in the following URL strings: 'gs://bucket//' : OBJECT 'gs://bucket//' : PREFIX If we right-strip all slashes from the prefix entry and add a wildcard suffix, we will get 'gs://bucket/*' which will produce identical results (and infinitely recurse). Example return values: ('gs://bucket/subdir/', '*') becomes 'gs://bucket/subdir/*' ('gs://bucket/', '*') becomes 'gs://bucket/*' ('gs://bucket/', None) becomes 'gs://bucket' ('gs://bucket/subdir//', '*') becomes 'gs://bucket/subdir//*' ('gs://bucket/subdir///', '**') becomes 'gs://bucket/subdir///**' ('gs://bucket/subdir/', '*') where 'subdir/' is an object becomes 'gs://bucket/subdir/*', but iterating on this will return 'subdir/' as a BucketListingObject, so we will not recurse on it as a subdir during listing. z CreatePrefixUrl not overriddenNr �r �wildcard_suffixr r r �CreatePrefixUrlE s %zStorageUrl.CreatePrefixUrlc C sZ | j s dS | j| j | j }|�dj| jd��}d�|�}d|v rVtj�d�tj �� dS )z-Warn if ** use may lead to undefined results.Nz{delim}**{delim})�delim� z**zw** behavior is undefined if directly preceeded or followed by with characters other than / in the cloud and {} locally.) � object_namer �split�format�join�sys�stderr�write�os�sep)r Zdelimiter_bounded_urlZ split_urlZ+removed_correct_double_wildcards_url_stringr r r � _WarnIfUnsupportedDoubleWildcardl s � ��z+StorageUrl._WarnIfUnsupportedDoubleWildcardc C s t d��d S )Nzurl_string not overriddenr r r r r � url_string� s zStorageUrl.url_stringc C s t d��d S )Nz%versionless_url_string not overriddenr r r r r �versionless_url_string� s z!StorageUrl.versionless_url_stringc C s t |t�o| j|jkS �N)� isinstancer r' )r �otherr r r �__eq__� s zStorageUrl.__eq__c C s t | j�S r) )�hashr' r r r r �__hash__� s zStorageUrl.__hash__)N)�__name__� __module__� __qualname__�__doc__r r r r r r r& �propertyr' r( r, r. r r r r r 3 s ' r c @ st e Zd ZdZddd�Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z ddd�Z e dd� �Z e dd� �Zdd� ZdS ) �_FileUrla� File URL class providing parsing and convenience methods. This class assists with usage and manipulation of an (optionally wildcarded) file URL string. Depending on the string contents, this class represents one or more directories or files. For File URLs, scheme is always file, bucket_name is always blank, and object_name contains the file/directory path. Fc C st d| _ tj| _d| _t�|�}|r:|jdkr:|�d�| _ n|| _ t j rV| j � dd�| _ d | _ || _|| _| �� d S )N�filer � �/�\)�schemer$ r% r � bucket_name�FILE_OBJECT_REGEX�match� lastindex�groupr r � IS_WINDOWS�replace� generation� is_stream�is_fifor& )r r' rB rC r< r r r �__init__� s z_FileUrl.__init__c C s t | j�S r) )r4 r' r r r r r � s z_FileUrl.Clonec C s dS �NTr r r r r r � s z_FileUrl.IsFileUrlc C s dS �NFr r r r r r � s z_FileUrl.IsCloudUrlc C s | j S r) )rB r r r r r � s z_FileUrl.IsStreamc C s | j S r) )rC r r r r r � s z_FileUrl.IsFifoc C s" | � � o | �� o tj�| j�S r) )r r r$ �path�isdirr r r r r � IsDirectory� s �z_FileUrl.IsDirectoryNc C s | j S r) �r' r r r r r � s z_FileUrl.CreatePrefixUrlc C s d| j | jf S )Nz%s://%s)r9 r r r r r r' � s z_FileUrl.url_stringc C s | j S r) rJ r r r r r( � s z _FileUrl.versionless_url_stringc C s | j S r) rJ r r r r �__str__� s z_FileUrl.__str__)FF)N)r/ r0 r1 r2 rD r r r r r rI r r3 r' r( rK r r r r r4 � s r4 c @ s� e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dd� Z d!dd�Zedd� �Zedd � �Zed d � �Zd d � ZdS )"� _CloudUrlak Cloud URL class providing parsing and convenience methods. This class assists with usage and manipulation of an (optionally wildcarded) cloud URL string. Depending on the string contents, this class represents a provider, bucket(s), or object(s). This class operates only on strings. No cloud storage API calls are made from this class. c C sn d | _ d| _d | _d | _d | _t�|�}t�|�}|rD|�d�| _ n�|rb|�d�| _ |�d�| _n�t �|�}|�r$|�d�| _ |�d�| _|�d�| _| jdks�| jdkr�t d| j ��| j dkr�t �| j�}|r�|�d�| _|�d �| _n6| j d k�r0t �| j�}|�r0|�d�| _|�d �| _n t d | ��|t | j �t d � d � �| j��rbt d�|���| �� d S )Nr7 �providerZbucket�objectr r z'%s is an invalid root-level object name�gsrA �s3Z version_idz/CloudUrl: URL string %s did not match URL regex�://zPCloud URL scheme should be followed by colon and two slashes: "://". Found: "{}")r9 r r: r rA �PROVIDER_REGEXr< � BUCKET_REGEXr> � OBJECT_REGEXr �GS_GENERATION_REGEX�S3_VERSION_REGEX�len� startswithr r& )r r' Zprovider_matchZ bucket_matchZ object_matchZgeneration_matchZ version_matchr r r rD � sR � �$��z_CloudUrl.__init__c C s t | j�S r) )rL r' r r r r r s z_CloudUrl.Clonec C s dS rF r r r r r r s z_CloudUrl.IsFileUrlc C s dS rE r r r r r r s z_CloudUrl.IsCloudUrlc C s t d��d S )Nz"IsStream not supported on CloudUrlr r r r r r s z_CloudUrl.IsStreamc C s t d��d S )Nz IsFifo not supported on CloudUrlr r r r r r s z_CloudUrl.IsFifoc C s t | jo| j �S r) ��boolr: r r r r r �IsBucket s z_CloudUrl.IsBucketc C s t | jo | j�S r) rY r r r r �IsObject s z_CloudUrl.IsObjectc C s t | j�S r) )rZ rA r r r r � HasGeneration s z_CloudUrl.HasGenerationc C s t | jo| j �S r) )rZ r9 r: r r r r � IsProvider" s z_CloudUrl.IsProviderNc C s t | j�}|rd||f }|S )Nz%s/%s)� StripOneSlashr( )r r �prefixr r r r % s z_CloudUrl.CreatePrefixUrlc C s d| j | jf S )Nz%s://%s/)r9 r: r r r r �bucket_url_string+ s z_CloudUrl.bucket_url_stringc C s | j }| �� r |d| j 7 }|S )Nz#%s)r( r] rA )r �url_strr r r r' / s z_CloudUrl.url_stringc C s4 | � � rd| j S | �� r | jS d| j| j| jf S )Nz%s://z %s://%s/%s)r^ r9 r[ ra r: r r r r r r( 6 s z _CloudUrl.versionless_url_stringc C s | j S r) rJ r r r r rK > s z_CloudUrl.__str__)N)r/ r0 r1 r2 rD r r r r r r[ r\ r] r^ r r3 ra r' r( rK r r r r rL � s& + rL c C s* | � d�}|dkrdS | d|� �� S dS )z)Returns scheme component of a URL string.rQ ���r5 r N)�find�lower�rb Zend_scheme_idxr r r �GetSchemeFromUrlStringB s rg c C s | dv S )N)r5 rP rO r )Z scheme_strr r r �IsKnownUrlSchemeM s rh c C s* | � d�}|dkr| S | |d d� S dS )z'Returns path component of a URL string.rQ rc � N)rd rf r r r �_GetPathFromUrlStringQ s rj c C s t t�| ��S )zChecks whether url_string contains a wildcard. Args: url_string: URL string to check. Returns: bool indicator. )rZ �WILDCARD_REGEX�searchrJ r r r �ContainsWildcard[ s rm c C s | j dkr|rt�|�S |S )aD Decodes a generation from a StorageURL and a generation string. This is used to represent gs and s3 versioning. Args: url: StorageUrl representing the object. generation: Long or string representing the object's generation or version. Returns: Valid generation string for use in URLs. rP )r9 r ZDecodeLongAsString)�urlrA r r r �GenerationFromUrlAndStringg s ro c C s$ | D ]}t |�}|�� r dS qdS )z�Checks whether args_to_check contain any file URLs. Args: args_to_check: Command-line argument subset to check. Returns: True if args_to_check contains any file URLs. TF)�StorageUrlFromStringr �Z args_to_checkrb � storage_urlr r r � HaveFileUrlsy s rs c C s, | D ]"}t |�}|�� r|�� r dS qdS )z�Checks whether args_to_check contains any provider URLs (like 'gs://'). Args: args_to_check: Command-line argument subset to check. Returns: True if args_to_check contains any provider URLs. TF)rp r r^ rq r r r �HaveProviderUrls� s rt c C sL | � � s dS | j}|�d�r dS |r6|�� r6|jj}nd}|dkoJ|�d�S )aX Determines if a StorageUrl is a cloud subdir placeholder. This function is needed because GUI tools (like the GCS cloud console) allow users to create empty "folders" by creating a placeholder object; and parts of gsutil need to treat those placeholder objects specially. For example, gsutil rsync needs to avoid downloading those objects because they can cause conflicts (see comments in rsync command for details). We currently detect two cases: - Cloud objects whose name ends with '_$folder$' - Cloud objects whose name ends with '/' Args: url: (gslib.storage_url.StorageUrl) The URL to be checked. blr: (gslib.BucketListingRef or None) The blr to check, or None if not available. If `blr` is None, size won't be checked. Returns: (bool) True if the URL is a cloud subdir placeholder, otherwise False. Fz _$folder$Tr r7 )r r' �endswithr\ Z root_object�size)rn Zblrrb rv r r r �IsCloudSubdirPlaceholder� s rw c C s t | �dkS )z'Returns whether a string is a file URL.r5 )rg �rb r r r �IsFileUrlString� s ry c C sx t | �}t|�s td| ��|dkrpt| �}|dk}d}zt�t�|�j�}W n ty` Y n0 t | ||d�S t | �S )z@Static factory function for creating a StorageUrl from a string.zUnrecognized scheme "%s"r5 �-F)rB rC ) rg rh r rj �stat�S_ISFIFOr$ �st_mode�OSErrorr4 rL )rb r9 rG rB rC r r r rp � s rp c C s | r| � d�r| d d� S | S )Nr7 rc )ru rx r r r r_ � s r_ c C s> d}d}| D ](}t |�}|s$|j}q |j|kr dS q |duS )aB Tests whether the URLs are all for a single provider. Args: url_args: (Iterable[str]) Collection of strings to check. Returns: True if all URLs are for single provider; False if `url_args` was empty (as this would not result in a single unique provider) or URLs targeted multiple unique providers. NF)rp r9 )Zurl_argsrM rn rb r r r �UrlsAreForSingleProvider� s r c C s: t dd� | D ��r6ttdd� | ��}t|�o4t |� S dS )a6 Tests whether the URLs are a mix of buckets and objects. Args: url_args: (Iterable[gslib.storage_url.StorageUrl]) Collection of URLs to check. Returns: True if URLs are a mix of buckets and objects. False if URLs are all buckets or all objects. None if invalid Cloud URLs are included. c s s | ]}|� � V qd S r) )r )�.0rn r r r � <genexpr>� � z0UrlsAreMixOfBucketsAndObjects.<locals>.<genexpr>c S s | � � S r) )r[ )�xr r r �<lambda>� r� z/UrlsAreMixOfBucketsAndObjects.<locals>.<lambda>N)�all�list�map�any)�urlsZ are_bucketsr r r � UrlsAreMixOfBucketsAndObjects� s r� c C s t | �r|std��dS )zBRaises error if mix of buckets and objects adjusted for recursion.z/Cannot operate on a mix of buckets and objects.N)r� r )r� Zrecursion_requestedr r r �)RaiseErrorIfUrlsAreMixOfBucketsAndObjects� s r� )N),r2 � __future__r r r r r$ �rer{ r! �gslib.exceptionr r � gslib.utilsr r �compilerR rS rT rU rV r; rk � frozensetZRELATIVE_PATH_SYMBOLSrN r r4 rL rg rh rj rm ro rs rt rw ry rp r_ r r� r� r r r r �<module> sN � ^Cn ! __pycache__/__init__.cpython-39.pyc 0000644 00000005700 15025013616 0013145 0 ustar 00 a +(Wg6 � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z e � \ZZes�e j�e��e�dd�Zer�z8ddlZejed d d gd d e �� d gd�Ze�� W n ey� Y n0 ej�ej�e ��Z ej�ej d �Z ej�e �Z d Z!dZ"e Z#ej�$ej�%e#d���sRej�&e e f�e k�rRdZ!e Z#ej�$ej�%e#d���s�ej�'ej�%e d��Z#d Z"dZ(dd� Z)e)� dd� Z*e*d�\Z+Z,e,�s�e j�d��e,Z-e*d�\Z.Z/e/�s�e j�d��dd� Z0dS ) zPackage marker file.� )�absolute_import)�division)�print_function)�unicode_literalsN)� check_python_version_supportZGSUTIL_COVERAGE_OUTPUT_FILET�gslibzgslib/third_party/*z gslib/tests/*�*)Z data_fileZ data_suffixZ auto_data�sourceZomitF�VERSIONz..� c C s^ t j�td�} ddg}|D ]"\}}tj�dt j�| ||�� qtj�t j�| dddd�� d S ) zFFix our Python path so that it correctly finds our vendored libraries.Zvendored)�botor )Z oauth2clientr r r ZtestsZ integrationZs3N)�os�path�join� GSLIB_DIR�sys�insert�append)Z vendored_pathZvendored_lib_dirs�libdir�subdir� r � /opt/gsutil/gslib/__init__.py� _AddVendoredDepsToPythonPath` s � �r c C s� t j�t| �}t j�|�rNt|d��}|�� }W d � q^1 sB0 Y nt�d| �}d}|dur�t j j dkr�t |t �r�|�d�}|�� }||fS )a Tries to find the given filename on disk or via pkgutil.get_data. Args: filename: String name of the file. Returns: A tuple containing the absolute path to the requested file and the file's contents as a string (or None if the file doesn't exist). �rNr � zutf-8)r r r �PROGRAM_FILES_DIR�isfile�open�read�pkgutil�get_datar � version_info�major� isinstance�bytes�decode�strip)�filenameZfpath�fZcontentr r r �_GetFileContents| s ( r) z<VERSION file not found. Please reinstall gsutil from scratch�CHECKSUMz=CHECKSUM file not found. Please reinstall gsutil from scratchc C s t sdS ttj�t ��S )zBReturns unix timestamp of when the VERSION file was last modified.r )� VERSION_FILE�intr r �getmtimer r r r � GetGsutilVersionModifiedTime� s r. )1�__doc__� __future__r r r r r r r ZtempfileZgslib.exceptionr Zgslib.utils.version_checkr Z supported�errZ exceptionZCommandException�getenvZcoverage_outfileZcoverageZ gettempdirZcoverage_controller�start� ImportErrorr �dirname�realpath�__file__r �argvZ GSUTIL_PATH� GSUTIL_DIRZIS_PACKAGE_INSTALLZIS_EDITABLE_INSTALLr r r � commonprefix�normpathZ USER_AGENTr r) r+ r � __version__Z CHECKSUM_FILEr* r. r r r r �<module> sv �� � � � __pycache__/devshell_auth_plugin.cpython-39.pyc 0000644 00000002365 15025013616 0015617 0 ustar 00 a +(Wg� � @ sr d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m m Z G d d � d e�Z dS ) zYDeveloper Shell auth bridge. This enables Boto API auth in Developer Shell environment. � )�absolute_import)�print_function)�division)�unicode_literals)� AuthHandler)�NotReadyToAuthenticateNc @ s& e Zd ZdZdgZdd� Zdd� ZdS )� DevshellAuthz+Developer Shell authorization plugin class.�s3c C s6 |j dkrt� �zt�� | _W n t� �Y n0 d S )N�google)�namer �devshellZDevshellCredentials�creds)�self�path�config�provider� r �)/opt/gsutil/gslib/devshell_auth_plugin.py�__init__# s zDevshellAuth.__init__c C s d| j j |jd< d S )Nz Bearer %s� Authorization)r � access_token�headers)r � http_requestr r r �add_auth. s �zDevshellAuth.add_authN)�__name__� __module__� __qualname__�__doc__� capabilityr r r r r r r s r ) r � __future__r r r r �boto.auth_handlerr r Z oauth2client.contrib.devshell�contribr r r r r r �<module> s __pycache__/daisy_chain_wrapper.cpython-39.pyc 0000644 00000020546 15025013616 0015426 0 ustar 00 a +(Wg�9 � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z ddl m Z dd lmZ dd lmZ dd lmZ d ZG d d� de�Zejdd� �ZG dd� de�ZdS )z(Wrapper for use in daisy-chained copies.� )�absolute_import)�division)�print_function)�unicode_literalsN)�BadRequestException)�CloudApi)� constants)�parallelism_framework_util)�CryptoKeyWrapperFromKeyi @c @ s" e Zd ZdZddd�Zdd� ZdS ) � BufferWrapperz<Wraps the download file pointer to use our in-memory buffer.�bc C s$ || _ t|d�r|j| _n|| _dS )z�Provides a buffered write interface for a file download. Args: daisy_chain_wrapper: DaisyChainWrapper instance to use for buffer and locking. �modeN)�daisy_chain_wrapper�hasattrr )�selfr r � r �(/opt/gsutil/gslib/daisy_chain_wrapper.py�__init__, s zBufferWrapper.__init__c C s� | j j�. | j j| j jk r(W d � qRW d � n1 s<0 Y t�d� q t|�}|r�| j j�. | j j�|� | j j|7 _W d � n1 s�0 Y dS )z>Waits for space in the buffer, then writes data to the buffer.Nr ) r �lock�bytes_buffered�max_buffer_size�time�sleep�len�buffer�append)r �data�data_lenr r r �write9 s �, zBufferWrapper.writeN)r )�__name__� __module__� __qualname__�__doc__r r r r r r r ) s r c c s" | j |d�}|V |r | �� d S )N)�timeout)�acquire�release)r r# �resultr r r �AcquireLockWithTimeoutI s r' c @ sT e Zd ZdZddedfdd�Zddd�Zdd d �Zd d � Ze j fd d�Z dd� Z dS )�DaisyChainWrappera� Wrapper class for daisy-chaining a cloud download to an upload. This class instantiates a BufferWrapper object to buffer the download into memory, consuming a maximum of max_buffer_size. It implements intelligent behavior around read and seek that allow for all of the operations necessary to copy a file. This class is coupled with the XML and JSON implementations in that it expects that small buffers (maximum of constants.TRANSFER_BUFFER_SIZE) in size will be used. FNc C s� d| _ t�� | _d| _d| _|| _d| _d| _t � � | _ t � � | _ || _ || _|| _t|�| _|| _d| _d| _|| _t�� | _t�� | _| j| jd� | j�d�s�t d��d| _ dS )a` Initializes the daisy chain wrapper. Args: src_url: Source CloudUrl to copy from. src_obj_size: Size of source object. gsutil_api: gsutil Cloud API to use for the copy. compressed_encoding: If true, source object has content-encoding: gzip. progress_callback: Optional callback function for progress notifications for the download thread. Receives calls with arguments (bytes_transferred, total_size). download_chunk_size: Integer number of bytes to download per GetObjectMedia request. This is the upper bound of bytes that may be unnecessarily downloaded if there is a break in the resumable upload. decryption_key: Base64-encoded decryption key for the source object, if any. Raises: Exception: if the download thread doesn't start within 60 seconds r i N)�progress_callback�< z1Could not start download thread after 60 seconds.� ) �position� collections�dequer r r �_download_chunk_size� last_position� last_datar � CreateLockr �download_exception_lock� src_obj_size�src_url�compressed_encodingr �decryption_tuple� gsutil_api�download_exception�download_threadr) � threading�Event�download_started� stop_download�StartDownloadThread�wait� Exception� _unused_data_from_previous_read)r r5 r4 r8 r6 r) Zdownload_chunk_size�decryption_keyr r r r ^ s. zDaisyChainWrapper.__init__r c s. � fdd�}t j|||fd�� _� j�� dS )zCStarts the download thread for the source object (from start_byte).c s, � j �� z�| � j � jk r�� jj� jj� jjt � �� j | | � j d � jj � jt j j� jj|� jd� � j�� r�� j�� W dS | � j7 } q � jj� jj� jjt � �� j | � jj � jt j j� jj|� jd� W nV t�y& } z<� j� |� _� W d � n1 �s0 Y W Y d}~n d}~0 0 dS )a" Downloads the source object in chunks. This function checks the stop_download event and exits early if it is set. It should be set when there is an error during the daisy-chain upload, then this function can be called again with the upload's current position as start_byte. Args: start_byte: Byte from which to begin the download. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). � ) r6 � start_byte�end_byte� generation� object_size�download_strategy�providerr) r7 N)r6 rE rG rH rI rJ r) r7 )r= �setr/ r4 r8 �GetObjectMediar5 � bucket_name� object_namer r6 rG r �DownloadStrategy�ONE_SHOT�schemer7 r> �is_set�clearrA r3 r9 )rE r) �e�r r r �PerformDownload� sL � � z>DaisyChainWrapper.StartDownloadThread.<locals>.PerformDownload)�target�argsN)r; �Threadr: �start)r rE r) rV r rU r r? � s :��z%DaisyChainWrapper.StartDownloadThreadc C sz | j | jks|dkrdS |du s*|tjkr<td|tjf ��| jrb| jd|� }| j|d� | _nvd}| j�P | jr�W d � q�t| j d�r�| j r�| j �n| j � � s�t d��W d � n1 s�0 Y t�d� qf| j�l |�s | j�� }|d|� }||d� | _| j | _|| _t|�}| j |7 _ | j|8 _W d � n1 �sR0 Y ||k�rvtd||f ��|S ) z9Exposes a stream from the in-memory buffer to the upload.r � NzGInvalid HTTP read size %s during daisy chain operation, expected <= %s.r+ � z Download thread died suddenly.zQInvalid read during daisy chain operation, got data of size %s, expected size %s.)r, r4 r �TRANSFER_BUFFER_SIZEr rB r r r' r3 r9 r: �is_aliverA r r �popleftr0 r1 r r )r �amtZ valid_datar r r r r �read� sN �� � & . ��zDaisyChainWrapper.readc C s0 | j � | jW d � S 1 s"0 Y d S )N)r r, rU r r r �tell" s zDaisyChainWrapper.tellc C s� d}|t jkr^|r td| ��| j�& | j| _d| _| j| _W d � n1 sP0 Y �nd|t jk�r�| j�\ || jkr~n@|| jkr�| j| _| jr�| j � | j� | j t | j�7 _ nd}W d � n1 s�0 Y |�r�| j �� | j�� �rP| j�4 | j �r$| j t | j �� �8 _ �q W d � n1 �s:0 Y t�d� q�| j�< || _t�� | _ d| _ d| _d| _| j �� W d � n1 �s�0 Y | j|| jd� n td| ��dS )z)Sets current read position of the stream.Fz_Invalid seek during daisy chain operation. Non-zero offset %s from os.SEEK_END is not supportedNTr )rE r) z:Daisy-chain download wrapper does not support seek mode %s)�os�SEEK_END�IOErrorr r, r0 r1 r4 �SEEK_SETr � appendleftr r r> rK r: r^ r_ r r r- r. rS r? r) )r �offset�whenceZrestart_downloadr r r �seek& sT ��* " < *��zDaisyChainWrapper.seekc C s dS )NTr rU r r r �seekable_ s zDaisyChainWrapper.seekable)r N)N) r r r! r" � _DEFAULT_DOWNLOAD_CHUNK_SIZEr r? ra rb rc rf rj rk r r r r r( Q s � H B :9r( )r" � __future__r r r r r- � contextlibrc r; r �gslib.cloud_apir r � gslib.utilsr r � gslib.utils.encryption_helperr rl �objectr �contextmanagerr' r( r r r r �<module> s& __pycache__/seek_ahead_thread.cpython-39.pyc 0000644 00000010131 15025013616 0015000 0 ustar 00 a +(Wgt � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z e j ZG d d � d e�ZG d d � d ej�ZdS )zIThreading code for estimating total work of long-running gsutil commands.� )�absolute_import)�print_function)�division)�unicode_literalsN)�thread_message)� constants)�parallelism_framework_utilc @ s e Zd ZdZddd�ZdS )�SeekAheadResultz�Result class for seek_ahead_iterator results. A class is used instead of a namedtuple, making it easier to document and use default keyword arguments. � r c C s || _ || _dS )a� Create a SeekAheadResult. Args: est_num_ops: Number of operations the iterated result represents. Operation is loosely defined as a single API call for a single object. The total number of API calls may not be known at the time of iteration, so this number is approximate. data_bytes: Number of data bytes that will be transferred (uploaded, downloaded, or rewritten) for this iterated result. N)� est_num_ops� data_bytes)�selfr r � r �&/opt/gsutil/gslib/seek_ahead_thread.py�__init__' s zSeekAheadResult.__init__N)r r )�__name__� __module__� __qualname__�__doc__r r r r r r s r c s( e Zd ZdZ� fdd�Zdd� Z� ZS )�SeekAheadThreada� Thread to estimate total work to be performed by all processes and threads. Because the ProducerThread can only buffer a certain number of tasks on the global task queue, it cannot reliably provide the total count or size of iterated results for operations involving many iterated arguments until it nears the end of iteration. This thread consumes an iterator that should be functionally identical to the ProducerThread, but iterates to the end without adding tasks to the global task queue in an effort to estimate the amount of total work that the call to Apply will perform. It should be used only for large operations, and thus it is created by the main ProducerThread only when the number of iterated arguments exceeds a threshold. This thread may produce an inaccurate estimate if its iterator produces different results than the iterator used by the ProducerThread. This can happen due to eventual listing consistency or due to the source being modified as iteration occurs. This thread estimates operations for top-level objects only; sub-operations (such as a parallel composite upload) should be reported via the iterator as a single object including the total number of bytes affected. c s2 t t| ��� || _|| _|| _d| _| �� dS )ay Initializes the seek ahead thread. Args: seek_ahead_iterator: Iterator matching the ProducerThread's args_iterator, but returning only object name and/or size in the result. cancel_event: threading.Event for signaling the seek-ahead iterator to terminate. status_queue: Status queue for posting summary of fully iterated results. FN)�superr r � status_queue�seek_ahead_iterator� cancel_event� terminate�start)r r r r �� __class__r r r O s zSeekAheadThread.__init__c C s� d}d}zR| j D ]F}| jr" W d S |tj dkrB| j�� rB W d S ||j7 }||j7 }qW n& ty� } zW Y d }~d S d }~0 0 | j�� r�d S t | j t � ||t � � �� d S )Nr )r r r �NUM_OBJECTS_PER_LIST_PAGEr �isSetr r �OSError�_PutToQueueWithTimeoutr r �SeekAheadMessage�time)r � num_objectsZnum_data_bytesZseek_ahead_result�er r r �runb s* ��zSeekAheadThread.run)r r r r r r& � __classcell__r r r r r 6 s r )r � __future__r r r r � threadingr# �gslibr � gslib.utilsr r �PutToQueueWithTimeoutr! �objectr �Threadr r r r r �<module> s __pycache__/pubsub_api.cpython-39.pyc 0000644 00000017277 15025013616 0013553 0 ustar 00 a +(Wg�, � @ sF d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl m"Z" ddl m#Z# ddl$m%Z% e j&Z'e �(� �r.dZ)ndZ)G dd� de*�Z+dS ) z>JSON gsutil Cloud API implementation for Google Cloud Storage.� )�absolute_import)�print_function)�division)�unicode_literalsN)� exceptions)�config)�AccessDeniedException)�BadRequestException)�NotFoundException)�PreconditionException)�ServiceException)� SetUpJsonCredentialsAndCache)�NoOpCredentials)�pubsub_v1_client)�pubsub_v1_messages)� system_util)� GetCertsFile)�GetMaxRetryDelay)� GetNewHttp)� GetNumRetries)�UTF8zVInsufficient OAuth2 scope to perform this operation. Please re-run `gcloud auth login`zRInsufficient OAuth2 scope to perform this operation. Please re-run `gsutil config`c sn e Zd ZdZd� fdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z ddd�Z dd� Z dd� Z ddd�Z � ZS )� PubsubApiz;Wraps calls to the Cloud Pub/Sub v1 interface via apitools.Nr c s� t t| ��� || _t� | _t� | _d| _t � ddd�| _ t � ddd�}|rTd| nd| _ | j| j | j | _ t| ||d � |d k}|d k}tj| j | j||| jd �| _t� | _| j| j_t� | _| j| j_t| jt�r�| j�d d � dS )a' Performs necessary setup for interacting with Google Cloud Pub/Sub. Args: logger: logging.logger for outputting log messages. credentials: Credentials to be used for interacting with Google Cloud Pub/Sub debug: Debug level for the API implementation (0..3). zhttps://� CredentialsZgs_pubsub_hostzpubsub.googleapis.com�gs_pubsub_portN�:� )� credentials� )�url�http� log_request� log_responser �key�'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM)�superr �__init__�loggerr � certs_filer r � http_baser �get� host_base� host_port�url_baser �apitools_clientZPubsubV1r � api_clientr � num_retriesr �max_retry_wait� isinstancer �AddGlobalParam)�selfr& r �debugr r r! �� __class__� � /opt/gsutil/gslib/pubsub_api.pyr% ; s8 �� �zPubsubApi.__init__c C sT t j|d�}z| jj�|�W S tyN } z| j||d� W Y d }~n d }~0 0 d S )N��topic�� topic_name)�apitools_messagesZ PubsubProjectsTopicsGetRequestr. �projects_topics�Get� TRANSLATABLE_APITOOLS_EXCEPTIONS�_TranslateExceptionAndRaise�r3 r< �request�er7 r7 r8 �GetTopicg s zPubsubApi.GetTopicc C sT t j|d�}z| jj�|�W S tyN } z| j||d� W Y d }~n d }~0 0 d S )N)�namer; )r= ZTopicr. r> �Creater@ rA )r3 r< r: rD r7 r7 r8 � CreateTopicn s zPubsubApi.CreateTopicc C sT t j|d�}z| jj�|�W S tyN } z| j||d� W Y d}~n d}~0 0 dS )zOnly used in system tests.r9 r; N)r= Z!PubsubProjectsTopicsDeleteRequestr. r> �Deleter@ rA rB r7 r7 r8 � DeleteTopicu s �zPubsubApi.DeleteTopicc C sT t j|d�}z| jj�|�W S tyN } z| j||d� W Y d }~n d }~0 0 d S )N)�resourcer; )r= Z'PubsubProjectsTopicsGetIamPolicyRequestr. r> � GetIamPolicyr@ rA rB r7 r7 r8 �GetTopicIamPolicy~ s �zPubsubApi.GetTopicIamPolicyc C sb t j|d�}t j||d�}z| jj�|�W S ty\ } z| j||d� W Y d }~n d }~0 0 d S )N)�policy)rK �setIamPolicyRequestr; )r= �SetIamPolicyRequestZ'PubsubProjectsTopicsSetIamPolicyRequestr. r> � SetIamPolicyr@ rA )r3 r< rN �policy_requestrC rD r7 r7 r8 �SetTopicIamPolicy� s �zPubsubApi.SetTopicIamPolicyc C s> | j �tj�r | j �dt�� � | j||d�}|r8|�n� dS )a+ Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. topic_name: Optional topic name in request that caused the exception. Raises: Translated CloudApi exception, or the original exception if it was not translatable. z TranslateExceptionAndRaise: %sr; N)r& � isEnabledFor�logging�DEBUGr4 � traceback� format_exc�_TranslateApitoolsException)r3 rD r< �translated_exceptionr7 r7 r8 rA � s ��z%PubsubApi._TranslateExceptionAndRaisec C sh t |tj�rdt|dd �rdz8t�|j�t��}d|v rNd|d v rN|d d W S W n t yb Y n0 d S )N�content�error�message) r1 �apitools_exceptions� HttpError�getattr�json�loadsr[ �decoder � Exception)r3 � http_error�json_objr7 r7 r8 �_GetMessageFromHttpError� s z"PubsubApi._GetMessageFromHttpErrorc C sX z@|j d }|�d�}|dkr>||d � �d�d }d| W S W n tyR Y n0 d S )Nzwww-authenticatezscope="r �"� zAcceptable scopes: %s)�response�find�splitrd )r3 re �www_authenticate� scope_idx�scopesr7 r7 r8 �!_GetAcceptableScopesFromHttpError� s z+PubsubApi._GetAcceptableScopesFromHttpErrorc C s� t |tj��r�| �|�}|jdkr4t|p*d|jd�S |jdkr�dt|�v r\t|pRd|jd�S dt|�v r~tt|j| � |�d�S �n|jd k�r<d t|�v r�t|p�d |jd�S d t|�v r�t|p�d |jd�S dt|�v r�td|jd�S dt|�v �r td|jd�S dt|�v �r$tt|j| � |�d�S t|�p0|j |jd�S nX|jdk�rVt ||jd�S |jdk�rz|�rzt d| |jd�S |jdk�r�t ||jd�S t ||jd�S dS )a- Translates apitools exceptions into their gsutil equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. topic_name: Optional topic name in request that caused the exception. Returns: ServiceException for translatable exceptions, None otherwise. i� z Bad Request)�statusi� zLogin Requiredz Access denied: login required.�insufficient_scope)rq �bodyi� z7The account for the specified project has been disabledzAccount disabled.z,Daily Limit for Unauthenticated Use Exceededz8Access denied: quota exceeded. Is your project ID valid?zUser Rate Limit Exceededz5Rate limit exceeded. Please retry this request later.zAccess Not Configuredz�Access Not Configured. Please go to the Google Cloud Platform Console (https://cloud.google.com/console#/project) for your project, select APIs and Auth and enable the Google Cloud Pub/Sub API.i� i� z The topic %s already exists.i� N)r1 r^ r_ rg � status_coder �strr �"_INSUFFICIENT_OAUTH2_SCOPE_MESSAGErp r] r r r )r3 rD r< r] r7 r7 r8 rY � sp � � � � � ��� � � z%PubsubApi._TranslateApitoolsException)NNr )N)N)�__name__� __module__� __qualname__�__doc__r% rE rH rJ rM rS rA rg rp rY � __classcell__r7 r7 r5 r8 r 8 s , r ),rz � __future__r r r r ra rU rW �apitools.base.pyr r^ �botor �gslib.cloud_apir r r r r �gslib.gcs_json_credentialsr �gslib.no_op_credentialsr Z!gslib.third_party.pubsub_apitoolsr r- r r= � gslib.utilsr Zgslib.utils.boto_utilr r r r �gslib.utils.constantsr r_ r@ �InvokedViaCloudSdkrv �objectr r7 r7 r7 r8 �<module> s> �� __pycache__/metrics_tuple.cpython-39.pyc 0000644 00000000753 15025013616 0014270 0 ustar 00 a +(Wg� � @ sR d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ edg d��Zd S ) zIDefines a Metric object used for sending information to Google Analytics.� )�absolute_import)�print_function)�division)�unicode_literals)� namedtuple�Metric)�endpoint�method�body� user_agentN) �__doc__� __future__r r r r � collectionsr r � r r �"/opt/gsutil/gslib/metrics_tuple.py�<module> s � __pycache__/gcs_json_media.cpython-39.pyc 0000644 00000046650 15025013616 0014363 0 ustar 00 a +(Wg�q � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddl mZ dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlm Z ddlm Z ddl m Z ddl Z ddl m!Z! e j"�re#Z$edd� �Z%G dd� de&�Z'G dd� de&�Z(dd� Z)G d d � d e&�Z*d d � Z+G d d!� d!e j,�Z-G d"d#� d#e j,�Z.dS )$zEMedia helper functions and classes for Google Cloud Storage JSON API.� )�absolute_import)�print_function)�division)�unicode_literalsN)� http_client)�urllib)� cStringIO)� exceptions)�BadRequestException)� LazyWrapper)�ProgressCallbackWithTimeout)�DEBUGLEVEL_DUMP_REQUESTS)�SSL_TIMEOUT_SEC)�TRANSFER_BUFFER_SIZE)�UTF8)� text_util)� parse_uric C s t �d�S )Nz\d+)�re�compile� r r �#/opt/gsutil/gslib/gcs_json_media.py�<lambda>2 � r c @ s2 e Zd ZdZdd� Zedd� �Zejdd� �ZdS )�BytesTransferredContainera Container class for passing number of bytes transferred to lower layers. For resumed transfers or connection rebuilds in the middle of a transfer, we need to rebuild the connection class with how much we've transferred so far. For uploads, we don't know the total number of bytes uploaded until we've queried the server, but we need to create the connection class to pass to httplib2 before we can query the server. This container object allows us to pass a reference into Upload/DownloadCallbackConnection. c C s d| _ d S )Nr �Z-_BytesTransferredContainer__bytes_transferred��selfr r r �__init__@ s z"BytesTransferredContainer.__init__c C s | j S �Nr r r r r �bytes_transferredC s z+BytesTransferredContainer.bytes_transferredc C s || _ d S r r �r �valuer r r r G s N)�__name__� __module__� __qualname__�__doc__r �propertyr �setterr r r r r 5 s r c @ s, e Zd ZdZeddddfdd�Zdd� ZdS )�$UploadCallbackConnectionClassFactoryz�Creates a class that can override an httplib2 connection. This is used to provide progress callbacks and disable dumping the upload payload during debug statements. It can later be used to provide on-the-fly hash digestion during upload. r Nc C s( || _ || _|| _|| _|| _|| _d S r )�bytes_uploaded_container� buffer_size� total_size�progress_callback�logger�debug)r r) r* r+ r, r- r. r r r r T s z-UploadCallbackConnectionClassFactory.__init__c sH | j �| j� | j�| j�| j�| j�G � �����fdd�dtj�}|S )z/Returns a connection class that overrides send.c sj e Zd ZdZ�ZdZ� ZdZ�ZdZ dZ dZ dZ dd� Z d��fdd � Z��fd d �Zd��fd d� ZdS )zYUploadCallbackConnectionClassFactory.GetConnectionClass.<locals>.UploadCallbackConnectionz&Connection class override for uploads.FN� g �?c _ s& t |d< tjj| g|�R i |�� d S �N�timeout�r �httplib2�HTTPSConnectionWithTimeoutr �r �args�kwargsr r r r z s zbUploadCallbackConnectionClassFactory.GetConnectionClass.<locals>.UploadCallbackConnection.__init__c s� | j �d� tjr| j }n6g }| j D ]*}t|t�r>|�|� q$|�|�t�� q$d� |�}t |�}� t kr|�r|�� d| � | j dd�= t|t �r�||7 }d}| j||d� |dur�| �|� dS )z�Send the currently buffered request and clear the buffer. Appends an extra \r\n to the buffer. Args: message_body: if specified, this is appended to the request. )r r s zsend: %sN)�num_metadata_bytes)�_buffer�extend�six�PY2� isinstance�bytes�append�encoder �join�lenr r. �str�send)r � message_body�encode_chunked�items�item�msgr8 �� outer_debug� outer_loggerr r � _send_output� s& zfUploadCallbackConnectionClassFactory.GetConnectionClass.<locals>.UploadCallbackConnection._send_outputc s^ |dkr:d� dd� |D ��}|| _� tkr�r��d� n�|dkrzz"td� dd� |D ���}|| _W q� tyv Y q�0 nj|dkr�zNd� d d� |D ��}t� �|�}t |�d kr�t|d �t|d � d | _ W n ty� Y n0 | jd k�rD| j�rD| j �rD| j t | j� | _ d| _d | _d | _ � tk�rD��rD��d| j � t jj| |g|�R � d S )a� Overrides HTTPConnection.putheader. Send a request header line to the server. For example: h.putheader('Accept', 'text/html'). This override records the content encoding, length, and range of the payload. For uploads where the content-range difference does not match the content-length, progress printing will under-report progress. These headers are used to calculate a multiplier to correct the progress. For example: the content-length for gzip transport encoded data represents the compressed size of the data while the content-range difference represents the uncompressed size. Dividing the content-range difference by the content-length gives the ratio to multiply the progress by to correctly report the relative progress. Args: header: The header. *values: A set of values for the header. zcontent-encodingr/ c S s g | ] }t |��qS r �rC ��.0�vr r r � <listcomp>� r zwUploadCallbackConnectionClassFactory.GetConnectionClass.<locals>.UploadCallbackConnection.putheader.<locals>.<listcomp>z4send: Using gzip transport encoding for the request.�content-lengthc S s g | ] }t |��qS r rN rO r r r rR � r z content-rangec S s g | ] }t |��qS r rN rO r r r rR � r � r �gzipNz&send: Setting progress modifier to %s.)rA �header_encodingr r. �int� header_length� ValueError� DECIMAL_REGEX�findallrB � header_range�float� size_modifierr �HTTPSConnection� putheader)r �header�valuesr! �rangesrJ r r r` � sB � " ��zcUploadCallbackConnectionClassFactory.GetConnectionClass.<locals>.UploadCallbackConnection.putheaderr c s | j s,d| _ � r,t�� �| _| j�| jj� t|tj�rBt |�}n t|tj �rZt� |�}n|}|� | j �}|�rtjr�tj�| |� n.t|t�r�tj�| |� ntj�| |�t�� t|�}|r�||kr�||8 }d}n ||8 }d}| j�rt|| j �}| j�|� |� | j �}qjdS )a Overrides HTTPConnection.send. Args: data: string or file-like object (implements read()) of data to send. num_metadata_bytes: number of bytes that consist of metadata (headers, etc.) not representing the data being uploaded. Tr N)�processed_initial_bytesr �callback_processor�Progressr) r r= r; � text_typer � binary_type�BytesIO�read�GCS_JSON_BUFFER_SIZEr< r3 r4 rD r> r@ r rB rW r^ )r �datar8 Z full_bufferZpartial_bufferZsent_data_bytes)�outer_progress_callback�outer_total_sizer r rD � sD �� � z^UploadCallbackConnectionClassFactory.GetConnectionClass.<locals>.UploadCallbackConnection.send)NF)r )r" r# r$ r% r) rd rk re �sizerV rX r\ r^ r rM r` rD r �Zouter_buffer_sizeZ outer_bytes_uploaded_containerrK rL rm rn r r �UploadCallbackConnectionk s '?rq )r) r* r+ r, r- r. r3 r4 )r rq r rp r �GetConnectionClassb s 2z7UploadCallbackConnectionClassFactory.GetConnectionClass�r" r# r$ r% r r rr r r r r r( L s � r( c s* | j � dddtjdf� fdd� }|| _ dS )z�Wraps upload_http so we only use our custom connection_type on PUTs. POSTs are used to refresh oauth tokens, and we don't want to process the data sent in those requests. Args: upload_http: httplib2.Http instance to wrap �GETNc s. |dks|dkr|}nd }� | |||||d�S )N�PUT�POST��method�body�headers� redirections�connection_typer )�urirx ry rz r{ r| Zoverride_connection_type�Z request_origr r � NewRequest* s �z)WrapUploadHttpRequest.<locals>.NewRequest)�requestr3 �DEFAULT_MAX_REDIRECTS)� upload_httpr r r~ r �WrapUploadHttpRequest s �r� c @ s* e Zd ZdZedddfdd�Zdd� ZdS )�&DownloadCallbackConnectionClassFactorya� Creates a class that can override an httplib2 connection. This is used to provide progress callbacks, disable dumping the download payload during debug statements, and provide on-the-fly hash digestion during download. On-the-fly digestion is particularly important because httplib2 will decompress gzipped content on-the-fly, thus this class provides our only opportunity to calculate the correct hash for an object that has a gzip hash in the cloud. r Nc C s" || _ || _|| _|| _|| _d S r )r* r+ r, � digesters�bytes_downloaded_container)r r� r* r+ r, r� r r r r J s z/DownloadCallbackConnectionClassFactory.__init__c s G � fdd�dt j�}|S )z6Returns a connection class that overrides getresponse.c sB e Zd ZdZ� jZ� jZ� jZ � j Z dZ dZ dd� Zddd�ZdS ) z]DownloadCallbackConnectionClassFactory.GetConnectionClass.<locals>.DownloadCallbackConnectionz(Connection class override for downloads.FNc _ s& t |d< tjj| g|�R i |�� d S r0 r2 r5 r r r r b s zfDownloadCallbackConnectionClassFactory.GetConnectionClass.<locals>.DownloadCallbackConnection.__init__c sB t j���}|jt jt jfvr"|S |j� d� �fdd� }||_|S )a Wraps an HTTPResponse to perform callbacks and hashing. In this function, self is a DownloadCallbackConnection. Args: buffering: Unused. This function uses a local buffer. Returns: HTTPResponse object with wrapped read function. Nc s� | r | t kr td| t f ��n| p$t } �jsXd�_�jrXt�j�j��_�j��jj � � | �}t |�}�jrz�j�|� �j r��j D ]}�j | � |� q�|S )a� Overrides HTTPConnection.getresponse.read. This function only supports reads of TRANSFER_BUFFER_SIZE or smaller. Args: amt: Integer n where 0 < n <= TRANSFER_BUFFER_SIZE. This is a keyword argument to match the read function it overrides, but it is required. Returns: Data read from HTTPConnection. z7Invalid HTTP read size %s during download, expected %s.T) r r rd rm r rn re rf � outer_bytes_downloaded_containerr rB �outer_digesters�update)�amtrl � read_length�alg�Zorig_read_funcr r r rj w s0 ���� zwDownloadCallbackConnectionClassFactory.GetConnectionClass.<locals>.DownloadCallbackConnection.getresponse.<locals>.read)N)r �HTTPConnection� getresponse�status�OK�PARTIAL_CONTENTrj )r � bufferingZ orig_responserj r r� r r� f s �%ziDownloadCallbackConnectionClassFactory.GetConnectionClass.<locals>.DownloadCallbackConnection.getresponse)F)r" r# r$ r% r+ rn r� r� r, rm r� r� rd re r r� r r r r �DownloadCallbackConnectionY s r� )r3 r4 )r r� r r r rr V s Gz9DownloadCallbackConnectionClassFactory.GetConnectionClassrs r r r r r� ? s � r� c s@ dd� }| j � dddtjdf� fdd� }t�|| �| _|| _ | S )z�Overrides download request functions for an httplib2.Http object. Args: download_http: httplib2.Http.object to wrap / override. Returns: Wrapped / overridden httplib2.Http object. c s� � �fdd�| j D �} | r*t| �d d p,d} | rB| �|�||� | �|�|||�\} } | r�| �| |�r�| �|�||� | �|�|||�\} } d| _| jdk�r| �� �|| | �D ]R}|�|�||� | �|�|||�\} } | jdkr�| j �|� |�| |� �qq�| j �s$|dv �s$| jdk�r�| j �r�| jd v �r�|�r�d | v�rb| jd k�rbt � d | | ��d | v �r�| d }t |�\}}}}}|du �r�tj�||�| d < | jd k�r�|dv �r�| d | d< d| v�r�|| d< t �|| | | j| � d|v �r�|d= d|v �r |d= d|v �r"| j�s"|d= d | v �r�| d }t�| �}d|v�rP||d< |}| jdv �rhd}d}| j|||||d |jd�\} } || _nt �d| | ��n<| jdv �r�|dv �r�d| v �r�|| d< t �|| | | j| � | | fS )zlDo the actual request using the connection object. Also follow one level of redirects if necessary. c s& g | ] }|� � ��r|���|f�qS r )�inscope�depth)rP �auth��host� request_urir r rR � s �zDWrapDownloadHttpRequest.<locals>.OverrideRequest.<locals>.<listcomp>r rT Ni� )rt �HEAD�/ )�, �- �. r� i3 �locationr� z:Redirected but the response is missing a Location: header.r� z-x-permanent-redirect-urlzcontent-locationz if-none-matchzif-modified-since� authorization)r� r� rt )ry rz r{ r| z4Redirected more times than redirection_limit allows.)�� �� )�authorizations�sortedr� � _conn_request�response� _stale_digestr� �_auth_from_challenger? �follow_all_redirects�follow_redirectsr3 �RedirectMissingLocationr r �parse�urljoin� _updateCache�cache� forward_authorization_headers�copy�deepcopy� __class__�previous� RedirectLimit)r �connr� � absolute_urir� rx ry rz r{ �cachekey�authsr� r� �contentr� r� �scheme� authority�path�query�fragment� old_response�redirect_methodr r� r �OverrideRequest� s� � � � � �� � � � � �z0WrapDownloadHttpRequest.<locals>.OverrideRequestrt Nc s4 |dkr � | ||||d d�S � | |||||d�S d S )Nrv rw r )r} rx ry rz r{ r| r~ r r r s ��z+WrapDownloadHttpRequest.<locals>.NewRequest)r� r3 r� �types� MethodType�_request)� download_httpr� r r r~ r �WrapDownloadHttpRequest� s _� r� c @ s e Zd ZdZdd� ZdS )�HttpWithNoRetriesa< httplib2.Http variant that does not retry. httplib2 automatically retries requests according to httplib2.RETRIES, but in certain cases httplib2 ignores the RETRIES value and forces a retry. Because httplib2 does not handle the case where the underlying request body is a stream, a retry may cause a non-idempotent write as the stream is partially consumed and not reset before the retry occurs. Here we override _conn_request to disable retries unequivocally, so that uploads may be retried at higher layers that properly handle stream request bodies. c C s� z0t |d�r |jd u r |�� |�||||� W n� tjyF � Y n� tjyp |�� t� d|j ��Y n� tj j y� |�� � Y nv tj y� } z>d}t |d�r�t|d�d }n|j}|tjjkrЂ W Y d }~n(d }~0 tj�y |�� � Y n0 z |�� }W n& tj tjf�y6 |�� � Y nB0 d} |dk�rP|�� n|�� } t�|�}|dk�rxt�|| �} || fS )N�sock� Unable to find the server at %sr r6 r/ r� )�hasattrr� �connectr� �socketr1 �gaierror�closer3 �ServerNotFoundErrorr� �ssl�SSLError�error�getattr�errno� ECONNREFUSEDr � HTTPExceptionr� rj �Response�_decompressContent) r r� r� rx ry rz �e�errr� r� r r r r� 9 sJ � z HttpWithNoRetries._conn_requestN)r" r# r$ r% r� r r r r r� + s r� c sB e Zd ZdZ� fdd�Zedd� �Zejdd� �Zdd� Z� Z S ) �HttpWithDownloadStreama� httplib2.Http variant that only pushes bytes through a stream. httplib2 handles media by storing entire chunks of responses in memory, which is undesirable particularly when multiple instances are used during multi-threaded/multi-process copy. This class copies and then overrides some httplib2 functions to use a streaming copy approach that uses small memory buffers. Also disables httplib2 retries (for reasons stated in the HttpWithNoRetries class doc). c s* d | _ t�� | _tt| �j|i |�� d S r )�_stream�logging� getLogger�_logger�superr� r )r r6 �kwds�r� r r r r s z HttpWithDownloadStream.__init__c C s | j S r �r� r r r r �streamw s z HttpWithDownloadStream.streamc C s || _ d S r r� r r r r r� { s c C sr z0t |d�r |jd u r |�� |�||||� W n� tjyF � Y n� tjyp |�� t� d|j ��Y n� tj j y� |�� � Y nv tj y� } z>d}t |d�r�t|d�d }n|j}|tjjkrЂ W Y d }~n(d }~0 tj�y |�� � Y n0 z |�� }W n: tj tjf�yJ } z|�� � W Y d }~�n(d }~0 0 d} |dk�rn|�� t�|�}n�|dk�rL|jtjtjfv �rLd } t |d��r�|�d �} |} d} | �t�} | �r�| jd u �r�t � d ��t � | j| � | t | �7 } n�q��q�| d u�r@t!| �t!| �k�r@| j"�#t$j%d | | |� |j&d = t'| �|j&d < t�|�}n |�� } t�|�}t�(|| �} || fS ) Nr� r� r r6 r/ r� rt rI rS z5Cannot exercise HttpWithDownloadStream with no streamzlOnly got %s bytes out of content-length %s for request URI %s. Resetting content-length to match bytes read.))r� r� r� r� r� r1 r� r� r3 r� r� r� r� r� r� r� r� r r� r� r� r� r� r� � getheaderrj r r� �apitools_exceptions�InvalidUserInputErrorr � write_to_fdrB �longr� �logr� �DEBUGrI rC r� )r r� r� rx ry rz r� r� r� r� �content_lengthZ http_streamZ bytes_readZnew_datar r r r� � s� � � � �� z$HttpWithDownloadStream._conn_request) r" r# r$ r% r r& r� r' r� � __classcell__r r r� r r� e s r� )/r% � __future__r r r r r� r� r r� r� r; � six.movesr r r �apitools.base.pyr r� �gslib.cloud_apir �gslib.lazy_wrapperr �gslib.progress_callbackr �gslib.utils.constantsr r r r � gslib.utilsr r3 r �PY3rW r� rZ �objectr r( r� r� r� �Httpr� r� r r r r �<module> sH T d : __pycache__/boto_resumable_upload.cpython-39.pyc 0000644 00000035563 15025013616 0015766 0 ustar 00 a +(Wg�b � @ s d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddl mZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z e j r�e Z!G dd� de"�Z#dS )ap Boto translation layer for resumable uploads. See https://cloud.google.com/storage/docs/resumable-uploads-xml for details. Resumable uploads will retry interrupted uploads, resuming at the byte count completed by the last upload attempt. If too many retries happen with no progress (per configurable num_retries param), the upload will be aborted in the current process. Unlike the boto implementation of resumable upload handler, this class does not directly interact with tracker files. Originally Google wrote and contributed this code to the boto project, then copied that code back into gsutil on the release of gsutil 4.0 which supports both boto and non-boto codepaths for resumable uploads. Any bug fixes made to this file should also be integrated to resumable_upload_handler.py in boto, where applicable. TODO: gsutil-beta: Add a similar comment to the boto code. � )�absolute_import)�print_function)�division)�unicode_literalsN)�urllib)� http_client)�config)� UserAgent)�AWSAuthConnection)� ResumableTransferDisposition)�ResumableUploadException)�InvalidUrlError)�GetMaxRetryDelay)� GetNumRetries)�XML_PROGRESS_CALLBACKS)�UTF8c @ s� e Zd ZdZdZejeej ej fZ dZ d dd�Z dd� Zd d d �Zd d � Zdd� Zd dd�Zdd� Zdd� Zdd� Zd dd�Zddefdd �ZdS )!�BotoResumableUploadz3Upload helper class for resumable uploads via boto.i )r ���Nc C s8 |r| � |� nd| _|| _d| _d| _|| _|| _dS )ae Constructor. Instantiate once for each uploaded file. Args: tracker_callback: Callback function that takes a string argument. Used by caller to track this upload across upload interruption. logger: logging.logger instance to use for debug messages. resume_url: If present, attempt to resume the upload at this URL. num_retries: Number of times to retry the upload making no progress. This count resets every time we make progress, so the upload can span many more than this number of retries. Nr )� _SetUploadUrl� upload_url� num_retries�service_has_bytes�upload_start_point�tracker_callback�logger)�selfr r � resume_urlr � r �*/opt/gsutil/gslib/boto_resumable_upload.py�__init__U s z BotoResumableUpload.__init__c C sd t j�|�}|j�� dvs |js,td| ��|| _t� ddd�pD|j| _ d|j |j f | _ d| _dS )z�Saves URL and resets upload state. Called when we start a new resumable upload or get a new tracker URL for the upload. Args: url: URL string for the upload. Raises InvalidUrlError if URL is syntactically invalid. )�http�httpszInvalid upload URL (%s)� Credentials�gs_hostNz%s?%sr )r �parse�urlparse�scheme�lower�netlocr r r �get�upload_url_host�path�query�upload_url_pathr )r �url� parse_resultr r r r r s � �z!BotoResumableUpload._SetUploadUrl�*c C s d||f S )Nz bytes %s/%sr )r � range_specZ length_specr r r �_BuildContentRangeHeader� s z,BotoResumableUpload._BuildContentRangeHeaderc C s. | � d|�dd�}tj|d| j| j|| jd�S )a� Queries service to find out state of given upload. Note that this method really just makes special case use of the fact that the upload service always returns the current start/end state whenever a PUT doesn't complete. Args: conn: HTTPConnection to use for the query. file_length: Total length of the file. Returns: HTTP response from sending request. Raises: ResumableUploadException if problem querying service. r0 �0)� Content-Range�Content-Length�PUT�r+ � auth_path�headers�host)r2 r � make_requestr- r* )r �conn� file_length� put_headersr r r �_QueryServiceState� s ��z&BotoResumableUpload._QueryServiceStatec C s� | � ||�}|jdkr"d|d fS |jdkr>td|j tj��d}|�d�}|r�t�d|�}|r�t|� d��}t|� d ��}d }n| j S |s�td t |� � � tj��|j dkr�| j� d ||� ||fS ) a� Queries service to find out what bytes it currently has. Args: conn: HTTPConnection to use for the query. file_length: Total length of the file. Returns: (service_start, service_end), where the values are inclusive. For example, (0, 2) would mean that the service has bytes 0, 1, *and* 2. Raises: ResumableUploadException if problem querying service. �� r � i4 z2Got non-308 response (%s) from service state queryF�rangezbytes=(\d+)-(\d+)� Tz7Couldn't parse upload service state query response (%s)z Service has: Range: %d - %d.)r? �statusr r � START_OVER� getheader�re�search�long�group�SERVICE_HAS_NOTHING�str� getheaders�debugr ) r r< r= �respZgot_valid_responser1 �m� service_start� service_endr r r �_QueryServicePos� s: � �� �z$BotoResumableUpload._QueryServicePosc C s� |j j}|jdkr | j�d� d| _i }|D ](}|�� dkrHtdtj��|| ||< q,d||j j < |� d|j j |j |�}|� � }|jdv r�td |j tj��n&|jd kr�|jd kr�td |j tj��|�d �}|s�td| tj��| �|� | �|� dS )z�Starts a new resumable upload. Args: key: Boto Key representing the object to upload. headers: Headers to use in the upload requests. Raises: ResumableUploadException if any errors occur. rA z Starting new resumable upload.r zcontent-lengthz5Attempt to specify Content-Length header (disallowed)�start�POST)� �� � zEGot status %d from attempt to start resumable upload. Will wait/retryr@ �� z>Got status %d from attempt to start resumable upload. Aborting�LocationzHNo resumable upload URL found in resumable initiation POST response (%s)N)�bucket� connectionrN r r r' r r �ABORT�provider�resumable_upload_headerr; �name�readrD �WAIT_BEFORE_RETRYrF r r ) r �keyr9 r< Z post_headers�krO �bodyr r r r �_StartNewResumableUpload� sN � ���� �� z,BotoResumableUpload._StartNewResumableUploadc C s@ |� | j�} |rL|dkr,|| j |d } n|dk r:d} nd} d} |||� |rX|�� ni } |r�||krv| �d|�} n| �d||d f |�} | | d< t|| �| d< tj|d | jd | | jd �}|� d |j � | D ]}|� || | � q�|� � |� d� | �r�tj�r |�| � |t| �7 }nDt| t��rD|�| � |t| �7 }n | �t�}|�|� |t|�7 }|�r�| d7 } | | k�s�| dk�r�|||� d} |� | j�} q�|� |j� |�r�|||� ||k�r�td ||f tj��|�� }|jd k�r |� d�|� d�|� d�fS |jdv �r tj }ntj}td|j|j f |��d S )ad Attempts to upload file bytes. Makes a single attempt using an existing resumable upload connection. Args: conn: HTTPConnection from the boto Key. http_conn: Separate HTTPConnection for the transfer. fp: File pointer containing bytes to upload. file_length: Total length of the file. total_bytes_uploaded: The total number of bytes uploaded. cb: Progress callback function that takes (progress, total_size). num_cb: Granularity of the callback (maximum number of times the callback will be called during the file transfer). If negative, perform callback with each buffer read. headers: Headers to be used in the upload requests. Returns: (etag, generation, metageneration) from service upon success. Raises: ResumableUploadException if any problems occur. rC r r r0 z%d-%drA r4 r5 r6 Nr7 z<File changed during upload: EOF at %d bytes of %d byte file.r@ �etagzx-goog-generationzx-goog-metageneration)i� rV rW rX z1Got response code %d while attempting upload (%s)) ra � BUFFER_SIZE�copyr2 rL r �build_base_http_requestr- r* � putrequestr+ � putheader� endheaders�set_debuglevel�six�PY2�send�len� isinstance�bytes�encoder rN r r r] � getresponserD rF rb �reason)r r< � http_conn�fpr= �total_bytes_uploaded�cb�num_cbr9 �buf�cb_count�ir> � range_header� http_requestrd Z buf_bytesrO � dispositionr r r �_UploadFileBytes s� �� �� � ��z$BotoResumableUpload._UploadFileBytesc C sx | j \}}|jj} | jr�z0| �| |�\}}|| _| jdkrF| j�d� W q� ty� } z2| jdkrr| j�d| j � | � ||� W Y d} ~ q�d} ~ 0 0 n | � ||� | j du r�|| _ |d } | |k r�|� | � |jj} | � | j| j| j�} | �| j� ztz$| �| | ||| |||�W W | �� S ttjf�y\ | �| |�} | jdk�rVtdtj��n� Y n0 W | �� n | �� 0 dS )a� Attempts a resumable upload. Args: key: Boto key representing object to upload. fp: File pointer containing upload bytes. file_length: Total length of the upload. headers: Headers to be used in upload requests. cb: Progress callback function that takes (progress, total_size). num_cb: Granularity of the callback (maximum number of times the callback will be called during the file transfer). If negative, perform callback with each buffer read. Returns: (etag, generation, metageneration) from service upon success. Raises: ResumableUploadException if any problems occur. rA zResuming transfer.z Unable to resume transfer (%s).Ni� z�Got 400 response from service state query after failed resumable upload attempt. This can happen for various reasons, including specifying an invalid request (e.g., an invalid canned ACL) or if the file size changed between upload attempts)rK r[ r\ r rS r rN r r �messagerf r �seek�new_http_connectionr* �port� is_securern r� �close�socket�errorr? rD r r] )r rc ry r= r9 r{ r| rQ rR r< �erz rx rO r r r �_AttemptResumableUpload� sP � $ � �� � z+BotoResumableUpload._AttemptResumableUploadc C s| |j tjkr(|dkr$| j�d|j� � nP|j tjkrP|dkrL| j�d|j� � n(|j tjkr`� n|dkrx| j�d|j� d S )NrA zWCaught non-retryable ResumableUploadException (%s); aborting but retaining tracker filezVCaught non-retryable ResumableUploadException (%s); aborting and removing tracker filez1Caught ResumableUploadException (%s) - will retry)r� r �ABORT_CUR_PROCESSr rN r� r] rE )r r� rN r r r � HandleResumableUploadException� s( � � �z2BotoResumableUpload.HandleResumableUploadExceptionr c C sz | j |krd| _n| jd7 _| j| jkr8tdtj��tt�� d| j t� �}|dkrl| j � d| j|� t � |� dS )a Tracks the number of iterations without progress. Performs randomized exponential backoff. Args: service_had_bytes_before_attempt: Number of bytes the service had prior to this upload attempt. debug: debug level 0..3 r rA zaToo many resumable upload attempts failed without progress. You might try this upload again laterrC zZGot retryable failure (%d progress-less in a row). Sleeping %3.1f seconds before re-tryingN) r �progress_less_iterationsr r r r� �min�randomr r rN �time�sleep)r � service_had_bytes_before_attemptrN �sleep_time_secsr r r �TrackProgressLessIterations� s" ���z/BotoResumableUpload.TrackProgressLessIterationsc C sZ |si }d}||v r&|| du r&||= |r6|||j j< t|d< |} |jjj} | jdu r^t� | _d| _| j } z@| � ||| |||�\} | _ | _ | j |_ | dkr�| j �d� W dS | j�y } zN| dkr�| j �d| �� � t| t��r | jtjk�r |jjj�� W Y d} ~ n:d} ~ 0 t�yD } z| �| | � W Y d} ~ n d} ~ 0 0 | j| | d� qddS ) a� Upload a file to a key into a bucket on GS, resumable upload protocol. Args: key: `boto.s3.key.Key` or subclass representing the upload destination. fp: File pointer to upload size: Size of the file to upload. headers: The headers to pass along with the PUT request canned_acl: Optional canned ACL to apply to object. cb: Callback function that will be called to report progress on the upload. The callback should accept two integer parameters, the first representing the number of bytes that have been successfully transmitted to GS, and the second representing the total number of bytes that need to be transmitted. num_cb: (optional) If a callback is specified with the cb parameter, this parameter determines the granularity of the callback by defining the maximum number of times the callback will be called during the file transfer. Providing a negative integer will cause your callback to be called with each buffer read. Raises: ResumableUploadException if a problem occurs during the transfer. z Content-TypeNz User-Agentr rA zResumable upload complete.zCaught exception (%s))rN )r^ � acl_headerr r[ r\ rN r r r� r r� � generation�metagenerationr �RETRYABLE_EXCEPTIONS�__repr__rs �IOError�errno�EPIPEr� r r� r� )r rc ry �sizer9 � canned_aclr{ r| � content_typer= rN r� �_r� r r r �SendFile s@ � ""�z BotoResumableUpload.SendFile)NN)r0 r0 )N)r )�__name__� __module__� __qualname__�__doc__rh r � HTTPExceptionr� r� r� �gaierrorr� rK r r r2 r? rS rf r� r� r� r� r r� r r r r r J s. � � : <wO � (�r )$r� � __future__r r r r r� r� rG r� r� ro � six.movesr r �botor r �boto.connectionr �boto.exceptionr r �gslib.exceptionr �gslib.utils.boto_utilr r �gslib.utils.constantsr r �PY3�intrI �objectr r r r r �<module> s2 __pycache__/metrics.cpython-39.pyc 0000644 00000065715 15025013616 0013070 0 ustar 00 a +(Wg� � @ s d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZdd lmZ dd lmZ ddlZdd l m Z dd l m Z dd l m!Z! ddl"m#Z# ddl"m$Z$ dZ%dZ&dZ'dZ(dZ)dZ*dZ+e�,d�Z-e j.�/e j.�0dd��Z1dZ2dd d d d d d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7� Z3G d8d9� d9e4�Z5d:d;� Z6d<d=� Z7ej8e6d>d?� ��Z9e6dLd@dA��Z:e6dBdC� �Z;e6dDdE� �Z<e6dFdG� �Z=e6dHdI� �Z>dMdJdK�Z?dS )Nz:Static data and helper functions for collecting user data.� )�absolute_import)�print_function)�division)�unicode_literalsN)� defaultdict��wraps)�input)�urllib)�VERSION)�Metric)� system_util)�CalculateThroughput)�HumanReadableToBytesz(https://ssl.google-analytics.com/collectzUA-36037335-16zUA-36037335-17ZCommandZRetryableErrorZ FatalErrorZPerformanceSummaryz.*google\.com$�~z.gsutil/analytics-uuidZDISABLED�ec�ea�el�evZcd1Zcd2Zcd3Zcd4Zcd5Zcd6Zcd7Zcd8Zcd9Zcd10Zcm1Zcm2Zcm3Zcm4Zcm5Zcm6Zcm7Zcm8Zcm9Zcm10Zcm11Zcm12Zcm13Zcm14) zEvent Categoryz Event Actionz Event Labelz Event Value� Command Name�Global Options�Command-Level Options�Config� Command Alias� Fatal Error�Parallelism Strategy�Source URL Type�Provider Types� Timestamp�Execution Time�Retryable Errors�Is Google Corp User� Num Processes� Num Threads�#Number of Files/Objects Transferred�!Size of Files/Objects Transferred�Average Overall Throughput� Num Retryable Service Errors� Num Retryable Network Errors�Thread Idle Time Percent�Slowest Thread Throughput�Fastest Thread Throughput� Disk I/O Timec @ s� e Zd ZdZeefdd�ZdZdZdd� Z e efdd��Z e d d � �Z e d d � �Ze d)dd��Ze d*dd��Ze dd� �Zdd� Zdd� Zeddfdd�ZG d d � d e�Zd d � Zd d!� Zd"d#� Zd$d%� Zd+d'd(�Z dS ),�MetricsCollectorz�A singleton class to handle metrics reporting to Google Analytics (GA). This class is not thread or process-safe, and logging directly to the MetricsCollector instance can only be done by a single thread. c C s� t � | _|| _t�� | _t�� }tj � dd�}|r6t }| � � }d}t �t�� �rTd}ddd|d|d d td |td |i| _d jt�� t�� d�| _g | _tt�| _d| _ dS )a Initialize a new MetricsCollector. This should only be invoked through the GetCollector or StartTestCollector functions. Args: ga_tid: The Google Analytics tracking ID to use for metrics collection. endpoint: The URL to send requests to. �GSUtilZuse_test_GA_propertyr � �v�1�tid�cid�t�eventr r! z{system}/{release})�system�releaseN) �_GetTimeInMillis� start_time�endpoint�logging� getLogger�loggerr- �_GetCID�boto�config�getbool�_GA_TID_TESTING�_ValidateAndGetConfigValues�_GOOGLE_CORP_HOST_RE�match�socket� gethostname� _GA_LABEL_MAP� ga_params�format�platformr6 r7 � user_agent�_metricsr �int�retryable_errors�perf_sum_params)�self�ga_tidr: r3 Zuse_test_property� config_valuesZ is_corp_user� rT � /opt/gsutil/gslib/metrics.py�__init__x s0 � � zMetricsCollector.__init__Nc sL g � d�� �fdd�}dD ]\}}|||dd� d� qd�d D ] \}}|||�fd d�d� q>d D ]\}}|||d d� d� qbd D ]V\}}t j�||�}|r�zt|�}� �||f� W q� ty� � �|�f� Y q�0 q�|dddd� d� |dddd� d� |dddd� d� |dddd� d� |dddd� d� d�tdd � � D ���S ) z�Parses the user's config file to aggregate non-PII config values. Returns: A comma-delimited string of config values explicitly set by the user in key:value pairs, sorted alphabetically by key. �INVALIDc s` z@t j�| |�}|r,||�r,� �||f� n|r>� �|�f� W n � �|�f� Y n0 d S �N)r? r@ � get_value�append)�section�category� validation_fn� config_value)rS �invalid_value_stringrT rU �GetAndValidateConfigValue� s zOMetricsCollector._ValidateAndGetConfigValues.<locals>.GetAndValidateConfigValue))�Boto�https_validate_certificates)r. �disable_analytics_prompt)r. Z use_magicfile)r. Ztab_completion_time_logsc S s t | ��� dv S )N)�true�false)�str�lower��valrT rT rU �<lambda>� s �z>MetricsCollector._ValidateAndGetConfigValues.<locals>.<lambda>)r[ r\ r] i� ) )ra �debug)ra Zhttp_socket_timeout)ra Z num_retries)ra Zmax_retry_delay)r. Zdefault_api_version)r. Z%sliced_object_download_max_components)r. Zparallel_process_count)r. Zparallel_thread_count)r. Z software_update_check_period)r. Ztab_completion_timeout)�OAuth2Zoauth2_refresh_retriesc s t | ��� ot| �� k S rX )rf �isdigitrN rh )�small_int_thresholdrT rU rj � s ))r. Zresumable_threshold)r. Zrsync_buffer_lines)r. Ztask_estimation_thresholdc S s t | ��� S rX )rf rm rh rT rT rU rj � � ))r. Z(parallel_composite_upload_component_size)r. Z#parallel_composite_upload_threshold)r. Z%sliced_object_download_component_size)r. Z sliced_object_download_thresholdr. Z check_hashesc S s | dv S )N)Zif_fast_else_failZif_fast_else_skip�always�neverrT rh rT rT rU rj s �Zcontent_languagec S s | � � ot| �dkS )N� )�isalpha�lenrh rT rT rU rj ro Zjson_api_versionc S s | d � � dko | dd � �� S )Nr r0 r/ )rg rm rh rT rT rU rj ro Z prefer_apic S s | dv S )N)�json�xmlrT rh rT rT rU rj ro rl Z token_cachec S s | dv S )N)Z file_systemZ in_memoryrT rh rT rT rU rj s ��,c S s g | ]}d � |d |d ��qS )z{0}:{1}r r/ )rJ )�.0r@ rT rT rU � <listcomp> s z@MetricsCollector._ValidateAndGetConfigValues.<locals>.<listcomp>)r? r@ rY r rZ � ValueError�join�sorted) rQ r` r[ Z bool_categoryZsmall_int_categoryZlarge_int_categoryZdata_size_categoryr^ Z size_in_bytesrT )rS r_ rn rU rC � sn � � � �������z,MetricsCollector._ValidateAndGetConfigValuesc C s" t �� r dS t js t | �t _t jS )zDReturns the singleton MetricsCollector instance or None if disabled.N)r- � IsDisabled� _instance)rR rT rT rU � GetCollector# s z MetricsCollector.GetCollectorc C s t jdu rt �� t jS )z6Returns whether metrics collection should be disabled.N)r- �_disabled_cache�_CheckAndSetDisabledCacherT rT rT rU r} - s zMetricsCollector.IsDisabledc C s� t j�d�dkrd| _n�t j�d�dkr8d| _| �� n|tj�ddd�rPd| _ndt� � rjt j�d� | _nJt j � t �r�t t �� }|�� tk| _W d � q�1 s�0 Y nd| _d S ) z1Sets _disabled_cache based on user opt-in or out.�GSUTIL_TEST_ANALYTICSr1 T�2Fr. Zuse_gcloud_storage�GA_CIDN)�os�environ�getr� �StartTestCollectorr? r@ rA r �InvokedViaCloudSdk�path�exists�_UUID_FILE_PATH�open�read�_DISABLED_TEXT)�cls�frT rT rU r� 4 s .z*MetricsCollector._CheckAndSetDisabledCache�https://example.com�user-agent-007c C s` | � � rdtjd< d| _| t|�| _|du r6ddd�}|| j_|| j_tjd dkr\d | j_dS ) aN Reset the singleton MetricsCollector with testing parameters. Should only be used for tests, where we want to change the default parameters. Args: endpoint: str, URL to post to user_agent: str, User-Agent string for header. ga_params: A list of two-dimensional string tuples to send as parameters. �0r� FN�b�d)�a�cr� r ) r} r� r� r� rB r~ rI rL r9 )r� r: rL rI rT rT rU r� L s z#MetricsCollector.StartTestCollectorc C s dt jd< d| _|| _dS )z�Reset the MetricsCollector with default parameters after testing. Args: original_instance: The original instance of the MetricsCollector so we can set the collector back to its original state. r1 r� N)r� r� r� r~ )r� Zoriginal_instancerT rT rU �StopTestCollectorg s z"MetricsCollector.StopTestCollectorc C sP t j�t�rDtt��} | �� }W d � n1 s20 Y |rD|S t j�d�S )z�Gets the client id from the UUID file or the SDK opt-in, or returns None. Returns: str, The hex string of the client id. Nr� )r� r� r� r� r� r� r� r� )r� r3 rT rT rU r> s s &zMetricsCollector._GetCIDc C s | j �|� dS )a Extends self.ga_params to include new parameters. This is only used to record parameters that are sent with every event type, such as global and command-level options. Args: new_params: A dictionary of key-value parameters to send. N)rI �update)rQ � new_paramsrT rT rU �ExtendGAParams� s z MetricsCollector.ExtendGAParamsc C s | j �t| �S )z�Convenience function for getting a ga_param of the collector. Args: param_name: The descriptive name of the param (e.g. 'Command Name'). Must be a key in _GA_LABEL_MAP. Returns: The GA parameter specified, or None. )rI r� rH )rQ � param_namerT rT rU � GetGAParam� s zMetricsCollector.GetGAParamr c K s� d|fd|fd|fd|ft d t� fg}|�dd� t�|�D �� |�dd� t�| j�D �� |d u rrt� | j }|�t d |f� tj � t |��}| j �t | jd || jd �� d S ) a� Adds a GA metric with the given parameters to the metrics queue. Args: category: str, the GA Event category. action: str, the GA Event action. label: str, the GA Event label. value: int, the GA Event value. execution_time: int, the execution time to record in ms. **custom_params: A dictionary of key, value pairs containing custom metrics and dimensions to send with the GA Event. r r r r r c S s g | ]\}}|d ur||f�qS rX rT �rx �kr0 rT rT rU ry � s z4MetricsCollector.CollectGAMetric.<locals>.<listcomp>c S s g | ]\}}|d ur||f�qS rX rT r� rT rT rU ry � s Nr �POST)r: �method�bodyrL )rH r8 �extend�six� iteritemsrI r9 rZ r �parse� urlencoder| rM r r: rL ) rQ r\ �action�label�value�execution_time� custom_params�params�datarT rT rU �CollectGAMetric� s( � � � ��z MetricsCollector.CollectGAMetricc @ s( e Zd ZdZdd� ZG dd� de�ZdS )z)MetricsCollector._PeformanceSummaryParamszEThis class contains information to create a PerformanceSummary event.c C s� d| _ d| _d| _d| _t� | _tjr0t�� | _ d| _ d| _ d| _ d| _ t| j�| _d | _d | _d | _d| _d| _d| _d| _d| _d| _d S )Nr F)� num_processes� num_threads� num_retryable_service_errors� num_retryable_network_errors�set�provider_typesr �IS_LINUX�GetDiskCounters�disk_counters_start�uses_fan� uses_slice�thread_idle_time�thread_execution_timer � _ThreadThroughputInformation�thread_throughputs�avg_throughput�total_elapsed_time�total_bytes_transferred�num_objects_transferred�is_daisy_chain� has_file_dst� has_cloud_dst� has_file_src� has_cloud_src�rQ rT rT rU rV � s* z2MetricsCollector._PeformanceSummaryParams.__init__c @ s0 e Zd ZdZdd� Zdd� Zdd� Zdd � Zd S ) zFMetricsCollector._PeformanceSummaryParams._ThreadThroughputInformationzDA class to keep track of throughput information for a single thread.c C s d| _ d| _d | _d | _d S )Nr )r� r� �task_start_time� task_sizer� rT rT rU rV � s zOMetricsCollector._PeformanceSummaryParams._ThreadThroughputInformation.__init__c C s || _ || _d S rX )r� r� )rQ r9 Zbytes_to_transferrT rT rU � LogTaskStart� s zSMetricsCollector._PeformanceSummaryParams._ThreadThroughputInformation.LogTaskStartc C s4 | j || j 7 _ | j| j7 _d | _d | _d S rX )r� r� r� r� )rQ Zend_timerT rT rU � LogTaskEnd s zQMetricsCollector._PeformanceSummaryParams._ThreadThroughputInformation.LogTaskEndc C s t | j| j�S rX )r r� r� r� rT rT rU � GetThroughput s �zTMetricsCollector._PeformanceSummaryParams._ThreadThroughputInformation.GetThroughputN)�__name__� __module__� __qualname__�__doc__rV r� r� r� rT rT rT rU r� � s r� N)r� r� r� r� rV �objectr� rT rT rT rU �_PeformanceSummaryParams� s 'r� c C s� | � d�dvrdS | jdu r&| �� | _d|v rB| j|d d� dS t�|�D ]�\}}|dv rzt| j|�}|szt| j||� |dv r�t| j|�}t| j||| � |dv r�t| j|�}||k r�t| j||� |d krL| jj� |� qLdS ) ag Updates the _PeformanceSummaryParams object. Args: params: A dictionary of keyword arguments. - uses_fan: True if the command uses fan parallelism. - uses_slice: True if the command uses slice parallelism. - avg_throughput: The average throughput of the data transfer. - is_daisy_chain: True if the transfer uses the daisy-chain method. - has_file_dst: True if the transfer's destination is a file URL. - has_cloud_dst: True if the transfer's destination is in the cloud. - has_file_src: True if the transfer has a file URL as a source. - has_cloud_src: True if the transfer has a cloud URL as a source. - total_elapsed_time: The total amount of time spent on Apply. - total_bytes_transferred: The total number of bytes transferred. - thread_idle_time: The additional amount of time that threads spent idle in Apply. - thread_execution_time: The additional amount of time that threads spent executing in Apply. - num_retryable_service_errors: The additional number of retryable service errors that occurred. - num_retryable_network_errors: The additional number of retryable network errors that occurred. - num_processes: The number of processes used in a call to Apply. - num_threads: The number of threads used in a call to Apply. - num_objects_transferred: The total number of objects transferred, as specified by a ProducerThreadMessage. - provider_types: A list of additional provider types used. - file_message: A FileMessage used to calculate thread throughput and number of objects transferred in the non-parallel case. r )�cp�rsyncN� file_message)r� ) r� r� r� r� r� r� r� r� r� r� r� )r� r� r� r� )r� r� r� ) r� rP r� �_ProcessFileMessager� r� �getattr�setattrr� r� )rQ r� r� �paramZ cur_valuerT rT rU � UpdatePerformanceSummaryParams s* z/MetricsCollector.UpdatePerformanceSummaryParamsc C s\ | j j|j|jf }|jrH| j js:| j js:| j jd7 _|�|j � n|� |j |j � dS )aT Processes FileMessages for thread throughput calculations. Update a thread's throughput based on the FileMessage, which marks the start or end of a file or component transfer. The FileMessage provides the number of bytes transferred as well as start and end time. Args: file_message: The FileMessage to process. r/ N) rP r� Z process_idZ thread_id�finishedr� r� r� r� �timer� �size)rQ r� � thread_inforT rT rU r� R s �z$MetricsCollector._ProcessFileMessagec C s� | � d�}|r6| jf t|d�td t| j�� �i�� t�| j�D ]&\}}| jf t |d�td |i�� qB| � d�}|r�| jt |d� dS )zDAggregates command and error info and adds them to the metrics list.r )r\ r� r r N) r� r� �_GA_COMMANDS_CATEGORYrH �sumrO �valuesr� r� �_GA_ERRORRETRY_CATEGORY�_GA_ERRORFATAL_CATEGORY)rQ � command_name� error_typeZ num_errorsZfatal_error_typerT rT rU � _CollectCommandAndErrorMetricsh s( � ��� � �z/MetricsCollector._CollectCommandAndErrorMetricsc C s� | j du rdS i }dD ] \}}t| j |�|t| < qtjr~| j j}t�� }tdd� |�� D ��tdd� |�� D �� |td < | j j r�| j j r�dnd}nd }||td < | j j r�| j j r�dnd }n| j j r�d nd }||td < | j j | j j }|�rt| j j �t|� |td < | j j�rPdd� | j j�� D �} t| �|td < t| �|td < d�t| j j��|td < | j j �oz| j j| j j �o�| j j| j j| j j �o�| j j| j j �o�| j jd�} d�tdd� t�| �D ���} t| j j �} | j f t | | d�|�� dS )zCAggregates PerformanceSummary info and adds the metric to the list.N))r� r" )r� r# )r� r' )r� r( )r� r& )r� r$ )r� r% c S s g | ]}|d |d �qS �� � rT �rx �statrT rT rU ry � ro zEMetricsCollector._CollectPerformanceSummaryMetric.<locals>.<listcomp>c S s g | ]}|d |d �qS r� rT r� rT rT rU ry � ro r, ZbothZcloud�filer Zfan�sliceZnoner r) c S s g | ] }|� � �qS rT )r� )rx �threadrT rT rU ry � s �r* r+ rw r )Z CloudToCloudZ CloudToFileZ DaisyChainZ FileToCloudZ FileToFilec S s g | ]\}}|r|�qS rT rT )rx Z transfer_typeZcondrT rT rU ry � s �)r\ r� r� ) rP r� rH r r� r� r� r� r� r� r� r� r� r� r� �floatr� �min�maxr{ r| r� r� r� r� r� r� r8 r� r� �_GA_PERFSUM_CATEGORY) rQ r� � attr_namer� Z disk_startZdisk_endZ src_url_typeZstrategyZ total_timeZ throughputsZtransfer_typesr� Zapply_execution_timerT rT rU � _CollectPerformanceSummaryMetric� s� ��� �� �� � � ��� ���z1MetricsCollector._CollectPerformanceSummaryMetricFc C s� | � � | �� | jsdS |s(| j�� }tj�d�dkr>tj }t j dd�}t � |j�}|� t�| j|� W d � n1 s~0 Y t�| j� g | _|dur�t � d| �}t � d�|||��}tjd|g}tj�� }tj�tj�|d < t� } t �|�D ] \} } t � | �| t � | �< q�zDtj || t j �o0t j d �} | j�d � |�r\| � � | j�d � W n> t!�y� | j�d � zt�"|j� W n Y n0 Y n0 dS )aZ Reports the collected metrics using a separate async process. Args: wait_for_report: bool, True if the main process should wait for the subprocess to exit for testing purposes. log_level: int, The subprocess logger's level of debugging for testing purposes. log_file_path: str, The file that the metrics_reporter module should write its logs to. If not supplied, the metrics_reporter module will use a predetermined default path. This parameter is intended for use by tests that need to evaluate the contents of the file at this path. Nr� r� F)�deletezr"%s"z_from gslib.metrics_reporter import ReportMetrics; ReportMetrics(r"{0}", {1}, log_file_path={2})z-cZ PYTHONPATH)�env�shellz$Metrics reporting process started...z#Metrics reporting process finished.z*Metrics reporting process failed to start.)#r� r� rM r= �getEffectiveLevelr� r� r� r; �WARN�tempfile�NamedTemporaryFiler� � ensure_str�name�pickle�dumprk rJ �sys� executable�copy�pathsepr{ r� �dictr� � subprocess�Popen�PY3r � IS_WINDOWS� communicate�OSError�unlink) rQ Zwait_for_reportZ log_levelZ log_file_pathZtemp_metrics_fileZtemp_metrics_file_nameZreporting_codeZexecution_argsZexec_envZsm_envr� r0 �prT rT rU � ReportMetrics� sT , �� � z MetricsCollector.ReportMetrics)r� r� N)N)FNN) r� r� r� r� �_GA_TID� _GA_ENDPOINTrV r~ r� rC � staticmethodr r} � classmethodr� r� r� r> r� r� r r� r� r� r� r� r� r� r rT rT rT rU r- q sF 8o � � )FHi �r- c s t � �� fdd��}|S )z�Function decorator to capture and log any exceptions. This is extra insurance that analytics collection will not hinder the command being run upon an error. Args: func: The function to wrap. Returns: The wrapped function. c sT z� | i |��W S t yN } z&t�d�}|�d� j|� W Y d }~n d }~0 0 d S )N�metricsz6Exception captured in %s during metrics collection: %s)� Exceptionr; r< rk r� )�args�kwds�er= ��funcrT rU �Wrapper? s �z'CaptureAndLogException.<locals>.Wrapperr �r r rT r rU �CaptureAndLogException2 s r c s t � �� fdd��}|S )a\ Function decorator to ignore an exception on collecting thread stats. An exception can happen if the thread_stats dictionary's manager gets shutdown before the thread's process is successfully killed. See _ThreadStat.AggregateStat for how we handle that case. Args: func: The function to wrap. Returns: The wrapped function. c s" z� | i |��W S Y n0 d S rX rT )r r r rT rU r Y s z+CaptureThreadStatException.<locals>.Wrapperr r rT r rU �CaptureThreadStatExceptionK s r c C s t �� } | r| �� dS )z9Reports the metrics that were collected upon termination.N)r- r r )� collectorrT rT rU �Shutdownf s r c C s� t �� }|sdS | r0|�d�s0|�td | i� |rh|�d�shd�tdd� |D ���}|�td |i� |�d�} | szdS |r�d�| d�|��}|�td |i� |r�|�d �s�d�td d� |D ���}|�td |i� |r�|�d �s�|�td |i� dS ) a� Logs info about the gsutil command being run. This only updates the collector's ga_params. The actual command metric will be collected once ReportMetrics() is called at shutdown. Args: command_name: str, The official command name (e.g. version instead of ver). subcommands: A list of subcommands as strings already validated by RunCommand. We do not log subcommands for the help or test commands. global_opts: A list of string tuples already parsed by __main__. sub_opts: A list of command-level options as string tuples already parsed by RunCommand. command_alias: str, The supported alias that the user inputed. Nr r rw c S s g | ]}|d � d��qS �r �-��strip�rx �optrT rT rU ry � ro z$LogCommandParams.<locals>.<listcomp>z{0} {1}� r c S s g | ]}|d � d��qS r! r# r% rT rT rU ry � ro r )r- r r� r� rH r{ r| rJ ) r� Z subcommands� global_optsZsub_optsZ command_aliasr Zglobal_opts_stringZfull_command_nameZsub_opts_stringrT rT rU �LogCommandParamso s2 � � �r) c C s@ t �� }|r<|j| j d7 < | jr2tdd� n tdd� dS )z�Logs that a retryable error was caught for a gsutil command. Args: message: The RetryableErrorMessage posted to the global status queue. r/ )r� )r� N)r- r rO r� Zis_service_error�LogPerformanceSummaryParams)�messager rT rT rU �LogRetryableError� s r, c C s& t �� }|r"|�td | jji� dS )z}Logs that a fatal error was caught for a gsutil command. Args: exception: The exception that the command failed on. r N)r- r r� rH � __class__r� )� exceptionr rT rT rU � LogFatalError� s �r/ c K s t �� }|r|�| � dS )a� Logs parameters necessary for a PerformanceSummary. gsutil periodically monitors its own threads; at the end of the execution of each cp/rsync command, it will present a performance summary of the command run. Args: **params_to_update: A dictionary. See UpdatePerformanceSummaryParams for details. The argument ambiguity at this level allows for flexibility in dealing with arguments that are processed similarly. N)r- r r� )Zparams_to_updater rT rT rU r* � s r* c C s� t j�dd�} tj�t�s�| s�t�� s�t dt � d� d �}t }|� � d dkrZt�� j}t�tj�t�� ttd��}|�|� W d � n1 s�0 Y d S ) z�Asks a user to opt-in to data collection if a UUID file does not exist. If the user agrees, generates a UUID file. Will not prompt if part of SDK. r. rc � z�gsutil developers rely on user feedback to make improvements to the tool. Would you like to send anonymous usage statistics to help improve gsutil? [y/N]r' r �y�wN)r? r@ rY r� r� r� r� r r� r �textwrap�fillr� rg �uuidZuuid4�hex�CreateDirIfNeeded�dirnamer� �write)Zdisable_promptZenable_analyticsZ text_to_writer� rT rT rU �'CheckAndMaybePromptForAnalyticsEnabling� s ��� r: c C s | d u rt � � } t| d �S )Ni� )r� rN )Z time_in_secrT rT rU r8 � s r8 )NNNNN)N)@r� � __future__r r r r �atexit� collectionsr � functoolsr r; r� r rK �rerF r r r� r3 r� r5 r� � six.movesr r r? �gslibr Zgslib.metrics_tupler � gslib.utilsr �gslib.utils.unit_utilr r r r rB r� r� r� r� �compilerD r� � expanduserr{ r� r� rH r� r- r r �registerr r) r, r/ r* r: r8 rT rT rT rU �<module> s� �� F � 1 __pycache__/help_provider.cpython-39.pyc 0000644 00000004563 15025013616 0014256 0 ustar 00 a +(Wg � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ dd gZ d Z d Z d e Z d Z dZG dd� de�Zdd� Zdd� ZdS )z=Module defining help types and providers for gsutil commands.� )�absolute_import)�print_function)�division)�unicode_literalsN)�CommandExceptionZ command_helpZadditional_help� � �P z <B>DESCRIPTION</B>z <B>SYNOPSIS</B>c @ s$ e Zd ZdZe�dg d��ZdZdS )� HelpProviderz Interface for providing help.�HelpSpec)� help_name�help_name_aliasesZ help_type�help_one_line_summary� help_text�subcommand_help_textN)�__name__� __module__� __qualname__�__doc__� collections� namedtupler � help_spec� r r �"/opt/gsutil/gslib/help_provider.pyr ( s �r c C s t | jj�}|dkr |tk s2J dj| jjtd���| jjD ] }|s:J �q:t | jj�}|tkstJ dj| jj|td���|tks�J dj| jj|td���t | jj �dks�J d j| jj d ���| jjg}|� | jj� |D ]&}||v r�t d || j || j f ��q�d S ) zGHelper for checking that a HelpProvider has minimally adequate content.� z.The help name "{text}" must be less then {max})�text�maxzZThe one line summary "{text}" with a length of {length} must be more then {min} characters)r �length�minzZThe one line summary "{text}" with a length of {length} must be less then {max} characters)r r r r z+The length of "{text}" must be less then 10)r zhDuplicate help name/alias "%s" found while loading help from %s. That name/alias was already taken by %sN) �lenr r �MAX_HELP_NAME_LEN�formatr r �MIN_ONE_LINE_SUMMARY_LEN�MAX_ONE_LINE_SUMMARY_LENr �extendr r )Z help_providerZ help_name_mapZ help_name_lenZhnaZone_line_summary_lenZname_check_listZ name_or_aliasr r r � SanityCheckD sR �� �� �� ��� ���r% c C s t | t | S )zCHelper for adding help text headers given synopsis and description.)�SYNOPSIS_PREFIX�DESCRIPTION_PREFIX)Zsynopsis� descriptionr r r �CreateHelpTextj s r) )r � __future__r r r r r �gslib.exceptionr ZALL_HELP_TYPESr r" r# r' r& �objectr r% r) r r r r �<module> s & __pycache__/gcs_json_api.cpython-39.pyc 0000644 00000152574 15025013616 0014060 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZddl m Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZdd lmZ dd lmZ dd lmZ dd lmZ dd lm Z ddl m Z ddl Z ddl Z ddl!m"Z" ddl#m$Z$ ddl#m%Z% ddl#m&Z& ddl#m'Z' ddl#m(Z( ddl#m)Z) ddl#m*Z* ddl#m+Z+ ddl#m,Z, ddl#m-Z- ddl#m.Z. ddl#m/Z/ dd l#m0Z0 dd l#m1Z1 dd l#m2Z2 dd l3m4Z4 dd l5m6Z6 dd!l5m7Z7 dd"l8m9Z9 dd#l8m:Z: dd$l8m;Z; dd%l8m<Z< dd&l8m=Z= dd'l8m>Z> dd(l8m?Z? dd)l@mAZA dd*lBmCZC dd+lDmEZE dd,lFmGZG dd-lHmIZJ dd.lHmKZL dd/lMmNZN dd0lMmOZO dd1lMmPZP dd2lMmQZQ dd3lMmRZR dd4lSmTZT dd5lSmUZU dd6lSmVZV dd7lSmWZW dd8lSmXZX dd9lSmYZY dd:lSmZZZ dd;l[m\Z\ dd<l[m]Z] dd=l^m_Z_ dd>l^m`Z` dd?l^maZa dd@l^mbZb ddAl^mcZc ddBldmeZe ddCldmfZf ddDldmgZg ddEldmhZh ddFlimjZj ddGlkmlZl ddHlmmnZnmoZo ddIlpmqZq ddJlrmsZs ddKlrmtZt ddLlrmuZu ddMlrmvZv ddNlrmwZw ddOlrmxZx ddPlymzZz ej{�re|Z}eLj~Z~e~jZ�e�j�Z�dQZ�ej�ej�ej�ej�fZ�ej�ej�ej�ej�e j�e j�e j�e j�e j�j�ej�ej�ej�ej�e�fZ�e�g dR��Z�ddSdTdUdVdWdX�Z�ddSdYdZdTdUdVd[�Z�e���� Z�e���e�� e�d\d]g�Z�d^Z�d_Z�d`Z�daZ�G dbdc� dce'�Z�dS )dz>JSON gsutil Cloud API implementation for Google Cloud Storage.� )�absolute_import)�print_function)�division)�unicode_literals)�contextmanagerN)� http_client)�encoding)� exceptions)� http_wrapper)�transfer)�CalculateWaitForRetry)�config)�context_config)�AccessDeniedException)�ArgumentException)�BadRequestException)�CloudApi)�EncryptionException)�NotEmptyException)�NotFoundException)�PreconditionException)� Preconditions)� PublishPermissionDeniedException)�ResumableDownloadException)� ResumableUploadAbortException)�ResumableUploadException)�!ResumableUploadStartOverException)�ServiceException)�CommandException)� SetUpJsonCredentialsAndCache)�isP12Credentials)�BytesTransferredContainer)�&DownloadCallbackConnectionClassFactory)�HttpWithDownloadStream)�HttpWithNoRetries)�$UploadCallbackConnectionClassFactory)�WrapDownloadHttpRequest)�WrapUploadHttpRequest)�ImpersonationCredentials)�NoOpCredentials)�ProgressCallbackWithTimeout)�PopulateProjectId)�storage_v1_client)�storage_v1_messages)�DeleteTrackerFile)�GetRewriteTrackerFilePath)�HashRewriteParameters)�ReadRewriteTrackerFile)�WriteRewriteTrackerFile)� GetCertsFile)�GetGcsJsonApiVersion)�GetJsonResumableChunkSize)�GetMaxRetryDelay)� GetNewHttp)� GetNumRetries)� JsonResumableChunkSizeDefined)�ListToGetFields)�ValidateDstObjectMetadata)�IAM_POLICY_VERSION)�NUM_OBJECTS_PER_LIST_PAGE)�REQUEST_REASON_ENV_VAR)�REQUEST_REASON_HEADER_KEY)�UTF8)�#Base64Sha256FromBase64EncryptionKey)� CryptoKeyType)�CryptoKeyWrapperFromKey)� FindMatchingCSEKInBotoConfig)� AddAcceptEncodingGzipIfNeeded)�LogAndHandleRetries)� CreatePayload� GetFinalUrl)�GetPrintableExceptionString)� CreateBucketNotFoundException)�%CreateNotFoundExceptionForObjectWrite)� CreateObjectNotFoundException)�DEFAULT_CONTENT_TYPE)�PRIVATE_DEFAULT_OBJ_ACL)�REMOVE_CORS_CONFIG)�ServiceAccountCredentialsi� )�acl�defaultObjectAclz items/aclzitems/defaultObjectAclz items/owner�owner�authenticatedRead�private�projectPrivate� publicRead�publicReadWrite)N�authenticated-readrU �project-private� public-readzpublic-read-write�bucketOwnerRead�bucketOwnerFullControl)NrY zbucket-owner-readzbucket-owner-full-controlrU rZ r[ �crc32c�md5Hash�skipz4Insufficient OAuth2 scope to perform this operation.zstorage.googleapis.comzstorage.mtls.googleapis.comc s� e Zd ZdZd�� fdd� Zdd� Zdd � Zd d � Zd d � Zdd� Z d�dd�Z d�dd�Z d�dd�Z d�dd�Z d�dd�Zd�dd�Zd d � Zd�d d �Zd d!� Zd"d#� Zd�d$d%�Zd�d&d'�Zd�d(d)�Zd�d*d+�Zd�d,d-�Zd.d/� Zd�d0d1�Zd�d2d3�Zd4d5� Z e d6d7� �Z d�d8d9�Z d�d:d;�Z d�d<d=�Z!d�d>d?�Z"d�d@dA�Z#d�dBdC�Z$dDdE� Z%d�dFdG�Z&d�dHdI�Z'd�dJdK�Z(ddddLe)j*j+ddddddf dMdN�Z,d�dOdP�Z-d�dQdR�Z.d�dSdT�Z/ddddddddde0j1ddLf dUdV�Z2dWdX� Z3d�dYdZ�Z4d�d[d\�Z5d�d]d^�Z6d�d_d`�Z7d�dadb�Z8d�dcdd�Z9d�dedf�Z:d�dgdh�Z;d�didj�Z<dkdl� Z=d�dmdn�Z>dodp� Z?dqdr� Z@dsdt� ZAdudv� ZBdwdx� ZCd�dzd{�ZDd|d}� ZEd~d� ZFd�d�� ZGd�d�� ZHd�d�d��ZId�d�� ZJd�d�� ZKd�d�� ZLd�d�d��ZM� ZNS )�� GcsJsonApiz=Google Cloud Storage JSON implementation of gsutil Cloud API.Nr c s� t t| �j|||d|||| | d� t� | _t� | _t| ||d� | �� | _ | � � | _ | j r|| j � | j �| _| j � | j �| _n| j | _| j | _t| j �r�t| jj� t| jj� nt| j� t| j� d| _t�ddd�} t�� �r t�� j�r | �rtd�t| ���t| _ n | �pt | _ | �s>t�dd d�} | �r>td | ��t�dd d�} | �szt�dd d�}|�rrtd | ��d| _ ndt�dd � | _ t � | _ | j| j | j d d | j d | _!|�r�t"j#d| d�nd| _$i }t�ddd�}|�r�| �r�||d< | �%|� |dk}|dk}t&j'| j!| j||| j | j | j$|d�| _(t)� | _*| j*| j(_*t+� | _,| j,| j(_,t-| j.d�| j(_/d| j(_0t1| j t2��r�| j(�3dd� dS ) a� Performs necessary setup for interacting with Google Cloud Storage. Args: bucket_storage_uri_class: Unused. logger: logging.logger for outputting log messages. status_queue: Queue for relaying status to UI. provider: Unused. This implementation supports only Google Cloud Storage. credentials: Credentials to be used for interacting with Google Cloud Storage. debug: Debug level for the API implementation (0..3). http_headers (dict|None): Arbitrary headers to be included in every request. trace_token: Trace token to pass to the API implementation. perf_trace_token: Performance trace token to use when making API calls. user_project: Project to be billed for this request. �gs)�provider�debug� http_headers� trace_token�perf_trace_token� user_project)� credentialszhttps://� Credentials� gs_json_hostNz�"use_client_certificate" is enabled, which sets gsutil to use the host {}. However, a custom host was set using "gs_json_host": {}. Please set "use_client_certificate" to "False" or comment out the "gs_json_host" line in the Boto config.�gs_hostz�JSON API is selected but gs_json_host is not configured, while gs_host is configured to %s. Please also configure gs_json_host and gs_json_port to match your desired endpoint.� gs_json_port�gs_portz�JSON API is selected but gs_json_port is not configured, while gs_port is configured to %s. Please also configure gs_json_host and gs_json_port to match your desired endpoint.� �:�/zstorage/ztoken:%s)�trace�gs_json_host_header�Host� )�url�http� log_request� log_responseri �version�default_global_params�additional_http_headers)� status_queueT�keyZ'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM)4�superra �__init__r3 � certs_filer7 rw r �_GetNewDownloadHttp� download_http�_GetNewUploadHttp� upload_httpri � authorize�authorized_download_http�authorized_upload_httpr r& r' Z http_baser �getr �get_context_config�use_client_certificater �format� MTLS_HOSTZ host_base� DEFAULT_HOST� host_portr4 � api_version�url_base�apitools_messages�StandardQueryParameters� global_params�_UpdateHeaders�apitools_clientZ StorageV1� api_clientr6 �max_retry_waitr8 � num_retriesrF r} � retry_func�(overwrite_transfer_urls_with_client_base� isinstancer) �AddGlobalParam)�self�bucket_storage_uri_class�loggerr} rc ri rd re rf rg rh rk rl rm rn r| rs rx ry �� __class__� �!/opt/gsutil/gslib/gcs_json_api.pyr� � s� � � ��� ���������� � � �zGcsJsonApi.__init__c C s4 t | jt�r| jjS t | jt�r(| jjS td��dS )z%Returns the service account email id.zBCannot get service account email id for the given credential type.N)r� ri r( �service_account_idrP �service_account_emailr �r� r� r� r� �GetServiceAccountIdY s �z GcsJsonApi.GetServiceAccountIdc C s> | j r|�| j � | jr"| j|d< tj�t�}|r:||t< d S )N�cookie)re �updaterg �os�environr� r> r? )r� �headers�request_reasonr� r� r� r� d s zGcsJsonApi._UpdateHeadersc C s t td�S )N�� http_class)r7 r# r� r� r� r� r� q s z GcsJsonApi._GetNewDownloadHttpc C s t td�S )zCReturns an upload-safe Http object (by disabling httplib2 retries).r� )r7 r$ r� r� r� r� r� t s z GcsJsonApi._GetNewUploadHttpc C sX t | jt�r4| jj}| jj}|�||�}|j|jfS t | jt�rL| j� |�S t d��dS )zReturns the Signed Content.zKAuthentication using a service account is required for signing the content.N) r� ri r( �apir� ZSignBlobZkeyIdZ signedBlobrP � sign_blobr )r� �string_to_signZ iam_cred_apir� �responser� r� r� �_GetSignedContentx s �z GcsJsonApi._GetSignedContentc C s |du pt �t|��S )a Checks Returns true if ACL related values are in fields set. Args: fields: list or set of fields. May be in GET ['acl'] or List ['items/acl'] call format. Returns: True if an ACL value is requested in the input fields, False otherwise. N)�_ACL_FIELDS_SET� intersection�set�r� �fieldsr� r� r� �_FieldsContainsAclField� s z"GcsJsonApi._FieldsContainsAclFieldc C s* |r&t �t|��r&d|vr&tdt ��dS )zEEnsures customerEncryption field is included if hashes are requested.�customerEncryptionzhgsutil client error: customerEncryption must be included when requesting potentially encrypted fields %sN)�_ENCRYPTED_HASHES_SETr� r� r r� r� r� r� �._ValidateEncryptionFieldsForMetadataGetRequest� s ���z9GcsJsonApi._ValidateEncryptionFieldsForMetadataGetRequestc C sz t j|| jtd�}t �� }|r.d�t|��|_z| jj j ||d�W S t yt } z| j ||d� W Y d }~n d }~0 0 d S )N)�bucket� userProject� optionsRequestedPolicyVersion�,�r� �� bucket_name) r� �!StorageBucketsGetIamPolicyRequestrh r< r� �joinr� r� r� �buckets� GetIamPolicy� TRANSLATABLE_APITOOLS_EXCEPTIONS�_TranslateExceptionAndRaise)r� r� rc r� �apitools_requestr� �er� r� r� �GetBucketIamPolicy� s � �z GcsJsonApi.GetBucketIamPolicyc C s� |d urt |�}tj|||| jd�}t�� }|r@d�t|��|_z| jj j ||d�W S t y� } z | j ||||d� W Y d }~n d }~0 0 d S )N�r� �object� generationr� r� r� �r� � object_namer� ) �longr� �!StorageObjectsGetIamPolicyRequestrh r� r� r� r� r� �objectsr� r� r� ) r� r� r� r� rc r� r� r� r� r� r� r� �GetObjectIamPolicy� s* � ��z GcsJsonApi.GetObjectIamPolicyc C s| |d urt |�}tj||||| jd�}t�� }z| jjj||d�W S tyv } z | j |||d� W Y d }~n d }~0 0 d S )N)r� r� r� �policyr� r� �r� r� ) r� r� �!StorageObjectsSetIamPolicyRequestrh r� r� r� � SetIamPolicyr� r� ) r� r� r� r� r� rc � api_requestr� r� r� r� r� �SetObjectIamPolicy� s& � ��z GcsJsonApi.SetObjectIamPolicyc C sf t j||| jd�}t �� }z| jjj||d�W S ty` } z| j||d� W Y d }~n d }~0 0 d S )N)r� r� r� r� r� ) r� �!StorageBucketsSetIamPolicyRequestrh r� r� r� r� r� r� )r� r� r� rc r� r� r� r� r� r� �SetBucketIamPolicy� s � �z GcsJsonApi.SetBucketIamPolicyc C sh | � � } t| ||||||||d� \} } tjr6| �t�} | �| �\} } |�d| | f � t| |d || �S )�,See CloudApi class for function doc strings.) � client_id�method�duration�pathr� r� �region�signed_headers�string_to_sign_debugz7Key ID used to sign blob for service account "%s": "%s"�host) r� rG �six�PY3�encoder@ r� rd rH )r� r� r� r� r� r� r� r� r� r� r� �canonical_query_string�key_idZ raw_signaturer� r� r� �SignUrl� s* � � �zGcsJsonApi.SignUrlc C s� t jjj}| �|�r t jjj}t j||| jd�}t �� }|rLd�t |��|_ z| j j j ||d�W S ty� } z| j||d� W Y d}~n d}~0 0 dS )r� )r� � projectionr� r� r� r� N)r� �StorageBucketsGetRequest�ProjectionValueValuesEnum�noAclr� �fullrh r� r� r� r� r� r� �Getr� r� )r� r� rc r� r� r� r� r� r� r� r� � GetBucket� s" � �zGcsJsonApi.GetBucketc C sT z t j|d�}| jj�|�W S tyN } z| j||d� W Y d}~n d}~0 0 dS )r� )r� r� N)r� �&StorageBucketAccessControlsListRequestr� ZbucketAccessControls�Listr� r� )r� r� �requestr� r� r� r� �ListBucketAccessControls s �z#GcsJsonApi.ListBucketAccessControlsc C sX z t j||d�}| jj�|�W S tyR } z | j|||d� W Y d}~n d}~0 0 dS )r� )r� r� r� N)r� �&StorageObjectAccessControlsListRequestr� ZobjectAccessControlsr� r� r� )r� r� r� r� r� r� r� r� �ListObjectAccessControls s ��z#GcsJsonApi.ListObjectAccessControlsc C s� t jjj}| �|�r t jjj}|} |s,t� }g } dD ]4} t| | d�} | r4t� | �s4t | | d� | � | � q4| j r�| j t kr�g | _ | � d� | jr�| jd tkr�g | _| � d� d} |r�| � d� t j�| �|��} d}|r�| � d� t j�| �|��}t j|| ||j| || jd�}t �� }|�r0d�t|��|_| j�| ��d z"| jj j ||d �W W d � S t �y� } z| � |� W Y d}~n d}~0 0 W d � n1 �s�0 Y dS ) r� )�billing� encryption� lifecycle�logging�metadata�retentionPolicy� versioning�websiteN�corsr rR rQ )r� �bucketResourcer� �ifMetagenerationMatch� predefinedAcl�predefinedDefaultObjectAclr� r� r� ) r� �StorageBucketsPatchRequestr� r� r� r� r �getattrr � MessageToDict�setattr�appendr rO rR rN � PredefinedAclValueValuesEnum� _BucketCannedAclToPredefinedAcl�)PredefinedDefaultObjectAclValueValuesEnum� _ObjectCannedAclToPredefinedAcl�meta_gen_matchrh r� r� r� r� r� � IncludeFieldsr� �Patchr� r� )r� r� r � canned_acl�canned_def_acl� preconditionsrc r� r� �bucket_metadata�apitools_include_fieldsZmetadata_field�attr�predefined_aclZpredefined_def_aclr� r� r� r� r� r� � PatchBucket+ sh � � ��� �zGcsJsonApi.PatchBucketc C s� z t |�}W n ty& td��Y n0 tj||| jd�}t�� }z| jjj ||d�W S t y� } z| j ||d� W Y d }~n d }~0 0 d S )Nz6LockRetentionPolicy Metageneration must be an integer.�r� r r� r� r� ) r� � ValueErrorr r� �(StorageBucketsLockRetentionPolicyRequestrh r� r� r� �LockRetentionPolicyr� r� )r� r� �metagenerationrc r� r� r� r� r� r� r | s& � ���z GcsJsonApi.LockRetentionPolicyc C s� t jjj}| �|�r t jjj}|s*t �� }||_|jrB|j� � |_|j rT|j � � |_ t |�}t j|||| j d�}t � � }|r�d�t|��|_z| jjj||d�W S ty� } z| j| |d� W Y d} ~ n d} ~ 0 0 dS )r� )r� �projectr� r� r� r� r� N)r� �StorageBucketsInsertRequestr� r� r� r� �Bucket�name�location�lower� storageClassr+ rh r� r� r� r� r� r� �Insertr� r� ) r� r� � project_idr rc r� r� r� r� r� r� r� r� � CreateBucket� s4 � �zGcsJsonApi.CreateBucketc C s� |s t � }tj||j| jd�}z| jj�|� W nr ty� } zZt | j ||d�t �r�| j |dgd�}|j r�|j jr�t d|jd��| j||d� W Y d}~n d}~0 0 dS )r� r r� r �r� ZVersionedBucketNotEmpty��statusN)r r� �StorageBucketsDeleteRequestr rh r� r� �Deleter� r� �_TranslateApitoolsExceptionr r� r �enabled� status_coder� )r� r� r rc r� r� r r� r� r� � DeleteBucket� s( � ��zGcsJsonApi.DeleteBucketc c sJ t jjj}| �|�r t jjj}t|�}t j|t|| jd�}t � � }|rhd|vrX|� d� d� t |��|_ z| jjj||d�}W n. ty� } z| �|� W Y d}~n d}~0 0 | �|�D ] } | V q�|j�rFt j||jt|| jd�}z| jjj||d�}W n0 t�y* } z| �|� W Y d}~n d}~0 0 | �|�D ] } | V �q6q�dS )r� )r" � maxResultsr� r� � nextPageTokenr� r� N)r" � pageTokenr5 r� r� )r� �StorageBucketsListRequestr� r� r� r� r+ �NUM_BUCKETS_PER_LIST_PAGErh r� �addr� r� r� r� r� r� r� r� � _YieldBucketsr6 ) r� r* rc r� r� r� r� � bucket_listr� r� r� r� r� � ListBuckets� sN � � � � zGcsJsonApi.ListBucketsc c s |j r|j D ] }|V q dS )z0Yields buckets from a list returned by apitools.N)�items)r� r< r� r� r� r� r; � s zGcsJsonApi._YieldBucketsc c s� t jjj}| �|�r t jjj}t j|||||t| jd�}t �� } |r�t |�}d|vr^|� d� t |d�� t �r�|� d� |� d� |� d� |� d� d�|�| _d } d } | �r�d } t j|||||| t| jd �}z| jjj|| d �} W n4 t�y } z| j| |d� W Y d } ~ n d } ~ 0 0 | j�r.d } | j} | �| �D ]P}|jtjjk�r�| j||j t |�d�}|t k�rr�q8n|�r�||_ n |V �q8q�d S )r� )r� �prefix� delimiter�versionsr� r5 r� r6 �� list_fieldszitems/customerEncryptionzitems/kmsKeyNamezitems/generationz items/namer� TNF)r� r? r@ rA r� r7 r5 r� r� r� r, ) r� �StorageObjectsListRequestr� r� r� r� r= rh r� r� r: r: r� r� r� r� r� r� r� r� r� r6 �_YieldObjectsAndPrefixes�datatyper �CsObjectOrPrefixType�OBJECT�_DecryptHashesIfPossible�data�_SKIP_LISTING_OBJECT)r� r� r? r@ � all_versionsrc r� r� r� r� Z list_pageZnext_page_token� object_listr� Zobject_or_prefixZdecrypted_metadatar� r� r� � ListObjects� s| � �� � � $� zGcsJsonApi.ListObjectsc C s� t |d�}| �|�s&| j||d� dS d}| j||d�rTtj| j||j|j|d�}n^| j ||d�r�|j j }t j r�t|t�s�|�d�}t|t�}|r�tj| j||j|j|t|�d�}|du r�dS z|� W S ty� t Y S 0 dS )a. Attempts to decrypt object metadata. Helps fetch encrypted object metadata fields (if requested) for API method calls that requested them but did not receive them; this occurs for ListObjects(), along with GetObjectMetadata() calls for which the object is CSEK-encrypted but no decryption information was supplied. This method issues a StorageObjectsGetRequest (if object hash fields were requested but not retrieved), formatted appropriately depending on the encryption method. Args: bucket_name: String of bucket containing the object. object_metadata: apitools Object describing the object. Must include name, generation, and customerEncryption fields. fields: ListObjects-format fields to return. Returns: - apitools_messages.Object with decrypted hash fields if decryption was performed. - None if the object was not encrypted, or CSEK-encrypted but we could not find a matching decryption key. - gcs_json_api._SKIP_LISTING_OBJECT if the object no longer exists. rB r, N�r� r� �ascii�r� r� �decryption_tuple)r: �_ObjectIsEncrypted�2_RaiseIfNoDesiredHashesPresentForUnencryptedObject� _ObjectKMSEncryptedAndNeedHashes� functools�partial�_GetObjectMetadataHelperr% r� �!_ObjectCSEKEncryptedAndNeedHashesr� � keySha256r� r� r� �bytesr� rD r rC r rK )r� r� �object_metadatar� � get_fieldsZget_metadata_func� key_sha256�decryption_keyr� r� r� rI L sP ���� � z#GcsJsonApi._DecryptHashesIfPossiblec c sL |j r$|j D ]}t�|tjj�V q |jrH|jD ]}t�|tjj�V q0d S �N)r> r �CsObjectOrPrefixrG rH �prefixes�PREFIX)r� rM Z cloud_objr? r� r� r� rE � s � �z#GcsJsonApi._YieldObjectsAndPrefixesc c s4 |r | j j�� }| j j�|� dV |r0|| j _dS )z@Wrapped code makes apitools requests with the specified headers.N)r� r| �copyr� )r� r� Z old_headersr� r� r� �_ApitoolsRequestHeaders� s z"GcsJsonApi._ApitoolsRequestHeadersc C s: i }|r6|j tjkr6|j|d< |j|d< t|j�|d< |S )z:Returns headers dict matching crypto_tuple, or empty dict.�x-goog-encryption-algorithm�x-goog-encryption-key� x-goog-encryption-key-sha256)� crypto_typerB �CSEK� crypto_alg� crypto_keyrA )r� � crypto_tupler� r� r� r� �_EncryptionHeadersFromTuple� s �z&GcsJsonApi._EncryptionHeadersFromTuplec C sd i }|r2|j tjkr2|j|d< |j|d< |j|d< |r`|j tjkr`|j|d< |j|d< |j|d< |S )zAReturns headers dict matching the provided tuples, or empty dict.z'x-goog-copy-source-encryption-algorithmz!x-goog-copy-source-encryption-keyz(x-goog-copy-source-encryption-key-sha256rf rg rh )ri rB rj rk rl �crypto_key_sha256)r� rR �encryption_tupler� r� r� r� � _RewriteCryptoHeadersFromTuples� s ��� �z*GcsJsonApi._RewriteCryptoHeadersFromTuplesc C s� | j ||||d�}| j|d�}zd|rl| �| j|d���$ | jjj||d�W d � W S 1 s`0 Y n| jjj||d�W S W n6 ty� } z | j||||d� W Y d}~n d}~0 0 dS )a� Get metadata, adding necessary headers for CSEK decryption if necessary. Args: bucket_name: (str) Bucket containing the object. object_name: (str) Object name. generation: (int) Generation of the object to retrieve. fields: (list<str>) If present, return only these Object metadata fields, for example: ['acl', 'updated']. decryption_tuple: (gslib.utils.encryption_helper.CryptoKeyWrapper) The CryptoKeyWrapper for the desired decryption key. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: (apitools_message.Object) The retrieved object metadata. rO r, �rm r� Nr� ) �'_CreateApitoolsObjectMetadataGetRequest�&_GetApitoolsObjectMetadataGlobalParamsre rn r� r� r� r� r� ) r� r� r� r� r� rR r� r� r� r� r� r� rX � s* � � �( � �z#GcsJsonApi._GetObjectMetadataHelperc C sL | j |d� tjjj}| �|�r*tjjj}|r6t|�}tj||||| jd�S )Nr, )r� r� r� r� r� ) r� r� �StorageObjectsGetRequestr� r� r� r� r� rh )r� r� r� r� r� r� r� r� r� rs � s �z2GcsJsonApi._CreateApitoolsObjectMetadataGetRequestc C s t �� }|r d�t|��|_|S )Nr� )r� r� r� r� r� )r� r� r� r� r� r� rt s z1GcsJsonApi._GetApitoolsObjectMetadataGlobalParamsc C s� z�| j ||||d�}| j||d�s(|W S |jj}tjrJt|t�sJ|�d�}t |t �}|snt d|| j ||f ��tjr�t|t�s�|�t �}| j ||||t|�d�W S ty� } z | j| |||d� W Y d} ~ n d} ~ 0 0 dS )r� rO r, rP zWMissing decryption key with SHA256 hash %s. No decryption key matches object %s://%s/%srQ r� N)rX rY r� rZ r� r� r� r[ r� rD r r rc r@ rC r� r� ) r� r� r� r� rc r� r\ r^ r_ r� r� r� r� �GetObjectMetadata sJ �� �� ��z GcsJsonApi.GetObjectMetadatac C s t |jp |j�S r` )�boolr� � kmsKeyName)r� r\ r� r� r� rS ; s �z GcsJsonApi._ObjectIsEncryptedc C sL |du r dS d|v o|j }d|v o*|j }|rH|jsH|rHtd|j ��dS )zBRaise an exception if none of the desired hash fields are present.Nr_ r^ zNService did not provide requested hashes in metadata for unencrypted object %s)r_ r^ �componentCountr r% )r� r\ r� Zmd5_requested_but_missingZ crc32c_requested_but_missingr� r� r� rT ? s �� ���z=GcsJsonApi._RaiseIfNoDesiredHashesPresentForUnencryptedObjectc C s, |durt �|�sdS |jo*|j o*|j S )a� Checks if an object is KMS-encrypted and lacking requested hash fields. Args: object_metadata: (apitools_messages.Object) Object metadata returned from an API call; may or may not contain hash-related metadata fields. fields: (list<str>) Requested object metadata fields. Returns: True if a GetObjectMetadata request should be issued on the KMS-encrypted object in order to retrieve encrypted hash fields, False otherwise. NF)r� r� rx r^ r_ �r� r\ r� r� r� r� rU R s �z+GcsJsonApi._ObjectKMSEncryptedAndNeedHashesc C s, |du st �|�sdS |jo*|j o*|j S )a� Checks if an object is CSEK-encrypted and lacking requested hash fields. Args: object_metadata: (apitools_messages.Object) Object metadata returned from an API call; may or may not contain hash-related metadata fields. fields: (list<str>) Requested object metadata fields. Returns: True if a GetObjectMetadata request including a decryption key should be issued on the CSEK-encrypted object in order to retrieve encrypted hash fields, False otherwise. NF)r� r� r� r^ r_ rz r� r� r� rY e s ��z,GcsJsonApi._ObjectCSEKEncryptedAndNeedHashesFc C s� |r t |�}|}| r | d }n| r6t�t�| ��d }| rT|du rJtd��| | |� t� }| |_t||| | d�}|� � }|| j _ d|i| j _ | r�| j r�t�t�| ��}|d d| j 7 < t�|�} tjj|| | jj| j| jd �}ntjj|d || jd �}| j|_tj|||| jd �}td | j d�|_ zX|t j j k�r\| j!||||||||| | | |d� W S | j"|||||||| | | |d� W S W n8 t#�y� } z | j$||||d� W Y d}~n d}~0 0 dS )r� � � total_sizeNz`Download size is required when callbacks are requested for a download, but no size was provided.)r| �progress_callback� digesters�httpsrv z&trace=token%%3A%s)r� �clientF)� auto_transferr| r� r� T�Zis_data_transferr} )�compressed_encodingr� � start_byte�end_byte�serialization_datarR �r� r� r� r� r� rR r� )%r� �json�loadsr� � ensure_strr r! �bytes_transferredr" �GetConnectionClassr� �stream� connectionsrf �dumps�apitools_transfer�Download�FromDatar� rw r� � FromStreamr� � bytes_httpr� ru rh rF r} r� r �DownloadStrategy� RESUMABLE�_PerformResumableDownload�_PerformDownloadr� r� )r� r� r� �download_streamrc r� � object_sizer� �download_strategyr� r� r} r� r~ rR Zouter_total_size�bytes_downloaded_container�callback_class_factoryZdownload_http_class�serialization_dict�apitools_downloadr� r� r� r� r� �GetObjectMediay s� �� � ��� ��� �zGcsJsonApi.GetObjectMediac C s� d} | }| | j kr�z"| j|||||||| | | | d� W S ty� } z�| �|� |�� } | |_| |krl| }d} | d7 } | | j kr�td| j t|�f ��t� t | | j d�� | j � tj�r�| j �d| t|�t�� � t�|j� W Y d }~qd }~0 0 qd S )Nr r� r{ �5Transfer failed after %d retries. Final exception: %s��max_waitz=Retrying download from byte %s after exception: %s. Trace: %s)r� r� �HTTP_TRANSFER_EXCEPTIONS�$_ValidateHttpAccessTokenRefreshError�tellr� r rI �time�sleepr r� r� � isEnabledForr� �DEBUGrd � traceback� format_exc�apitools_http_wrapper�RebuildHttpConnectionsr� )r� r� r� r� r� r� r� r� r� r� r� r� rR �retries�last_progress_byter� r� r� r� r� � sP � ����z$GcsJsonApi._PerformResumableDownloadc C s� | sRz| j jj||d� W n6 tyP } z | j| |||d� W Y d } ~ n d } ~ 0 0 dd� } d| j ji}t||d� | �|� |�| � | �� |s�| d ur�|j ||| dd� n|j | | |dd � |j S ) N)�downloadr� c S s d S r` r� )�unused_responseZunused_download_objectr� r� r� � _NoOpCallbackT s z2GcsJsonApi._PerformDownload.<locals>._NoOpCallback� user-agent)r� F)�additional_headers�start�end� use_chunks)�callback�finish_callbackr� r� ) r� r� r� r� r� � user_agentrE r� r� rn �GetRange� StreamMediar )r� r� r� r� r� r� r� r� r� r� r� rR r� r� r� r� r� r� r� = sB � � �� � ��zGcsJsonApi._PerformDownloadc C sn t jjj} | �|�r t jjj} |s(t� }|r4t|�}d} g } |r\| �d� t j� | � |��} |j dkrp| �d� |j dkr�| �d� |j dkr�| �d� |jdkr�| �d� t j||||| |j|j| | jd� } t �� }|r�d �t|��|_zF| j�| ��$ | jjj| |d �W d � W S 1 �s$0 Y W n8 t�yh } z | j||||d � W Y d}~n d}~0 0 dS ) r� NrQ ro � cacheControl�contentDisposition�contentEncoding�contentLanguage) r� r� �objectResourcer� r� �ifGenerationMatchr r r� r� r� r� ) r� �StorageObjectsPatchRequestr� r� r� r� r r� r r r r� r� r� r� � gen_matchr rh r� r� r� r� r� r r� r r� r� )r� r� r� r r r� r rc r� rh r� r r r� r� r� r� r� r� �PatchObjectMetadatax s^ � � �,�z GcsJsonApi.PatchObjectMetadatac C s t |� d}|r"tj�| �|��}t� }| r>|r>|} | d|� t|| | | j| jd�}|� � }||d�| j _ d| j j i}| �|� �z.d}d}d}| s�|j}|s�t}|s�t� }tj|j||j|j|| jd�}t�� }|r�d�t|��|_| j|d�}| tj k�rhtj |||d | j |d �}| |_ | j |_!| �"|��& | j j#j$|||d �W d � W S 1 �s\0 Y nFd| j j i}|�%|� | �|� | �&|| j ||| | |||| || |� W S W nJ t'�y� } z0t(| j)|j�}| j*||j|j+|d � W Y d}~n d}~0 0 dS ) a Upload implementation. Cloud API arguments, plus two more. Additional args: apitools_strategy: SIMPLE_UPLOAD or RESUMABLE_UPLOAD. total_size: Total size of the upload; None if it is unknown (streaming). Returns: Uploaded object metadata. Nr )r| r} r� rd )rw r r� )r� r� r� r r r� r� rr T)r| r� r� � gzip_encoded��uploadr� �r� r� �not_found_exception),r; r� �StorageObjectsInsertRequestr r r! r% r� rd r� r� r� r� r� r� � contentTyperM r r� r� r rh r� r� r� r� rn r� � SIMPLE_UPLOAD�Uploadr� �strategyr� r� re r� r) r� �_PerformResumableUploadr� rK rc r� r% )r� � upload_streamr\ r �sizer rp rc r� r� �tracker_callbackr} �apitools_strategyr| r� r �bytes_uploaded_containerr� Zupload_http_classr� � content_typer� r� �encryption_headers�apitools_uploadr� r� r� r� r� � _UploadObject� s� � �� � �� � �*� � ��zGcsJsonApi._UploadObjectc C s �z�|r8t jj||| jj| j| | jd�}t� |_||_nft j|||t� d| j| d�}||_ ||_| � | ��$ | jj j |||d� W d � n1 s�0 Y t d| jd�|_dd� }|j| _| r�| t�|j�� d }|j}|| jk�r�zv| �s|�rt� �s|j||| d �}n|j||| d �}| jj �| jj �d �|�}|d u �rX| �rX| |j|j� |W W S t�y� } �zD| � |� t � |j� || jk�r.z |� � |j}|| _W �q.W nv t�y( } z\| � |� t � |j� |d 7 }|| jk�r t d | j|f ��t!�"t#|| j$d�� W Y d }~n d }~0 0 �q�||k�rB|}d }n>|d 7 }|| jk�rlt d | jt%|�f ��t!�"t#|| j$d�� | j&�'t(j)��r�| j&�*d|t%|�t+�,� � W Y d }~q�d }~0 0 q�W n> t-�y } z$| �.|�}|�r�|�n� W Y d }~n d }~0 0 d S )N)r� r� r� F)r| � chunksizer� r� r� r� Tr� c S s d S r` r� )r� Zunused_upload_objectr� r� r� r� ] s z9GcsJsonApi._PerformResumableUpload.<locals>._NoOpCallbackr )r� r� r� r) r{ r� r� z;Retrying upload from byte %s after exception: %s. Trace: %s)/r� r� r� r� rw r� r5 r� r� r� re r� r) rF r} r� �progressr� r� r� r� r9 r� �StreamInChunks�ProcessHttpResponse�GetMethodConfigr| r� r� r� r� �RefreshResumableUploadStater r� r� r r� rI r� r� r� r� rd r� r� r� �*_TranslateApitoolsResumableUploadException)r� r� r� r� r� r� r� r� r� r� r� Z addl_headersr} r� r� r� r� r� � http_responseZprocessed_responser� r� �e2Z resumable_exr� r� r� r� 6 s� �� �$� � ��� � � �� � �� �� z"GcsJsonApi._PerformResumableUploadc C s | j |||||||| tj| d� S )r� )r r� r r} rp r� r� r� )r� r� r� ) r� r� r\ r r� r r} rp rc r� r� r� r� r� � UploadObject� s �zGcsJsonApi.UploadObjectc C s | j |||||||tjd| d� S )r� N)r r r} rp r� r� r| r� �r� r� �RESUMABLE_UPLOAD) r� r� r\ r r r} rp rc r� r� r� r� r� �UploadObjectStreaming� s �z GcsJsonApi.UploadObjectStreamingc C s$ | j ||||| ||||| tj| d� S )r� ) r r r� r� r� r� r} rp r� r� r� ) r� r� r\ r r r� r� r� r} rp rc r� r� r� r� r� �UploadObjectResumable� s �z GcsJsonApi.UploadObjectResumablec C s� t |� d} |r"tj�| �|��} |r.t|�}|s8t� }tjjj} | � | �rVtjjj } t� � }| r�t g d��}| D ]}|� d| � qrd�t |��|_d}| r�| jtjkr�| j}d}|r�|jtjkr�|j}n|jtjkr�|j|_t|j|j|j|jd�}t||| ||j|j | ||||jd� }t ||�}| j | |d�}d}z�td�}| � |��X tj|j|j|j|j|j| |||j|j | ||| j d �}| j!j"j#||d �}W d � n1 �s�0 Y t|j$�}|�r�|�s�t%t|j&�|�}|�r�|�'|| � |j(�r��q n|�s|j)}t*|||j)� |}�qBt+|� |j,W S t-�y� } z@t.| j/|j| j/|j|j|d �} | j0| |j|j| d � W Y d} ~ n d} ~ 0 0 dS ) r� N)�done� objectSize� rewriteToken�totalBytesRewrittenz resource/r� �JSON)�src_generationr� r r �max_bytes_per_callZsrc_dec_key_sha256Zdst_enc_key_sha256r� )rR rp r )� sourceBucket� sourceObject�destinationBucket�destinationKmsKeyName�destinationObjectr� r� �sourceGenerationr� r �destinationPredefinedAclr� �maxBytesRewrittenPerCallr� r� )� src_provider�src_bucket_name�src_object_namer� r� )1r; r� � StorageObjectsRewriteRequest�'DestinationPredefinedAclValueValuesEnumr r� r r� r� r� r� r� r� r: r� r� ri rB rj ro �CMEKrl rx r/ r� r% r0 r� r r1 rq re rh r� r� ZRewriter� r* r� �Progressr� r� r2 r. �resourcer� rK rc r� ) r� �src_obj_metadata�dst_obj_metadatar� r r r} r� rp rR rc r� r r� r� Z new_fields�fieldZdec_key_sha256Zenc_key_sha256Ztracker_file_nameZrewrite_params_hashZresume_rewrite_tokenZcrypto_headersZprogress_cb_with_timeoutZlast_bytes_writtenr� Zrewrite_responseZ bytes_writtenr� r� r� r� r� � CopyObject s� � �� �� ��& ����zGcsJsonApi.CopyObjectc C s~ |s t � }|rt|�}tj||||j|j| jd�}z| jj� |�W S t yx } z | j ||||d� W Y d}~n d}~0 0 dS )r� )r� r� r� r� r r� r� N) r r� r� �StorageObjectsDeleteRequestr� r rh r� r� r0 r� r� )r� r� r� r r� rc r� r� r� r� r� � DeleteObject� s( ��zGcsJsonApi.DeleteObjectc C sL t |� |j}d|_|j}d|_|js,t|_|s6t� }t�� } |rRd�t |��| _ tj ||d�} | j |d�} d} |r�|j tjkr�|j} | �| ��� tj| |||j|j| j| d�} z"| jjj| | d�W W d � S t�y& } z:t|tj ��r|j dk�rt d��n | � |� W Y d}~n d}~0 0 W d � n1 �s>0 Y dS ) r� Nr� )� sourceObjects� destinationrr )�composeRequestr� r� r� r r� rx r� � z)One of the source objects does not exist.) r; r% r� r� rM r r� r� r� r� r� �ComposeRequestrn ri rB r� rl re � StorageObjectsComposeRequestr� r rh r� r� ZComposer� r� �apitools_exceptions� HttpErrorr3 r r� )r� �src_objs_metadatar r rp rc r� � dst_obj_name�dst_bucket_namer� Zsrc_objs_compose_requestr� rx r� r� r� r� r� � ComposeObject� sT �� � �� zGcsJsonApi.ComposeObjectc C s� t jjj}t j|||dd�}t j|||| jd�} t �� } |rLd�t|��| _ z| j j j | | d�W S t y� } z| j| |d� W Y d} ~ n d} ~ 0 0 dS )r� ZWEB_HOOK)�address�id�token�type)r� �channelr� r� r� r� r� N)r� � StorageObjectsWatchAllRequestr� r� �Channelrh r� r� r� r� r� r� ZWatchAllr� r� ) r� r� r � channel_idr rc r� r� r r� r� r� r� r� r� � WatchBucket� s* �� �zGcsJsonApi.WatchBucketc C sT t j||d�}z| jj�|� W n. tyN } z| �|� W Y d}~n d}~0 0 dS )r� )r � resourceIdN)r� r r� ZchannelsZStopr� r� )r� r � resource_idrc r r� r� r� r� � StopChannel s zGcsJsonApi.StopChannelc C sX t j|| jd�}z| jj�|�W S tyR } z| j||d� W Y d}~n d}~0 0 dS )r� �r� r� r� N)r� �!StorageBucketsListChannelsRequestrh r� r� � ListChannelsr� r� )r� r� rc r� r� r� r� r� r� r s �zGcsJsonApi.ListChannelsc C sV z"t jt�|�d�}| jj�|�W S tyP } z| �|� W Y d}~n d}~0 0 dS )r� )� projectIdN) r� �'StorageProjectsServiceAccountGetRequestr� � text_typer� Zprojects_serviceAccountr� r� r� )r� �project_numberr� r� r� r� r� �GetProjectServiceAccount s �z#GcsJsonApi.GetProjectServiceAccountc C s� zft j||d�}|r||_|r<dd� |�� D �}t|d�|_|rF||_t j||| jd�} | j j � | �W S t y� } z| � | � W Y d} ~ n d} ~ 0 0 dS )r� )�topic�payload_formatc S s g | ]\}}t ||d ��qS ))r~ �value)� NotificationAdditionalProperty)�.0r~ r( r� r� r� � <listcomp>) s �z7GcsJsonApi.CreateNotificationConfig.<locals>.<listcomp>)�additionalProperties�r� � notificationr� N)r� � Notification� event_typesr> �!NotificationCustomAttributesValue�custom_attributes�object_name_prefix�!StorageNotificationsInsertRequestrh r� � notificationsr) r� r� ) r� r� � pubsub_topicr' r0 r2 r3 r. �additional_propertiesr� r� r� r� r� �CreateNotificationConfig s. ����z#GcsJsonApi.CreateNotificationConfigc C sV z"t j||| jd�}| jj�|�W S tyP } z| �|� W Y d}~n d}~0 0 dS )r� r- N)r� �!StorageNotificationsDeleteRequestrh r� r5 r0 r� r� )r� r� r. r� r� r� r� r� �DeleteNotificationConfig: s �z#GcsJsonApi.DeleteNotificationConfigc c sh z4t j|| jd�}| jj�|�}|jD ] }|V q&W n. tyb } z| �|� W Y d}~n d}~0 0 dS )r� r N) r� � StorageNotificationsListRequestrh r� r5 r� r> r� r� )r� r� r� r� r. r� r� r� r� �ListNotificationConfigsE s � z"GcsJsonApi.ListNotificationConfigsc C sR z t j||d�}| jj�|�W S tyL } z| �|� W Y d}~n d}~0 0 dS )r� )r! �serviceAccountEmailN)r� �$StorageProjectsHmacKeysCreateRequestr� �hmacKeysZCreater� r� )r� r* r� r� r� r� r� r� � CreateHmacKeyP s �zGcsJsonApi.CreateHmacKeyc C sR z t j||d�}| jj�|�W S tyL } z| �|� W Y d}~n d}~0 0 dS �r� )r! �accessIdN)r� �$StorageProjectsHmacKeysDeleteRequestr� r? r0 r� r� �r� r* � access_idr� r� r� r� r� � DeleteHmacKeyY s �zGcsJsonApi.DeleteHmacKeyc C sR z t j||d�}| jj�|�W S tyL } z| �|� W Y d}~n d}~0 0 dS rA )r� �!StorageProjectsHmacKeysGetRequestr� r? r� r� r� rD r� r� r� � GetHmacKeyb s �zGcsJsonApi.GetHmacKeyTc c s� z6t j|||td�}| jj�|�}|jD ] }|V q(W n. tyd } z| �|� W Y d}~n d}~0 0 |j r�t j||||j td�}z| jj�|�}W n. ty� } z| �|� W Y d}~n d}~0 0 |jD ] }|V q�qfdS )r� )r! r= �showDeletedKeysr5 N)r! r= rI r7 r5 ) r� �"StorageProjectsHmacKeysListRequestr9 r� r? r� r> r� r� r6 )r� r* r� �show_deleted_keysr� r� r~ r� r� r� r� � ListHmacKeysk s4 � � zGcsJsonApi.ListHmacKeysc C sp z<t j||d�}t �� }||_|r(||_||_| jj�|�W S t yj } z| � |� W Y d}~n d}~0 0 dS rA ) r� �$StorageProjectsHmacKeysUpdateRequest�HmacKeyMetadata�state�etagr� r� r? ZUpdater� r� )r� r* rE rO rP r� r r� r� r� r� � UpdateHmacKey� s �zGcsJsonApi.UpdateHmacKeyc C s |t v rt | S td| ��dS )ai Translates the input string to a bucket PredefinedAcl string. Args: canned_acl_string: Canned ACL string. Returns: String that can be used as a query parameter with the JSON API. This corresponds to a flavor of *PredefinedAclValueValuesEnum and can be used as input to apitools requests that affect bucket access controls. �Invalid canned ACL %sN)�._BUCKET_PREDEFINED_ACL_XML_TO_JSON_TRANSLATIONr �r� Zcanned_acl_stringr� r� r� r � s z*GcsJsonApi._BucketCannedAclToPredefinedAclc C s |t v rt | S td| ��dS )aj Translates the input string to an object PredefinedAcl string. Args: canned_acl_string: Canned ACL string. Returns: String that can be used as a query parameter with the JSON API. This corresponds to a flavor of *PredefinedAclValueValuesEnum and can be used as input to apitools requests that affect object access controls. rR N)�._OBJECT_PREDEFINED_ACL_XML_TO_JSON_TRANSLATIONr rT r� r� r� r � s z*GcsJsonApi._ObjectCannedAclToPredefinedAclc C s( t |tjj�r$|jdks$|jdks$� d S )N� �� )r� � oauth2clientr� �HttpAccessTokenRefreshErrorr. )r� r� r� r� r� r� � s ��z/GcsJsonApi._ValidateHttpAccessTokenRefreshErrorc C sD | j �tj�r | j �dt�� � | j|||||d�}|r>|�n� dS )a Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. bucket_name: Optional bucket name in request that caused the exception. object_name: Optional object name in request that caused the exception. generation: Optional generation in request that caused the exception. not_found_exception: Optional exception to raise in the not-found case. Raises: Translated CloudApi exception, or the original exception if it was not translatable. z TranslateExceptionAndRaise: %s)r� r� r� r� N)r� r� r� r� rd r� r� r1 )r� r� r� r� r� r� �translated_exceptionr� r� r� r� � s ��z&GcsJsonApi._TranslateExceptionAndRaisec C sh t |tj�rdt|dd �rdz8t�t�|j��}d|v rNd|d v rN|d d W S W n t yb Y n0 d S )N�content�error�message) r� r r r r� r� r� r� r[ � Exception)r� � http_errorZjson_objr� r� r� �_GetMessageFromHttpError� s z#GcsJsonApi._GetMessageFromHttpErrorc C sX z@|j d }|�d�}|dkr>||d � �d�d }d| W S W n tyR Y n0 d S )Nzwww-authenticatezscope="r �"r{ zAcceptable scopes: %s)r� �find�splitr^ )r� r_ �www_authenticateZ scope_idx�scopesr� r� r� �!_GetAcceptableScopesFromHttpError� s z,GcsJsonApi._GetAcceptableScopesFromHttpErrorc C s t |tj�r�| �|�}|jdkr2t|p(d|jd�S |jdkrNt|pDd|jd�S |jdkrjt|p`d|jd�S |jdkr�t|p|d|jd�S |jd kr�t|p�d|jd�S t |tj�r�td t |� �S t |tj ��rd t |�v s�d t |�v r�tt |��S d t |�v �rtdt |� �S d S )NrW z Server Errorr- rV zToo Many Requestsi� � Bad Requestr � za%s; if this issue persists, try deleting the tracker files present under ~/.gsutil/tracker-files/zAborting transferzNot enough bytes in streamz additional bytes left in streamz?%s; this can happen if a file changes size while being uploaded) r� r r r` r3 r r r �StreamExhausted�str� TransferError)r� r� r] r� r� r� r� � sN � � � � � �� � ��z5GcsJsonApi._TranslateApitoolsResumableUploadExceptionc C s� t |tj��rn| �|�}|jdkr4t|p*d|jd�S |jdkr�dt|�v r\t|pRd|jd�S dt|�v r~tt|j| � |�d�S �n�|jd k�r�d t|�v r�t|p�d |jd�S d t|�v r�t|p�d |jd�S dt|�v r�t d| |jd�S dt|�v �rtd|jd�S dt|�v �r td|jd�S dt|�v �r<td|jd�S dt|�v �r`tt|j| � |�d�S dt|�v �r|t ||jd�S t|�p�t|�|jd�S n�|jdk�r|�r�t |d|j� |S |�r�|�r�t |j| j|||d�S t|j| j|�S t|�p�|j|jd�S |jdk�rF|�rFdt|�v �r4t d| |jd�S td | |jd�S |jd k�r`t||jd�S t||jd�S t |tj��r�td t|� �S d S ) a! Translates apitools exceptions into their gsutil Cloud Api equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. bucket_name: Optional bucket name in request that caused the exception. object_name: Optional object name in request that caused the exception. generation: Optional generation in request that caused the exception. not_found_exception: Optional exception to raise in the not-found case. Returns: ServiceException for translatable exceptions, None otherwise. rh rg r- i� zLogin Requiredz Access denied: login required.Zinsufficient_scope)r. �bodyi� z7The account for the specified project has been disabledzAccount disabled.z,Daily Limit for Unauthenticated Use Exceededz8Access denied: quota exceeded. Is your project ID valid?z-The bucket you tried to delete was not empty.zBucketNotEmpty (%s)zFThe bucket you tried to create requires domain ownership verification.z�The bucket you tried to create requires domain ownership verification. Please see https://cloud.google.com/storage/docs/naming?hl=en#verification for more details.zUser Rate Limit Exceededz5Rate limit exceeded. Please retry this request later.zAccess Not Configuredz�Access Not Configured. Please go to the Google Cloud Platform Console (https://cloud.google.com/console#/project) for your project, select APIs and Auth and enable the Google Cloud Storage JSON API.z,does not have permission to publish messagesr r. )r� i� z,The bucket you tried to delete is not empty.z�A Cloud Storage bucket named '%s' already exists. Try another name. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization.i� z.Transfer invalid (possible encoding error: %s)N)r� r r r` r3 r rj r �"_INSUFFICIENT_OAUTH2_SCOPE_MESSAGErf r r r rL rc rJ r r] r r �TransferInvalidError)r� r� r� r� r� r� r] r� r� r� r1 s� � � � � � ������ � ���� �z&GcsJsonApi._TranslateApitoolsException)NNr NNNN)N)N)NN)NNN)NN)N)NN)NNNNN)N)NNNN)NN)NNN)NNNNN)N)N)NN)NNN)NN)N)NNN)N)N)N)NFr NNN)NFr NNN)NNNNNN)NNNNNNNF)NNNNNNF) NNNNNNNNNF) NNNNNNNNN)NNN)NNNN)NNN)N)NN)NNN)T)NNNN)NNNN)O�__name__� __module__� __qualname__�__doc__r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r+ r4 r= r; rN rI rE r re rn rq rX rs rt rv rS rT rU rY r r� �ONE_SHOTr� r� r� r� r� r� r� r� r� r� r� r r r r r r r% r8 r: r< r@ rF rH rL rQ r r r� r� r` rf r� r1 � __classcell__r� r� r� r� ra � s� � � � � Q � % , � N H � � / � � , � � � J � ? � E� | � � � � � � ; � � � ! � ! ! �ra )�rr � __future__r r r r � contextlibr rV � six.movesr r� r� r� �socket�sslr� r� r� �apitools.base.pyr r r r r� r r� Zapitools.base.py.utilr �botor �httplib2rX �gslibr �gslib.cloud_apir r r r r r r r r r r r r r r �gslib.exceptionr Zgslib.gcs_json_credentialsr r Zgslib.gcs_json_mediar! r" r# r$ r% r& r' Z gslib.impersonation_credentialsr( �gslib.no_op_credentialsr) Zgslib.progress_callbackr* �gslib.project_idr+ �"gslib.third_party.storage_apitoolsr, r� r- r� Zgslib.tracker_filer. r/ r0 r1 r2 Zgslib.utils.boto_utilr3 r4 r5 r6 r7 r8 r9 � gslib.utils.cloud_api_helperr: r; �gslib.utils.constantsr< r= r> r? r@ Z gslib.utils.encryption_helperrA rB rC rD �gslib.utils.metadata_utilrE Zgslib.utils.retry_utilrF Zgslib.utils.signurl_helperrG rH �gslib.utils.text_utilrI � gslib.utils.translation_helperrJ rK rL rM rN rO Z oauth2client.service_accountrP r� �intr� r/ �CustomAttributesValuer1 �AdditionalPropertyr) r9 r ri rk rn r� �TransferRetryError�BadStatusCodeError�RetryAfterError� RequestError� BadStatusLine�IncompleteRead�ResponseNotReady�ServerNotFoundErrorr� rY r\ �gaierror�timeout�SSLErrorr r� r� r� rS rU rd Z+FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATIONr� r� rK rm r� r� ra r� r� r� r� �<module> s �� � � � � � __pycache__/file_part.cpython-39.pyc 0000644 00000010045 15025013616 0013351 0 ustar 00 a +(Wg� � @ sZ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZG dd� dej�Z dS ) z8FilePart implementation for representing part of a file.� )�absolute_import)�print_function)�division)�unicode_literalsNc @ s� e Zd ZdZdd� Zdd� Zdd� Zdd � Zd&d d �Ze j fd d�Z dd� Z d'dd�Z d(dd�Zd)dd�Zd*dd�Zd+dd�Zd,d d �Zd-d d �Zd.d d!�Zd/d"d#�Zd0d$d%�ZdS )1�FileParta Subclass of the file API for representing part of a file. This class behaves as a contiguous subset of a given file (e.g., this object will behave as though the desired part of the file was written to another file, and the second file was opened). c C s8 t |d�| _|| _|| _| j| j | _| j�| j� dS )aF Initializes the FilePart. Args: filename: The name of the existing file, of which this object represents a part. offset: The position (in bytes) in the original file that corresponds to the first byte of the FilePart. length: The total number of bytes in the FilePart. �rbN)�open�_fp�length�_start�_end�seek)�self�filename�offsetr � r � /opt/gsutil/gslib/file_part.py�__init__# s zFilePart.__init__c C s d S �Nr �r r r r � __enter__3 s zFilePart.__enter__c C s | � � d S r )�close)r �type�value� tracebackr r r �__exit__7 s zFilePart.__exit__c C s | j �� | j S r )r �tellr r r r r r : s z FilePart.tell���c C s6 |dk r| j }t|| j| j�� �}| j�td|��S )Nr )r �minr r r �read�max�r �sizer r r r = s z FilePart.readc C sJ |t jkr | j�|| j �S |t jkr4| j�||�S | j�| j| �S d S r )�os�SEEK_ENDr r r �SEEK_CURr )r r �whencer r r r C s z FilePart.seekc C s | j �� d S r )r r r r r r r K s zFilePart.closeNc C s t d��d S )Nz%flush is not implemented in FilePart.��NotImplementedErrorr! r r r �flushN s zFilePart.flushc C s t d��d S )Nz&fileno is not implemented in FilePart.r' r! r r r �filenoQ s zFilePart.filenoc C s t d��d S )Nz&isatty is not implemented in FilePart.r' r! r r r �isattyT s zFilePart.isattyc C s t d��d S )Nz$next is not implemented in FilePart.r' r! r r r �nextW s z FilePart.nextc C s t d��d S )Nz(readline is not implemented in FilePart.r' r! r r r �readlineZ s zFilePart.readlinec C s t d��d S )Nz)readlines is not implemented in FilePart.r' r! r r r � readlines] s zFilePart.readlinesc C s t d��d S )Nz*xreadlines is not implemented in FilePart.r' r! r r r � xreadlines` s zFilePart.xreadlinesc C s t d��d S )Nz(truncate is not implemented in FilePart.r' r! r r r �truncatec s zFilePart.truncatec C s t d��d S )Nz%write is not implemented in FilePart.r' r! r r r �writef s zFilePart.writec C s t d��d S )Nz*writelines is not implemented in FilePart.r' r! r r r � writelinesi s zFilePart.writelines)r )N)N)N)N)N)N)N)N)N)N)�__name__� __module__� __qualname__�__doc__r r r r r r# �SEEK_SETr r r) r* r+ r, r- r. r/ r0 r1 r2 r r r r r s$ r ) r6 � __future__r r r r r# �io�IOBaser r r r r �<module> s __pycache__/project_id.cpython-39.pyc 0000644 00000001511 15025013616 0013524 0 ustar 00 a +(Wg� � @ s^ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ dZ dZ d d d �Z dS ) z3Helper module for Google Cloud Storage project ids.� )�absolute_import)�print_function)�division)�unicode_literalsN)�ProjectIdExceptionzx-goog-project-idc C s2 | r| S t j�dd�}|r |S tr&tS td��dS )zHReturns the project_id from the boto config file if one is not provided.�GSUtil�default_project_idZMissingProjectIdN)�boto�config� get_value�UNIT_TEST_PROJECT_IDr )� project_idZ default_id� r � /opt/gsutil/gslib/project_id.py�PopulateProjectId s r )N) �__doc__� __future__r r r r r �gslib.cloud_apir �GOOG_PROJ_ID_HDRr r r r r r �<module> s __pycache__/exception.cpython-39.pyc 0000644 00000013537 15025013616 0013413 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZdZed Zed Z ej rZe Z G d d � d e �Z G d d � d e �ZG dd� de �ZG dd� de �ZG dd� de �ZG dd� de �ZG dd� de �ZG dd� de �ZdS )zagsutil exceptions. The exceptions in this module are for use across multiple different classes. � )�absolute_import)�print_function)�division)�unicode_literalsNzNo URLs matchedz). Do the files you're operating on exist?z: %sc @ s( e Zd ZdZdd� Zdd� Zdd� ZdS ) �AbortExceptionzGException raised when a user aborts a command that needs to do cleanup.c C s t �| � || _d S �N)� StandardError�__init__�reason)�selfr � r � /opt/gsutil/gslib/exception.pyr / s zAbortException.__init__c C s d| j S �NzAbortException: %s�r �r r r r �__repr__3 s zAbortException.__repr__c C s d| j S r r r r r r �__str__6 s zAbortException.__str__N��__name__� __module__� __qualname__�__doc__r r r r r r r r , s r c @ s* e Zd ZdZd dd�Zdd� Zdd� Zd S ) �CommandExceptional Exception raised when a problem is encountered running a gsutil command. This exception should be used to signal user errors or system failures (like timeouts), not bugs (like an incorrect param value). For the latter you should raise Exception so we can see where/how it happened via gsutil -D (which will include a stack trace for raised Exceptions). Fc C s t �| � || _|| _dS )z�Instantiate a CommandException. Args: reason: Text describing the problem. informational: Indicates reason should be printed as FYI, not a failure. N)r r r � informational)r r r r r r r C s zCommandException.__init__c C s t | �S r ��strr r r r r N s zCommandException.__repr__c C s d| j S )NzCommandException: %sr r r r r r Q s zCommandException.__str__N)Fr r r r r r : s r c @ s e Zd ZdZdS )�ControlCExceptionz�Exception to report to analytics when the user exits via ctrl-C. This exception is never actually raised, but is used by analytics collection to provide a more descriptive name for user exit. N�r r r r r r r r r U s r c @ s e Zd ZdZdS )� GcloudStorageTranslationErrorzEException raised when a gsutil command can't be translated to gcloud.Nr r r r r r ^ s r c @ s e Zd ZdZdS )�HashMismatchExceptionz6Exception raised when data integrity validation fails.Nr r r r r r c s r c @ s( e Zd ZdZdd� Zdd� Zdd� ZdS ) �&IamChOnResourceWithConditionsExceptiona� Raised when trying to use "iam ch" on an IAM policy with conditions. Because the syntax for conditions is fairly complex, it doesn't make sense to specify them on the command line using a colon-delimited set of values in the same way you'd specify simple bindings - it would be a complex and potentially surprising interface, which isn't what you want when dealing with permissions. Additionally, providing partial functionality -- e.g. if a policy contains bindings with conditions, still allow users to interact with bindings that don't contain conditions -- might sound tempting, but results in a bad user experience. Bindings can be thought of as a mapping from (role, condition) -> [members]. Thus, a user might think they're editing the binding for (role1, condition1), but they'd really be editing the binding for (role1, None). Thus, we just raise an error if we encounter a binding with conditions present, and encourage users to use "iam {get,set}" instead. c C s t �| |� || _d S r �� Exceptionr �message�r r# r r r r z s z/IamChOnResourceWithConditionsException.__init__c C s t | �S r r r r r r r ~ s z/IamChOnResourceWithConditionsException.__repr__c C s d| j S )Nz*IamChOnResourceWithConditionsException: %s�r# r r r r r � s z.IamChOnResourceWithConditionsException.__str__Nr r r r r r h s r c @ s( e Zd ZdZdd� Zdd� Zdd� ZdS ) �InvalidUrlErrorz%Exception raised when URL is invalid.c C s t �| |� || _d S r r! r$ r r r r � s zInvalidUrlError.__init__c C s t | �S r r r r r r r � s zInvalidUrlError.__repr__c C s d| j S )NzInvalidUrlError: %sr% r r r r r � s zInvalidUrlError.__str__Nr r r r r r& � s r& c @ s e Zd ZdZdd� Zdd� ZdS )�ExternalBinaryErrorzCException raised when gsutil runs an external binary, and it fails.c C s t �| |� || _d S r r! r$ r r r r � s z ExternalBinaryError.__init__c C s d| j S )NzExternalBinaryError: %sr% r r r r r � s z ExternalBinaryError.__repr__N)r r r r r r r r r r r' � s r' )r � __future__r r r r �sixZNO_URLS_MATCHED_PREFIXZNO_URLS_MATCHED_GENERICZNO_URLS_MATCHED_TARGETZPY3r" r r r r r r r r& r' r r r r �<module> s( � __pycache__/thread_message.cpython-39.pyc 0000644 00000030253 15025013616 0014362 0 ustar 00 a +(Wg�6 � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z G d d � d e�ZG d d � d e�ZG d d� de�ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�ZdS )zAThread message classes. Messages are added to the status queue. � )�absolute_import)�print_function)�division)�unicode_literalsN)�Error)�errorc @ s" e Zd ZdZddd�Zdd� ZdS )� StatusMessagezXGeneral StatusMessage class. All Message classes inherit this StatusMessage class. Nc C s( || _ |pt�� | _|p t�� j| _dS )a Creates a Message. Args: message_time: Time that this message was created (since Epoch). process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). N)�time�os�getpid� process_id� threading�current_thread�ident� thread_id)�self� message_timer r � r �#/opt/gsutil/gslib/thread_message.py�__init__& s zStatusMessage.__init__c C s d| j j| j| j| jf S )�;Returns a string with a valid constructor for this message.z#%s(%s, process_id=%s, thread_id=%s))� __class__�__name__r r r �r r r r �__str__3 s ��zStatusMessage.__str__)NN)r � __module__� __qualname__�__doc__r r r r r r r s r c s* e Zd ZdZd� fdd� Zdd� Z� ZS ) �RetryableErrorMessagez�Message class for retryable errors encountered by the JSON API. This class contains information about the retryable error encountered to report to analytics collection and to display in the UI. r Nc sp t t| �j|||d� |jj| _|jjdv r>d|jj�� | _t|t �sRt|t �rZd| _ nd| _ || _ || _ dS )a� Creates a RetryableErrorMessage. Args: exception: The retryable error that was thrown. message_time: Float representing when message was created (seconds since Epoch). num_retries: The number of retries consumed so far. total_wait_sec: The total amount of time waited so far in retrying. process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). �r r )�socket�_socketZSocketTFN)�superr r r r � error_typer � capitalize� isinstance�apitools_service_error�six_service_error�is_service_error� num_retries�total_wait_sec)r � exceptionr r) r* r r �r r r r A s � �z RetryableErrorMessage.__init__c C s& d| j j| j| j| j| j| j| jf S )r zQ%s(%s(), num_retries=%s, total_wait_sec=%s, time=%s, process_id=%s, thread_id=%s))r r r# r) r* r r r r r r r r k s ��z RetryableErrorMessage.__str__)r r NN�r r r r r r � __classcell__r r r, r r : s �*r c s e Zd ZdZ� fdd�Z� ZS )� FinalMessagezbCreates a FinalMessage. A FinalMessage simply indicates that we have finished our operation. c s t t| ��|� dS )z�Creates a FinalMessage. Args: message_time: Float representing when message was created (seconds since Epoch). N)r" r/ r �r r r, r r r y s zFinalMessage.__init__�r r r r r r. r r r, r r/ s s r/ c s e Zd ZdZ� fdd�Z� ZS )�MetadataMessagez�Creates a MetadataMessage. A MetadataMessage simply indicates that a metadata operation on a given object has been successfully done. The only passed argument is the time when such operation has finished. c s t t| ��|� dS )z�Creates a MetadataMessage. Args: message_time: Float representing when message was created (seconds since Epoch). N)r" r2 r r0 r, r r r � s zMetadataMessage.__init__r1 r r r, r r2 � s r2 c sV e Zd ZdZdZdZdZdZdZdZ dZ d Z d Z d Z d Zd� fdd� Zdd� Z� ZS )� FileMessagez�Marks start or end of an operation for a file, cloud object or component. This class contains general information about each file/component. With that, information such as total size and estimated time remaining may be calculated beforehand. � � � � � � � � � � � NFc sD t t| �j|| | d� || _|| _|| _|| _|| _|| _|| _ dS )a� Creates a FileMessage. Args: src_url: FileUrl/CloudUrl representing the source file. dst_url: FileUrl/CloudUrl representing the destination file. message_time: Float representing when message was created (seconds since Epoch). size: Total size of this file/component, in bytes. finished: Boolean to indicate whether this is starting or finishing a file/component transfer. component_num: Component number, if dealing with a component. message_type: Type of the file/component. bytes_already_downloaded: Specific field for resuming downloads. When starting a component download, it tells how many bytes were already downloaded. process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). r N) r" r3 r �src_url�dst_url�size� component_num�finished� message_type�bytes_already_downloaded) r r? r@ r rA rC rB rD rE r r r, r r r � s �zFileMessage.__init__c C s6 d| j j| j| j| j| j| j| j| j| j | j | j f S )r z�%s('%s', '%s', %s, size=%s, finished=%s, component_num=%s, message_type=%s, bytes_already_downloaded=%s, process_id=%s, thread_id=%s)) r r r? r@ r rA rC rB rD rE r r r r r r r � s ��zFileMessage.__str__)NFNNNNN)r r r r Z FILE_DOWNLOADZ FILE_UPLOADZFILE_CLOUD_COPYZFILE_LOCAL_COPYZFILE_DAISY_COPYZ FILE_REWRITEZ FILE_HASHZCOMPONENT_TO_UPLOADZEXISTING_COMPONENTZCOMPONENT_TO_DOWNLOADZEXISTING_OBJECT_TO_DELETEr r r. r r r, r r3 � s* �*r3 c s* e Zd ZdZd� fdd� Zdd� Z� ZS )�ProgressMessagez�Message class for a file/object/component progress. This class contains specific information about the current progress of a file, cloud object or single component. Nc sB t t| ��|� || _|| _|| _|| _|| _||k| _|| _ dS )aa Creates a ProgressMessage. Args: size: Integer for total size of this file/component, in bytes. processed_bytes: Integer for number of bytes already processed from that specific component, which means processed_bytes <= size. src_url: FileUrl/CloudUrl representing the source file. message_time: Float representing when message was created (seconds since Epoch). dst_url: FileUrl/CloudUrl representing the destination file, or None for unary operations like hashing. component_num: Indicates the component number, if any. operation_name: Name of the operation that is being made over that component. process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). N) r" rF r rA �processed_bytesrB r? r@ rC �operation_name) r rA rG r? r r@ rB rH r r r, r r r � s zProgressMessage.__init__c C sV | j rd| j nd}| jr$d| j nd}d| jj| j| j| j| j|| j|| j | j f S )r z'%s'Nzb%s(%s, %s, '%s', %s, dst_url=%s, component_num=%s, operation_name=%s, process_id=%s, thread_id=%s)) r@ rH r r rA rG r? r rB r r )r �dst_url_stringZoperation_name_stringr r r r s ��� ��zProgressMessage.__str__)NNNNNr- r r r, r rF � s �%rF c s( e Zd ZdZ� fdd�Zdd� Z� ZS )�SeekAheadMessagea Message class for results obtained by SeekAheadThread(). It estimates the number of objects and total size in case the task_queue cannot hold all tasks at once (only used in large operations). This class contains information about all the objects yet to be processed. c s t t| ��|� || _|| _dS )aI Creates a SeekAheadMessage. Args: num_objects: Number of total objects that the SeekAheadThread estimates. size: Total size corresponding to the sum of the size of objects iterated by SeekAheadThread. message_time: Float representing when message was created (seconds since Epoch). N)r" rJ r � num_objectsrA )r rK rA r r, r r r $ s zSeekAheadMessage.__init__c C s" d| j j| j| j| j| j| jf S )r z+%s(%s, %s, %s, process_id=%s, thread_id=%s))r r rK rA r r r r r r r r 2 s ��zSeekAheadMessage.__str__r- r r r, r rJ s rJ c s* e Zd ZdZd� fdd� Zdd� Z� ZS )�ProducerThreadMessagea% Message class for results obtained by calculations made on ProducerThread. It estimates the number of objects and total size currently dealty by task_queue. If the task_queue cannot support all objects at once, the SeekAheadThread will be responsible for sending an accurate message. Fc s& t t| ��|� || _|| _|| _dS )a^ Creates a SeekAheadMessage. Args: num_objects: Number of total objects that the task_queue has. size: Total size corresponding to the sum of the size of objects iterated by the task_queue message_time: Float representing when message was created (seconds since Epoch). finished: Boolean to indicate whether this is the final message from the ProducerThread. The difference is that this message displays the correct total size and number of objects, whereas the previous ones were periodic (on the number of files) updates. N)r" rL r rK rA rC )r rK rA r rC r, r r r A s z ProducerThreadMessage.__init__c C s d| j j| j| j| j| jf S )r z%s(%s, %s, %s, finished=%s))r r rK rA r rC r r r r r T s ��z ProducerThreadMessage.__str__)Fr- r r r, r rL 9 s rL c s( e Zd ZdZ� fdd�Zdd� Z� ZS )�PerformanceSummaryMessagez�Message class to log PerformanceSummary parameters. This class acts as a relay between a multiprocess/multithread situation and the global status queue, from which the PerformanceSummary info gets consumed. c s t t| �j|ddd� || _dS )z�Creates a PerformanceSummaryMessage. Args: message_time: Float representing when message was created (seconds since Epoch). uses_slice: True if the command uses slice parallelism. Nr )r" rM r � uses_slice)r r rN r, r r r b s �z"PerformanceSummaryMessage.__init__c C s d| j j| j| jf S )r z %s(%s, %s))r r r rN r r r r r o s �z!PerformanceSummaryMessage.__str__r- r r r, r rM [ s rM )r � __future__r r r r r r �apitools.base.py.exceptionsr r& Zsix.moves.http_clientr r' �objectr r r/ r2 r3 rF rJ rL rM r r r r �<module> s" 9N9 " __pycache__/kms_api.cpython-39.pyc 0000644 00000022537 15025013616 0013040 0 ustar 00 a +(Wg�4 � @ s: d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl m"Z" ddl m#Z# e j$Z%e �&� �r"dZ'ndZ'G dd� de(�Z)dS )z>JSON gsutil Cloud API implementation for Google Cloud Storage.� )�absolute_import)�print_function)�division)�unicode_literalsN)� exceptions)�config)�AccessDeniedException)�BadRequestException)�NotFoundException)�PreconditionException)�ServiceException)� SetUpJsonCredentialsAndCache)�NoOpCredentials)�cloudkms_v1_client)�cloudkms_v1_messages)� system_util)� GetCertsFile)�GetMaxRetryDelay)� GetNewHttp)� GetNumRetrieszVInsufficient OAuth2 scope to perform this operation. Please re-run `gcloud auth login`zRInsufficient OAuth2 scope to perform this operation. Please re-run `gsutil config`c sh e Zd ZdZd� fdd� Zdd� Zdd � Zdd d �Zd d� Zddd�Z dd� Z dd� Z ddd�Z � Z S )�KmsApiz7Wraps calls to the Cloud KMS v1 interface via apitools.Nr c s� t t| ��� || _t� | _t� | _d| _t � ddd�| _ t � ddd�}|rTd| nd| _ | j| j | j | _ t| ||d � |d k}|d k}tj| j | j||| jd �| _t� | _| j| j_t� | _| j| j_t| jt�r�| j�d d � dS )a Performs necessary setup for interacting with Google Cloud KMS. Args: logger: logging.logger for outputting log messages. credentials: Credentials to be used for interacting with Cloud KMS debug: Debug level for the API implementation (0..3). zhttps://� CredentialsZ gs_kms_hostzcloudkms.googleapis.com� gs_kms_portN�:� )� credentials� )�url�http� log_request� log_responser �key�'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM)�superr �__init__�loggerr � certs_filer r � http_baser �get� host_base� host_port�url_baser �apitools_clientZ CloudkmsV1r � api_clientr � num_retriesr �max_retry_wait� isinstancer �AddGlobalParam)�selfr% r �debugr r r �� __class__� � /opt/gsutil/gslib/kms_api.pyr$ : s8 �� �zKmsApi.__init__c C sT t j|d�}z| jj�|�W S tyN } z| j||d� W Y d }~n d }~0 0 d S )N)�resource��key_name)�apitools_messagesZ>CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequestr- �&projects_locations_keyRings_cryptoKeys� GetIamPolicy� TRANSLATABLE_APITOOLS_EXCEPTIONS�_TranslateExceptionAndRaise)r2 r: �request�er6 r6 r7 �GetKeyIamPolicye s ��zKmsApi.GetKeyIamPolicyc C sb t j|d�}t j||d�}z| jj�|�W S ty\ } z| j||d� W Y d }~n d }~0 0 d S )N)�policy)r8 ZsetIamPolicyRequestr9 )r; ZSetIamPolicyRequestZ>CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequestr- r< � SetIamPolicyr> r? )r2 r: rC Zpolicy_requestr@ rA r6 r6 r7 �SetKeyIamPolicyo s ��zKmsApi.SetKeyIamPolicy�globalc C s� t jd|||f d�}t j||d||f d�}z| jj�|� W n0 typ } z|jdkr\� W Y d}~n d}~0 0 d|||f S )a� Attempts to create the specified keyRing. Args: project: (str) The project id in which to create the keyRing and key. keyring_name: (str) The name of the keyRing, e.g. my-keyring. Note that this must be unique within the location. location: (str) The location in which to create the keyRing. Defaults to 'global'. Returns: (str) The fully-qualified name of the keyRing, e.g.: projects/my-project/locations/global/keyRings/my-keyring Raises: Translated CloudApi exception if we were unable to create the keyRing. Note that in the event of a 409 status code (resource already exists) when attempting creation, we continue and treat this as a success. z$projects/%s/locations/%s/keyRings/%s)�namezprojects/%s/locations/%s)ZkeyRingZ keyRingId�parent� N)r; ZKeyRingZ.CloudkmsProjectsLocationsKeyRingsCreateRequestr- Zprojects_locations_keyRings�Creater> � status_code)r2 �project� keyring_name�locationZ keyring_msgZkeyring_create_requestrA r6 r6 r7 � CreateKeyRingz s& �� �� �zKmsApi.CreateKeyRingc C sx t jt jjjd�}t j|||d�}z| jj�|� W n0 tyd } z|j dkrP� W Y d}~n d}~0 0 d|� d�|f S )a� Attempts to create the specified cryptoKey. Args: keyring_fqn: (str) The fully-qualified name of the keyRing, e.g. projects/my-project/locations/global/keyRings/my-keyring. key_name: (str) The name of the desired key, e.g. my-key. Note that this must be unique within the keyRing. Returns: (str) The fully-qualified name of the cryptoKey, e.g.: projects/my-project/locations/global/keyRings/my-keyring/cryptoKeys/my-key Raises: Translated CloudApi exception if we were unable to create the cryptoKey. Note that in the event of a 409 status code (resource already exists) when attempting creation, we continue and treat this as a success. )�purpose)Z cryptoKeyZ cryptoKeyIdrH rI Nz%s/cryptoKeys/%s�/) r; Z CryptoKeyZPurposeValueValuesEnumZENCRYPT_DECRYPTZ8CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequestr- r< rJ r> rK �rstrip)r2 Z keyring_fqnr: Z cryptokey_msgZcryptokey_create_requestrA r6 r6 r7 �CreateCryptoKey� s ���� zKmsApi.CreateCryptoKeyc C s> | j �tj�r | j �dt�� � | j||d�}|r8|�n� dS )a' Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. key_name: Optional key name in request that caused the exception. Raises: Translated CloudApi exception, or the original exception if it was not translatable. z TranslateExceptionAndRaise: %sr9 N)r% � isEnabledFor�logging�DEBUGr3 � traceback� format_exc�_TranslateApitoolsException)r2 rA r: �translated_exceptionr6 r6 r7 r? � s ��z"KmsApi._TranslateExceptionAndRaisec C sb t |tj�r^t|dd �r^z2t�|j�}d|v rHd|d v rH|d d W S W n ty\ Y n0 d S )N�content�error�message)r0 �apitools_exceptions� HttpError�getattr�json�loadsr[ � Exception)r2 � http_error�json_objr6 r6 r7 �_GetMessageFromHttpError� s z KmsApi._GetMessageFromHttpErrorc C sX z@|j d }|�d�}|dkr>||d � �d�d }d| W S W n tyR Y n0 d S )Nzwww-authenticatezscope="r �"� zAcceptable scopes: %s)�response�find�splitrc )r2 rd �www_authenticate� scope_idx�scopesr6 r6 r7 �!_GetAcceptableScopesFromHttpError� s z(KmsApi._GetAcceptableScopesFromHttpErrorc C s� t |tj��r�| �|�}|jdkr4t|p*d|jd�S |jdkr�dt|�v r\t|pRd|jd�S dt|�v r~tt|j| � |�d�S �n |jd k�rBd t|�v r�t|p�d |jd�S d t|�v r�t|p�d |jd�S dt|�v r�td|jd�S dt|�v �r td|jd�S dt|�v �r$tt|j| � |�d�S t|�p6|j �p6||jd�S n`|jdk�rdt |�pZ|j |jd�S |jdk�r�|�r�t d| |jd�S |jdk�r�t ||jd�S t ||jd�S dS )a8 Translates apitools exceptions into their gsutil equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. key_name: Optional key name in request that caused the exception. Returns: CloudStorageApiServiceException for translatable exceptions, None otherwise. i� z Bad Request)�statusi� zLogin Requiredz Access denied: login required.�insufficient_scope)rp �bodyi� z7The account for the specified project has been disabledzAccount disabled.z,Daily Limit for Unauthenticated Use Exceededz8Access denied: quota exceeded. Is your project ID valid?zUser Rate Limit Exceededz5Rate limit exceeded. Please retry this request later.zAccess Not Configuredz�Access Not Configured. Please go to the Google Cloud Platform Console (https://cloud.google.com/console#/project) for your project, select APIs & services, and enable the Google Cloud KMS API.i� rI zThe key %s already exists.i� N)r0 r^ r_ rf rK r �strr �"_INSUFFICIENT_OAUTH2_SCOPE_MESSAGEro r] r r r )r2 rA r: r] r6 r6 r7 rY � sp � � � � � ���� � z"KmsApi._TranslateApitoolsException)NNr )rF )N)N)�__name__� __module__� __qualname__�__doc__r$ rB rE rO rS r? rf ro rY � __classcell__r6 r6 r4 r7 r 7 s + # r )*rx � __future__r r r r ra rU rW �apitools.base.pyr r^ �botor �gslib.cloud_apir r r r r �gslib.gcs_json_credentialsr �gslib.no_op_credentialsr Z gslib.third_party.kms_apitoolsr r, r r; � gslib.utilsr Zgslib.utils.boto_utilr r r r r_ r> �InvokedViaCloudSdkrt �objectr r6 r6 r6 r7 �<module> s< �� __pycache__/iamcredentials_api.cpython-39.pyc 0000644 00000017176 15025013616 0015235 0 ustar 00 a +(Wg�. � @ s. d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z ddl m Z ddl m Z ddl m!Z! e j"Z#e �$� �rdZ%ndZ%G dd� de&�Z'dS )z>JSON gsutil Cloud API implementation for Google Cloud Storage.� )�absolute_import)�print_function)�division)�unicode_literalsN)� exceptions)�config)�AccessDeniedException)�BadRequestException)�NotFoundException)�PreconditionException)�ServiceException)�NoOpCredentials)�iamcredentials_v1_client)�iamcredentials_v1_messages)� system_util)� GetCertsFile)�GetMaxRetryDelay)� GetNewHttp)� GetNumRetrieszVInsufficient OAuth2 scope to perform this operation. Please re-run `gcloud auth login`zRInsufficient OAuth2 scope to perform this operation. Please re-run `gsutil config`c sV e Zd ZdZd� fdd� Zdd� Zdd � Zdd d �Zdd d �Zdd� Z dd� Z � Z S )�IamcredentailsApizCWraps calls to the Cloud IAM Credentials v1 interface via apitools.Nr c s� t t| ��� || _|| _t� | _t� | _d| _ d}t � ddt � d|d��| _ t � ddd�}|rhd| nd | _ | j | j | j | _|d k}|d k}tj| j| j||| jd �| _t� | _| j| j_t� | _| j| j_t| jt�r�| j�d d � dS )a Performs necessary setup for interacting with Google Cloud IAM Credentials. Args: logger: logging.logger for outputting log messages. credentials: Credentials to be used for interacting with Cloud IAM debug: Debug level for the API implementation (0..3). zhttps://Zgs_iamcredentails_host� CredentialsZgs_iamcredentials_hostz iamcredentials.googleapis.comZgs_iamcredentails_portN�:� � )�url�http� log_request� log_response� credentials�key�'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM)�superr �__init__�loggerr r � certs_filer r � http_baser �get� host_base� host_port�url_base�apitools_clientZIamcredentialsV1� api_clientr � num_retriesr �max_retry_wait� isinstancer �AddGlobalParam)�selfr# r �debugZ legacy_typoZgs_iamcred_portr r �� __class__� �'/opt/gsutil/gslib/iamcredentials_api.pyr" 9 s> �� �zIamcredentailsApi.__init__c C sh d| }t j|d�}t j||d�}z| jj�|�W S tyb } z| �||� W Y d}~n d}~0 0 dS )z0Sign the blob using iamcredentials.SignBlob API.� projects/-/serviceAccounts/%s)�payload)�nameZsignBlobRequestN)�apitools_messagesZSignBlobRequestZ4IamcredentialsProjectsServiceAccountsSignBlobRequestr+ �projects_serviceAccounts�SignBlob� TRANSLATABLE_APITOOLS_EXCEPTIONS�_TranslateExceptionAndRaise)r0 �service_account_id�messager8 Zsign_blob_request�request�er4 r4 r5 r; j s ��zIamcredentailsApi.SignBlobc C sh d| }t j|d�}t j||d�}z| jj�|�W S tyb } z| �||� W Y d}~n d}~0 0 dS )z8Generates an access token for the given service account.r6 )�scope)r8 ZgenerateAccessTokenRequestN)r9 ZGenerateAccessTokenRequestZ?IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequestr+ r: �GenerateAccessTokenr< r= )r0 r> �scopesr8 Z generate_access_token_requestr@ rA r4 r4 r5 rC v s ���z%IamcredentailsApi.GenerateAccessTokenc C s> | j �tj�r | j �dt�� � | j||d�}|r8|�n� dS )a8 Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. service_account_id: Optional service account in request that caused the exception. Raises: Translated CloudApi exception, or the original exception if it was not translatable. z TranslateExceptionAndRaise: %s)r> N)r# � isEnabledFor�logging�DEBUGr1 � traceback� format_exc�_TranslateApitoolsException)r0 rA r> �translated_exceptionr4 r4 r5 r= � s ��z-IamcredentailsApi._TranslateExceptionAndRaisec C s� t |tj��r�| �|�}|jdkr4t|p*d|jd�S |jdkr�dt|�v r\t|pRd|jd�S dt|�v r~tt|j| � |�d�S �nJ|jd k�rld t|�v r�td �S d t|�v r�t|�S d t|�v r�t|p�d|jd�S dt|�v r�t|p�d|jd�S dt|�v �rtd|jd�S dt|�v �r*td|jd�S dt|�v �rNtt|j| � |�d�S t|�p`|j �p`||jd�S n`|jdk�r�t |�p�|j |jd�S |jdk�r�|�r�t d| |jd�S |jdk�r�t ||jd�S t ||jd�S dS )aA Translates apitools exceptions into their gsutil equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. service_account_id: Optional service account ID that caused the exception. Returns: CloudStorageApiServiceException for translatable exceptions, None otherwise. i� z Bad Request)�statusi� zLogin Requiredz Access denied: login required.�insufficient_scope)rL �bodyi� z#The caller does not have permissiona Service account impersonation failed. Please go to the Google Cloud Platform Console (https://cloud.google.com/console), select IAM & admin, then Service Accounts, and grant your originating account the Service Account Token Creator role on the target service account.z5IAM Service Account Credentials API has not been usedz7The account for the specified project has been disabledzAccount disabled.z,Daily Limit for Unauthenticated Use Exceededz8Access denied: quota exceeded. Is your project ID valid?zUser Rate Limit Exceededz5Rate limit exceeded. Please retry this request later.zAccess Not Configuredz�Access Not Configured. Please go to the Google Cloud Platform Console (https://cloud.google.com/console#/project) for your project, select APIs & services, and enable the Google Cloud IAM Credentials API.i� i� zThe key %s already exists.i� N)r. �apitools_exceptions� HttpError�_GetMessageFromHttpError� status_coder �strr �"_INSUFFICIENT_OAUTH2_SCOPE_MESSAGE�!_GetAcceptableScopesFromHttpErrorr? r r r )r0 rA r> r? r4 r4 r5 rJ � s� � � � � � ����� �� z-IamcredentailsApi._TranslateApitoolsExceptionc C sb t |tj�r^t|dd �r^z2t�|j�}d|v rHd|d v rH|d d W S W n ty\ Y n0 d S )N�content�errorr? )r. rO rP �getattr�json�loadsrV � Exception)r0 � http_error�json_objr4 r4 r5 rQ � s z*IamcredentailsApi._GetMessageFromHttpErrorc C sX z@|j d }|�d�}|dkr>||d � �d�d }d| W S W n tyR Y n0 d S )Nzwww-authenticatezscope="r �"� zAcceptable scopes: %s)�response�find�splitr[ )r0 r\ �www_authenticate� scope_idxrD r4 r4 r5 rU � s z3IamcredentailsApi._GetAcceptableScopesFromHttpError)NNr )N)N) �__name__� __module__� __qualname__�__doc__r" r; rC r= rJ rQ rU � __classcell__r4 r4 r2 r5 r 6 s 1 X r )(rh � __future__r r r r rY rF rH �apitools.base.pyr rO �botor �gslib.cloud_apir r r r r �gslib.no_op_credentialsr Z)gslib.third_party.iamcredentials_apitoolsr r* r r9 � gslib.utilsr �gslib.utils.boto_utilr r r r rP r< �InvokedViaCloudSdkrT �objectr r4 r4 r4 r5 �<module> s: �� __pycache__/discard_messages_queue.cpython-39.pyc 0000644 00000001735 15025013616 0016116 0 ustar 00 a +(Wg+ � @ sH d Z ddlmZ ddlmZ ddlmZ ddlmZ G dd� de�ZdS ) z4Provides a message queue that discards all messages.� )�absolute_import)�print_function)�division)�unicode_literalsc @ s e Zd ZdZddd�ZdS )�DiscardMessagesQueuea$ Emulates a Cloud API status queue but drops all messages. This is useful when you want to perform some operations but not have the UI thread display information about those ops (e.g. running a test or fetching the public gsutil tarball object's metadata to perform a version check). Nc C s d S )N� )�self�message�timeoutr r �+/opt/gsutil/gslib/discard_messages_queue.py�put s zDiscardMessagesQueue.put)NN)�__name__� __module__� __qualname__�__doc__r r r r r r s r N)r � __future__r r r r �objectr r r r r �<module> s __pycache__/cred_types.cpython-39.pyc 0000644 00000001523 15025013616 0013546 0 ustar 00 a +(Wg� � @ sH d Z ddlmZ ddlmZ ddlmZ ddlmZ G dd� de�ZdS ) z*Defines gsutil-supported credential types.� )�absolute_import)�print_function)�division)�unicode_literalsc @ s, e Zd ZdZdZdZdZdZdZdZ dZ d S ) � CredTypes�HMACz Service Account ImpersonationzOAuth 2.0 Service AccountzOauth 2.0 User Account�GCEzGoogle Developer ShellzExternal Accountz External Account Authorized UserN) �__name__� __module__� __qualname__r � IMPERSONATION�OAUTH2_SERVICE_ACCOUNT�OAUTH2_USER_ACCOUNTr �DEVSHELL�EXTERNAL_ACCOUNT� EXTERNAL_ACCOUNT_AUTHORIZED_USER� r r � /opt/gsutil/gslib/cred_types.pyr s r N)�__doc__� __future__r r r r �objectr r r r r �<module> s __pycache__/cs_api_map.cpython-39.pyc 0000644 00000006404 15025013616 0013503 0 ustar 00 a +(Wg> � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z G dd � d e �Z G d d � d e �Z G d d � d e �Z G dd� de �ZdS )z6API map classes used with the CloudApiDelegator class.� )�absolute_import)�print_function)�division)�unicode_literals)�BotoTranslation)� GcsJsonApic @ s e Zd ZdZdZdZdS )� ApiSelectorzEnum class for API.�XML�JSONN)�__name__� __module__� __qualname__�__doc__r r � r r � /opt/gsutil/gslib/cs_api_map.pyr s r c @ s e Zd ZdZdZdZdZdS )�ApiMapConstantsz Enum class for API map entries.Zapiclass� supported�defaultN)r r r r �API_MAP� SUPPORT_MAP� DEFAULT_MAPr r r r r s r c @ s e Zd ZdZedd� �ZdS )�GsutilApiClassMapFactorya Factory for generating gsutil API class maps. A valid class map is defined as: { (key) Provider prefix used in URI strings. (value) { (key) ApiSelector describing the API format. (value) CloudApi child class that implements this API. } } c C s( t jtt jti}t jti}||d�}|S )z%Returns the default gsutil class map.)�gs�s3)r r r r r )�clsZ gs_class_mapZ s3_class_mapZ class_mapr r r � GetClassMap4 s ���z$GsutilApiClassMapFactory.GetClassMapN)r r r r � classmethodr r r r r r ' s r c @ s e Zd ZdZedd� �ZdS )�GsutilApiMapFactorya� Factory the generates the default gsutil API map. The API map determines which Cloud API implementation is used for a given command. A valid API map is defined as: { (key) ApiMapConstants.API_MAP : (value) Gsutil API class map (as described in GsutilApiClassMapFactory comments). (key) ApiMapConstants.SUPPORT_MAP : (value) { (key) Provider prefix used in URI strings. (value) list of ApiSelectors supported by the command for this provider. } (key) ApiMapConstants.DEFAULT_MAP : (value) { (key) Provider prefix used in URI strings. (value) Default ApiSelector for this command and provider. } } c C s t j|�� t j|t j|iS )a� Creates a GsutilApiMap for use by the command from the inputs. Args: gsutil_api_class_map_factory: Factory defining a GetClassMap() function adhering to GsutilApiClassMapFactory semantics. support_map: Entries for ApiMapConstants.SUPPORT_MAP as described above. default_map: Entries for ApiMapConstants.DEFAULT_MAP as described above. Returns: GsutilApiMap generated from the inputs. )r r r r r )r � gsutil_api_class_map_factory� support_map� default_mapr r r � GetApiMapX s �z GsutilApiMapFactory.GetApiMapN)r r r r r r! r r r r r E s r N)r � __future__r r r r �gslib.boto_translationr �gslib.gcs_json_apir �objectr r r r r r r r �<module> s __pycache__/wildcard_iterator.cpython-39.pyc 0000644 00000057701 15025013616 0015120 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl mZ ddl mZ dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z ddl m!Z! ddl"m#Z# ddl"m$Z$ e j%�r8e&Z'e �(d�Z)dZ*G dd� de+�Z,G dd � d e,�Z-d d � Z.G d d � d e,�Z/G d!d"� d"e'�Z0d&d$d%�Z1dS )'z1Wildcard iterator class and supporting functions.� )�absolute_import)�print_function)�division)�unicode_literalsN)�BucketListingBucket)�BucketListingObject)�BucketListingPrefix)�AccessDeniedException)�CloudApi)�NotFoundException)�CommandException)�ContainsWildcard)�GenerationFromUrlAndString)�StorageUrlFromString)� StripOneSlash)�WILDCARD_REGEX)�storage_v1_messages)�UTF8)�FixWindowsEncodingIfNeeded)� PrintableStrz (?P<before>.*?)\*\*(?P<after>.*)a\ Invalid Unicode path encountered (%s). gsutil cannot proceed with such files present. Please remove or rename this file and try again. NOTE: the path printed above replaces the problematic characters with a hex-encoded printable representation. For more details (including how to convert to a gsutil-compatible encoding) see `gsutil help encoding`.c @ s e Zd ZdZdd� ZdS )�WildcardIteratora Class for iterating over Google Cloud Storage strings containing wildcards. The base class is abstract; you should instantiate using the wildcard_iterator() static factory method, which chooses the right implementation depending on the base string. c C s d| j j S )z2Returns string representation of WildcardIterator.zWildcardIterator(%s))� wildcard_url� url_string��self� r �&/opt/gsutil/gslib/wildcard_iterator.py�__repr__I s zWildcardIterator.__repr__N)�__name__� __module__� __qualname__�__doc__r r r r r r ? s r c @ s� e Zd ZdZd dd�Zd dd�Zdd � Zd d � Zd d � Zd dd�Z d dd�Z d dd�Z dd� Z d!dd�Z d"dd�Zd#dd�ZdS )$�CloudWildcardIteratoraV WildcardIterator subclass for buckets, bucket subdirs and objects. Iterates over BucketListingRef matching the Url string wildcard. It's much more efficient to first get metadata that's available in the Bucket (for example to get the name and size of each object), because that information is available in the object list results. FNc C s* || _ || _|| _|| _|p"t�� | _dS )a� Instantiates an iterator that matches the wildcard URL. Args: wildcard_url: CloudUrl that contains the wildcard to iterate. gsutil_api: Cloud storage interface. Passed in for thread safety, also settable for testing/mocking. all_versions: If true, the iterator yields all versions of objects matching the wildcard. If false, yields just the live object version. project_id: Project ID to use for bucket listings. logger: logging.Logger used for outputting debug messages during iteration. If None, the root logger will be used. N)r � all_versions� gsutil_api� project_id�logging� getLogger�logger)r r r$ r# r% r( r r r �__init__W s z CloudWildcardIterator.__init__c c sF | j �� }d}|rxt� }|D ]}|�|� q | j|d�}|�ddg� |�dg� |s\| jrx|�ddg� |�dd g� | jd gd �D �]�}|j}| j � � �r|�rt |�}| j j |j d | j| j j|d �D ]:} | jtjjkr�| j|| j| jd�V q�| �|| j�V q�n|V q�t| j j��s�| j �� �r�| j�s�zL| j j| j j | j j| j j| j j|d�} | j| j j| | j�pp|d�V W dS t t f�y� Y n0 |�r�d|| j j| j jf } nd|t | j j��p�d f } | g} | r�t | � d��}| � |j�\} }}}| �!|�}|�rtdg�n|}| j j |j | || j�p2|| j j|d�D ]�} |D ]�}| jtjjk�r�| j}|�"|j#��r4|�r�t |j#�|k�r�|�r�| �$|j��r�| j||| j�p�|d�V �q@nx| j} t| ��r�t%d|| f ��t | �}|�"|��rH|�r ||k�r d||d | f }| �&|� n| �|| �V �q@�qH�q@�q�q�dS )a! Iterator that gets called when iterating over the cloud wildcard. In the case where no wildcard is present, returns a single matching object, single matching prefix, or one of each if both exist. Args: bucket_listing_fields: Iterable fields to include in bucket listings. Ex. ['name', 'acl']. Iterator is responsible for converting these to list-style format ['items/name', 'items/acl'] as well as adding any fields necessary for listing such as prefixes. API implementation is responsible for adding pagination fields. If this is None, all fields are returned. expand_top_level_buckets: If true, yield no BUCKET references. Instead, expand buckets into top-level objects and prefixes. Yields: BucketListingRef of type BUCKET, OBJECT or PREFIX. N)� get_fieldsz items/name�prefixes�namezitems/generationzitems/metageneration� generation�metageneration�id�� bucket_fields�/)� delimiterr# �provider�fields)� with_version)r- r4 r5 �%s%s#%s�%s%sr )�prefixr3 r# r4 r5 znCloud folder %s%s contains a wildcard; gsutil does not currently support objects with wildcards in their name.)'r � HasGeneration�set�add�_GetToListFields�updater# �_ExpandBucketWildcardsr �IsBucketr r$ Z ListObjects� bucket_name�scheme�datatyper ZCsObjectOrPrefixTypeZOBJECT� _GetObjectRef�data� _GetPrefixRefr �IsObjectZGetObjectMetadata� object_namer- �bucket_url_stringr r r �pop�_BuildBucketFilterStrings�_GetRegexPatterns�matchr, �_SingleVersionMatchesr �append)r �bucket_listing_fields�expand_top_level_bucketsZsingle_version_requestr* �field�bucket_listing_refrI �urlZ obj_or_prefix� get_objectr Zurls_needing_expansionr9 r3 �prefix_wildcard�suffix_wildcardZregex_patternsZlisting_fields�pattern� gcs_objectZrstripped_prefixZurl_append_stringr r r �__iter__p s� � � � � ��� ��� � �� � � �� �� �� z CloudWildcardIterator.__iter__c C sb |g}d|v r&|� dd�}|�|� n|}||fD ] }|�d�r2|�|dd� � q2dd� |D �S )a� Returns list of regex patterns derived from the wildcard patterns. Args: wildcard_pattern (str): A wilcard_pattern to filter the resources. Returns: List of compiled regex patterns. This translates the wildcard_pattern and also creates some additional patterns so that we can treat ** in a/b/c/**/d.txt as zero or more folders. This means, a/b/c/d.txt will also be returned along with a/b/c/e/f/d.txt. z/**/r2 �**/� Nc S s g | ]}t �t�|���qS r )�re�compile�fnmatch� translate)�.0�pr r r � <listcomp>7 � z;CloudWildcardIterator._GetRegexPatterns.<locals>.<listcomp>)�replacerO � startswith)r Zwildcard_patternZwildcard_patternsZupdated_patternrX r r r rL s z'CloudWildcardIterator._GetRegexPatternsc C s( t �|�}|s |}d}|}d}n�|�� dkrN|d|�� � }||�� d� }nd}|}|�d�}|dkrx|d|d � }|p~d| }|�d�s�t|�}||�� d� }|�d�}|dkr�d}n||d d� }|�d�dkr�d}||7 }d}nd}| j�d t |�t |�t |�t |�t |�� ||||fS ) a� Builds strings needed for querying a bucket and filtering results. This implements wildcard object name matching. Args: wildcard: The wildcard string to match to objects. Returns: (prefix, delimiter, prefix_wildcard, suffix_wildcard) where: prefix is the prefix to be sent in bucket GET request. delimiter is the delimiter to be sent in bucket GET request. prefix_wildcard is the wildcard to be used to filter bucket GET results. suffix_wildcard is wildcard to be appended to filtered bucket GET results for next wildcard expansion iteration. For example, given the wildcard gs://bucket/abc/d*e/f*.txt we would build prefix= abc/d, delimiter=/, prefix_wildcard=d*e, and suffix_wildcard=f*.txt. Using this prefix and delimiter for a bucket listing request will then produce a listing result set that can be filtered using this prefix_wildcard; and we'd use this suffix_wildcard to feed into the next call(s) to _BuildBucketFilterStrings(), for the next iteration of listing/filtering. Raises: AssertionError if wildcard doesn't contain any wildcard chars. r2 � r N���� r[ z**zMwildcard=%s, prefix=%s, delimiter=%s, prefix_wildcard=%s, suffix_wildcard=%s ) r �search�start�find�endswithr �endr( �debugr ) r �wildcardrM r9 r3 rV rW Z wildcard_partrn r r r rK 9 sD �z/CloudWildcardIterator._BuildBucketFilterStringsc C s t | j|�}t| jj�t|�kS �N)r r �strr- )r Zlisted_generationZdecoded_generationr r r rN � s �z+CloudWildcardIterator._SingleVersionMatchesc c s� t | jj�}|r:t|�tdg�kr:t| jj�s:t|�V n�| j�� rvt| jj�svt|| jj | jj| jj |d�d�V n~t � | jj�}t �|�}| �|�}|r�|�d� | jj|| j| jj d�D ]4}|�|j�r�t d| jj |jf �}t||d�V q�dS )aG Expands bucket and provider wildcards. Builds a list of bucket url strings that can be iterated on. Args: bucket_fields: If present, populate only these metadata fields for buckets. Example value: ['acl', 'defaultObjectAcl'] Yields: BucketListingRefereneces of type BUCKET. r/ )r4 r5 �� root_objectzitems/id)r5 r% r4 z%s://%s/N)r r rI r; r rA r r@ r$ Z GetBucketrB r_ r` r] r^ r= r< Z ListBucketsr% rM r/ )r r1 Z bucket_url�regex�progr5 �bucketrT r r r r? � s< � ��� � �z,CloudWildcardIterator._ExpandBucketWildcardsc C s* |r&t � }|D ]}|�d| � q|S dS )a� Prepends 'items/' to the input fields and converts it to a set. This way field sets requested for GetBucket can be used in ListBucket calls. Note that the input set must contain only bucket or object fields; listing fields such as prefixes or nextPageToken should be added after calling this function. Args: get_fields: Iterable fields usable in GetBucket/GetObject calls. Returns: Set of fields usable in ListBuckets/ListObjects calls. zitems/N)r; r< )r r* Z list_fieldsrR r r r r= � s z&CloudWildcardIterator._GetToListFieldsc C sP |r.|j dur.t| j|j �}d||j|f }nd||jf }t|�}t||d�S )ah Creates a BucketListingRef of type OBJECT from the arguments. Args: bucket_url_string: Wildcardless string describing the containing bucket. gcs_object: gsutil_api root Object for populating the BucketListingRef. with_version: If true, return a reference with a versioned string. Returns: BucketListingRef of type OBJECT. Nr7 r8 rs )r- r r r, r r )r rI rY r6 Zgeneration_strZ object_stringZ object_urlr r r rD � s ��z#CloudWildcardIterator._GetObjectRefc C s t d||f �}t||d�S )a Creates a BucketListingRef of type PREFIX from the arguments. Args: bucket_url_string: Wildcardless string describing the containing bucket. prefix: gsutil_api Prefix for populating the BucketListingRef Returns: BucketListingRef of type PREFIX. r8 rs )r r )r rI r9 Z prefix_urlr r r rF � s z#CloudWildcardIterator._GetPrefixRefc c s | j |d�D ] }|V q dS )ah Iterates over the wildcard, returning refs for each expanded bucket. This ignores the object part of the URL entirely and expands only the the bucket portion. It will yield BucketListingRefs of type BUCKET only. Args: bucket_fields: Iterable fields to include in bucket listings. Ex. ['defaultObjectAcl', 'logging']. This function is responsible for converting these to listing-style format ['items/defaultObjectAcl', 'items/logging'], as well as adding any fields necessary for listing such as 'items/id'. API implemenation is responsible for adding pagination fields. If this is None, all fields are returned. Yields: BucketListingRef of type BUCKET, or empty iterator if no matches. r0 N)r? )r r1 �blrr r r � IterBuckets� s z!CloudWildcardIterator.IterBucketsc c s | j ||d�D ] }|V qdS )a} Iterates over the wildcard, yielding bucket, prefix or object refs. Args: bucket_listing_fields: If present, populate only these metadata fields for listed objects. expand_top_level_buckets: If true and the wildcard expands only to Bucket(s), yields the expansion of each bucket into a top-level listing of prefixes and objects in that bucket instead of a BucketListingRef to that bucket. Yields: BucketListingRef, or empty iterator if no matches. �rP rQ N�rZ )r rP rQ rx r r r �IterAll s � z CloudWildcardIterator.IterAllc c s& | j |dd�D ]}|�� r|V qdS )a& Iterates over the wildcard, yielding only object BucketListingRefs. Args: bucket_listing_fields: If present, populate only these metadata fields for listed objects. Yields: BucketListingRefs of type OBJECT or empty iterator if no matches. Trz N)rZ rG )r rP rx r r r � IterObjects s � z!CloudWildcardIterator.IterObjects)FNN)NF)N)N)F)N)NF)N)r r r r! r) rZ rL rK rN r? r= rD rF ry r| r} r r r r r" N s& � � )!U * r" c C s t jtj�| �d�S )a0 Returns an apitools Object class with supported file attributes. To provide size estimates for local to cloud file copies, we need to retrieve expose the local file's size. Args: filepath: Path to the file. Returns: apitools Object that with file name and size attributes filled-in. )�size)�apitools_messagesZObject�os�path�getsize)�filepathr r r �_GetFileObject. s r� c @ sR e Zd ZdZddd�Zddd�Zdd � Zd d � Zdd d �Zddd�Z ddd�Z dS )�FileWildcardIteratora� WildcardIterator subclass for files and directories. If you use recursive wildcards ('**') only a single such wildcard is supported. For example you could use the wildcard '**/*.txt' to list all .txt files in any subdirectory of the current directory, but you couldn't use a wildcard like '**/abc/**/*.txt' (which would, if supported, let you find .txt files in any subdirectory named 'abc'). NFc C s$ || _ || _|| _|p t�� | _dS )a' Instantiates an iterator over BucketListingRefs matching wildcard URL. Args: wildcard_url: FileUrl that contains the wildcard to iterate. exclude_tuple: (base_url, exclude_pattern), where base_url is top-level URL to list; exclude_pattern is a regex of paths to ignore during iteration. ignore_symlinks: If True, ignore symlinks during iteration. logger: logging.Logger used for outputting debug messages during iteration. If None, the root logger will be used. N)r � exclude_tuple�ignore_symlinksr& r' r( )r r r� r� r( r r r r) I s z FileWildcardIterator.__init__c c s0 |odt |�v }| jj}t�|�}|rz|�d�dd� }|�d�}|�d�rXtd| ��|s`d}|�t j �}| � ||�}n t � |�}|D ]�}t|�} zd| jr�t j�|�r�| jr�| j�d|� W q�t j�|�r�t| �V n |r�t|�nd} t| | d �V W q� t�y( td �t� t t |� ����Y q�0 q�dS ) a@ Iterator that gets called when iterating over the file wildcard. In the case where no wildcard is present, returns a single matching file or directory. Args: bucket_listing_fields: Iterable fields to include in listings. Ex. ['size']. Currently only 'size' is supported. If present, will populate yielded BucketListingObject.root_object with the file name and size. Raises: WildcardException: if invalid wildcard found. Yields: BucketListingRef of type OBJECT (for files) or PREFIX (for directories) r~ �beforeNrh Zafter�*z5Invalid wildcard with more than 2 consecutive *s (%s)z Skipping symbolic link %s...rs � ) r; r rH �FLAT_LIST_REGEXrM �grouprf �WildcardException�lstripr� �sep�_IterDir�glob�iglobr r� r� �islinkr( �info�isdirr r� r �UnicodeEncodeErrorr �join�textwrap�wrap�_UNICODE_EXCEPTION_TEXT�repr) r rP Z include_sizerp rM �base_dirZremaining_wildcardZ filepathsr� Z expanded_urlZ blr_objectr r r rZ ^ s@ � � �z FileWildcardIterator.__iter__c c s t j�|�d |kr |d7 }t jt�|�dd�D ]�\}}}g }|D ]J}t j�||�}| �|�sB|�|� nqB| j rBt j� |�rB| j � d|� qB||dd�< t � ||�D ]V} zt j�|t| ��V W q� ty� td�t�ttt j�|| �� ����Y q�0 q�q0dS )a� An iterator over the specified dir and wildcard. Args: directory (unicode): The path of the directory to iterate over. wildcard (str): The wildcard characters used for filename pattern matching. Yields: (str) A string containing the path to a file somewhere under the directory hierarchy of `directory`. Raises: ComandException: If this method encounters a file path that it cannot decode as UTF-8. r �\T)�topdownz Skipping symlink directory "%s"Nr� )r� r� � splitdrive�walk�six� ensure_textr� � _ExcludeDirrO r( r� r� r_ �filterr �UnicodeDecodeErrorr r� r� r� r� ) r � directoryrp �dirpath�dirnames� filenamesZfiltered_dirnames�dirnameZ full_dir_path�fr r r r� � s. � ��z FileWildcardIterator._IterDirc C sz | j du rdS | j \}}}|s"dS t|�jt|j�d� }|�| jj�rT|dd� }|�|�rv| jrr| j� d|� dS dS )z�Check a directory to see if it should be excluded from os.walk. Args: dir: String representing the directory to check. Returns: True if the directory should be excluded. NFri z!Skipping excluded directory %s...T) r� r r �lenrf r �delimrM r( r� )r �dir�base_urlZ exclude_dirsZexclude_patternZ str_to_checkr r r r� � s � � z FileWildcardIterator._ExcludeDirc c s$ | j |d�D ]}|�� r |V q dS )a� Iterates over the wildcard, yielding only object (file) refs. Args: bucket_listing_fields: Iterable fields to include in listings. Ex. ['size']. Currently only 'size' is supported. If present, will populate yielded BucketListingObject.root_object with the file name and size. Yields: BucketListingRefs of type OBJECT or empty iterator if no matches. �rP N)r| rG )r rP rS r r r r} s � z FileWildcardIterator.IterObjectsc c s | j |d�D ] }|V q dS )a� Iterates over the wildcard, yielding BucketListingRefs. Args: bucket_listing_fields: Iterable fields to include in listings. Ex. ['size']. Currently only 'size' is supported. If present, will populate yielded BucketListingObject.root_object with the file name and size. expand_top_level_buckets: Ignored; filesystems don't have buckets. Yields: BucketListingRefs of type OBJECT (file) or PREFIX (directory), or empty iterator if no matches. r� Nr{ )r rP rQ rS r r r r| s � z FileWildcardIterator.IterAllc C s t d��dS )z�Placeholder to allow polymorphic use of WildcardIterator. Args: unused_bucket_fields: Ignored; filesystems don't have buckets. Raises: WildcardException: in all cases. z6Iterating over Buckets not possible for file wildcardsN)r� )r Zunused_bucket_fieldsr r r ry * s �z FileWildcardIterator.IterBuckets)NFN)N)N)NF)N) r r r r! r) rZ r� r� r} r| ry r r r r r� ? s � <T r� c @ s( e Zd ZdZdd� Zdd� Zdd� ZdS ) r� z+Exception raised for invalid wildcard URLs.c C s t �| � || _d S rq )� StandardErrorr) �reason)r r� r r r r) : s zWildcardException.__init__c C s d| j S �NzWildcardException: %s�r� r r r r r > s zWildcardException.__repr__c C s d| j S r� r� r r r r �__str__A s zWildcardException.__str__N)r r r r! r) r r� r r r r r� 7 s r� Fc C s@ t | �}|pt�� }|�� r,t||||d�S t||||d�S dS )a� Instantiate a WildcardIterator for the given URL string. Args: url_str: URL string naming wildcard object(s) to iterate. gsutil_api: Cloud storage interface. Passed in for thread safety, also settable for testing/mocking. all_versions: If true, the iterator yields all versions of objects matching the wildcard. If false, yields just the live object version. project_id: Project id to use for bucket listings. exclude_tuple: (base_url, exclude_pattern), where base_url is top-level URL to list; exclude_pattern is a regex of paths to ignore during iteration. ignore_symlinks: For FileUrls, ignore symlinks during iteration if true. logger: logging.Logger used for outputting debug messages during iteration. If None, the root logger will be used. Returns: A WildcardIterator that handles the requested iteration. )r� r� r( )r# r% N)r r& r' � IsFileUrlr� r" )�url_strr$ r# r% r� r� r( rT r r r �CreateWildcardIteratorE s ��r� )FNNFN)2r! � __future__r r r r r_ r� r& r� r] r� r� Zgslib.bucket_listing_refr r r �gslib.cloud_apir r r �gslib.exceptionr �gslib.storage_urlr r r r r Z"gslib.third_party.storage_apitoolsr r �gslib.utils.constantsr Zgslib.utils.text_utilr r �PY3� Exceptionr� r^ r� r� �objectr r" r� r� r� r� r r r r �<module> s^ � c y � __pycache__/gcs_json_credentials.cpython-39.pyc 0000644 00000033541 15025013616 0015574 0 ustar 00 a +(Wg9Q � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl mZ ddl mZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlm Z ddlm Z ddl m Z ddl m Z ddl m!Z! ddl m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z+ ddl,m-Z- ddl.m/Z/ ddl0Z0ddl1m2Z2 ddl3m4Z4 dd l5m6Z6 dd l7m8Z8 dd l3m9Z9 dd l m:Z: d Z;d!Z<e j=j>e j=j?e j=j@e j=jAe j=jBgZCd"ZDd#d$� ZEG d%d&� d&e+jFe+jG�ZHG d'd(� d(e/jI�ZJdKd)d*�ZKd+d,� ZLdLd-d.�ZMd/d0� ZNd1d2� ZOd3d4� ZPd5d6� ZQd7d8� ZRd9d:� ZSd;d<� ZTd=d>� ZUd?d@� ZVdAdB� ZWdCdD� ZXdEdF� ZYdGdH� ZZdIdJ� Z[dS )Mz3Credentials logic for JSON CloudApi implementation.� )�absolute_import)�print_function)�division)�unicode_literalsN)�credentials_lib)� exceptions)�config)� CredTypes)�CommandException)�ImpersonationCredentials)�NoOpCredentials)� constants)� system_util)�GetFriendlyConfigFilePaths)�GetCredentialStoreFilename)� GetGceCredentialCacheFilename)�GetGcsJsonApiVersion)�UTF8)�WrappedCredentials)�_helpers)�base��AuthorizedHttp)�service_account)� HAS_CRYPTO)�devshell)�ServiceAccountCredentials)� reauth_creds)�multiprocess_file_storage)�BytesIOz)https://accounts.google.com/o/oauth2/authz#https://oauth2.googleapis.com/token� notasecretc C s t | t�S �N)� isinstance�P12Credentials)� credentials� r% �)/opt/gsutil/gslib/gcs_json_credentials.py�isP12CredentialsO s r' c @ s: e Zd ZdZdd� Zedd� �Zdd� Zed d d ��Z dS ) � PKCS12Signerz%Signer for a p12 service account key.c C s || _ d S r! )�_key)�self�keyr% r% r& �__init__U s zPKCS12Signer.__init__c C s d S r! r% )r* r% r% r& �key_idZ s zPKCS12Signer.key_idc C s* t �|�}ddlm} | j�||j|j�S )Nr )�_cryptography_rsa)r �to_bytes�google.auth.cryptr. r) �sign�_PADDING�_SHA256)r* �messager. r% r% r& r1 ^ s �zPKCS12Signer.signNc C sV ~dd� |D �\}}ddl m} z |�||�\}}}| |�W S td��Y n0 d S )Nc s s | ]}t �|�V qd S r! )r r/ )�.0�kr% r% r& � <genexpr>i � z+PKCS12Signer.from_string.<locals>.<genexpr>r )�pkcs12z<Unable to load the keyfile, Invalid password or PKCS12 data.)�,cryptography.hazmat.primitives.serializationr9 Zload_key_and_certificatesr )�clsZ key_stringsr- � key_string�passwordr9 r+ �_r% r% r& � from_stringf s zPKCS12Signer.from_string)N) �__name__� __module__� __qualname__�__doc__r, �propertyr- r1 � classmethodr? r% r% r% r& r( R s r( c @ s* e Zd ZdZdZdd� Zeddd��ZdS ) r# a google-auth service account credentials for p12 keys. p12 keys are not supported by the google-auth service account credentials. gsutil uses oauth2client to support p12 key users. Since oauth2client was deprecated and bundling it is security concern, we decided to support p12 in gsutil codebase. We prefer not adding it to the google-auth library because p12 is not supported from the beginning by google-auth. GCP strongly suggests users to use the JSON format. gsutil has to support it to not break users. )�service_account_email� token_uri�scopesc C s t | |d�S )N)�httpr )r* rI r% r% r& � authorize� s zP12Credentials.authorizeNc sV |pt }t�||f�}� fdd�| jD �}|rBtd�d�|����| |fi � ��}|S )Nc s g | ]}|� vr|�qS r% r% )r5 �f��kwargsr% r& � <listcomp>� r8 zHP12Credentials.from_service_account_pkcs12_keystring.<locals>.<listcomp>zMissing fields: {}.z, )�#GOOGLE_OAUTH2_DEFAULT_FILE_PASSWORDr( r? �_REQUIRED_FIELDSr �format�join)r; r< r= rM �signer�missing_fields�credsr% rL r& �%from_service_account_pkcs12_keystring� s �z4P12Credentials.from_service_account_pkcs12_keystring)N)r@ rA rB rC rP rJ rE rV r% r% r% r& r# s s �r# c K s6 zt j| |fi |��W S ty0 td��Y n0 dS )zCCreates a service account from a p12 key and handles import errors.zapyca/cryptography is not available. Either install it, or please consider using the .json keyfileN)r# rV � ImportErrorr )r< r= rM r% r% r& �CreateP12ServiceAccount� s �� �rX c C s� |g}t | tj�r |�| j� n�t | t�rx|�| j� t| dd�rP|�| j� q�t| dd�r�|�t � | j �dd� � nFt | t j j�r�| jr�| jdkr�|�| j� n |�d� |�| jp�d� t| dd�r�|�| j� d d � |D �}d �|�S ) a� Disambiguates a credential for caching in a credential store. Different credential types have different fields that identify them. This function assembles relevant information in a string to be used as the key for accessing a credential. Note that in addition to uniquely identifying the entity to which a credential corresponds, we must differentiate between two or more of that entity's credentials that have different attributes such that the credentials should not be treated as interchangeable, e.g. if they target different API versions (happens for developers targeting different test environments), have different private key IDs (for service account JSON keyfiles), or target different provider token (refresh) URIs. Args: credentials: An OAuth2Credentials object. api_version: JSON API version being used. Returns: A string that can be used as the key to identify a credential, e.g. "v1-909320924072.apps.googleusercontent.com-1/rEfrEshtOkEn-https://..." �_private_key_idN�_private_key_pkcs12� �nullZ noclientidZnorefreshtokenrG c S s g | ]}t �|��qS r% )�six� ensure_text)r5 �partr% r% r& rN � r8 z)GetCredentialStoreKey.<locals>.<listcomp>�-)r" r �DevshellCredentials�append� user_emailr �_service_account_email�getattrrY �base64� b64encoderZ � oauth2client�client�OAuth2Credentials� client_id� refresh_tokenrG rR )r$ � api_versionZ key_partsr% r% r& �GetCredentialStoreKey� s$ rn c C s� |pt |�pt� | _t| jt�r.|�dt� � t| j�s�t| jt � �}| j� t � t � |�� d}t| jt�sx| jj�� }|r�t|�tjjkr�|| _dS )zAHelper to ensure each GCS API client shares the same credentials.zeWARNING: This command is using service account impersonation. All API calls will be executed as [%s].N)�_CheckAndGetCredentialsr r$ r" r �warn� _GetImpersonateServiceAccountr' rn r � set_storer �MultiprocessFileStorager �store�get�typerh ri rj )�api�loggerr$ Zcredential_store_keyZ cached_credr% r% r& � SetUpJsonCredentialsAndCache� s0 � � ��� �ry c C sV g }d}z�t � r |�tj� t� r.|�tj� t|�dkrLtd|t� f ��tj}t � }tj}t � }tj }t � }tj }t� }tj}t� }tj}t� }|p�|p�|p�|p�|p�|} t� r�| r�tj}t| | �W S | W S W n� t�yP } zf|�r:| �tj��r| �t� � � |tjk�r| �n$t � � �r.| � d|� n | � d|� � W Y d} ~ n d} ~ 0 0 dS )z�Returns credentials from the configuration file, if any are present. Args: logger: logging.Logger instance for outputting messages. Returns: OAuth2Credentials object if any valid ones are found, otherwise None. N� a& You have multiple types of configured credentials (%s), which is not supported. One common way this happens is if you run gsutil config to create credentials and later run gcloud auth, and create a second set of credentials. Your boto config path is: %s. For more help, see "gsutil help creds".zCYour "%s" credentials are invalid. Please run $ gcloud auth loginz�Your "%s" credentials are invalid. For more help, see "gsutil help creds", or re-run the gsutil config command (see "gsutil help config").) �_HasOauth2UserAccountCredsrb r ZOAUTH2_USER_ACCOUNT� _HasOauth2ServiceAccountCredsZOAUTH2_SERVICE_ACCOUNT�lenr r � _GetOauth2UserAccountCredentials�#_GetOauth2ServiceAccountCredentialsZEXTERNAL_ACCOUNT� _GetExternalAccountCredentialsZ EXTERNAL_ACCOUNT_AUTHORIZED_USER�,_GetExternalAccountAuthorizedUserCredentialsZGCE� _GetGceCredsZDEVSHELL�_GetDevshellCreds� _HasImpersonateServiceAccountZ IMPERSONATION� _GetImpersonationCredentials� Exception� isEnabledFor�logging�DEBUG�debug� traceback� format_excr �InvokedViaCloudSdkrp ) rx Zconfigured_cred_typesZfailed_cred_typeZ user_credsZservice_account_credsZexternal_account_credsZ&external_account_authorized_user_credsZ gce_credsZdevshell_credsrU �er% r% r& ro s\ �� �� ro c C s t �ddt�S )N�OAuth2�provider_token_uri)r ru �(DEFAULT_GOOGLE_OAUTH2_PROVIDER_TOKEN_URIr% r% r% r& �_GetProviderTokenUriU s �r� c C s t �dd�S )N� Credentials�gs_service_key_file�r � has_optionr% r% r% r& r| Z s r| c C s t �dd�S )Nr� �gs_oauth2_refresh_tokenr� r% r% r% r& r{ ^ s r{ c C s t �dd�S )N� GoogleComputer r� r% r% r% r& � _HasGceCredsb s r� c C s t � dvS )N)N� )rq r% r% r% r& r� f s r� c C s t �ddd �} | sd S t�| �S )Nr� �gs_external_account_file)r ru r Zfor_external_account)Zexternal_account_filenamer% r% r& r� j s �r� c C s t �ddd �} | sd S t�| �S )Nr� �(gs_external_account_authorized_user_file)r ru r Z$for_external_account_authorized_user)Z)external_account_authorized_user_filenamer% r% r& r� s s ��r� c C s t jpt�ddtj�d��S )Nr� Z gs_impersonate_service_accountZ)CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT)r �IMPERSONATE_SERVICE_ACCOUNTr ru �os�environr% r% r% r& rq } s �rq c C s t � s dS t� } t�ddd�}t�ddd�}t�|d��}|�� }W d � n1 sV0 Y d}z|�t�}d}W n t y� Y n0 |r�zt � |�}W n t y� t d | ��Y n0 d D ] }||vr�t d ||f ��q�tj|t| d �S t�dd t�}t||t|| d�S dS )zDRetrieves OAuth2 service account credentials for a private key file.Nr� �gs_service_client_idr� r� �rbFTz/Could not parse JSON keyfile "%s" as valid JSON)rk � client_email�private_key_id� private_keyzFThe JSON private key file at %s did not contain the required entry: %s)rH rG Z gs_service_key_file_password)rH rF rG )r| r� r ru �io�open�read�decoder �UnicodeDecodeError�json�loads� ValueErrorr� r �from_json_keyfile_dict�DEFAULT_SCOPESrO rX ) r� Zservice_client_id�private_key_filenameZprivate_key_filer� Zkeyfile_is_utf8Z json_key_dictZ json_entryZ key_file_passr% r% r& r � s@ & � ���r c C s~ t � s dS t� } t�� \}}t�ddtj�d|��}t�ddtj�d|��}tj j tj j g}t j d||t�dd�d| d|d �S ) zARetrieves OAuth2 service account credentials for a refresh token.Nr� rk ZOAUTH2_CLIENT_ID� client_secretZOAUTH2_CLIENT_SECRETr� r� )rH )r{ r� r �GetGsutilClientIdAndSecretr ru r� r� r �Scopes�CLOUD_PLATFORM�REAUTHr �Oauth2WithReauthCredentials)r� �gsutil_client_id�gsutil_client_secretrk r� Zscopes_for_reauth_challenger% r% r& r~ � s0 � � � � �r~ c C sz t � s d S z tjt�ddd�t� d�W S tjyt } z4dt| �v r^dt| �v r^W Y d } ~ d S � W Y d } ~ n d } ~ 0 0 d S )Nr� r �default)�service_account_name�cache_filenamezservice accountzdoes not exist) r� r �GceAssertionCredentialsr ru r �apitools_exceptions�ResourceUnavailableError�str)r� r% r% r& r� � s ��r� c C s2 z t �� W S t jy Y d S � Y n0 d S r! )r ra �NoDevshellServerr% r% r% r& r� � s r� c C s$ t | t�rdS tt� tjjg| |�S )z?Retrieves temporary credentials impersonating a service accountN)r" r rq r r� r� )r$ rx r% r% r& r� � s �r� )N)N)\rC � __future__r r r r rf r� r� r� r� r] r� �apitools.base.pyr r r� �botor Zgslib.cred_typesr �gslib.exceptionr � gslib.impersonation_credentialsr �gslib.no_op_credentialsr � gslib.utilsr r �gslib.utils.boto_utilr r r r �gslib.utils.constantsr Z gslib.utils.wrapped_credentialsr � google.authr r0 r Z crypt_baseZgoogle_auth_httplib2r � google.oauth2r rh �oauth2client.clientr �oauth2client.contribr � oauth2client.service_accountr � google_reauthr r r Z0DEFAULT_GOOGLE_OAUTH2_PROVIDER_AUTHORIZATION_URIr� r� r� �CLOUD_PLATFORM_READ_ONLY� FULL_CONTROL� READ_ONLY� READ_WRITEr� rO r' �SignerZFromServiceAccountMixinr( r� r# rX rn ry ro r� r| r{ r� r� r� r� rq r r~ r� r� r� r% r% r% r& �<module> s� ���! = *S * __pycache__/cloud_api_delegator.cpython-39.pyc 0000644 00000053303 15025013616 0015375 0 ustar 00 a +(Wg}w � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlm Z dd l m Z dd l m Z dd l mZ dd l mZ dd lmZ ddlmZ G dd� de �ZdS )zBGsutil API delegator for interacting with cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�config)�context_config)�ArgumentException)�CloudApi)�ApiMapConstants)� ApiSelector)�CommandException)� boto_utilc s e Zd ZdZdj� fdd� Zdd� Zdd � Zdkd d �Zdld d �Zdmdd�Z dndd�Z dodd�Z dpdd�Z dqdd�Z drdd�Zdsdd�Zdtd d �Zdud d �Zdvd d!�Zdwd"d#�Zdxd$d%�Zdyd&d'�Zdddd(ejjddddddf d)d*�Zdzd+d,�Zd{d-d.�Zd|d/d0�Z d}d1d2�Z d~d3d4�Z dd5d6�Z d�d7d8�Z d�d9d:�Z!d�d;d<�Z"d�d=d>�Z#d�d?d@�Z$d�dAdB�Z%d�dCdD�Z&d�dEdF�Z'd�dGdH�Z(d�dIdJ�Z)d�dKdL�Z*d�dMdN�Z+d�dOdP�Z,dQdR� Z-d�dSdT�Z.d�dUdV�Z/d�dXdY�Z0d�dZd[�Z1d�d\d]�Z2d�d^d_�Z3d�d`da�Z4d�dbdc�Z5d�ddde�Z6d�dfdg�Z7d�dhdi�Z8� Z9S )��CloudApiDelegatora� Class that handles delegating requests to gsutil Cloud API implementations. This class is responsible for determining at runtime which gsutil Cloud API implementation should service the request based on the Cloud storage provider, command-level API support, and configuration file override. During initialization it takes as an argument a gsutil_api_map which maps providers to their default and supported gsutil Cloud API implementations (see comments in cs_api_map for details). Instantiation of multiple delegators per-thread is required for multiprocess and/or multithreaded operations. Calling methods on the same delegator in multiple threads is unsafe. Nr c s\ t t| �j|||||||| | d� || _tj�ddd��� | _i | _ | jt j sXt d��dS )a3 Performs necessary setup for delegating cloud storage requests. This function has different arguments than the gsutil Cloud API __init__ function because of the delegation responsibilties of this class. Args: bucket_storage_uri_class: boto storage_uri class, used by APIs that provide boto translation or mocking. gsutil_api_map: Map of providers and API selector tuples to api classes which can be used to communicate with those providers. logger: logging.logger for outputting log messages. status_queue: Queue for relaying status to UI. provider: Default provider prefix describing cloud storage provider to connect to. debug: Debug level for the API implementation (0..3). http_headers (dict|None): Arbitrary headers to be included in every request. trace_token: Apiary trace token to pass to API. perf_trace_token: Performance trace token to use when making API calls. user_project: Project to be billed for this project. ��provider�debug� http_headers� trace_token�perf_trace_token� user_project�GSUtil� prefer_api� z.No apiclass supplied for gsutil Cloud API map.N) �superr �__init__�api_map�botor �get�upperr � loaded_apisr �API_MAPr ) �self�bucket_storage_uri_class�gsutil_api_map�logger� status_queuer r r r r r �� __class__� �(/opt/gsutil/gslib/cloud_api_delegator.pyr 1 s � zCloudApiDelegator.__init__c C sd |p| j }|std��t|�}|| jvr2i | j|< | �|�}|| j| vrV| �||� | j| | S )a� Returns a valid CloudApi for use by the caller. This function lazy-loads connection and credentials using the API map and credential store provided during class initialization. Args: provider: Provider to load API for. If None, class-wide default is used. Raises: ArgumentException if there is no matching API available in the API map. Returns: Valid API instance that can be used to communicate with the Cloud Storage provider. z No provider selected for _GetApi)r r �strr �GetApiSelector�_LoadApi�r! r Z api_selectorr( r( r) �_GetApi` s zCloudApiDelegator._GetApic C s� || j tj vr td| ��|| j tj | vr@td||f ��| j tj | | | j| j| j|| j| j| j | j | j d� | j | |< dS )z�Loads a CloudApi into the loaded_apis map for this class. Args: provider: Provider to load the API for. api_selector: cs_api_map.ApiSelector defining the API type. z7gsutil Cloud API map contains no entry for provider %s.z=gsutil Cloud API map does not support API %s for provider %s.r N) r r r r r" r$ r% r r r r r r r- r( r( r) r, ~ s, �����zCloudApiDelegator._LoadApic C s" |p| j }|std��|| jtj vsF| jtj | | jtj | vrRtd| ��|| jtj vrntd| ��| jtj | }|dko�t�� }|dko�t � dd�p�t � dd�}|r�|r�t d��n4|r�t j }n(|r�t j}n | j| jtj | v r�| j}|t j k�r t�� �r t�� j�r td ��|S ) a1 Returns a cs_api_map.ApiSelector based on input and configuration. Args: provider: Provider to return the ApiSelector for. If None, class-wide default is used. Returns: cs_api_map.ApiSelector that will be used for calls to the delegator for this provider. z!No provider selected for CloudApiz(No default api available for provider %sz+No supported apis available for provider %s�gsr Zencryption_keyZdecryption_key1a gsutil does not support HMAC credentials with customer-supplied encryption keys (CSEK) or customer-managed KMS encryption keys (CMEK). Please generate and include non-HMAC credentials in your .boto configuration file, or to access public encrypted objects, remove your HMAC credentials.z�User enabled mTLS by setting "use_client_certificate", but mTLS is not supported for the selected XML API. Try configuring for the GCS JSON API or setting "use_client_certificate" to "False" in the Boto config.)r r r r Z DEFAULT_MAPr Z SUPPORT_MAPr � UsingGsHmacr � has_optionr r �XML�JSONr r �get_context_config�use_client_certificate)r! r Zselected_provider�apiZ using_gs_hmacZconfigured_encryptionr( r( r) r+ � sP ���� �� ���z CloudApiDelegator.GetApiSelectorc C s | � |��� S �N)r. �GetServiceAccountId)r! r r( r( r) r8 � s z%CloudApiDelegator.GetServiceAccountIdc C s | � |�j||d�S �N)�fields)r. � GetBucket�r! � bucket_namer r: r( r( r) r; � s zCloudApiDelegator.GetBucketc C s | � |�j||d�S r9 )r. �GetBucketIamPolicyr< r( r( r) r> � s z$CloudApiDelegator.GetBucketIamPolicyc C s | � |��||�S r7 )r. �SetBucketIamPolicy)r! r= �policyr r( r( r) r? � s z$CloudApiDelegator.SetBucketIamPolicyc C s | � |�j||d�S )N)� project_idr: )r. � ListBuckets)r! rA r r: r( r( r) rB � s �z CloudApiDelegator.ListBucketsc C s | � |�j||||||d�S )N)� canned_acl�canned_def_acl� preconditionsr: )r. � PatchBucket)r! r= �metadatarC rD rE r r: r( r( r) rF � s �z CloudApiDelegator.PatchBucketc C s | � |�j|||d�S )N)r )r. �LockRetentionPolicy)r! r= �metagenerationr r( r( r) rH � s �z%CloudApiDelegator.LockRetentionPolicyc C s | � |�j||||d�S )N)rA rG r: )r. � CreateBucket)r! r= rA rG r r: r( r( r) rJ s �z CloudApiDelegator.CreateBucketc C s | � |�j||d�S )N)rE )r. � DeleteBucket)r! r= rE r r( r( r) rK s �z CloudApiDelegator.DeleteBucketc C s | � |�j||||d�S r9 )r. �GetObjectIamPolicy�r! r= � object_name� generationr r: r( r( r) rL s �z$CloudApiDelegator.GetObjectIamPolicyc C s | � |��||||�S r7 )r. �SetObjectIamPolicy)r! r= rN r@ rO r r( r( r) rP s �z$CloudApiDelegator.SetObjectIamPolicyc C s | � |�j|||||d�S )N)�prefix� delimiter� all_versionsr: )r. � ListObjects)r! r= rQ rR rS r r: r( r( r) rT ' s �z CloudApiDelegator.ListObjectsc C s | � |�j||||d�S )N)rO r: )r. �GetObjectMetadatarM r( r( r) rU 4 s �z#CloudApiDelegator.GetObjectMetadatac C s | � |�j|||||||d�S )N)rC rO rE r: )r. �PatchObjectMetadata) r! r= rN rG rC rO rE r r: r( r( r) rV ? s �z%CloudApiDelegator.PatchObjectMetadataFc C s* | � |�j|||||| | ||| | | |d� S )N) �compressed_encoding�download_strategy� start_byte�end_byterO � object_size�progress_callback�serialization_data� digesters�decryption_tuple)r. �GetObjectMedia)r! r= rN �download_streamr rO r[ rW rX rY rZ r\ r] r^ r_ r( r( r) r` Q s �z CloudApiDelegator.GetObjectMediac C s" | � |�j|||||||| | d� S )N)�sizerC rE r\ �encryption_tupler: � gzip_encoded)r. � UploadObject) r! � upload_stream�object_metadatarb rC rE r\ rc r r: rd r( r( r) re o s �z CloudApiDelegator.UploadObjectc C s | � |�j|||||||| d�S )N)rC rE r\ rc r: rd )r. �UploadObjectStreaming) r! rf rg rC rE r\ rc r r: rd r( r( r) rh � s �z'CloudApiDelegator.UploadObjectStreamingc C s& | � | �j||||||||| | | d� S )N) rC rE rb r] �tracker_callbackr\ rc r: rd )r. �UploadObjectResumable) r! rf rg rC rE rb r] ri r\ rc r r: rd r( r( r) rj � s �z'CloudApiDelegator.UploadObjectResumablec C s$ | � | �j||||||||| | d� S )N)�src_generationrC rE r\ �max_bytes_per_callrc r_ r: )r. � CopyObject) r! �src_obj_metadata�dst_obj_metadatark rC rE r\ rl rc r_ r r: r( r( r) rm � s �z CloudApiDelegator.CopyObjectc C s | � |�j|||||d�S )N)rE rc r: )r. � ComposeObject)r! �src_objs_metadataro rE rc r r: r( r( r) rp � s �z CloudApiDelegator.ComposeObjectc C s | � |�j||||d�S )N)rE rO )r. � DeleteObject)r! r= rN rE rO r r( r( r) rr � s �z CloudApiDelegator.DeleteObjectc C s | � |�j|||||d�S )N)�tokenr: )r. � WatchBucket)r! r= �address� channel_idrs r r: r( r( r) rt � s �z CloudApiDelegator.WatchBucketc C s | � |��||�S r7 )r. � StopChannel)r! rv � resource_idr r( r( r) rw � s z CloudApiDelegator.StopChannelc C s | � |��|�S r7 )r. � ListChannels�r! r= r r( r( r) ry � s z CloudApiDelegator.ListChannelsc C s | � |��|�S r7 )r. �GetProjectServiceAccount)r! �project_numberr r( r( r) r{ � s z*CloudApiDelegator.GetProjectServiceAccountc C s | � |��||||||�S r7 )r. �CreateNotificationConfig)r! r= � pubsub_topic�payload_format� event_types�custom_attributes�object_name_prefixr r( r( r) r} � s �z*CloudApiDelegator.CreateNotificationConfigc C s | � |��||�S r7 )r. �DeleteNotificationConfig)r! r= � notificationr r( r( r) r� s �z*CloudApiDelegator.DeleteNotificationConfigc C s | � |��|�S r7 )r. �ListNotificationConfigsrz r( r( r) r� s z)CloudApiDelegator.ListNotificationConfigsc C s | � |��|�S r7 )r. �ListBucketAccessControlsrz r( r( r) r� s z*CloudApiDelegator.ListBucketAccessControlsc C s | � |��||�S r7 )r. �ListObjectAccessControls)r! r= rN r r( r( r) r� s �z*CloudApiDelegator.ListObjectAccessControlsc C s | � |��||�S r7 )r. � CreateHmacKey)r! rA �service_account_emailr r( r( r) r� s �z CloudApiDelegator.CreateHmacKeyc C s | � |��||�S r7 )r. � DeleteHmacKey�r! rA � access_idr r( r( r) r� s z CloudApiDelegator.DeleteHmacKeyc C s | � |��||�S r7 )r. � GetHmacKeyr� r( r( r) r� s z CloudApiDelegator.GetHmacKeyc C s | � |��|||�S r7 )r. � ListHmacKeys)r! rA r� �show_deleted_keysr r( r( r) r� s �z CloudApiDelegator.ListHmacKeysc C s | � |�j|||||||| d�S )N)�method�duration�pathrO r$ �region�signed_headers�string_to_sign_debug)r. �SignUrl) r! r r� r� r� rO r$ r� r� r� r( r( r) r� ' s �zCloudApiDelegator.SignUrlc C s | � |��||||�S r7 )r. � UpdateHmacKey)r! rA r� �state�etagr r( r( r) r� 3 s �z CloudApiDelegator.UpdateHmacKeyc C s | � |�j||d�S )a� XML compatibility function for getting ACLs. Args: storage_url: StorageUrl object. def_obj_acl: If true, get the default object ACL on a bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: ACL XML for the resource specified by storage_url. )� def_obj_acl)r. �XmlPassThroughGetAcl)r! � storage_urlr� r r( r( r) r� 7 s �z&CloudApiDelegator.XmlPassThroughGetAclTc C s | � |�j||||d� dS )a9 XML compatibility function for setting ACLs. Args: acl_text: XML ACL or canned ACL string. storage_url: StorageUrl object. canned: If true, acl_text is treated as a canned ACL string. def_obj_acl: If true, set the default object ACL on a bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. )�cannedr� N)r. �XmlPassThroughSetAcl)r! Zacl_textr� r� r� r r( r( r) r� J s �z&CloudApiDelegator.XmlPassThroughSetAclc C s | � |��|�S )a� XML compatibility function for getting CORS configuration on a bucket. Args: storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: CORS configuration XML for the bucket specified by storage_url. )r. �XmlPassThroughGetCors�r! r� r r( r( r) r� f s z'CloudApiDelegator.XmlPassThroughGetCorsc C s | � |��||� dS )a� XML compatibility function for setting CORS configuration on a bucket. Args: cors_text: Raw CORS XML string. storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. N)r. �XmlPassThroughSetCors)r! Z cors_textr� r r( r( r) r� w s z'CloudApiDelegator.XmlPassThroughSetCorsc C s | � |��|�S )a� XML compatibility function for getting lifecycle config on a bucket. Args: storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Lifecycle configuration XML for the bucket specified by storage_url. )r. �XmlPassThroughGetLifecycler� r( r( r) r� � s z,CloudApiDelegator.XmlPassThroughGetLifecyclec C s | � |��||� dS )a� XML compatibility function for setting lifecycle config on a bucket. Args: lifecycle_text: Raw lifecycle configuration XML string. storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. N)r. �XmlPassThroughSetLifecycle)r! Zlifecycle_textr� r r( r( r) r� � s �z,CloudApiDelegator.XmlPassThroughSetLifecyclec C s | � |��|�S )a� XML compatibility function for getting logging configuration on a bucket. Args: storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Logging configuration XML for the bucket specified by storage_url. )r. �XmlPassThroughGetLoggingr� r( r( r) r� � s z*CloudApiDelegator.XmlPassThroughGetLoggingc C s | � |��||�S )a` XML compatibility function for setting tagging configuration on a bucket. This passthrough provides support for setting a tagging configuration (equivalent to a label configuration) on a cloud bucket. Args: tags_text: Raw tagging configuration XML string. storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. )r. �XmlPassThroughSetTagging)r! Z tags_textr� r r( r( r) r� � s �z*CloudApiDelegator.XmlPassThroughSetTaggingc C s | � |��|�S )a� XML compatibility function for getting tagging configuration on a bucket. Args: storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Tagging configuration XML for the bucket specified by storage_url. )r. �XmlPassThroughGetTaggingr� r( r( r) r� � s z*CloudApiDelegator.XmlPassThroughGetTaggingc C s | � |��|�S )a� XML compatibility function for getting website configuration on a bucket. Args: storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Website configuration XML for the bucket specified by storage_url. )r. �XmlPassThroughGetWebsiter� r( r( r) r� � s z*CloudApiDelegator.XmlPassThroughGetWebsite)Nr NNNN)N)N)NN)NN)N)NNN)NNNNN)N)NNNN)NN)NNN)NN)NNNNN)NNN)NNNNN)NNNNNNNF)NNNNNNF) NNNNNNNNNF) NNNNNNNNN)NNNN)NNN)NNN)N)N)N)NNNN)N)N)N)N)N)N)N)FN)N)FN)TFN)N)N)N)N)N)N)N)N):�__name__� __module__� __qualname__�__doc__r r. r, r+ r8 r; r> r? rB rF rH rJ rK rL rP rT rU rV r �DownloadStrategy�ONE_SHOTr` re rh rj rm rp rr rt rw ry r{ r} r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� � __classcell__r( r( r& r) r ! s* �/ G � � � � � � � � ! � � � � � � � � � � � r )r� � __future__r r r r r r �gslibr �gslib.cloud_apir r �gslib.cs_api_mapr r �gslib.exceptionr � gslib.utilsr r r( r( r( r) �<module> s __pycache__/progress_callback.cpython-39.pyc 0000644 00000010545 15025013616 0015071 0 ustar 00 a +(WgQ � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z d Z d Z d Z d ZG d d� de�ZG dd� de�ZdS )z(Helper functions for progress callbacks.� )�absolute_import)�print_function)�division)�unicode_literalsN)�ProgressMessage)�parallelism_framework_utili i @� �A c @ s& e Zd ZdZeefdd�Zdd� ZdS )�ProgressCallbackWithTimeoutz�Makes progress callbacks at least once every _TIMEOUT_SECONDS. This prevents wrong throughput calculation while not being excessively overwhelming. c C s8 || _ || _|| _t�� | _|| _d| _d| _d| _dS )a� Initializes the callback with timeout. Args: total_size: Total bytes to process. If this is None, size is not known at the outset. callback_func: Func of (int: processed_so_far, int: total_bytes) used to make callbacks. start_bytes_per_callback: Lower bound of bytes per callback. timeout: Number maximum of seconds without a callback. r N) �_bytes_per_callback�_callback_func� _total_size�time� _last_time�_timeout� _bytes_processed_since_callback�_callbacks_made�_total_bytes_processed)�self� total_sizeZ callback_funcZstart_bytes_per_callback�timeout� r �&/opt/gsutil/gslib/progress_callback.py�__init__, s z$ProgressCallbackWithTimeout.__init__c C s� | j |7 _ t�� }| j | jksN| jdur>| j| j | jksN| j| | jkr�| j| j 7 _| jdurxt| j| j�}n| j}| �|| j� d| _ | j d7 _ || _dS )z@Tracks byte processing progress, making a callback if necessary.Nr r ) r r r r r r r �minr r )r Zbytes_processedZcur_timeZ bytes_sentr r r �ProgressE s* ����� z$ProgressCallbackWithTimeout.ProgressN)�__name__� __module__� __qualname__�__doc__�_START_BYTES_PER_CALLBACK�_TIMEOUT_SECONDSr r r r r r r % s � r c @ s" e Zd ZdZddd�Zdd� ZdS ) �FileProgressCallbackHandlerz�Tracks progress info for large operations like file copy or hash. Information is sent to the status_queue, which will print it in the UI Thread. r Nc C s4 || _ || _|| _|| _|| _|| _|| _d| _dS )aC Initializes the callback handler. Args: status_queue: Queue for posting status messages for UI display. start_byte: The beginning of the file component, if one is being used. override_total_size: The size of the file component, if one is being used. src_url: FileUrl/CloudUrl representing the source file. component_num: Indicates the component number, if any. dst_url: FileUrl/CloudUrl representing the destination file, or None for unary operations like hashing. operation_name: String representing the operation name FN)� _status_queue� _start_byte�_override_total_size�_component_num�_src_url�_dst_url�_operation_name�_last_byte_written)r � status_queue� start_byteZoverride_total_size�src_url� component_num�dst_url�operation_namer r r r c s z$FileProgressCallbackHandler.__init__c C sd | j r dS | jr| j}t�| jt||| j | jt�� | j | j | j d�� |r`|| j |kr`d| _ dS )a� Gathers information describing the operation progress. Actual message is printed to stderr by UIThread. Args: last_byte_processed: The last byte processed in the file. For file components, this number should be in the range [start_byte:start_byte + override_total_size]. total_size: Total size of the ongoing operation. N)r. r0 r/ T) r* r% r �PutToQueueWithTimeoutr# r r$ r' r r& r) r( )r Zlast_byte_processedr r r r �call� s" �� z FileProgressCallbackHandler.call)r NNNNN)r r r r r r2 r r r r r" \ s � r" )r � __future__r r r r r Zgslib.thread_messager � gslib.utilsr r Z_MAX_BYTES_PER_CALLBACKr! Z MAX_PROGRESS_INDICATOR_COLUMNS�objectr r" r r r r �<module> s 7 __pycache__/cloud_api.cpython-39.pyc 0000644 00000122414 15025013616 0013347 0 ustar 00 a +(Wgޣ � @ s\ d Z ddlmZ ddlmZ ddlmZ G dd� de�ZG dd� de�ZG d d � d e�Z G d d � d e�Z G d d� de �Z G dd� de�Z G dd� de �Z G dd� de �ZG dd� de �ZG dd� de �ZG dd� de �ZG dd � d e �ZG d d � d e �ZG d d � d e �ZG d!d"� d"e�ZG d#d$� d$e �ZG d%d&� d&e �ZG d'd(� d(e �ZG d)d*� d*e �Zd+S ),z8Gsutil API for interacting with cloud storage providers.� )�absolute_import)�division)�unicode_literalsc @ s� e Zd ZdZdQdd�Zdd� ZdRdd �ZdSd d �ZdTd d �Zdd� Z dUdd�Z dVdd�Z dWdd�Z dXdd�Z dYdd�ZG dd� de�ZG d d � d e�ZdZd d �Zd[d d!�Zd\d"d#�Zd]d$d%�Zd^d&d'�ZG d(d)� d)e�Zdddd*ejddddddf d+d,�Zd_d-d.�Zd`d/d0�Zdad1d2�Z dbd3d4�Z dcd5d6�Z ddd7d8�Z ded9d:�Z dfd;d<�Z!dgd=d>�Z"dhd?d@�Z#didAdB�Z$djdCdD�Z%dkdEdF�Z&dGdH� Z'dIdJ� Z(dKdL� Z)dldMdN�Z*dOdP� Z+dS )m�CloudApiay Abstract base class for interacting with cloud storage providers. Implementations of the gsutil Cloud API are not guaranteed to be thread-safe. Behavior when calling a gsutil Cloud API instance simultaneously across threads is undefined and doing so will likely cause errors. Therefore, a separate instance of the gsutil Cloud API should be instantiated per-thread. Nr c C s: || _ || _|| _|| _|| _|| _|| _|| _| | _dS )aF Performs necessary setup for interacting with the cloud storage provider. Args: bucket_storage_uri_class: boto storage_uri class, used by APIs that provide boto translation or mocking. logger: logging.logger for outputting log messages. status_queue: Queue for relaying status to UI. provider: Default provider prefix describing cloud storage provider to connect to. debug: Debug level for the API implementation (0..3). http_headers (dict|None): Arbitrary headers to be included in every request. trace_token: Google internal trace token to pass to the API implementation (string). perf_trace_token: Performance trace token to use when making API calls. user_project: Project to be billed for this request. N) �bucket_storage_uri_class�logger� status_queue�provider�debug� http_headers� trace_token�perf_trace_token� user_project) �selfr r r r r r r r r � r � /opt/gsutil/gslib/cloud_api.py�__init__ s zCloudApi.__init__c C s t d��dS )z%Returns the service account email id.z'GetServiceAccountId must be overridden.N��NotImplementedError�r r r r �GetServiceAccountIdC s z CloudApi.GetServiceAccountIdc C s t d��dS )a� Gets Bucket metadata. Args: bucket_name: Name of the bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields, for example, ['logging', 'defaultObjectAcl'] Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object. z GetBucket must be overloadedNr �r � bucket_namer �fieldsr r r � GetBucketG s zCloudApi.GetBucketc C s t d��dS )a� Returns an IAM policy for the specified Bucket. Args: bucket_name: Name of the bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only the IAM policy fields specified. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with the cloud storage providers. Returns: Policy object of the bucket. z%GetBucketIamPolicy must be overloadedNr r r r r �GetBucketIamPolicyZ s zCloudApi.GetBucketIamPolicyc C s t d��dS )a� Sets an IAM policy for the specified Bucket. Args: bucket_name: Name of the bucket. policy: A Policy object describing the IAM policy. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with the cloud storage providers. Returns: Policy object of the bucket. May differ from input Policy. z%SetBucketIamPolicy must be overloadedNr )r r �policyr r r r �SetBucketIamPolicyl s zCloudApi.SetBucketIamPolicyc C s t d��dS )aW Sign a url using service account's system managed private key. Args: method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. path: String path to the bucket or object for signing, in the form 'bucket' or 'bucket/object'. generation: If not None, specifies a version of an object for signing. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. signed_headers: Dict containing the header info like host content-type etc. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. CommandException for errors because of invalid account used for signing. Returns: The signed url. zSignUrl must be overloadedNr ) r �methodZduration�path� generationr �region�signed_headersZstring_to_sign_debugr r r �SignUrl~ s zCloudApi.SignUrlc C s t d��dS )a� Lists bucket metadata for the given project. Args: project_id: Project owning the buckets, default from config if None. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these metadata fields for the listing, for example: ['items/logging', 'items/defaultObjectAcl']. Note that the WildcardIterator class should be used to list buckets instead of calling this function directly. It amends the fields definition from get-like syntax such as ['logging', 'defaultObjectAcl'] so that the caller does not need to prepend 'items/' or specify fields necessary for listing (like nextPageToken). Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Iterator over Bucket objects. z ListBuckets must be overloadedNr )r � project_idr r r r r � ListBuckets� s zCloudApi.ListBucketsc C s t d��dS )a� Updates bucket metadata for the bucket with patch semantics. Args: bucket_name: Name of bucket to update. metadata: Bucket object defining metadata to be updated. canned_acl: Canned ACL to apply to the bucket. canned_def_acl: Canned default object ACL to apply to the bucket. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object describing new bucket metadata. z PatchBucket must be overloadedNr )r r �metadata� canned_aclZcanned_def_acl� preconditionsr r r r r � PatchBucket� s zCloudApi.PatchBucketc C s t d��dS )a� Locks the Retention Policy on the bucket. Args: bucket_name: Name of bucket to update. metageneration: Bucket metageneration to use as a precondition. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ServiceException for errors interacting with cloud storage providers. Returns: None z&LockRetentionPolicy must be overloadedNr )r r �metagenerationr r r r �LockRetentionPolicy� s z CloudApi.LockRetentionPolicyc C s t d��dS )a� Creates a new bucket with the specified metadata. Args: bucket_name: Name of the new bucket. project_id: Project owner of the new bucket, default from config if None. metadata: Bucket object defining new bucket metadata. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object describing new bucket metadata. z CreateBucket must be overloadedNr )r r r$ r& r r r r r � CreateBucket� s zCloudApi.CreateBucketc C s t d��dS )a� Deletes a bucket. Args: bucket_name: Name of the bucket to delete. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. z DeleteBucket must be overloadedNr )r r r( r r r r � DeleteBucket� s zCloudApi.DeleteBucketc @ s e Zd ZdZdZdZdS )z CloudApi.CsObjectOrPrefixTypez1Enum class for describing CsObjectOrPrefix types.�object�prefixN)�__name__� __module__� __qualname__�__doc__�OBJECT�PREFIXr r r r �CsObjectOrPrefixType s r6 c @ s e Zd ZdZdd� ZdS )zCloudApi.CsObjectOrPrefixz(Container class for ListObjects results.c C s || _ || _dS )z�Stores a ListObjects result. Args: data: Root object, either an apitools Object or a string Prefix. datatype: CsObjectOrPrefixType of data. N)�data�datatype)r r7 r8 r r r r s z"CloudApi.CsObjectOrPrefix.__init__N�r0 r1 r2 r3 r r r r r �CsObjectOrPrefix s r: c C s t d��dS )a� Lists objects (with metadata) and prefixes in a bucket. Args: bucket_name: Bucket containing the objects. prefix: Prefix for directory-like behavior. delimiter: Delimiter for directory-like behavior. all_versions: If true, list all object versions. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these metadata fields for the listing, for example: ['items/acl', 'items/updated', 'prefixes']. Note that the WildcardIterator class should be used to list objects instead of calling this function directly. It amends the fields definition from get-like syntax such as ['acl', 'updated'] so that the caller does not need to prepend 'items/' or specify any fields necessary for listing (such as prefixes or nextPageToken). Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Iterator over CsObjectOrPrefix wrapper class. z ListObjects must be overloadedNr )r r r/ � delimiter� all_versionsr r r r r � ListObjects s !zCloudApi.ListObjectsc C s t d��dS )a; Gets IAM policy for specified Object. Args: bucket_name: Bucket containing the object. object_name: Name of the object. generation: Generation of the object to retrieve. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only the IAM policy fields specified. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object IAM policy. z%GetObjectIamPolicy must be overloadedNr �r r � object_namer r r r r r �GetObjectIamPolicyB s zCloudApi.GetObjectIamPolicyc C s t d��dS )aS Sets IAM policy for specified Object. Args: bucket_name: Bucket containing the object. object_name: Name of the object. policy: IAM Policy object. generation: Generation of the object to which the IAM policy will apply. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Policy object of the object. May differ from input Policy. z%SetObjectIamPolicy must be overloadedNr )r r r? r r r r r r �SetObjectIamPolicy[ s zCloudApi.SetObjectIamPolicyc C s t d��dS )a Gets object metadata. If decryption is supported by the implementing class, this function will read decryption keys from configuration and appropriately retry requests to encrypted objects with the correct key. Args: bucket_name: Bucket containing the object. object_name: Object name. generation: Generation of the object to retrieve. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields, for example, ['acl', 'updated']. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object. z$GetObjectMetadata must be overloadedNr r> r r r �GetObjectMetadatat s zCloudApi.GetObjectMetadatac C s t d��dS )a� Updates object metadata with patch semantics. Args: bucket_name: Bucket containing the object. object_name: Object name for object. metadata: Object object defining metadata to be updated. canned_acl: Canned ACL to be set on the object. generation: Generation (or version) of the object to update. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Updated object metadata. z&PatchObjectMetadata must be overloadedNr ) r r r? r& r' r r( r r r r r �PatchObjectMetadata� s z CloudApi.PatchObjectMetadatac @ s e Zd ZdZdZdZdS )zCloudApi.DownloadStrategyz,Enum class for specifying download strategy.ZoneshotZ resumableN)r0 r1 r2 r3 �ONE_SHOTZ RESUMABLEr r r r �DownloadStrategy� s rE Fc C s t d��dS )a Gets object data. Args: bucket_name: Bucket containing the object. object_name: Object name. download_stream: Stream to send the object data to. provider: Cloud storage provider to connect to. If not present, class-wide default is used. generation: Generation of the object to retrieve. object_size: Total size of the object being downloaded. compressed_encoding: If true, object is stored with a compressed encoding. download_strategy: Cloud API download strategy to use for download. start_byte: Starting point for download (for resumable downloads and range requests). Can be set to negative to request a range of bytes (python equivalent of [:-3]) end_byte: Ending byte number, inclusive, for download (for range requests). If None, download the rest of the object. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). serialization_data: Implementation-specific JSON string of a dict containing serialization information for the download. digesters: Dict of {string : digester}, where string is a name of a hash algorithm, and digester is a validation digester that supports update(bytes) and digest() using that algorithm. Implementation can set the digester value to None to indicate bytes were not successfully digested on-the-fly. decryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for decrypting an encrypted object. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Content-encoding string if it was detected that the server sent an encoded object during transfer, None otherwise. z!GetObjectMedia must be overloadedNr )r r r? Zdownload_streamr r Z object_sizeZcompressed_encodingZdownload_strategyZ start_byteZend_byte�progress_callback�serialization_data� digesters�decryption_tupler r r �GetObjectMedia� s 5zCloudApi.GetObjectMediac C s t d��dS )a� Uploads object data and metadata. Args: upload_stream: Seekable stream of object data. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. size: Optional object size. preconditions: Preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. gzip_encoded: Whether to use gzip transport encoding for the upload. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. z UploadObject must be overloadedNr ) r � upload_stream�object_metadatar' �sizer( rF �encryption_tupler r � gzip_encodedr r r � UploadObject� s &zCloudApi.UploadObjectc C s t d��dS )a� Uploads object data and metadata. Args: upload_stream: Stream of object data. May not be seekable. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. preconditions: Preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size), but fills in only bytes_transferred. encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. gzip_encoded: Whether to use gzip transport encoding for the upload. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. z(UploadObjectStreaming must be overloadedNr ) r rK rL r' r( rF rN r r rO r r r �UploadObjectStreaming s %z CloudApi.UploadObjectStreamingc C s t d��dS )a� Uploads object data and metadata using a resumable upload strategy. Args: upload_stream: Seekable stream of object data. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. size: Total size of the object. preconditions: Preconditions for the request. serialization_data: Dict of {'url' : UploadURL} allowing for uploads to be resumed. tracker_callback: Callback function taking a upload URL string. Guaranteed to be called when the implementation gets an upload URL, allowing the caller to resume the upload across process breaks by saving the upload URL in a tracker file. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields when the upload is complete. gzip_encoded: Whether to use gzip transport encoding for the upload. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. z(UploadObjectResumable must be overloadedNr ) r rK rL r' rM r( rG Ztracker_callbackrF rN r r rO r r r �UploadObjectResumable> s 0z CloudApi.UploadObjectResumablec C s t d��dS )a� Copies an object in the cloud. Args: src_obj_metadata: Object metadata for source object. Must include bucket name, object name, and etag. dst_obj_metadata: Object metadata for new object. Must include bucket and object name. src_generation: Generation of the source object to copy. canned_acl: Optional canned ACL to apply to destination object. Overrides ACL set in dst_obj_metadata. preconditions: Destination object preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). max_bytes_per_call: Integer describing maximum number of bytes to rewrite per service call. encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the destination object. decryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for decrypting the source object. If supplied without encryption_tuple, destination object will be written without customer-supplied encryption. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. z CopyObject must be overloadedNr ) r Zsrc_obj_metadata�dst_obj_metadata�src_generationr' r( rF Zmax_bytes_per_callrN rI r r r r r � CopyObjectr s -zCloudApi.CopyObjectc C s t d��dS )a� Composes an object in the cloud. Args: src_objs_metadata: List of ComposeRequest.SourceObjectsValueListEntries specifying the objects to compose. dst_obj_metadata: Metadata for the destination object including bucket and object name. preconditions: Destination object preconditions for the request. encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for decrypting source objects and encrypting the destination object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Composed object metadata. z ComposeObject must be overloadedNr )r Zsrc_objs_metadatarS r( rN r r r r r � ComposeObject� s zCloudApi.ComposeObjectc C s t d��dS )aR Deletes an object. Args: bucket_name: Name of the containing bucket. object_name: Name of the object to delete. preconditions: Preconditions for the request. generation: Generation (or version) of the object to delete; if None, deletes the live object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. z DeleteObject must be overloadedNr )r r r? r( r r r r r � DeleteObject� s zCloudApi.DeleteObjectc C s t d��dS )a� Creates a notification subscription for changes to objects in a bucket. Args: bucket_name: Bucket containing the objects. address: Address to which to send notifications. channel_id: Unique ID string for the channel. token: If present, token string is delivered with each notification. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Channel metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Channel object describing the notification subscription. z WatchBucket must be overloadedNr )r r �address� channel_id�tokenr r r r r � WatchBucket� s zCloudApi.WatchBucketc C s t d��dS )a� Stops a notification channel. Args: channel_id: Unique ID string for the channel. resource_id: Version-agnostic ID string for the channel. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. z StopChannel must be overloadedNr )r rY Z resource_idr r r r � StopChannel� s zCloudApi.StopChannelc C s t d��dS )a� Lists object change notifications for a bucket. Args: bucket_name: Bucket containing the objects provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. z ListChannels must be overloadedNr �r r r r r r � ListChannels s zCloudApi.ListChannelsc C s t d��dS )a� Get the GCS-owned service account representing this project. Args: project_number: the project in question. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ServiceException for errors interacting with cloud storage providers. Returns: Service account object (with email_address string field) z+GetProjectServiceAccount must be overloadedNr )r Zproject_numberr r r r �GetProjectServiceAccount s z!CloudApi.GetProjectServiceAccountc C s t d��dS )a$ Creates a new notification with the specified parameters. Args: bucket_name: (Required) Name of the bucket. pubsub_topic: (Required) Cloud Pub/Sub topic to which to publish. payload_format: (Required) payload format, must be 'JSON' or 'NONE'. event_types: (Opt) List of event type filters, e.g. 'OBJECT_FINALIZE'. custom_attributes: (Opt) Dictionary of custom attributes. object_name_prefix: (Opt) Filter on object name. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Notification object describing new notificationConfig z+CreateNotificationConfig must be overloadedNr )r r Z pubsub_topicZpayload_formatZ event_typesZcustom_attributesZobject_name_prefixr r r r �CreateNotificationConfig( s z!CloudApi.CreateNotificationConfigc C s t d��dS )a� Deletes a notification. Args: bucket_name: (Required) Name of the bucket. notification: (Required) Integer name of the notification. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None z+DeleteNotificationConfig must be overloadedNr )r r Z notificationr r r r �DeleteNotificationConfigE s z!CloudApi.DeleteNotificationConfigc C s t d��dS )a� Lists notification configs in a bucket. Args: bucket_name: Name of the bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Yields: List of notification objects. z)ListNotificationConfig must be overloadedNr r] r r r �ListNotificationConfigsW s z CloudApi.ListNotificationConfigsc C s t d��dS )a� Creates a new HMAC key for the specified service account. Args: project_id: Project ID owning the service account for which the key is to be created. service_account_email: Email address of the service account for which to create a key. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Returns The key metadata and the secret key material. z CreateHmacKey must be overloadedNr )r r$ �service_account_emailr r r � CreateHmacKeyh s zCloudApi.CreateHmacKeyc C s t d��dS )z�Deletes an HMAC key. Args: project_id: Project ID owning the requested key. access_id: Name of the HMAC key to be deleted. Raises: NotImplementedError: not implemented TODO(tuckerkirven) z DeleteHmacKey must be overloadedNr �r r$ � access_idr r r � DeleteHmacKeyy s zCloudApi.DeleteHmacKeyc C s t d��dS )aQ Retrieves an HMAC key's metadata. Args: project_id: Project ID owning the service account of the requested key. access_id: Name of the HMAC key for which the metadata is being requested. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Returns: Metadata for the specified key. z GetHmacKey must be overloadedNr re r r r � GetHmacKey� s zCloudApi.GetHmacKeyc C s t d��dS )a� Lists HMAC keys matching the criteria. Args: project_id: Name of the project from which to list HMAC keys. service_account_email: If present, only keys for the given service account will be returned. show_deleted_keys: If set, show keys in the DELETED state. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Yields: List of HMAC key metadata objects. z ListHmacKeys must be overloadedNr )r r$ rc Zshow_deleted_keysr r r � ListHmacKeys� s zCloudApi.ListHmacKeysc C s t d��dS )a� Updates the state of an HMAC key. Args: project_id: Project ID owning the service account of the updated key. access_id: Name of the HMAC key being updated. state: The state to which the key should be updated. etag: None or a string matching the key's etag to ensure the appropriate version of the key is updated. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Returns: The updated key metadata. z UpdateHmacKey must be overloadedNr )r r$ rf �state�etagr r r � UpdateHmacKey� s zCloudApi.UpdateHmacKey)Nr NNNN)NN)NN)N)NNN)NNNNN)N)NNNN)NN)NNNNN)NNN)NN)NNN)NNNNN)NNNNNNNF)NNNNNNF) NNNNNNNNNF) NNNNNNNNN)NNNN)NNN)NNN)N)N)N)NNNN)N)N)F),r0 r1 r2 r3 r r r r r r# r% r) r+ r, r- r. r6 r: r= r@ rA rB rC rE rD rJ rP rQ rR rU rV rW r[ r\ r^ r_ r` ra rb rd rg rh ri rl r r r r r s � $ � � � & � � � " � � : � + � * � 7 � 2 � " � � � � r c @ s e Zd ZdZddd�ZdS )� PreconditionszGPreconditions class for specifying preconditions to cloud API requests.Nc C s || _ || _dS )aH Instantiates a Preconditions object. Args: gen_match: Perform request only if generation of target object matches the given integer. Ignored for bucket requests. meta_gen_match: Perform request only if metageneration of target object/bucket matches the given integer. N)� gen_match�meta_gen_match)r rn ro r r r r � s zPreconditions.__init__)NNr9 r r r r rm � s rm c @ s e Zd ZdZdS )�EncryptionExceptionz@Exception raised when an encrypted resource cannot be decrypted.N�r0 r1 r2 r3 r r r r rp � s rp c @ s( e Zd ZdZdd� Zdd� Zdd� ZdS ) �ArgumentExceptionz�Exception raised when arguments to a Cloud API method are invalid. This exception is never raised as a result of a failed call to a cloud storage provider. c C s t �| � || _d S �N)� Exceptionr �reason)r ru r r r r � s zArgumentException.__init__c C s t | �S rs ��strr r r r �__repr__� s zArgumentException.__repr__c C s d| j j| jf S )Nz%s: %s)� __class__r0 ru r r r r �__str__� s zArgumentException.__str__N�r0 r1 r2 r3 r rx rz r r r r rr � s rr c @ s e Zd ZdZdS )�ProjectIdExceptionzHException raised when a Project ID argument is required but not present.Nrq r r r r r| � s r| c @ s* e Zd ZdZd dd�Zdd� Zdd� ZdS ) �ServiceExceptionz�Exception raised when a cloud storage provider request fails. This exception is raised only as a result of a failed remote call. Nc C s t �| � || _|| _|| _d S rs )rt r ru �status�body)r ru r~ r r r r r � s zServiceException.__init__c C s t | �S rs rv r r r r rx � s zServiceException.__repr__c C sF d| j j }| jr |d| j 7 }|d| j 7 }| jrB|d| j 7 }|S )Nz%s:z %sz %s)ry r0 r~ ru r )r �messager r r rz � s zServiceException.__str__)NNr{ r r r r r} � s r} c @ s e Zd ZdZdS )�RetryableServiceExceptionz)Exception class for retryable exceptions.Nrq r r r r r� � s r� c @ s e Zd ZdZdS )�ResumableDownloadExceptionz>Exception raised for res. downloads that can be retried later.Nrq r r r r r� s r� c @ s e Zd ZdZdS )�ResumableUploadExceptionzHException raised for res. uploads that can be retried w/ same upload ID.Nrq r r r r r� s r� c @ s e Zd ZdZdS )�!ResumableUploadStartOverExceptionzGException raised for res. uploads that can be retried w/ new upload ID.Nrq r r r r r� s r� c @ s e Zd ZdZdS )� ResumableUploadAbortExceptionzDException raised for resumable uploads that cannot be retried later.Nrq r r r r r� s r� c @ s e Zd ZdZdS )�AuthenticationExceptionz>Exception raised for errors during the authentication process.Nrq r r r r r� s r� c @ s e Zd ZdZdS )�PreconditionExceptionz+Exception raised for precondition failures.Nrq r r r r r� s r� c @ s e Zd ZdZdS )�NotFoundExceptionz4Exception raised when a resource is not found (404).Nrq r r r r r� s r� c s" e Zd ZdZd� fdd� Z� ZS )�BucketNotFoundExceptionz;Exception raised when a bucket resource is not found (404).Nc s t t| �j|||d� || _d S )N)r~ r )�superr� r r )r ru r r~ r �ry r r r s �z BucketNotFoundException.__init__)NN)r0 r1 r2 r3 r � __classcell__r r r� r r� s r� c @ s e Zd ZdZdS )�NotEmptyExceptionz=Exception raised when trying to delete a bucket is not empty.Nrq r r r r r� ' s r� c @ s e Zd ZdZdS )�BadRequestExceptionz�Exception raised for malformed requests. Where it is possible to detect invalid arguments prior to sending them to the server, an ArgumentException should be raised instead. Nrq r r r r r� + s r� c @ s e Zd ZdZdS )�AccessDeniedExceptionz�Exception raised when authenticated user has insufficient access rights. This is raised when the authentication process succeeded but the authenticated user does not have access rights to the requested resource. Nrq r r r r r� 3 s r� c @ s e Zd ZdZdS )� PublishPermissionDeniedExceptiona Exception raised when bucket does not have publish permission to a topic. This is raised when a custom attempts to set up a notification config to a Cloud Pub/Sub topic, but their GCS bucket does not have permission to publish to the specified topic. Nrq r r r r r� ; s r� N)r3 � __future__r r r r. r rm rt rp rr r| r} r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r �<module> s: + __pycache__/sig_handling.cpython-39.pyc 0000644 00000011172 15025013616 0014034 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZ dd lmZ dd lmZ er�ddlZi Zi Zdd d �Zdd� Zdd� Zdd� Zdd� Zdd� Z dd� Z dS )zSignal handling functions.� )�absolute_import)�print_function)�division)�unicode_literalsN)�metrics)�ControlCException)�UTF8)� IS_WINDOWSFc C s: | t � vrtd|| f ��|r(|t| < nt| �|� dS )a� Registers a handler for signal signal_num. Unlike calling signal.signal(): - This function can be called from any thread (and will cause the handler to be run by the main thread when the signal is received). - Handlers are cumulative: When a given signal is received, all registered handlers will be executed (with the exception that only the last handler to register with is_final_handler=True will be called). Handlers should make no ordering assumptions, other than that the last handler to register with is_final_handler=True will be called after all the other handlers. Args: signal_num: The signal number with which to associate handler. handler: The handler. is_final_handler: Bool indicator whether handler should be called last among all the handlers for this signal_num. The last handler to register this way survives; other handlers registered with is_final_handler=True will not be called when the signal is received. Raises: RuntimeError: if attempt is made to register a signal_num not in GetCaughtSignals. zPAttempt to register handler (%s) for signal %d, which is not in GetCaughtSignalsN)�GetCaughtSignals� RuntimeError�_final_signal_handlers�_non_final_signal_handlers�append)� signal_num�handler�is_final_handler� r �!/opt/gsutil/gslib/sig_handling.py�RegisterSignalHandler, s � r c C s: | t v r t | D ]}|| |� q| tv r6t| | |� dS )z�Global signal handler. When a signal is caught we execute each registered handler for that signal. Args: signal_num: Signal that was caught. cur_stack_frame: Unused. N)r r )r �cur_stack_framer r r r �_SignalHandlerO s r c C s$ t � D ]} g t| < t�| t� qdS )zbInitializes global signal handling. Sets up global signal handler for each signal we handle. N)r r �signalr )r r r r �InitializeSignalHandling_ s r c C s t jt jg} ts | �t j� | S )zCReturns terminating signals that can be caught on this OS platform.)r �SIGINT�SIGTERMr r �SIGQUIT)Zsignalsr r r r j s r c C sn z\t rLdtjdd� kr$dk rLn n$tjj}|�dd| �}|�|d� nt� | t j � W n Y n0 dS )z�Make best effort to kill the given process. We ignore all exceptions so a caller looping through a list of processes will continue attempting to kill each, even if one encounters a problem. Args: pid: The process ID. )� r Nr )r � � r ) r �sys� version_info�ctypes�windll�kernel32Z OpenProcessZTerminateProcess�os�killr �SIGKILL)�pidr# �handler r r � KillProcesss s &r) c C s� | t jkr�t�� �tj�r�d�t�t� � ��}d| t � dd|�f }zt j �|�t�� W q� ttfy� } zt j �|� W Y d}~q�d}~0 0 nt j �d| � tjt� d� t�� tt�� � dS )z5Final signal handler for multi-threaded main process.� z@DEBUG: Caught CTRL-C (signal %d) - Exception stack trace: %sz\nz Nz$Caught CTRL-C (signal %d) - exiting )� exception)r r �logging� getLogger� isEnabledFor�DEBUG�join� traceback� format_list� extract_stack�re�subr �stderr�write�encoder �UnicodeDecodeError� TypeErrorr � LogFatalErrorr �Shutdownr) r$ �getpid)r r � stack_trace�err�er r r � MultithreadedMainSignalHandler� s �$rA c C s t t�� � dS )z)Final signal handler for child processes.N)r) r$ r= )r r r r r �ChildProcessSignalHandler� s rB )F) �__doc__� __future__r r r r r, r$ r4 r r r1 �gslibr �gslib.exceptionr �gslib.utils.constantsr �gslib.utils.system_utilr r! r r r r r r r) rA rB r r r r �<module> s2 # __pycache__/context_config.cpython-39.pyc 0000644 00000015755 15025013616 0014432 0 ustar 00 a +(Wgp! � @ s d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl Z ddl m Z dd l mZ dd l mZ daej�ej�d d d ��ZdZdZG dd� de�ZG dd� de�Zdd� Zdd� Zdd� Z dd� Z d d � Z G d d � d e �Z d d!� Z!d"d#� Z"dS )$z)Manages device context mTLS certificates.� )�absolute_import)�print_function)�division)�unicode_literalsN)�config)� exception)� boto_util)�execution_util�~z.secureConnectzcontext_aware_metadata.json�cert_provider_commandz--with_passphrasec @ s e Zd ZdZdS )�CertProvisionErrorz9Represents errors when provisioning a client certificate.N��__name__� __module__� __qualname__�__doc__� r r �#/opt/gsutil/gslib/context_config.pyr + s r c @ s e Zd ZdZdS )�(ContextConfigSingletonAlreadyExistsErrorz>Error for when create_context_config is called multiple times.Nr r r r r r 0 s r c C sT | � d�r&| �d�r&dd| dd� fS | � d�rL| �d�rLdd| dd� fS d S d S ) z)Returns (begin:bool, end:bool, name:str).z -----BEGIN z-----TF� ���z -----END � )FF� N)� startswith�endswith)�liner r r �_is_pem_section_marker5 s r c C s� i }g }d}| � � D ]�}|�� }|s&qt|�\}}}|rn|rJ|�d| � ||�� v rd|�d| � |}g }n6|r�|s�|�d| � n ||kr�|�d||f � d}|r|�|� |rd�|�d ||< d}q|r�|�d| � |S )a Returns dict with {name: section} by parsing contents in PEM format. A simple parser for PEM file. Please see RFC 7468 for the format of PEM file. Not using regex to improve performance catching nested matches. Note: This parser requires the post-encapsulation label of a section to match its pre-encapsulation label. It ignores a section without a matching label. Args: contents (str): Contents of a PEM file. logger (logging.logger): gsutil logger. Returns: A dict of the PEM file sections. Nz0Section %s missing end line and will be ignored.zASection %s already exists, and the older section will be ignored.z8Section %s missing a beginning line and will be ignored.z1Section %s missing a matching end line. Found: %s� z Section %s missing an end line.)� splitlines�stripr �warning�keys�append�join) �contents�logger�result� pem_linesZpem_section_namer �begin�end�namer r r �_split_pem_into_sections? sJ � ��� r+ c C s t j�t�} t j�| �s dS | S )z�Checks for content aware metadata. If content aware metadata exists, return its absolute path; otherwise, returns None. Returns: str: Absolute path if exists. Otherwise, None. N)�os�path� expanduser�_DEFAULT_METADATA_PATH�exists)� metadata_pathr r r � _check_pathw s r2 c C sj z8t | �� }t�|�W d � W S 1 s,0 Y W n, tyd } zt|��W Y d}~n d}~0 0 dS )z�Loads context aware metadata from the given path. Returns: dict: The metadata JSON. Raises: CertProvisionError: If failed to parse metadata as JSON. N)�open�json�load� ValueErrorr )r1 �f�er r r �_read_metadata_file� s .r9 c C sH t � } | std��t| �}t|vr*td��|t }t|vrD|�t� |S )z�Loads default cert provider command. Returns: str: The default command. Raises: CertProvisionError: If command cannot be found. z+Client certificate provider file not found.z.Client certificate provider command not found.)r2 r r9 �_CERT_PROVIDER_COMMAND�(_CERT_PROVIDER_COMMAND_PASSPHRASE_OPTIONr" )r1 Z metadata_json�commandr r r �_default_command� s r= c @ s( e Zd ZdZdd� Zdd� Zdd� ZdS ) �_ContextConfigz�Represents the configurations associated with context aware access. Only one instance of Config can be created for the program. c C s� || _ t�dd�| _d| _| js$dS t�| j� tj � t � � d�| _z| � | j� W n4 ty� } z | j �d| � W Y d}~n d}~0 0 dS )zQInitializes config. Args: logger (logging.logger): gsutil logger. � Credentials�use_client_certificateNz caa_cert.pemz*Failed to provision client certificate: %s)r% r �getboolr@ �client_cert_path�atexit�register�_unprovision_client_certr, r- r# r ZGetGsutilStateDir�_provision_client_certr �error)�selfr% r8 r r r �__init__� s � �z_ContextConfig.__init__c C s t �ddd�}|r |�d�}nt� }z�t�|�\}}t|| j�}t|d��\}|� |d � d|v r�|� |d � |d � � d | _ n|� |d � d| _ W d � n1 s�0 Y W nb t j tfy� } zt|��W Y d}~n:d}~0 t�y } ztd | ��W Y d}~n d}~0 0 dS ) zDExecutes certificate provider to obtain client certificate and keys.r? r N� zw+� CERTIFICATEzENCRYPTED PRIVATE KEYZ PASSPHRASE� z PRIVATE KEYz6Invalid output format from certificate provider, no %s)r �get�splitr= r ZExecuteExternalCommandr+ r% r3 �writer Zclient_cert_passwordr �ExternalBinaryError�OSErrorr �KeyError) rH Z cert_pathZcert_command_stringZ cert_commandZcommand_stdout_string�_�sectionsr7 r8 r r r rF � s0 � � ( �z%_ContextConfig._provision_client_certc C sf | j durbz"t�| j � | j�d| j � W n4 ty` } z | j�d| � W Y d}~n d}~0 0 dS )z@Cleans up any files or resources provisioned during config init.Nz Unprovisioned client cert: %sz'Failed to remove client certificate: %s)rB r, �remover% �debugrQ rG )rH r8 r r r rE � s � z'_ContextConfig._unprovision_client_certN)r r r r rI rF rE r r r r r> � s r> c C s t st| �a t S t�dS )z�Should be run once at gsutil startup. Creates global singleton. Args: logger (logging.logger): For logging during config functions. Returns: New ContextConfig singleton. Raises: Exception if singleton already exists. N)�_singleton_configr> r )r% r r r �create_context_config� s rX c C s t S )zuRetrieves ContextConfig global singleton. Returns: ContextConfig or None if global singleton doesn't exist. )rW r r r r �get_context_config s rY )#r � __future__r r r r rC r4 r, �botor �gslibr � gslib.utilsr r rW r- r. r# r/ r: r; � Exceptionr r r r+ r2 r9 r= �objectr> rX rY r r r r �<module> s8 � 8H __pycache__/command_runner.cpython-39.pyc 0000644 00000037402 15025013616 0014421 0 ustar 00 a +(Wg�\ � @ s8 d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZ ddlZddlmZ dd lmZ ddlZdd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlm Z ddlm Z ddl Zddl m Z ddl m!Z! ddl m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl-m/Z/ ddl-m0Z0 dd l1m2Z2 dd l1m3Z3 dd l4m5Z5 dd l6m7Z7 dd l8m9Z9 dd!l8m:Z: dd"l8m;Z; dd#l8m<Z< dd$l=m>Z> dd%l?m@Z@ dd&l?mAZA d'd(� ZBd)d*� ZCd+d,� ZDG d-d.� d.eE�ZFdS )/z'Class that runs a named gsutil command.� )�absolute_import)�print_function)�division)�unicode_literalsN)�input)�config)�BucketStorageUri)�metrics)�CloudApiDelegator)�Command��CreateOrGetGsutilLogger)�GetFailureCount)� OLD_ALIAS_MAP)�ShutDownGsutil)� ApiSelector)�GsutilApiClassMapFactory)�GsutilApiMapFactory)�DiscardMessagesQueue)�CommandException)� GcsJsonApi)�NoOpCredentials)� MakeCompleter)� boto_util)� shim_util)� system_util)�RELEASE_NOTES_URL)�UTF8)�IsCustomMetadataHeader)�$CheckMultiprocessingAvailableAndInit)�CompareVersions)�InsistAsciiHeader)�InsistAsciiHeaderValue)� print_to_fd)�SECONDS_PER_DAY)�LookUpGsutilVersion)�GsutilPubTarballc C s� | sdS | D ]�}t |� t|�r�t| | tj�s�z| | �t�| |< W q� ty� td� t � dt |�t | | �f ����Y q�0 q t || | � q dS )a� Handles coding of headers and their values. Alters the dict in-place. Converts a dict of headers and their values to their appropriate types. We ensure that all headers and their values will contain only ASCII characters, with the exception of custom metadata header values; these values may contain Unicode characters, and thus if they are not already unicode-type objects, we attempt to decode them to Unicode using UTF-8 encoding. Args: headers: A dict mapping headers to their values. All keys and values must be either str or unicode objects. Raises: CommandException: If a header or its value cannot be encoded in the required encoding. N� a' Invalid encoding for header value (%s: %s). Values must be decodable as Unicode. NOTE: the value printed above replaces the problematic characters with a hex-encoded printable representation. For more details (including how to convert to a gsutil-compatible encoding) see `gsutil help encoding`.)r! r � isinstance�six� text_type�decoder �UnicodeDecodeErrorr �join�textwrap�wrap�reprr" )�headers�key� r3 �#/opt/gsutil/gslib/command_runner.py�HandleHeaderCodingD s" ��� r5 c C sl t t| ��D ]Z}| | }t|tj�s z|�t�| |< W q tyd td� t � dt |� ����Y q 0 q dS )z�Handles coding of command-line args. Alters the list in-place. Args: args: A list of command-line args. Raises: CommandException: if errors encountered. r' a% Invalid encoding for argument (%s). Arguments must be decodable as Unicode. NOTE: the argument printed above replaces the problematic characters with a hex-encoded printable representation. For more details (including how to convert to a gsutil-compatible encoding) see `gsutil help encoding`.N) �range�lenr( r) r* r+ r r, r r- r. r/ r0 )�args�i�argr3 r3 r4 �HandleArgCodingl s ���r; c C s t jr| �t�S | S )zDConverts a string literal (unicode) to the same type as sys.argv[0].)r) �PY2�encoder )Z unicode_strr3 r3 r4 �_StringToSysArgType� s r> c @ sj e Zd ZdZeedfdd�Zdd� Zdd� Zd d � Z d d � Z d d� Z ddd�Z dd� Z dd� Zdd� ZdS )� CommandRunnerz?Runs gsutil commands and does some top-level argument handling.Nc C s& || _ || _|r|| _n | �� | _dS )a� Instantiates a CommandRunner. Args: bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. gsutil_api_class_map_factory: Creates map of cloud storage interfaces. Settable for testing/mocking. command_map: Map of command names to their implementations for testing/mocking. If not set, the map is built dynamically. N)�bucket_storage_uri_class� gsutil_api_class_map_factory� command_map�_LoadCommandMap)�selfr@ rA rB r3 r3 r4 �__init__� s zCommandRunner.__init__c C s^ t �tjj�D ]\}}}td| � qi }t�� D ]&}|||jj < |jj D ] }|||< qJq2|S )z=Returns dict mapping each command_name to implementing class.zgslib.commands.%s) �pkgutil� iter_modules�gslib�commands�__path__� __import__r �__subclasses__� command_spec� command_name�command_name_aliases)rD �_� module_namerB �commandrO r3 r3 r4 rC � s z CommandRunner._LoadCommandMapc C s t d�S )z$Returns a logger for tab completion.Z tab_completer )rD r3 r3 r4 �_GetTabCompleteLogger� s z#CommandRunner._GetTabCompleteLoggerc sl �� � �� ��fdd�}� ����fdd�}t�t�r>|� n*t�t�rP|� nd}t|��t�����dS )a� Creates parsers recursively for potentially nested subcommands. Args: parser: argparse parser object. subcommands_or_arguments: list of CommandArgument objects, or recursive dict with subcommand names as keys. gsutil_api: gsutil Cloud API instance to use. Raises: RuntimeError: if argument is configured with unsupported completer TypeError: if subcommands_or_arguments is not a dict or list c s6 �D ],} �j | ji | j��}| jrt| j� �|_qd S )N)� add_argumentr8 �kwargsZ completerr )Zcommand_argument�action)� gsutil_api�parser�subcommands_or_argumentsr3 r4 � HandleList� s ��zJCommandRunner._ConfigureCommandArgumentParserArguments.<locals>.HandleListc sJ �� � } ��� D ]4\}}| j|dd�}��d�|�� ��||� � qd S )NF)�add_helpz$Constructing argument parsers for {})�add_subparsers�items� add_parser�info�format�(_ConfigureCommandArgumentParserArguments)Z subparsersZsubcommand_nameZsubcommand_valueZcur_subcommand_parser�rW �loggerrX rD rY r3 r4 � HandleDict� s ���zJCommandRunner._ConfigureCommandArgumentParserArguments.<locals>.HandleDictzAsubcommands_or_arguments {} should be list or dict, found type {}N)rS r( �list�dict� TypeErrorr` �type)rD rX rY rW rZ rd Z error_formatr3 rb r4 ra � s ��z6CommandRunner._ConfigureCommandArgumentParserArgumentsc C sR t jt jgt jgd�}t jt jd�}t�| j||�}t| j|| �� t � dd�}|S )zxBuilds and returns a gsutil_api based off gsutil_api_class_map_factory. Returns: the gsutil_api instance )�gs�s3r )�debug) r ZXMLZJSONr Z GetApiMaprA r r@ rS r )rD Z support_mapZ default_mapZgsutil_api_maprW r3 r3 r4 �GetGsutilApiForTabComplete� s ���z(CommandRunner.GetGsutilApiForTabCompletec C s. | � � }dd� | j�� D �}| �|||� dS )z�Configures argparse arguments and argcomplete completers for commands. Args: main_parser: argparse object that can be called to get subparsers to add subcommands (called just 'commands' in gsutil) c S s i | ]}|j j|j j�qS r3 )rM rN Zargparse_arguments)�.0rR r3 r3 r4 � <dictcomp> s �zACommandRunner.ConfigureCommandArgumentParsers.<locals>.<dictcomp>N)rl rB �valuesra )rD Z main_parserrW Z command_to_argparse_argumentsr3 r3 r4 � ConfigureCommandArgumentParsers s ��z-CommandRunner.ConfigureCommandArgumentParsersr FTc C s d} |s:| � ||�r:d}d} td�g}t�� r:| r:t�� | �|� |sLg }tj� ddd�}|sdi }||d< || j vr�t j || j � � d d �}|r�t�|d |�d }td tjd � td| tjd � n|dkr�tjr�tj�d� td| ��td�|v �rD|g}| j | }|jj� � }|D ] }||v �r|�|� �q<�q|}d}t|� t|� | j | }|| |||||| j | j ||| | d� }| �r�tj |j |j |d� |�!� �r�|�"� }n|�#� }t$� j%�r�| �r�t&� t'� d k�r�d }| �r�d }td�(t)�*d��� |S )a� Runs the named command. Used by gsutil main, commands built atop other commands, and tests. Args: command_name: The name of the command being run. args: Command-line args (arg0 = actual arg, not command name ala bash). headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). trace_token: Trace token to pass to the underlying API. parallel_operations: Should command operations be executed in parallel? skip_update_check: Set to True to disable checking for gsutil updates. logging_filters: Optional list of logging.Filters to apply to this command's logger. do_shutdown: Stop all parallelism framework workers iff this is True. perf_trace_token: Performance measurement trace token to pass to the underlying API. user_project: The project to bill this request to. collect_analytics: Set to True to collect an analytics metric logging this command. Raises: CommandException: if errors encountered. Returns: Return value(s) from Command that was run. F�updateTz-n�GSUtil�default_api_version�1zx-goog-api-version� )�nr zDid you mean this?)�filez %szgUpdate command is not supported for package installs; please instead update using your package manager.zInvalid command "%s".z--help�help)Zcommand_alias_used�perf_trace_token� user_project)rN �sub_opts� command_aliasr' z�Update was successful. Exiting with code 1 as the original command issued prior to the update was not executed and should be re-run.)+�#MaybeCheckForAndOfferSoftwareUpdater> r �IsRunningInteractivelyr �'CheckAndMaybePromptForAnalyticsEnabling�MaybePromptForPythonUpdate�botor � get_valuerB �difflibZget_close_matches�keysr �get�print�sys�stderrrH �IS_PACKAGE_INSTALL�writer Z help_specZsubcommand_help_text�appendr; r5 r@ rA �LogCommandParamsrN r{ Z(translate_to_gcloud_storage_if_requestedZrun_gcloud_storageZ RunCommandr � is_availabler r r- r. r/ )rD rN r8 r1 rk � trace_token�parallel_operationsZskip_update_checkZlogging_filtersZ do_shutdownry rz �collect_analyticsZcommand_changed_to_update� api_versionZ close_matchesZtranslated_command_name�new_argsZoriginal_command_class� subcommandsr: Z command_classZ command_inst� return_coder3 r3 r4 �RunNamedCommand' s� ( � � ��� �� ��z CommandRunner.RunNamedCommandc C s, t �� }t�� r$|�t j�r$t�� r(dS dS )a� Helper function that will determine if update checks should be skipped. Args: command_name: The name of the command being run. Returns: True if: - gsutil is not connected to a tty (e.g., if being run from cron); - user is running gsutil -q - user specified gs_host (which could be a non-production different service instance, in which case credentials won't work for checking gsutil tarball).TF)�logging� getLoggerr r~ � isEnabledFor�INFOr �HasUserSpecifiedGsHost)rD rc r3 r3 r4 �SkipUpdateCheck� s ��z CommandRunner.SkipUpdateCheckc C s< t jjdks,| �� s,|dvs,tj�ddd�r0dS td� dS )z�Alert the user that they should install Python 3. Args: command_name: The name of the command being run. Returns: True if a prompt was output. � )rq �ver�versionrr Zskip_python_update_promptFzxGsutil 5 drops Python 2 support. Please install Python 3 to update to the latest version of gsutil. https://goo.gle/py3 T)r� � version_info�majorr� r� r �getboolr# )rD rN r3 r3 r4 r� � s ���z(CommandRunner.MaybePromptForPythonUpdatec C sN t �� }| �� s |dv s t�� r$dS tj�ddd�}|dkr@dS t� � }t t � � �}t j �|�s�t�� }t|d�� }|�t|�� W d � q�1 s�0 Y nTz:t|d �� }t |�� �}W d � n1 s�0 Y W n ttfy� Y dS 0 || |t k�rJt| j|t� t� |d �} tj } zt | t � �} W n t �yL Y dS 0 t|d�� }|�t|�� W d � n1 �s~0 Y t | tj �\} } | �r�t!d �"t#�$d | tj t%f ��� tj&�r�dS t!d � t'd �} | �o�| �(� d dkS | �rJt!d �"t#�$d| tj t%f ��� tj&�r"dS t!d � t'd�} | �pH| �(� d dkS dS )a� Checks the last time we checked for an update and offers one if needed. Offer is made if the time since the last update check is longer than the configured threshold offers the user to update gsutil. Args: command_name: The name of the command being run. debug: Debug level to pass in to boto connection (range 0..3). Returns: True if the user decides to update. )r rq r� r� Frr � software_update_check_period� r �wN�r)� credentialsrk r' a A newer version of gsutil (%s) is available than the version you are running (%s). NOTE: This is a major new version, so it is strongly recommended that you review the release note details at %s before updating to this version, especially if you use gsutil in scripts.z Would you like to update [y/N]? �yz�A newer version of gsutil (%s) is available than the version you are running (%s). A detailed log of gsutil release changes is available at %s if you would like to read them before updating.z Would you like to update [Y/n]? rv ))r� r� r� r �InvokedViaCloudSdkr� r �getintr �*GetLastCheckedForGsutilUpdateTimestampFile�int�time�os�path�isfilerH � GetGsutilVersionModifiedTime�openr� �str�readlinerg � ValueErrorr$ r r@ r r �VERSIONr% r& � Exceptionr r# r- r. r/ r r� r �lower)rD rN rk rc r� Z-last_checked_for_gsutil_update_timestamp_fileZcur_tsZlast_checked_ts�frW Zcur_ver�g�m�answerr3 r3 r4 r} � s� ���� . .�� . ��� ���z1CommandRunner.MaybeCheckForAndOfferSoftwareUpdate) NNr NFFNTNNF)�__name__� __module__� __qualname__�__doc__r r rE rC rS ra rl rp r� r� r� r} r3 r3 r3 r4 r? � s4 � 0) � r? )Gr� � __future__r r r r r� r� r� rF r� r. r� r) � six.movesr r� r �boto.storage_urir rH r Zgslib.cloud_api_delegatorr � gslib.commandr r r r r Zgslib.commandsZgslib.cs_api_mapr r r Z gslib.discard_messages_queuer �gslib.exceptionr Zgslib.gcs_json_apir Zgslib.no_op_credentialsr Zgslib.tab_completer � gslib.utilsr r r �gslib.utils.constantsr r Zgslib.utils.metadata_utilr �&gslib.utils.parallelism_framework_utilr Zgslib.utils.text_utilr r! r" r# �gslib.utils.unit_utilr$ Zgslib.utils.update_utilr% r& r5 r; r> �objectr? r3 r3 r3 r4 �<module> sf ( __pycache__/lazy_wrapper.cpython-39.pyc 0000644 00000004420 15025013616 0014123 0 ustar 00 a +(Wg � @ sH d Z ddlmZ ddlmZ ddlmZ ddlmZ G dd� de�ZdS ) zHProvides a LazyWrapper class to defer loading a value until it's needed.� )�absolute_import)�print_function)�division)�unicode_literalsc @ sP e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dS )� LazyWrapperz(Wrapper for lazily instantiated objects.c C s || _ dS )zqThe init method for LazyWrapper. Args: func: A function (lambda or otherwise) to lazily evaluate. N)�_func)�self�func� r �!/opt/gsutil/gslib/lazy_wrapper.py�__init__ s zLazyWrapper.__init__c C s8 z t | j�W S ty2 | �� | _t | j� Y S 0 d S �N)�int�_value�AttributeErrorr �r r r r �__int__" s zLazyWrapper.__int__c C s8 z | j |kW S ty2 | �� | _ | j |k Y S 0 d S r �r r r )r �otherr r r �__eq__) s zLazyWrapper.__eq__c C s8 z t | j�W S ty2 | �� | _t | j� Y S 0 d S r ��strr r r r r r r �__repr__0 s zLazyWrapper.__repr__c C s8 z t | j�W S ty2 | �� | _t | j� Y S 0 d S r r r r r r �__str__7 s zLazyWrapper.__str__c C s0 z| j W S ty* | �� | _ | j Y S 0 dS )z)The call method for a LazyWrapper object.Nr r r r r �__call__> s zLazyWrapper.__call__c C s6 z t | j�W S ty0 | �� t | j� Y S 0 dS )z(The len method for a LazyWrapper object.N)�lenr r r r r r r �__len__F s zLazyWrapper.__len__c C s6 z | j �� W S ty0 | �� | j �� Y S 0 dS )z)The iter method for a LazyWrapper object.N)r �__iter__r r r r r r r N s zLazyWrapper.__iter__N) �__name__� __module__� __qualname__�__doc__r r r r r r r r r r r r r s r N)r! � __future__r r r r �objectr r r r r �<module> s gcs_json_api.py 0000644 00000320026 15025013616 0007556 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """JSON gsutil Cloud API implementation for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from contextlib import contextmanager import functools from six.moves import http_client import json import logging import os import socket import ssl import time import traceback import six from apitools.base.py import encoding from apitools.base.py import exceptions as apitools_exceptions from apitools.base.py import http_wrapper as apitools_http_wrapper from apitools.base.py import transfer as apitools_transfer from apitools.base.py.util import CalculateWaitForRetry from boto import config import httplib2 import oauth2client from gslib import context_config from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ArgumentException from gslib.cloud_api import BadRequestException from gslib.cloud_api import CloudApi from gslib.cloud_api import EncryptionException from gslib.cloud_api import NotEmptyException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.cloud_api import PublishPermissionDeniedException from gslib.cloud_api import ResumableDownloadException from gslib.cloud_api import ResumableUploadAbortException from gslib.cloud_api import ResumableUploadException from gslib.cloud_api import ResumableUploadStartOverException from gslib.cloud_api import ServiceException from gslib.exception import CommandException from gslib.gcs_json_credentials import SetUpJsonCredentialsAndCache from gslib.gcs_json_credentials import isP12Credentials from gslib.gcs_json_media import BytesTransferredContainer from gslib.gcs_json_media import DownloadCallbackConnectionClassFactory from gslib.gcs_json_media import HttpWithDownloadStream from gslib.gcs_json_media import HttpWithNoRetries from gslib.gcs_json_media import UploadCallbackConnectionClassFactory from gslib.gcs_json_media import WrapDownloadHttpRequest from gslib.gcs_json_media import WrapUploadHttpRequest from gslib.impersonation_credentials import ImpersonationCredentials from gslib.no_op_credentials import NoOpCredentials from gslib.progress_callback import ProgressCallbackWithTimeout from gslib.project_id import PopulateProjectId from gslib.third_party.storage_apitools import storage_v1_client as apitools_client from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.tracker_file import DeleteTrackerFile from gslib.tracker_file import GetRewriteTrackerFilePath from gslib.tracker_file import HashRewriteParameters from gslib.tracker_file import ReadRewriteTrackerFile from gslib.tracker_file import WriteRewriteTrackerFile from gslib.utils.boto_util import GetCertsFile from gslib.utils.boto_util import GetGcsJsonApiVersion from gslib.utils.boto_util import GetJsonResumableChunkSize from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNewHttp from gslib.utils.boto_util import GetNumRetries from gslib.utils.boto_util import JsonResumableChunkSizeDefined from gslib.utils.cloud_api_helper import ListToGetFields from gslib.utils.cloud_api_helper import ValidateDstObjectMetadata from gslib.utils.constants import IAM_POLICY_VERSION from gslib.utils.constants import NUM_OBJECTS_PER_LIST_PAGE from gslib.utils.constants import REQUEST_REASON_ENV_VAR from gslib.utils.constants import REQUEST_REASON_HEADER_KEY from gslib.utils.constants import UTF8 from gslib.utils.encryption_helper import Base64Sha256FromBase64EncryptionKey from gslib.utils.encryption_helper import CryptoKeyType from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.encryption_helper import FindMatchingCSEKInBotoConfig from gslib.utils.metadata_util import AddAcceptEncodingGzipIfNeeded from gslib.utils.retry_util import LogAndHandleRetries from gslib.utils.signurl_helper import CreatePayload, GetFinalUrl from gslib.utils.text_util import GetPrintableExceptionString from gslib.utils.translation_helper import CreateBucketNotFoundException from gslib.utils.translation_helper import CreateNotFoundExceptionForObjectWrite from gslib.utils.translation_helper import CreateObjectNotFoundException from gslib.utils.translation_helper import DEFAULT_CONTENT_TYPE from gslib.utils.translation_helper import PRIVATE_DEFAULT_OBJ_ACL from gslib.utils.translation_helper import REMOVE_CORS_CONFIG from oauth2client.service_account import ServiceAccountCredentials if six.PY3: long = int # pylint: disable=invalid-name Notification = apitools_messages.Notification NotificationCustomAttributesValue = Notification.CustomAttributesValue NotificationAdditionalProperty = ( NotificationCustomAttributesValue.AdditionalProperty) # Implementation supports only 'gs' URLs, so provider is unused. # pylint: disable=unused-argument NUM_BUCKETS_PER_LIST_PAGE = 1000 TRANSLATABLE_APITOOLS_EXCEPTIONS = (apitools_exceptions.HttpError, apitools_exceptions.StreamExhausted, apitools_exceptions.TransferError, apitools_exceptions.TransferInvalidError) # TODO: Distribute these exceptions better through apitools and here. # Right now, apitools is configured not to handle any exceptions on # uploads/downloads. # oauth2_client tries to JSON-decode the response, which can result # in a ValueError if the response was invalid. Until that is fixed in # oauth2_client, need to handle it here. HTTP_TRANSFER_EXCEPTIONS = ( apitools_exceptions.TransferRetryError, apitools_exceptions.BadStatusCodeError, # TODO: Honor retry-after headers. apitools_exceptions.RetryAfterError, apitools_exceptions.RequestError, http_client.BadStatusLine, http_client.IncompleteRead, http_client.ResponseNotReady, httplib2.ServerNotFoundError, oauth2client.client.HttpAccessTokenRefreshError, socket.error, socket.gaierror, socket.timeout, ssl.SSLError, ValueError, ) # Fields requiring projection=full across all API calls. _ACL_FIELDS_SET = set([ 'acl', 'defaultObjectAcl', 'items/acl', 'items/defaultObjectAcl', 'items/owner', 'owner', ]) _BUCKET_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION = { None: None, 'authenticated-read': 'authenticatedRead', 'private': 'private', 'project-private': 'projectPrivate', 'public-read': 'publicRead', 'public-read-write': 'publicReadWrite' } _OBJECT_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION = { None: None, 'authenticated-read': 'authenticatedRead', 'bucket-owner-read': 'bucketOwnerRead', 'bucket-owner-full-control': 'bucketOwnerFullControl', 'private': 'private', 'project-private': 'projectPrivate', 'public-read': 'publicRead' } FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION = _BUCKET_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION.copy( ) FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION.update( _OBJECT_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION) # Fields that may be encrypted. _ENCRYPTED_HASHES_SET = set(['crc32c', 'md5Hash']) # During listing, if we attempt to decrypt an object but it has been removed, # we skip including the object in the listing. _SKIP_LISTING_OBJECT = 'skip' _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation.') DEFAULT_HOST = 'storage.googleapis.com' MTLS_HOST = 'storage.mtls.googleapis.com' class GcsJsonApi(CloudApi): """Google Cloud Storage JSON implementation of gsutil Cloud API.""" def __init__(self, bucket_storage_uri_class, logger, status_queue, provider=None, credentials=None, debug=0, http_headers=None, trace_token=None, perf_trace_token=None, user_project=None): """Performs necessary setup for interacting with Google Cloud Storage. Args: bucket_storage_uri_class: Unused. logger: logging.logger for outputting log messages. status_queue: Queue for relaying status to UI. provider: Unused. This implementation supports only Google Cloud Storage. credentials: Credentials to be used for interacting with Google Cloud Storage. debug: Debug level for the API implementation (0..3). http_headers (dict|None): Arbitrary headers to be included in every request. trace_token: Trace token to pass to the API implementation. perf_trace_token: Performance trace token to use when making API calls. user_project: Project to be billed for this request. """ # TODO: Plumb host_header for perfdiag / test_perfdiag. # TODO: Add jitter to apitools' http_wrapper retry mechanism. super(GcsJsonApi, self).__init__(bucket_storage_uri_class, logger, status_queue, provider='gs', debug=debug, http_headers=http_headers, trace_token=trace_token, perf_trace_token=perf_trace_token, user_project=user_project) self.certs_file = GetCertsFile() self.http = GetNewHttp() SetUpJsonCredentialsAndCache(self, logger, credentials=credentials) # Re-use download and upload connections. This class is only called # sequentially, but we can share TCP warmed-up connections across calls. self.download_http = self._GetNewDownloadHttp() self.upload_http = self._GetNewUploadHttp() if self.credentials: self.authorized_download_http = self.credentials.authorize( self.download_http) self.authorized_upload_http = self.credentials.authorize(self.upload_http) else: self.authorized_download_http = self.download_http self.authorized_upload_http = self.upload_http if isP12Credentials(self.credentials): WrapDownloadHttpRequest(self.authorized_download_http.http) WrapUploadHttpRequest(self.authorized_upload_http.http) else: WrapDownloadHttpRequest(self.authorized_download_http) WrapUploadHttpRequest(self.authorized_upload_http) self.http_base = 'https://' gs_json_host = config.get('Credentials', 'gs_json_host', None) if (context_config.get_context_config() and context_config.get_context_config().use_client_certificate): if gs_json_host: raise ArgumentException( '"use_client_certificate" is enabled, which sets gsutil to use the' ' host {}. However, a custom host was set using' ' "gs_json_host": {}. Please set "use_client_certificate" to' ' "False" or comment out the "gs_json_host" line in the Boto' ' config.'.format(MTLS_HOST, gs_json_host)) self.host_base = MTLS_HOST else: self.host_base = gs_json_host or DEFAULT_HOST if not gs_json_host: gs_host = config.get('Credentials', 'gs_host', None) if gs_host: raise ArgumentException( 'JSON API is selected but gs_json_host is not configured, ' 'while gs_host is configured to %s. Please also configure ' 'gs_json_host and gs_json_port to match your desired endpoint.' % gs_host) gs_json_port = config.get('Credentials', 'gs_json_port', None) if not gs_json_port: gs_port = config.get('Credentials', 'gs_port', None) if gs_port: raise ArgumentException( 'JSON API is selected but gs_json_port is not configured, ' 'while gs_port is configured to %s. Please also configure ' 'gs_json_host and gs_json_port to match your desired endpoint.' % gs_port) self.host_port = '' else: self.host_port = ':' + config.get('Credentials', 'gs_json_port') self.api_version = GetGcsJsonApiVersion() self.url_base = (self.http_base + self.host_base + self.host_port + '/' + 'storage/' + self.api_version + '/') self.global_params = apitools_messages.StandardQueryParameters( trace='token:%s' % trace_token) if trace_token else None additional_http_headers = {} gs_json_host_header = config.get('Credentials', 'gs_json_host_header', None) if gs_json_host_header and gs_json_host: additional_http_headers['Host'] = gs_json_host_header self._UpdateHeaders(additional_http_headers) log_request = (debug >= 3) log_response = (debug >= 3) self.api_client = apitools_client.StorageV1( url=self.url_base, http=self.http, log_request=log_request, log_response=log_response, credentials=self.credentials, version=self.api_version, default_global_params=self.global_params, additional_http_headers=additional_http_headers) self.max_retry_wait = GetMaxRetryDelay() self.api_client.max_retry_wait = self.max_retry_wait self.num_retries = GetNumRetries() self.api_client.num_retries = self.num_retries self.api_client.retry_func = LogAndHandleRetries( status_queue=self.status_queue) self.api_client.overwrite_transfer_urls_with_client_base = True if isinstance(self.credentials, NoOpCredentials): # This API key is not secret and is used to identify gsutil during # anonymous requests. self.api_client.AddGlobalParam('key', 'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM') def GetServiceAccountId(self): """Returns the service account email id.""" if isinstance(self.credentials, ImpersonationCredentials): return self.credentials.service_account_id elif isinstance(self.credentials, ServiceAccountCredentials): return self.credentials.service_account_email else: raise CommandException( 'Cannot get service account email id for the given ' 'credential type.') def _UpdateHeaders(self, headers): if self.http_headers: headers.update(self.http_headers) # The following functional headers potentially overwrite # arbitrary ones provided by -h. if self.perf_trace_token: headers['cookie'] = self.perf_trace_token request_reason = os.environ.get(REQUEST_REASON_ENV_VAR) if request_reason: headers[REQUEST_REASON_HEADER_KEY] = request_reason def _GetNewDownloadHttp(self): return GetNewHttp(http_class=HttpWithDownloadStream) def _GetNewUploadHttp(self): """Returns an upload-safe Http object (by disabling httplib2 retries).""" return GetNewHttp(http_class=HttpWithNoRetries) def _GetSignedContent(self, string_to_sign): """Returns the Signed Content.""" if isinstance(self.credentials, ImpersonationCredentials): iam_cred_api = self.credentials.api service_account_id = self.credentials.service_account_id response = iam_cred_api.SignBlob(service_account_id, string_to_sign) return response.keyId, response.signedBlob elif isinstance(self.credentials, ServiceAccountCredentials): return self.credentials.sign_blob(string_to_sign) else: raise CommandException( 'Authentication using a service account is required for signing ' 'the content.') def _FieldsContainsAclField(self, fields=None): """Checks Returns true if ACL related values are in fields set. Args: fields: list or set of fields. May be in GET ['acl'] or List ['items/acl'] call format. Returns: True if an ACL value is requested in the input fields, False otherwise. """ return fields is None or _ACL_FIELDS_SET.intersection(set(fields)) def _ValidateEncryptionFieldsForMetadataGetRequest(self, fields=None): """Ensures customerEncryption field is included if hashes are requested.""" # In the context of fetching object metadata, we only need to check for # customerEncryption, the field present if an object is CSEK-encrypted. Note # that if the object is KMS-encrypted, no field checks are necessary; a # metadata get request will be able to access the object's stored kmsKeyName # field to access the key needed to decrypt hash fields). if (fields and _ENCRYPTED_HASHES_SET.intersection(set(fields)) and 'customerEncryption' not in fields): raise ArgumentException( 'gsutil client error: customerEncryption must be included when ' 'requesting potentially encrypted fields %s' % _ENCRYPTED_HASHES_SET) def GetBucketIamPolicy(self, bucket_name, provider=None, fields=None): apitools_request = apitools_messages.StorageBucketsGetIamPolicyRequest( bucket=bucket_name, userProject=self.user_project, optionsRequestedPolicyVersion=IAM_POLICY_VERSION) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) try: return self.api_client.buckets.GetIamPolicy(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def GetObjectIamPolicy(self, bucket_name, object_name, generation=None, provider=None, fields=None): if generation is not None: generation = long(generation) apitools_request = apitools_messages.StorageObjectsGetIamPolicyRequest( bucket=bucket_name, object=object_name, generation=generation, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) try: return self.api_client.objects.GetIamPolicy(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def SetObjectIamPolicy(self, bucket_name, object_name, policy, generation=None, provider=None): if generation is not None: generation = long(generation) api_request = apitools_messages.StorageObjectsSetIamPolicyRequest( bucket=bucket_name, object=object_name, generation=generation, policy=policy, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() try: return self.api_client.objects.SetIamPolicy(api_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name) def SetBucketIamPolicy(self, bucket_name, policy, provider=None): apitools_request = apitools_messages.StorageBucketsSetIamPolicyRequest( bucket=bucket_name, policy=policy, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() try: return self.api_client.buckets.SetIamPolicy(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def SignUrl(self, method, duration, path, generation, logger, region, signed_headers, string_to_sign_debug): """See CloudApi class for function doc strings.""" service_account_id = self.GetServiceAccountId() string_to_sign, canonical_query_string = CreatePayload( client_id=service_account_id, method=method, duration=duration, path=path, generation=generation, logger=logger, region=region, signed_headers=signed_headers, string_to_sign_debug=string_to_sign_debug) if six.PY3: string_to_sign = string_to_sign.encode(UTF8) key_id, raw_signature = self._GetSignedContent(string_to_sign) logger.debug('Key ID used to sign blob for service account "%s": "%s"' % (service_account_id, key_id)) return GetFinalUrl(raw_signature, signed_headers['host'], path, canonical_query_string) def GetBucket(self, bucket_name, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageBucketsGetRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageBucketsGetRequest. ProjectionValueValuesEnum.full) apitools_request = apitools_messages.StorageBucketsGetRequest( bucket=bucket_name, projection=projection, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) # Here and in list buckets, we have no way of knowing # whether we requested a field and didn't get it because it didn't exist # or because we didn't have permission to access it. try: return self.api_client.buckets.Get(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def ListBucketAccessControls(self, bucket_name): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageBucketAccessControlsListRequest( bucket=bucket_name) return self.api_client.bucketAccessControls.List(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def ListObjectAccessControls(self, bucket_name, object_name): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageObjectAccessControlsListRequest( bucket=bucket_name, object=object_name) return self.api_client.objectAccessControls.List(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name) def PatchBucket(self, bucket_name, metadata, canned_acl=None, canned_def_acl=None, preconditions=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageBucketsPatchRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageBucketsPatchRequest. ProjectionValueValuesEnum.full) bucket_metadata = metadata if not preconditions: preconditions = Preconditions() # For blank metadata objects, we need to explicitly call # them out to apitools so it will send/erase them. apitools_include_fields = [] for metadata_field in ( 'billing', 'encryption', 'lifecycle', 'logging', 'metadata', 'retentionPolicy', 'versioning', 'website', ): attr = getattr(bucket_metadata, metadata_field, None) if attr and not encoding.MessageToDict(attr): setattr(bucket_metadata, metadata_field, None) apitools_include_fields.append(metadata_field) if bucket_metadata.cors and bucket_metadata.cors == REMOVE_CORS_CONFIG: bucket_metadata.cors = [] apitools_include_fields.append('cors') if (bucket_metadata.defaultObjectAcl and bucket_metadata.defaultObjectAcl[0] == PRIVATE_DEFAULT_OBJ_ACL): bucket_metadata.defaultObjectAcl = [] apitools_include_fields.append('defaultObjectAcl') predefined_acl = None if canned_acl: # Must null out existing ACLs to apply a canned ACL. apitools_include_fields.append('acl') predefined_acl = (apitools_messages.StorageBucketsPatchRequest. PredefinedAclValueValuesEnum( self._BucketCannedAclToPredefinedAcl(canned_acl))) predefined_def_acl = None if canned_def_acl: # Must null out existing default object ACLs to apply a canned ACL. apitools_include_fields.append('defaultObjectAcl') predefined_def_acl = ( apitools_messages.StorageBucketsPatchRequest. PredefinedDefaultObjectAclValueValuesEnum( self._ObjectCannedAclToPredefinedAcl(canned_def_acl))) apitools_request = apitools_messages.StorageBucketsPatchRequest( bucket=bucket_name, bucketResource=bucket_metadata, projection=projection, ifMetagenerationMatch=preconditions.meta_gen_match, predefinedAcl=predefined_acl, predefinedDefaultObjectAcl=predefined_def_acl, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) with self.api_client.IncludeFields(apitools_include_fields): try: return self.api_client.buckets.Patch(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def LockRetentionPolicy(self, bucket_name, metageneration, provider=None): try: metageneration = long(metageneration) except ValueError: raise ArgumentException( 'LockRetentionPolicy Metageneration must be an integer.') apitools_request = ( apitools_messages.StorageBucketsLockRetentionPolicyRequest( bucket=bucket_name, ifMetagenerationMatch=metageneration, userProject=self.user_project)) global_params = apitools_messages.StandardQueryParameters() try: return self.api_client.buckets.LockRetentionPolicy( apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def CreateBucket(self, bucket_name, project_id=None, metadata=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageBucketsInsertRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageBucketsInsertRequest. ProjectionValueValuesEnum.full) if not metadata: metadata = apitools_messages.Bucket() metadata.name = bucket_name if metadata.location: metadata.location = metadata.location.lower() if metadata.storageClass: metadata.storageClass = metadata.storageClass.lower() project_id = PopulateProjectId(project_id) apitools_request = apitools_messages.StorageBucketsInsertRequest( bucket=metadata, project=project_id, projection=projection, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) try: return self.api_client.buckets.Insert(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def DeleteBucket(self, bucket_name, preconditions=None, provider=None): """See CloudApi class for function doc strings.""" if not preconditions: preconditions = Preconditions() apitools_request = apitools_messages.StorageBucketsDeleteRequest( bucket=bucket_name, ifMetagenerationMatch=preconditions.meta_gen_match, userProject=self.user_project) try: self.api_client.buckets.Delete(apitools_request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: if isinstance( self._TranslateApitoolsException(e, bucket_name=bucket_name), NotEmptyException): # If bucket is not empty, check to see if versioning is enabled and # signal that in the exception if it is. bucket_metadata = self.GetBucket(bucket_name, fields=['versioning']) if bucket_metadata.versioning and bucket_metadata.versioning.enabled: raise NotEmptyException('VersionedBucketNotEmpty', status=e.status_code) self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def ListBuckets(self, project_id=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageBucketsListRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageBucketsListRequest. ProjectionValueValuesEnum.full) project_id = PopulateProjectId(project_id) apitools_request = apitools_messages.StorageBucketsListRequest( project=project_id, maxResults=NUM_BUCKETS_PER_LIST_PAGE, projection=projection, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: if 'nextPageToken' not in fields: fields.add('nextPageToken') global_params.fields = ','.join(set(fields)) try: bucket_list = self.api_client.buckets.List(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) for bucket in self._YieldBuckets(bucket_list): yield bucket while bucket_list.nextPageToken: apitools_request = apitools_messages.StorageBucketsListRequest( project=project_id, pageToken=bucket_list.nextPageToken, maxResults=NUM_BUCKETS_PER_LIST_PAGE, projection=projection, userProject=self.user_project) try: bucket_list = self.api_client.buckets.List(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) for bucket in self._YieldBuckets(bucket_list): yield bucket def _YieldBuckets(self, bucket_list): """Yields buckets from a list returned by apitools.""" if bucket_list.items: for bucket in bucket_list.items: yield bucket def ListObjects(self, bucket_name, prefix=None, delimiter=None, all_versions=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageObjectsListRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageObjectsListRequest. ProjectionValueValuesEnum.full) apitools_request = apitools_messages.StorageObjectsListRequest( bucket=bucket_name, prefix=prefix, delimiter=delimiter, versions=all_versions, projection=projection, maxResults=NUM_OBJECTS_PER_LIST_PAGE, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: fields = set(fields) if 'nextPageToken' not in fields: fields.add('nextPageToken') if ListToGetFields( list_fields=fields).intersection(_ENCRYPTED_HASHES_SET): # We may need to make a follow-up request to decrypt the hashes, # so ensure the required fields are present. fields.add('items/customerEncryption') fields.add('items/kmsKeyName') fields.add('items/generation') fields.add('items/name') global_params.fields = ','.join(fields) list_page = True next_page_token = None while list_page: list_page = False apitools_request = apitools_messages.StorageObjectsListRequest( bucket=bucket_name, prefix=prefix, delimiter=delimiter, versions=all_versions, projection=projection, pageToken=next_page_token, maxResults=NUM_OBJECTS_PER_LIST_PAGE, userProject=self.user_project) try: object_list = self.api_client.objects.List(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) if object_list.nextPageToken: list_page = True next_page_token = object_list.nextPageToken for object_or_prefix in self._YieldObjectsAndPrefixes(object_list): if object_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT: decrypted_metadata = self._DecryptHashesIfPossible( bucket_name, object_or_prefix.data, fields=ListToGetFields(fields)) if decrypted_metadata == _SKIP_LISTING_OBJECT: continue elif decrypted_metadata: object_or_prefix.data = decrypted_metadata else: # Object metadata was unencrypted or we did not have a matching # key; yield what we received in the initial listing. pass yield object_or_prefix def _DecryptHashesIfPossible(self, bucket_name, object_metadata, fields=None): """Attempts to decrypt object metadata. Helps fetch encrypted object metadata fields (if requested) for API method calls that requested them but did not receive them; this occurs for ListObjects(), along with GetObjectMetadata() calls for which the object is CSEK-encrypted but no decryption information was supplied. This method issues a StorageObjectsGetRequest (if object hash fields were requested but not retrieved), formatted appropriately depending on the encryption method. Args: bucket_name: String of bucket containing the object. object_metadata: apitools Object describing the object. Must include name, generation, and customerEncryption fields. fields: ListObjects-format fields to return. Returns: - apitools_messages.Object with decrypted hash fields if decryption was performed. - None if the object was not encrypted, or CSEK-encrypted but we could not find a matching decryption key. - gcs_json_api._SKIP_LISTING_OBJECT if the object no longer exists. """ get_fields = ListToGetFields(list_fields=fields) if not self._ObjectIsEncrypted(object_metadata): self._RaiseIfNoDesiredHashesPresentForUnencryptedObject(object_metadata, fields=get_fields) return None # If we reach this point, we know the object is encrypted. get_metadata_func = None if self._ObjectKMSEncryptedAndNeedHashes(object_metadata, fields=get_fields): # For CMK-encrypted objects, we can just call GetObjectMetadata. get_metadata_func = functools.partial( self._GetObjectMetadataHelper, bucket_name, object_metadata.name, generation=object_metadata.generation, fields=get_fields) elif self._ObjectCSEKEncryptedAndNeedHashes(object_metadata, fields=get_fields): # For CSEK-encrypted objects, we can do almost the same thing as above, # but we also need to add some info about the decryption key (if it's # available) into the request. key_sha256 = object_metadata.customerEncryption.keySha256 if six.PY3: if not isinstance(key_sha256, bytes): key_sha256 = key_sha256.encode('ascii') decryption_key = FindMatchingCSEKInBotoConfig(key_sha256, config) if decryption_key: get_metadata_func = functools.partial( self._GetObjectMetadataHelper, bucket_name, object_metadata.name, generation=object_metadata.generation, fields=get_fields, decryption_tuple=CryptoKeyWrapperFromKey(decryption_key)) if get_metadata_func is None: return try: return get_metadata_func() except NotFoundException: # If the object was deleted between the listing request and # the get request, do not include it in the list. return _SKIP_LISTING_OBJECT def _YieldObjectsAndPrefixes(self, object_list): # ls depends on iterating objects before prefixes for proper display. if object_list.items: for cloud_obj in object_list.items: yield CloudApi.CsObjectOrPrefix(cloud_obj, CloudApi.CsObjectOrPrefixType.OBJECT) if object_list.prefixes: for prefix in object_list.prefixes: yield CloudApi.CsObjectOrPrefix(prefix, CloudApi.CsObjectOrPrefixType.PREFIX) @contextmanager def _ApitoolsRequestHeaders(self, headers): """Wrapped code makes apitools requests with the specified headers.""" if headers: old_headers = self.api_client.additional_http_headers.copy() self.api_client.additional_http_headers.update(headers) yield if headers: self.api_client.additional_http_headers = old_headers def _EncryptionHeadersFromTuple(self, crypto_tuple=None): """Returns headers dict matching crypto_tuple, or empty dict.""" headers = {} if crypto_tuple and crypto_tuple.crypto_type == CryptoKeyType.CSEK: headers['x-goog-encryption-algorithm'] = crypto_tuple.crypto_alg headers['x-goog-encryption-key'] = crypto_tuple.crypto_key headers['x-goog-encryption-key-sha256'] = ( Base64Sha256FromBase64EncryptionKey(crypto_tuple.crypto_key)) return headers def _RewriteCryptoHeadersFromTuples(self, decryption_tuple=None, encryption_tuple=None): """Returns headers dict matching the provided tuples, or empty dict.""" headers = {} if decryption_tuple and decryption_tuple.crypto_type == CryptoKeyType.CSEK: headers['x-goog-copy-source-encryption-algorithm'] = ( decryption_tuple.crypto_alg) headers['x-goog-copy-source-encryption-key'] = ( decryption_tuple.crypto_key) headers['x-goog-copy-source-encryption-key-sha256'] = ( decryption_tuple.crypto_key_sha256) if encryption_tuple and encryption_tuple.crypto_type == CryptoKeyType.CSEK: headers['x-goog-encryption-algorithm'] = encryption_tuple.crypto_alg headers['x-goog-encryption-key'] = encryption_tuple.crypto_key headers['x-goog-encryption-key-sha256'] = ( encryption_tuple.crypto_key_sha256) return headers def _GetObjectMetadataHelper(self, bucket_name, object_name, generation=None, fields=None, decryption_tuple=None): """Get metadata, adding necessary headers for CSEK decryption if necessary. Args: bucket_name: (str) Bucket containing the object. object_name: (str) Object name. generation: (int) Generation of the object to retrieve. fields: (list<str>) If present, return only these Object metadata fields, for example: ['acl', 'updated']. decryption_tuple: (gslib.utils.encryption_helper.CryptoKeyWrapper) The CryptoKeyWrapper for the desired decryption key. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: (apitools_message.Object) The retrieved object metadata. """ apitools_request = self._CreateApitoolsObjectMetadataGetRequest( bucket_name, object_name, generation=generation, fields=fields) global_params = self._GetApitoolsObjectMetadataGlobalParams(fields=fields) try: # Add special headers if fetching metadata for a CSEK-encrypted object. if decryption_tuple: with self._ApitoolsRequestHeaders( self._EncryptionHeadersFromTuple(crypto_tuple=decryption_tuple)): return self.api_client.objects.Get(apitools_request, global_params=global_params) else: return self.api_client.objects.Get(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def _CreateApitoolsObjectMetadataGetRequest(self, bucket_name, object_name, generation=None, fields=None): self._ValidateEncryptionFieldsForMetadataGetRequest(fields=fields) projection = (apitools_messages.StorageObjectsGetRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageObjectsGetRequest. ProjectionValueValuesEnum.full) if generation: generation = long(generation) return apitools_messages.StorageObjectsGetRequest( bucket=bucket_name, object=object_name, projection=projection, generation=generation, userProject=self.user_project) def _GetApitoolsObjectMetadataGlobalParams(self, fields=None): global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) return global_params def GetObjectMetadata(self, bucket_name, object_name, generation=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" try: object_metadata = self._GetObjectMetadataHelper(bucket_name, object_name, generation=generation, fields=fields) if not self._ObjectCSEKEncryptedAndNeedHashes(object_metadata, fields=fields): return object_metadata # CSEK-encrypted objects require corresponding decryption information to # be added to the StorageObjectsGetRequest in order to retrieve hash # fields. key_sha256 = object_metadata.customerEncryption.keySha256 if six.PY3: if not isinstance(key_sha256, bytes): key_sha256 = key_sha256.encode('ascii') decryption_key = FindMatchingCSEKInBotoConfig(key_sha256, config) if not decryption_key: raise EncryptionException( 'Missing decryption key with SHA256 hash %s. No decryption key ' 'matches object %s://%s/%s' % (key_sha256, self.provider, bucket_name, object_name)) if six.PY3: if not isinstance(decryption_key, bytes): decryption_key = decryption_key.encode(UTF8) return self._GetObjectMetadataHelper( bucket_name, object_name, generation=generation, fields=fields, decryption_tuple=CryptoKeyWrapperFromKey(decryption_key)) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def _ObjectIsEncrypted(self, object_metadata): return bool(object_metadata.customerEncryption or object_metadata.kmsKeyName) def _RaiseIfNoDesiredHashesPresentForUnencryptedObject( self, object_metadata, fields=None): """Raise an exception if none of the desired hash fields are present.""" if fields is None: return md5_requested_but_missing = ('md5Hash' in fields and not object_metadata.md5Hash) crc32c_requested_but_missing = ('crc32c' in fields and not object_metadata.crc32c) # md5 not available for composite objects; in this case, don't raise an # exception if that was the only hash we requested. if (md5_requested_but_missing and not object_metadata.componentCount and crc32c_requested_but_missing): raise ServiceException( 'Service did not provide requested hashes in metadata for ' 'unencrypted object %s' % object_metadata.name) def _ObjectKMSEncryptedAndNeedHashes(self, object_metadata, fields=None): """Checks if an object is KMS-encrypted and lacking requested hash fields. Args: object_metadata: (apitools_messages.Object) Object metadata returned from an API call; may or may not contain hash-related metadata fields. fields: (list<str>) Requested object metadata fields. Returns: True if a GetObjectMetadata request should be issued on the KMS-encrypted object in order to retrieve encrypted hash fields, False otherwise. """ if fields is not None and not _ENCRYPTED_HASHES_SET.intersection(fields): # No potentially encrypted metadata fields were requested. return False return (object_metadata.kmsKeyName and not object_metadata.crc32c and not object_metadata.md5Hash) def _ObjectCSEKEncryptedAndNeedHashes(self, object_metadata, fields=None): """Checks if an object is CSEK-encrypted and lacking requested hash fields. Args: object_metadata: (apitools_messages.Object) Object metadata returned from an API call; may or may not contain hash-related metadata fields. fields: (list<str>) Requested object metadata fields. Returns: True if a GetObjectMetadata request including a decryption key should be issued on the CSEK-encrypted object in order to retrieve encrypted hash fields, False otherwise. """ if fields is None or not _ENCRYPTED_HASHES_SET.intersection(fields): # No potentially encrypted metadata fields were requested. return False return (object_metadata.customerEncryption and not object_metadata.crc32c and not object_metadata.md5Hash) def GetObjectMedia(self, bucket_name, object_name, download_stream, provider=None, generation=None, object_size=None, compressed_encoding=False, download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, start_byte=0, end_byte=None, progress_callback=None, serialization_data=None, digesters=None, decryption_tuple=None): """See CloudApi class for function doc strings.""" # This implementation will get the object metadata first if we don't pass it # in via serialization_data. if generation: generation = long(generation) # 'outer_total_size' is only used for formatting user output, and is # expected to be one higher than the last byte that should be downloaded. # TODO: Change DownloadCallbackConnectionClassFactory and progress callbacks # to more elegantly handle total size for components of files. outer_total_size = object_size if end_byte: outer_total_size = end_byte + 1 elif serialization_data: outer_total_size = json.loads( six.ensure_str(serialization_data))['total_size'] if progress_callback: if outer_total_size is None: raise ArgumentException('Download size is required when callbacks are ' 'requested for a download, but no size was ' 'provided.') progress_callback(start_byte, outer_total_size) bytes_downloaded_container = BytesTransferredContainer() bytes_downloaded_container.bytes_transferred = start_byte callback_class_factory = DownloadCallbackConnectionClassFactory( bytes_downloaded_container, total_size=outer_total_size, progress_callback=progress_callback, digesters=digesters) download_http_class = callback_class_factory.GetConnectionClass() # Point our download HTTP at our download stream. self.download_http.stream = download_stream self.download_http.connections = {'https': download_http_class} if serialization_data: # If we have an apiary trace token, add it to the URL. # TODO: Add query parameter support to apitools downloads so there is # a well-defined way to express query parameters. Currently, we assume # the URL ends in ?alt=media, and this will break if that changes. if self.trace_token: serialization_dict = json.loads(six.ensure_str(serialization_data)) serialization_dict['url'] += '&trace=token%%3A%s' % self.trace_token serialization_data = json.dumps(serialization_dict) apitools_download = apitools_transfer.Download.FromData( download_stream, serialization_data, self.api_client.http, num_retries=self.num_retries, client=self.api_client) else: apitools_download = apitools_transfer.Download.FromStream( download_stream, auto_transfer=False, total_size=object_size, num_retries=self.num_retries) apitools_download.bytes_http = self.authorized_download_http apitools_request = apitools_messages.StorageObjectsGetRequest( bucket=bucket_name, object=object_name, generation=generation, userProject=self.user_project) # Disable retries in apitools. We will handle them explicitly for # resumable downloads; one-shot downloads are not retryable as we do # not track how many bytes were written to the stream. apitools_download.retry_func = LogAndHandleRetries( is_data_transfer=True, status_queue=self.status_queue) try: if download_strategy == CloudApi.DownloadStrategy.RESUMABLE: return self._PerformResumableDownload( bucket_name, object_name, download_stream, apitools_request, apitools_download, bytes_downloaded_container, compressed_encoding=compressed_encoding, generation=generation, start_byte=start_byte, end_byte=end_byte, serialization_data=serialization_data, decryption_tuple=decryption_tuple) else: return self._PerformDownload(bucket_name, object_name, download_stream, apitools_request, apitools_download, generation=generation, compressed_encoding=compressed_encoding, start_byte=start_byte, end_byte=end_byte, serialization_data=serialization_data, decryption_tuple=decryption_tuple) # If you are fighting a redacted exception spew in multiprocess/multithread # calls, add your exception to TRANSLATABLE_APITOOLS_EXCEPTIONS and put # something like this immediately after the following except statement: # import sys, traceback; sys.stderr.write('\n{}\n'.format( # traceback.format_exc())); sys.stderr.flush() # This may hang, but you should get a stack trace spew after Ctrl-C. except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def _PerformResumableDownload(self, bucket_name, object_name, download_stream, apitools_request, apitools_download, bytes_downloaded_container, generation=None, compressed_encoding=False, start_byte=0, end_byte=None, serialization_data=None, decryption_tuple=None): retries = 0 last_progress_byte = start_byte while retries <= self.num_retries: try: return self._PerformDownload(bucket_name, object_name, download_stream, apitools_request, apitools_download, generation=generation, compressed_encoding=compressed_encoding, start_byte=start_byte, end_byte=end_byte, serialization_data=serialization_data, decryption_tuple=decryption_tuple) except HTTP_TRANSFER_EXCEPTIONS as e: self._ValidateHttpAccessTokenRefreshError(e) start_byte = download_stream.tell() bytes_downloaded_container.bytes_transferred = start_byte if start_byte > last_progress_byte: # We've made progress, so allow a fresh set of retries. last_progress_byte = start_byte retries = 0 retries += 1 if retries > self.num_retries: ##### DEBUG # If you are fighting a redacted exception spew in # multiprocess/multithread calls, add your exception to # HTTP_TRANSFER_EXCEPTIONS and uncomment the following block. This # may hang, but you should get a stack trace spew after Ctrl-C. ##### # import re, sys, traceback # from gslib.utils import text_util # message = 'some exception happened' # stack_trace = traceback.format_exc() # err = ('DEBUG: Exception stack trace:\n %s\n%s\n' % ( # re.sub('\\n', '\n ', stack_trace), # message, # )) # dest_fd = sys.stderr # text_util.print_to_fd(err, end='', file=dest_fd) # dest_fd.flush() ##### END DEBUG raise ResumableDownloadException( 'Transfer failed after %d retries. Final exception: %s' % (self.num_retries, GetPrintableExceptionString(e))) time.sleep(CalculateWaitForRetry(retries, max_wait=self.max_retry_wait)) if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug( 'Retrying download from byte %s after exception: %s. Trace: %s', start_byte, GetPrintableExceptionString(e), traceback.format_exc()) apitools_http_wrapper.RebuildHttpConnections( apitools_download.bytes_http) def _PerformDownload(self, bucket_name, object_name, download_stream, apitools_request, apitools_download, generation=None, compressed_encoding=False, start_byte=0, end_byte=None, serialization_data=None, decryption_tuple=None): if not serialization_data: try: self.api_client.objects.Get(apitools_request, download=apitools_download) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) # Disable apitools' default print callbacks. def _NoOpCallback(unused_response, unused_download_object): pass # TODO: If we have a resumable download with accept-encoding:gzip # on a object that is compressible but not in gzip form in the cloud, # on-the-fly compression may gzip the object. In this case if our # download breaks, future requests will ignore the range header and just # return the object (gzipped) in its entirety. Ideally, we would unzip # the bytes that we have locally and send a range request without # accept-encoding:gzip so that we can download only the (uncompressed) bytes # that we don't yet have. # Since bytes_http is created in this function, we don't get the # user-agent header from api_client's http automatically. additional_headers = { 'user-agent': self.api_client.user_agent, } AddAcceptEncodingGzipIfNeeded(additional_headers, compressed_encoding=compressed_encoding) self._UpdateHeaders(additional_headers) additional_headers.update( self._EncryptionHeadersFromTuple(decryption_tuple)) if start_byte or end_byte is not None: apitools_download.GetRange(additional_headers=additional_headers, start=start_byte, end=end_byte, use_chunks=False) else: apitools_download.StreamMedia(callback=_NoOpCallback, finish_callback=_NoOpCallback, additional_headers=additional_headers, use_chunks=False) return apitools_download.encoding def PatchObjectMetadata(self, bucket_name, object_name, metadata, canned_acl=None, generation=None, preconditions=None, provider=None, fields=None, user_project=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageObjectsPatchRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageObjectsPatchRequest. ProjectionValueValuesEnum.full) if not preconditions: preconditions = Preconditions() if generation: generation = long(generation) predefined_acl = None apitools_include_fields = [] if canned_acl: # Must null out existing ACLs to apply a canned ACL. apitools_include_fields.append('acl') predefined_acl = (apitools_messages.StorageObjectsPatchRequest. PredefinedAclValueValuesEnum( self._ObjectCannedAclToPredefinedAcl(canned_acl))) # Provide the ability to delete response headers from metadata. if metadata.cacheControl == '': apitools_include_fields.append('cacheControl') if metadata.contentDisposition == '': apitools_include_fields.append('contentDisposition') if metadata.contentEncoding == '': apitools_include_fields.append('contentEncoding') if metadata.contentLanguage == '': apitools_include_fields.append('contentLanguage') apitools_request = apitools_messages.StorageObjectsPatchRequest( bucket=bucket_name, object=object_name, objectResource=metadata, generation=generation, projection=projection, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match, predefinedAcl=predefined_acl, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) try: with self.api_client.IncludeFields(apitools_include_fields): return self.api_client.objects.Patch(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def _UploadObject(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, encryption_tuple=None, provider=None, fields=None, serialization_data=None, tracker_callback=None, progress_callback=None, apitools_strategy=apitools_transfer.SIMPLE_UPLOAD, total_size=0, gzip_encoded=False): # pylint: disable=g-doc-args """Upload implementation. Cloud API arguments, plus two more. Additional args: apitools_strategy: SIMPLE_UPLOAD or RESUMABLE_UPLOAD. total_size: Total size of the upload; None if it is unknown (streaming). Returns: Uploaded object metadata. """ # pylint: enable=g-doc-args ValidateDstObjectMetadata(object_metadata) predefined_acl = None if canned_acl: predefined_acl = (apitools_messages.StorageObjectsInsertRequest. PredefinedAclValueValuesEnum( self._ObjectCannedAclToPredefinedAcl(canned_acl))) bytes_uploaded_container = BytesTransferredContainer() if progress_callback and size: total_size = size progress_callback(0, size) callback_class_factory = UploadCallbackConnectionClassFactory( bytes_uploaded_container, total_size=total_size, progress_callback=progress_callback, logger=self.logger, debug=self.debug) upload_http_class = callback_class_factory.GetConnectionClass() self.upload_http.connections = { 'http': upload_http_class, 'https': upload_http_class, } # Since bytes_http is created in this function, we don't get the # user-agent header from api_client's http automatically. additional_headers = { 'user-agent': self.api_client.user_agent, } self._UpdateHeaders(additional_headers) try: content_type = None apitools_request = None global_params = None if not serialization_data: # This is a new upload, set up initial upload state. content_type = object_metadata.contentType if not content_type: content_type = DEFAULT_CONTENT_TYPE if not preconditions: preconditions = Preconditions() apitools_request = apitools_messages.StorageObjectsInsertRequest( bucket=object_metadata.bucket, object=object_metadata, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match, predefinedAcl=predefined_acl, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) encryption_headers = self._EncryptionHeadersFromTuple( crypto_tuple=encryption_tuple) if apitools_strategy == apitools_transfer.SIMPLE_UPLOAD: # One-shot upload. apitools_upload = apitools_transfer.Upload(upload_stream, content_type, total_size=size, auto_transfer=True, num_retries=self.num_retries, gzip_encoded=gzip_encoded) apitools_upload.strategy = apitools_strategy apitools_upload.bytes_http = self.authorized_upload_http with self._ApitoolsRequestHeaders(encryption_headers): return self.api_client.objects.Insert(apitools_request, upload=apitools_upload, global_params=global_params) else: # Resumable upload. # Since bytes_http is created in this function, we don't get the # user-agent header from api_client's http automatically. additional_headers = { 'user-agent': self.api_client.user_agent, } additional_headers.update(encryption_headers) self._UpdateHeaders(additional_headers) return self._PerformResumableUpload( upload_stream, self.authorized_upload_http, content_type, size, serialization_data, apitools_strategy, apitools_request, global_params, bytes_uploaded_container, tracker_callback, additional_headers, progress_callback, gzip_encoded) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: not_found_exception = CreateNotFoundExceptionForObjectWrite( self.provider, object_metadata.bucket) self._TranslateExceptionAndRaise(e, bucket_name=object_metadata.bucket, object_name=object_metadata.name, not_found_exception=not_found_exception) def _PerformResumableUpload(self, upload_stream, authorized_upload_http, content_type, size, serialization_data, apitools_strategy, apitools_request, global_params, bytes_uploaded_container, tracker_callback, addl_headers, progress_callback, gzip_encoded): try: if serialization_data: # Resuming an existing upload. apitools_upload = apitools_transfer.Upload.FromData( upload_stream, serialization_data, self.api_client.http, num_retries=self.num_retries, gzip_encoded=gzip_encoded, client=self.api_client) apitools_upload.chunksize = GetJsonResumableChunkSize() apitools_upload.bytes_http = authorized_upload_http else: # New resumable upload. apitools_upload = apitools_transfer.Upload( upload_stream, content_type, total_size=size, chunksize=GetJsonResumableChunkSize(), auto_transfer=False, num_retries=self.num_retries, gzip_encoded=gzip_encoded) apitools_upload.strategy = apitools_strategy apitools_upload.bytes_http = authorized_upload_http with self._ApitoolsRequestHeaders(addl_headers): self.api_client.objects.Insert(apitools_request, upload=apitools_upload, global_params=global_params) # Disable retries in apitools. We will handle them explicitly here. apitools_upload.retry_func = LogAndHandleRetries( is_data_transfer=True, status_queue=self.status_queue) # Disable apitools' default print callbacks. def _NoOpCallback(unused_response, unused_upload_object): pass # If we're resuming an upload, apitools has at this point received # from the server how many bytes it already has. Update our # callback class with this information. bytes_uploaded_container.bytes_transferred = apitools_upload.progress if tracker_callback: tracker_callback(json.dumps(apitools_upload.serialization_data)) retries = 0 last_progress_byte = apitools_upload.progress while retries <= self.num_retries: try: # TODO: On retry, this will seek to the bytes that the server has, # causing the hash to be recalculated. Make HashingFileUploadWrapper # save a digest according to json_resumable_chunk_size. if not gzip_encoded and size and not JsonResumableChunkSizeDefined(): # If size is known and the request doesn't need to be compressed, # we can send it all in one request and avoid making a # round-trip per chunk. Compression is not supported for # non-chunked streaming uploads because supporting resumability # for that feature results in degraded upload performance and # adds significant complexity to the implementation. http_response = apitools_upload.StreamMedia( callback=_NoOpCallback, finish_callback=_NoOpCallback, additional_headers=addl_headers) else: # Otherwise it's a streaming request and we need to ensure that we # send the bytes in chunks so that we can guarantee that we never # need to seek backwards more than our buffer (and also that the # chunks are aligned to 256KB). http_response = apitools_upload.StreamInChunks( callback=_NoOpCallback, finish_callback=_NoOpCallback, additional_headers=addl_headers) processed_response = self.api_client.objects.ProcessHttpResponse( self.api_client.objects.GetMethodConfig('Insert'), http_response) if size is None and progress_callback: # Make final progress callback; total size should now be known. # This works around the fact the send function counts header bytes. # However, this will make the progress appear to go slightly # backwards at the end. progress_callback(apitools_upload.total_size, apitools_upload.total_size) return processed_response except HTTP_TRANSFER_EXCEPTIONS as e: self._ValidateHttpAccessTokenRefreshError(e) apitools_http_wrapper.RebuildHttpConnections( apitools_upload.bytes_http) while retries <= self.num_retries: try: # TODO: Simulate the refresh case in tests. Right now, our # mocks are not complex enough to simulate a failure. apitools_upload.RefreshResumableUploadState() start_byte = apitools_upload.progress bytes_uploaded_container.bytes_transferred = start_byte break except HTTP_TRANSFER_EXCEPTIONS as e2: self._ValidateHttpAccessTokenRefreshError(e2) apitools_http_wrapper.RebuildHttpConnections( apitools_upload.bytes_http) retries += 1 if retries > self.num_retries: raise ResumableUploadException( 'Transfer failed after %d retries. Final exception: %s' % (self.num_retries, e2)) time.sleep( CalculateWaitForRetry(retries, max_wait=self.max_retry_wait)) if start_byte > last_progress_byte: # We've made progress, so allow a fresh set of retries. last_progress_byte = start_byte retries = 0 else: retries += 1 if retries > self.num_retries: raise ResumableUploadException( 'Transfer failed after %d retries. Final exception: %s' % (self.num_retries, GetPrintableExceptionString(e))) time.sleep( CalculateWaitForRetry(retries, max_wait=self.max_retry_wait)) if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug( 'Retrying upload from byte %s after exception: %s. Trace: %s', start_byte, GetPrintableExceptionString(e), traceback.format_exc()) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: resumable_ex = self._TranslateApitoolsResumableUploadException(e) if resumable_ex: raise resumable_ex else: raise def UploadObject(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """See CloudApi class for function doc strings.""" return self._UploadObject(upload_stream, object_metadata, canned_acl=canned_acl, size=size, preconditions=preconditions, progress_callback=progress_callback, encryption_tuple=encryption_tuple, fields=fields, apitools_strategy=apitools_transfer.SIMPLE_UPLOAD, gzip_encoded=gzip_encoded) def UploadObjectStreaming(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """See CloudApi class for function doc strings.""" # Streaming indicated by not passing a size. # Resumable capabilities are present up to the resumable chunk size using # a buffered stream. return self._UploadObject( upload_stream, object_metadata, canned_acl=canned_acl, preconditions=preconditions, progress_callback=progress_callback, encryption_tuple=encryption_tuple, fields=fields, apitools_strategy=apitools_transfer.RESUMABLE_UPLOAD, total_size=None, gzip_encoded=gzip_encoded) def UploadObjectResumable(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, size=None, serialization_data=None, tracker_callback=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """See CloudApi class for function doc strings.""" return self._UploadObject( upload_stream, object_metadata, canned_acl=canned_acl, preconditions=preconditions, fields=fields, size=size, serialization_data=serialization_data, tracker_callback=tracker_callback, progress_callback=progress_callback, encryption_tuple=encryption_tuple, apitools_strategy=apitools_transfer.RESUMABLE_UPLOAD, gzip_encoded=gzip_encoded) def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, canned_acl=None, preconditions=None, progress_callback=None, max_bytes_per_call=None, encryption_tuple=None, decryption_tuple=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" ValidateDstObjectMetadata(dst_obj_metadata) predefined_acl = None if canned_acl: predefined_acl = (apitools_messages.StorageObjectsRewriteRequest. DestinationPredefinedAclValueValuesEnum( self._ObjectCannedAclToPredefinedAcl(canned_acl))) if src_generation: src_generation = long(src_generation) if not preconditions: preconditions = Preconditions() projection = (apitools_messages.StorageObjectsRewriteRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageObjectsRewriteRequest. ProjectionValueValuesEnum.full) global_params = apitools_messages.StandardQueryParameters() if fields: # Rewrite returns the resultant object under the 'resource' field. new_fields = set([ 'done', 'objectSize', 'rewriteToken', 'totalBytesRewritten', ]) for field in fields: new_fields.add('resource/' + field) global_params.fields = ','.join(set(new_fields)) dec_key_sha256 = None if decryption_tuple and decryption_tuple.crypto_type == CryptoKeyType.CSEK: dec_key_sha256 = decryption_tuple.crypto_key_sha256 enc_key_sha256 = None if encryption_tuple: if encryption_tuple.crypto_type == CryptoKeyType.CSEK: enc_key_sha256 = encryption_tuple.crypto_key_sha256 elif encryption_tuple.crypto_type == CryptoKeyType.CMEK: dst_obj_metadata.kmsKeyName = encryption_tuple.crypto_key # Check to see if we are resuming a rewrite. tracker_file_name = GetRewriteTrackerFilePath(src_obj_metadata.bucket, src_obj_metadata.name, dst_obj_metadata.bucket, dst_obj_metadata.name, 'JSON') rewrite_params_hash = HashRewriteParameters( src_obj_metadata, dst_obj_metadata, projection, src_generation=src_generation, gen_match=preconditions.gen_match, meta_gen_match=preconditions.meta_gen_match, canned_acl=predefined_acl, max_bytes_per_call=max_bytes_per_call, src_dec_key_sha256=dec_key_sha256, dst_enc_key_sha256=enc_key_sha256, fields=global_params.fields) resume_rewrite_token = ReadRewriteTrackerFile(tracker_file_name, rewrite_params_hash) crypto_headers = self._RewriteCryptoHeadersFromTuples( decryption_tuple=decryption_tuple, encryption_tuple=encryption_tuple) progress_cb_with_timeout = None try: last_bytes_written = long(0) while True: with self._ApitoolsRequestHeaders(crypto_headers): apitools_request = apitools_messages.StorageObjectsRewriteRequest( sourceBucket=src_obj_metadata.bucket, sourceObject=src_obj_metadata.name, destinationBucket=dst_obj_metadata.bucket, # TODO(KMS): Remove the destinationKmsKeyName parameter once the # API begins pulling key name from the dest obj metadata. destinationKmsKeyName=dst_obj_metadata.kmsKeyName, destinationObject=dst_obj_metadata.name, projection=projection, object=dst_obj_metadata, sourceGeneration=src_generation, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match, destinationPredefinedAcl=predefined_acl, rewriteToken=resume_rewrite_token, maxBytesRewrittenPerCall=max_bytes_per_call, userProject=self.user_project) rewrite_response = self.api_client.objects.Rewrite( apitools_request, global_params=global_params) bytes_written = long(rewrite_response.totalBytesRewritten) if progress_callback and not progress_cb_with_timeout: progress_cb_with_timeout = ProgressCallbackWithTimeout( long(rewrite_response.objectSize), progress_callback) if progress_cb_with_timeout: progress_cb_with_timeout.Progress(bytes_written - last_bytes_written) if rewrite_response.done: break elif not resume_rewrite_token: # Save the token and make a tracker file if they don't already exist. resume_rewrite_token = rewrite_response.rewriteToken WriteRewriteTrackerFile(tracker_file_name, rewrite_params_hash, rewrite_response.rewriteToken) last_bytes_written = bytes_written DeleteTrackerFile(tracker_file_name) return rewrite_response.resource except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: not_found_exception = CreateNotFoundExceptionForObjectWrite( self.provider, dst_obj_metadata.bucket, src_provider=self.provider, src_bucket_name=src_obj_metadata.bucket, src_object_name=src_obj_metadata.name, src_generation=src_generation) self._TranslateExceptionAndRaise(e, bucket_name=dst_obj_metadata.bucket, object_name=dst_obj_metadata.name, not_found_exception=not_found_exception) def DeleteObject(self, bucket_name, object_name, preconditions=None, generation=None, provider=None): """See CloudApi class for function doc strings.""" if not preconditions: preconditions = Preconditions() if generation: generation = long(generation) apitools_request = apitools_messages.StorageObjectsDeleteRequest( bucket=bucket_name, object=object_name, generation=generation, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match, userProject=self.user_project) try: return self.api_client.objects.Delete(apitools_request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def ComposeObject(self, src_objs_metadata, dst_obj_metadata, preconditions=None, encryption_tuple=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" ValidateDstObjectMetadata(dst_obj_metadata) dst_obj_name = dst_obj_metadata.name dst_obj_metadata.name = None dst_bucket_name = dst_obj_metadata.bucket dst_obj_metadata.bucket = None if not dst_obj_metadata.contentType: dst_obj_metadata.contentType = DEFAULT_CONTENT_TYPE if not preconditions: preconditions = Preconditions() global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) src_objs_compose_request = apitools_messages.ComposeRequest( sourceObjects=src_objs_metadata, destination=dst_obj_metadata) encryption_headers = self._EncryptionHeadersFromTuple( crypto_tuple=encryption_tuple) kmsKeyName = None if encryption_tuple: if encryption_tuple.crypto_type == CryptoKeyType.CMEK: kmsKeyName = encryption_tuple.crypto_key with self._ApitoolsRequestHeaders(encryption_headers): apitools_request = apitools_messages.StorageObjectsComposeRequest( composeRequest=src_objs_compose_request, destinationBucket=dst_bucket_name, destinationObject=dst_obj_name, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match, userProject=self.user_project, kmsKeyName=kmsKeyName) try: return self.api_client.objects.Compose(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: # We can't be sure which object was missing in the 404 case. if (isinstance(e, apitools_exceptions.HttpError) and e.status_code == 404): raise NotFoundException('One of the source objects does not exist.') else: self._TranslateExceptionAndRaise(e) def WatchBucket(self, bucket_name, address, channel_id, token=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageObjectsWatchAllRequest. ProjectionValueValuesEnum.full) channel = apitools_messages.Channel(address=address, id=channel_id, token=token, type='WEB_HOOK') apitools_request = apitools_messages.StorageObjectsWatchAllRequest( bucket=bucket_name, channel=channel, projection=projection, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) try: return self.api_client.objects.WatchAll(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def StopChannel(self, channel_id, resource_id, provider=None): """See CloudApi class for function doc strings.""" channel = apitools_messages.Channel(id=channel_id, resourceId=resource_id) try: self.api_client.channels.Stop(channel) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def ListChannels(self, bucket_name, provider=None, fields=None): """See CloudApi class for function doc strings.""" apitools_request = apitools_messages.StorageBucketsListChannelsRequest( bucket=bucket_name, userProject=self.user_project) try: return self.api_client.buckets.ListChannels(apitools_request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def GetProjectServiceAccount(self, project_number): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsServiceAccountGetRequest( projectId=six.text_type(project_number)) return self.api_client.projects_serviceAccount.Get(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def CreateNotificationConfig(self, bucket_name, pubsub_topic, payload_format, event_types=None, custom_attributes=None, object_name_prefix=None): """See CloudApi class for function doc strings.""" try: notification = apitools_messages.Notification( topic=pubsub_topic, payload_format=payload_format) if event_types: notification.event_types = event_types if custom_attributes: additional_properties = [ NotificationAdditionalProperty(key=key, value=value) for key, value in custom_attributes.items() ] notification.custom_attributes = NotificationCustomAttributesValue( additionalProperties=additional_properties) if object_name_prefix: notification.object_name_prefix = object_name_prefix request = apitools_messages.StorageNotificationsInsertRequest( bucket=bucket_name, notification=notification, userProject=self.user_project) return self.api_client.notifications.Insert(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def DeleteNotificationConfig(self, bucket_name, notification): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageNotificationsDeleteRequest( bucket=bucket_name, notification=notification, userProject=self.user_project) return self.api_client.notifications.Delete(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def ListNotificationConfigs(self, bucket_name): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageNotificationsListRequest( bucket=bucket_name, userProject=self.user_project) response = self.api_client.notifications.List(request) for notification in response.items: yield notification except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def CreateHmacKey(self, project_id, service_account_email): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsHmacKeysCreateRequest( projectId=project_id, serviceAccountEmail=service_account_email) return self.api_client.hmacKeys.Create(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def DeleteHmacKey(self, project_id, access_id): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsHmacKeysDeleteRequest( projectId=project_id, accessId=access_id) return self.api_client.hmacKeys.Delete(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def GetHmacKey(self, project_id, access_id): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsHmacKeysGetRequest( projectId=project_id, accessId=access_id) return self.api_client.hmacKeys.Get(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def ListHmacKeys(self, project_id, service_account_email, show_deleted_keys=True): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsHmacKeysListRequest( projectId=project_id, serviceAccountEmail=service_account_email, showDeletedKeys=show_deleted_keys, maxResults=NUM_BUCKETS_PER_LIST_PAGE) response = self.api_client.hmacKeys.List(request) for key in response.items: yield key except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) while response.nextPageToken: request = apitools_messages.StorageProjectsHmacKeysListRequest( projectId=project_id, serviceAccountEmail=service_account_email, showDeletedKeys=show_deleted_keys, pageToken=response.nextPageToken, maxResults=NUM_BUCKETS_PER_LIST_PAGE) try: response = self.api_client.hmacKeys.List(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) for key in response.items: yield key def UpdateHmacKey(self, project_id, access_id, state, etag): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsHmacKeysUpdateRequest( projectId=project_id, accessId=access_id) metadata = apitools_messages.HmacKeyMetadata() metadata.state = state if etag: metadata.etag = etag request.resource = metadata return self.api_client.hmacKeys.Update(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def _BucketCannedAclToPredefinedAcl(self, canned_acl_string): """Translates the input string to a bucket PredefinedAcl string. Args: canned_acl_string: Canned ACL string. Returns: String that can be used as a query parameter with the JSON API. This corresponds to a flavor of *PredefinedAclValueValuesEnum and can be used as input to apitools requests that affect bucket access controls. """ if canned_acl_string in _BUCKET_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION: return _BUCKET_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION[canned_acl_string] raise ArgumentException('Invalid canned ACL %s' % canned_acl_string) def _ObjectCannedAclToPredefinedAcl(self, canned_acl_string): """Translates the input string to an object PredefinedAcl string. Args: canned_acl_string: Canned ACL string. Returns: String that can be used as a query parameter with the JSON API. This corresponds to a flavor of *PredefinedAclValueValuesEnum and can be used as input to apitools requests that affect object access controls. """ if canned_acl_string in _OBJECT_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION: return _OBJECT_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION[canned_acl_string] raise ArgumentException('Invalid canned ACL %s' % canned_acl_string) def _ValidateHttpAccessTokenRefreshError(self, e): if (isinstance(e, oauth2client.client.HttpAccessTokenRefreshError) and not (e.status == 429 or e.status >= 500)): raise def _TranslateExceptionAndRaise(self, e, bucket_name=None, object_name=None, generation=None, not_found_exception=None): """Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. bucket_name: Optional bucket name in request that caused the exception. object_name: Optional object name in request that caused the exception. generation: Optional generation in request that caused the exception. not_found_exception: Optional exception to raise in the not-found case. Raises: Translated CloudApi exception, or the original exception if it was not translatable. """ if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('TranslateExceptionAndRaise: %s', traceback.format_exc()) translated_exception = self._TranslateApitoolsException( e, bucket_name=bucket_name, object_name=object_name, generation=generation, not_found_exception=not_found_exception) if translated_exception: raise translated_exception else: raise def _GetMessageFromHttpError(self, http_error): if isinstance(http_error, apitools_exceptions.HttpError): if getattr(http_error, 'content', None): try: json_obj = json.loads(six.ensure_str(http_error.content)) if 'error' in json_obj and 'message' in json_obj['error']: return json_obj['error']['message'] except Exception: # pylint: disable=broad-except # If we couldn't decode anything, just leave the message as None. pass def _GetAcceptableScopesFromHttpError(self, http_error): try: www_authenticate = http_error.response['www-authenticate'] # In the event of a scope error, the www-authenticate field of the HTTP # response should contain text of the form # # 'Bearer realm="https://oauth2.googleapis.com/", # error=insufficient_scope, scope="${space separated list of acceptable # scopes}"' # # Here we use a quick string search to find the scope list, just looking # for a substring with the form 'scope="${scopes}"'. scope_idx = www_authenticate.find('scope="') if scope_idx >= 0: scopes = www_authenticate[scope_idx:].split('"')[1] return 'Acceptable scopes: %s' % scopes except Exception: # pylint: disable=broad-except # Return None if we have any trouble parsing out the acceptable scopes. pass def _TranslateApitoolsResumableUploadException(self, e): if isinstance(e, apitools_exceptions.HttpError): message = self._GetMessageFromHttpError(e) if e.status_code >= 500: return ResumableUploadException(message or 'Server Error', status=e.status_code) elif e.status_code == 429: return ResumableUploadException(message or 'Too Many Requests', status=e.status_code) elif e.status_code == 410: return ResumableUploadStartOverException(message or 'Bad Request', status=e.status_code) elif e.status_code == 404: return ResumableUploadStartOverException(message or 'Bad Request', status=e.status_code) elif e.status_code >= 400: return ResumableUploadAbortException(message or 'Bad Request', status=e.status_code) if isinstance(e, apitools_exceptions.StreamExhausted): return ResumableUploadAbortException( '%s; if this issue persists, try deleting the tracker files present' ' under ~/.gsutil/tracker-files/' % str(e)) if isinstance(e, apitools_exceptions.TransferError): if ('Aborting transfer' in str(e) or 'Not enough bytes in stream' in str(e)): return ResumableUploadAbortException(str(e)) elif 'additional bytes left in stream' in str(e): return ResumableUploadAbortException( '%s; this can happen if a file changes size while being uploaded' % str(e)) def _TranslateApitoolsException(self, e, bucket_name=None, object_name=None, generation=None, not_found_exception=None): """Translates apitools exceptions into their gsutil Cloud Api equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. bucket_name: Optional bucket name in request that caused the exception. object_name: Optional object name in request that caused the exception. generation: Optional generation in request that caused the exception. not_found_exception: Optional exception to raise in the not-found case. Returns: ServiceException for translatable exceptions, None otherwise. """ if isinstance(e, apitools_exceptions.HttpError): message = self._GetMessageFromHttpError(e) if e.status_code == 400: # It is possible that the Project ID is incorrect. Unfortunately the # JSON API does not give us much information about what part of the # request was bad. return BadRequestException(message or 'Bad Request', status=e.status_code) elif e.status_code == 401: if 'Login Required' in str(e): return AccessDeniedException(message or 'Access denied: login required.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) elif e.status_code == 403: if 'The account for the specified project has been disabled' in str(e): return AccessDeniedException(message or 'Account disabled.', status=e.status_code) elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e): return AccessDeniedException(message or 'Access denied: quota exceeded. ' 'Is your project ID valid?', status=e.status_code) elif 'The bucket you tried to delete was not empty.' in str(e): return NotEmptyException('BucketNotEmpty (%s)' % bucket_name, status=e.status_code) elif ('The bucket you tried to create requires domain ownership ' 'verification.' in str(e)): return AccessDeniedException( 'The bucket you tried to create requires domain ownership ' 'verification. Please see ' 'https://cloud.google.com/storage/docs/naming' '?hl=en#verification for more details.', status=e.status_code) elif 'User Rate Limit Exceeded' in str(e): return AccessDeniedException( 'Rate limit exceeded. Please retry this ' 'request later.', status=e.status_code) elif 'Access Not Configured' in str(e): return AccessDeniedException( 'Access Not Configured. Please go to the Google Cloud Platform ' 'Console (https://cloud.google.com/console#/project) for your ' 'project, select APIs and Auth and enable the ' 'Google Cloud Storage JSON API.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) elif 'does not have permission to publish messages' in str(e): return PublishPermissionDeniedException(message, status=e.status_code) else: return AccessDeniedException(message or str(e), status=e.status_code) elif e.status_code == 404: if not_found_exception: # The exception is pre-constructed prior to translation; the HTTP # status code isn't available at that time. setattr(not_found_exception, 'status', e.status_code) return not_found_exception elif bucket_name: if object_name: return CreateObjectNotFoundException(e.status_code, self.provider, bucket_name, object_name, generation=generation) return CreateBucketNotFoundException(e.status_code, self.provider, bucket_name) return NotFoundException(message or e.message, status=e.status_code) elif e.status_code == 409 and bucket_name: if 'The bucket you tried to delete is not empty.' in str(e): return NotEmptyException('BucketNotEmpty (%s)' % bucket_name, status=e.status_code) return ServiceException( 'A Cloud Storage bucket named \'%s\' already exists. Try another ' 'name. Bucket names must be globally unique across all Google Cloud ' 'projects, including those outside of your ' 'organization.' % bucket_name, status=e.status_code) elif e.status_code == 412: return PreconditionException(message, status=e.status_code) return ServiceException(message, status=e.status_code) elif isinstance(e, apitools_exceptions.TransferInvalidError): return ServiceException('Transfer invalid (possible encoding error: %s)' % str(e)) progress_callback.py 0000644 00000014121 15025013616 0010574 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for progress callbacks.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import time from gslib.thread_message import ProgressMessage from gslib.utils import parallelism_framework_util # Default upper and lower bounds for progress callback frequency. _START_BYTES_PER_CALLBACK = 1024 * 256 _MAX_BYTES_PER_CALLBACK = 1024 * 1024 * 100 _TIMEOUT_SECONDS = 1 # Max width of URL to display in progress indicator. Wide enough to allow # 15 chars for x/y display on an 80 char wide terminal. MAX_PROGRESS_INDICATOR_COLUMNS = 65 class ProgressCallbackWithTimeout(object): """Makes progress callbacks at least once every _TIMEOUT_SECONDS. This prevents wrong throughput calculation while not being excessively overwhelming. """ def __init__(self, total_size, callback_func, start_bytes_per_callback=_START_BYTES_PER_CALLBACK, timeout=_TIMEOUT_SECONDS): """Initializes the callback with timeout. Args: total_size: Total bytes to process. If this is None, size is not known at the outset. callback_func: Func of (int: processed_so_far, int: total_bytes) used to make callbacks. start_bytes_per_callback: Lower bound of bytes per callback. timeout: Number maximum of seconds without a callback. """ self._bytes_per_callback = start_bytes_per_callback self._callback_func = callback_func self._total_size = total_size self._last_time = time.time() self._timeout = timeout self._bytes_processed_since_callback = 0 self._callbacks_made = 0 self._total_bytes_processed = 0 def Progress(self, bytes_processed): """Tracks byte processing progress, making a callback if necessary.""" self._bytes_processed_since_callback += bytes_processed cur_time = time.time() if (self._bytes_processed_since_callback > self._bytes_per_callback or (self._total_size is not None and self._total_bytes_processed + self._bytes_processed_since_callback >= self._total_size) or (self._last_time - cur_time) > self._timeout): self._total_bytes_processed += self._bytes_processed_since_callback # TODO: We check if >= total_size and truncate because JSON uploads count # multipart metadata during their send progress. If the size is unknown, # we can't do this and the progress message will make it appear that we # send more than the original stream. if self._total_size is not None: bytes_sent = min(self._total_bytes_processed, self._total_size) else: bytes_sent = self._total_bytes_processed self._callback_func(bytes_sent, self._total_size) self._bytes_processed_since_callback = 0 self._callbacks_made += 1 self._last_time = cur_time class FileProgressCallbackHandler(object): """Tracks progress info for large operations like file copy or hash. Information is sent to the status_queue, which will print it in the UI Thread. """ def __init__(self, status_queue, start_byte=0, override_total_size=None, src_url=None, component_num=None, dst_url=None, operation_name=None): """Initializes the callback handler. Args: status_queue: Queue for posting status messages for UI display. start_byte: The beginning of the file component, if one is being used. override_total_size: The size of the file component, if one is being used. src_url: FileUrl/CloudUrl representing the source file. component_num: Indicates the component number, if any. dst_url: FileUrl/CloudUrl representing the destination file, or None for unary operations like hashing. operation_name: String representing the operation name """ self._status_queue = status_queue self._start_byte = start_byte self._override_total_size = override_total_size self._component_num = component_num self._src_url = src_url self._dst_url = dst_url self._operation_name = operation_name # Ensures final newline is written once even if we get multiple callbacks. self._last_byte_written = False # Function signature is in boto callback format, which cannot be changed. def call( self, # pylint: disable=invalid-name last_byte_processed, total_size): """Gathers information describing the operation progress. Actual message is printed to stderr by UIThread. Args: last_byte_processed: The last byte processed in the file. For file components, this number should be in the range [start_byte:start_byte + override_total_size]. total_size: Total size of the ongoing operation. """ if self._last_byte_written: return if self._override_total_size: total_size = self._override_total_size parallelism_framework_util.PutToQueueWithTimeout( self._status_queue, ProgressMessage(total_size, last_byte_processed - self._start_byte, self._src_url, time.time(), component_num=self._component_num, operation_name=self._operation_name, dst_url=self._dst_url)) if total_size and last_byte_processed - self._start_byte == total_size: self._last_byte_written = True tests/test_web.py 0000644 00000015044 15025013616 0010077 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for the webcfg command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import os from unittest import mock from gslib.commands import web import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.utils import shim_util WEBCFG_FULL = json.loads('{"notFoundPage": "404", "mainPageSuffix": "main"}\n') WEBCFG_MAIN = json.loads('{"mainPageSuffix": "main"}\n') WEBCFG_ERROR = json.loads('{"notFoundPage": "404"}\n') WEBCFG_EMPTY = 'has no website configuration' @SkipForS3('Web set not supported for S3, web get returns XML.') class TestWeb(testcase.GsUtilIntegrationTestCase): """Integration tests for the web command.""" _set_web_cmd = ['web', 'set'] _get_web_cmd = ['web', 'get'] def test_full(self): bucket_uri = self.CreateBucket() self.RunGsUtil( self._set_web_cmd + ['-m', 'main', '-e', '404', suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_web_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), WEBCFG_FULL) def test_main(self): bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_web_cmd + ['-m', 'main', suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_web_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), WEBCFG_MAIN) def test_error(self): bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_web_cmd + ['-e', '404', suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_web_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), WEBCFG_ERROR) def test_empty(self): bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_web_cmd + [suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_web_cmd + [suri(bucket_uri)], return_stdout=True) self.assertIn(WEBCFG_EMPTY, stdout) def testTooFewArgumentsFails(self): """Ensures web commands fail with too few arguments.""" # No arguments for get, but valid subcommand. stderr = self.RunGsUtil(self._get_web_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for set, but valid subcommand. stderr = self.RunGsUtil(self._set_web_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Neither arguments nor subcommand. stderr = self.RunGsUtil(['web'], return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) class TestWebShim(testcase.ShimUnitTestBase): @mock.patch.object(web.WebCommand, '_GetWeb', new=mock.Mock()) def test_shim_translates_get_command(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('web', [ 'get', 'gs://bucket', ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage buckets describe' ' --format=gsutiljson[key=website_config,empty=\' has no website' ' configuration.\',empty_prefix_key=storage_url]' ' gs://bucket').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(web.WebCommand, '_SetWeb', new=mock.Mock()) def test_shim_translates_set_command(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('web', [ 'set', '-e', '404', '-m', 'main', 'gs://bucket', ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage buckets update' ' --web-error-page 404 --web-main-page-suffix main gs://bucket' ).format(shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(web.WebCommand, '_SetWeb', new=mock.Mock()) def test_shim_translates_clear_command(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('web', ['set', 'gs://bucket'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage buckets update' ' --clear-web-error-page --clear-web-main-page-suffix' ' gs://bucket').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) class TestWebOldAlias(TestWeb): _set_web_cmd = ['setwebcfg'] _get_web_cmd = ['getwebcfg'] tests/__pycache__/util.cpython-39.pyc 0000644 00000065455 15025013616 0013542 0 ustar 00 a +(Wg t � @ s� d dl mZ d dl mZ d dl mZ d dl mZ d dlmZ d dlZd dlZd dl Z d dl Z d dl Z d dl Z d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlZd dlZd dlZd d l m Z d d l m Z d dl Z d d l!m"Z" d d l!m#Z# d d l$m%Z% d dl&m'Z( d dl)m*Z* d dl+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5 e3�sd dl6Z6d dl7Z7dd� Z8dd� Z9dd� Z:dd� Z;e*�<� e=e*j>d �Z?e �@� ZAe%d d � �ZBe%d d � �ZCe%d d � �ZDe%d!d � �ZEd�d#d$�ZFd�d%d&�ZGd�d'd(�ZHd)ZIe1eI�ZJd*ZKe1eK�ZLd+ZMe1eM�ZNd,ZOe1eO�ZPd-ZQd.ZRd/ZSd0ZTd1ZUd2ZVd3ZWd4ZXd5ZYd6ZZd7Z[d8Z\d9Z]d:Z^d;Z_d<Z`d=Zad>Zbd>Zcd"Zdd"Zed?Zfd@ZgdAZhdBZidCZjdDZkG dEdF� dFel�ZmemjnfdGdH�ZodIdJ� ZpdKdL� Zqejr�sdMdNd��o ejr�sdMdOd�Ztejr�sdMdPdQ��udR�Zve-� Zwejr�sdMdSd�duZxejr�sdMdTd�duZyejr�sdUdVdW��z� dXkZ{dYdZ� Z|d[d\� Z}e}� Z~d]d^� Zd_d`� Z�dadb� Z�G dcdd� dde j��Z�e�� Z�G dedf� dfe j��Z�dgZ�dhdi� Z�djdk� Z�dldm� Z�dndo� Z�dpdq� Z�edrds� �Z�ed�dtdu��Z�edvdw� �Z�edxdy� �Z�dzd{� Z�d|d}� Z�d~d� Z�ed�d�� �Z�d�d�� Z�d�d�� Z�G d�d�� d�el�Z�G d�d�� d�el�Z�G d�d�� d�el�Z�G d�d�� d�e��Z�dS )�� )�absolute_import)�print_function)�division)�unicode_literals)�contextmanagerN)�urllib)� cStringIO)�KmsApi)�PopulateProjectId)�ResumableDownloadException)�ResumableUploadException)� LazyWrapper)� posix_util)�UsingCrcmodExtension�HasUserSpecifiedGsHost)�UTF8)�#Base64Sha256FromBase64EncryptionKey)� IS_WINDOWS)�MakeHumanReadablec C s� t dd� t�� D ��} d}|dk rx|| v r4|d7 }qzt�|� |d7 }W q ty` | Y S tyt Y qxY q0 qtd��d S )Nc S s g | ] }|j �qS � )�gr_gid)�.0�groupr r � /opt/gsutil/gslib/tests/util.py� <listcomp>B � z!GetInvalidGid.<locals>.<listcomp>i� l r T � zUnable to generate GID for )�sorted�grp�getgrall�getgrgid�KeyError� OverflowError� Exception)Zall_gid�gidr r r � GetInvalidGid@ s r% c s6 t � � t� } t| �dkr � S � fdd�t| �D �d S )Nr c s g | ]}|� kr|�qS r r �r �g�Z primary_gidr r r g r z$GetNonPrimaryGid.<locals>.<listcomp>r )� GetPrimaryGid� GetUserGroups�len�list)Z user_groupsr r( r �GetNonPrimaryGid\ s r- c C s t �� S �N)�os�getgidr r r r r) i s r) c C s t t� gdd� t�� D � �S )Nc S s g | ]}t � |jv r|j�qS r )� USER_NAME�gr_memr r& r r r r n r z!GetUserGroups.<locals>.<listcomp>)�setr) r r r r r r r* l s �r* � c C s t �t�jS r. )�pwd�getpwuid�USER_ID�pw_namer r r r �<lambda>s r r9 c C s t dd� t�� D ��d d S )Nc S s g | ] }|j �qS r )�pw_uid)r �userr r r r x r z <lambda>.<locals>.<listcomp>���r )r r5 �getpwallr r r r r9 x r c C s t � S r. )r% r r r r r9 } r c C s t � S r. )r* r r r r r9 � r Fc C sr |r d}nt j�ddd�}dd�|�ddg}tjdddt� g| | }t� s`tt j �g| }n|}d d � |D �S ) zFAdds config options to a list of strings defining a gsutil subcommand.F�GSUtil�use_gcloud_storagez-oz GSUtil:use_gcloud_storage={}z#GSUtil:hidden_shim_mode=no_fallbackz--testexceptiontraceszGSUtil:default_project_id=c S s g | ]}t �|��qS r )�six� ensure_str)r �partr r r r � r z$GetGsutilCommand.<locals>.<listcomp>) �boto�config�getbool�format�gslib� GSUTIL_PATHr �InvokedFromParFile�str�sys� executable)Z raw_command� force_gsutilr? Zgcloud_storage_settingZgsutil_commandZ#gsutil_command_with_executable_pathr r r �GetGsutilCommand� s* �����rN c C sv t j�� }|r|�|� t� }t�|�D ] \}}t�|�|t�|�< q(tt d�rVt j nd}t j | t j t j t j ||d�S )zCReturns a subprocess.Popen object for for running a gsutil command.�setsidN)�stdout�stderr�stdin�env� preexec_fn) r/ �environ�copy�update�dictr@ � iteritemsrA �hasattrrO � subprocess�Popen�PIPE)�cmd�env_varsrS �envstr�k�vrT r r r �GetGsutilSubprocess� s �rc c s� |d ur.t jr$t|t�s.|�t�}n |�t�}d|i}� fdd�}t jrRd|d< nt�d|�}|�� � j f i |��}t js�|� � zdd� |D �}W n t y� dd� |D �}Y n0 |S ) N�inputc s t �t �� j�tj� d S r. )r/ �killpg�getpgid�pid�signal�SIGKILLr ��processr r �Kill� s z$CommunicateWithTimeout.<locals>.Kill� �timeoutc S s g | ]}t �|��qS r �r@ � ensure_text�r �outputr r r r � r z*CommunicateWithTimeout.<locals>.<listcomp>c S s g | ]}t �|t�d ���qS )F)r@ rp �locale�getpreferredencodingrq r r r r � s �) r@ �PY3� isinstance�bytes�encoder � threading�Timer�start� communicate�cancel�UnicodeDecodeError)rk rR Z comm_kwargsrl �timerZc_outr rj r �CommunicateWithTimeout� s* � r� s, iMSM9eeXliDZHSBJZO71R98tfeW/+87VXTpk5chGd6Y=s, 4TSaQ3S4U+5oxAbByA7HgIigD51zfzGed/c03Ts2TXc=s, HO4Q2X28N/6SmuAJ1v1CTuJjf5emQcXf7YriKzT1gj0=s, U6zIErjZCK/IpIeDS0pJrDayqlZurY8M9dvPJU0SXI8=s, MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=s dnF5x6K/8ZZRzpfSlMMM+w==s barzN7UdGUp1E+RbVvZSTy1R8g==zCrcTMQ==s bar2zIk4lOfUiA+szcorNIotEMg==zQScXtg==s bar3z9iW6smjfu9hm0A//VQTQfw==zs0yUtQ==s bar4zkPCx6uZiUOU7W6E+cDCZFg==zZ4bwXg==s bar5z758XbXQOVkp8fTKMm83NXA==zle1zXQ==Tz&/tmp/.boto.parallel_upload_test_configzPThis sync will orphan file(s), please fix their permissions before trying again.z$Mode for %s won't allow read access.z+GID for %s doesn't exist on current system.z+UID for %s doesn't exist on current system.z,Insufficient access with uid/gid/mode for %sc @ s( e Zd ZdZdZdZdZdZdZdZ dS ) �KmsTestingResourceszCConstants for KMS resource names to be used in integration testing.z us-central1z$keyring-for-gsutil-integration-testsz key-for-gsutil-integration-testsz!key-for-gsutil-integration-tests2zkey-for-gsutil-no-authz-cryptokey-for-gsutil-integration-tests-%d%d%dN) �__name__� __module__� __qualname__�__doc__�KEYRING_LOCATION� KEYRING_NAME�CONSTANT_KEY_NAMEZCONSTANT_KEY_NAME2Z"CONSTANT_KEY_NAME_DO_NOT_AUTHORIZEZMUTABLE_KEY_NAME_TEMPLATEr r r r r� s r� c C sX t t�� �}|jtd�tjtjd�}|�|| �}t ddd|gdd�}t |�}t |� |S )zAEnsures test keys exist and that the service agent is authorized.N)�location�kms� authorizez-kT)rM ) r �logging� getLogger� CreateKeyRingr r� r� r� �CreateCryptoKeyrN rc r� )�key_name�kms_apiZ keyring_fully_qualified_nameZkey_fully_qualified_namer^ rk r r r �"AuthorizeProjectToUseTestingKmsKey) s �� �r� c C s t �|t| � �S )a Builds a regex to match a file name for a file that would be orphaned. Args: obj: Object uri. err_str: The error string to search for. Returns: A regex that will match the file name and with the error text for a file that would be orphaned. )�re�compile� ObjectToURI)�objZerr_strr r r �BuildErrorRegex< s r� c s t � fdd�|�� �d�D ��S )a� Returns set of object name tails. Tails can be compared between source and dest, past the point at which the command was done. For example if test ran {cp,mv,rsync} gs://bucket1/dir gs://bucket2/dir2, the tails for listings from bucket1 would start after "dir", while the tails for listings from bucket2 would start after "dir2". Args: start_point: The target of the cp command, e.g., for the above command it would be gs://bucket1/dir for the bucket1 listing results and gs://bucket2/dir2 for the bucket2 listing results. listing: The listing over which to compute tail. Returns: Object name tails. c 3 s | ]}|t � �d � V qd S r. )r+ )r �l�� start_pointr r � <genexpr>\ r zTailSet.<locals>.<genexpr>� )r3 �strip�split)r� �listingr r� r �TailSetJ s r� � Credentials�aws_access_key_id�aws_secret_access_key�gs_service_key_file� z.p12�gs_host�gs_portr> � prefer_api�json�XMLc C s t rt�| �S dd� S d S )Nc S s | S r. r ��funcr r r r9 r r z!SkipForP12Creds.<locals>.<lambda>)� HAS_P12_CREDS�unittest�skip��reasonr r r �SkipForP12Credsn s r� c C s0 d } t s(z dd l} W n ty& Y n0 | d uS )Nr )r � argcomplete� ImportError)r� r r r �_ArgcompleteAvailablet s r� c C sl | � dd�} ttj�| ��}t�|d �|d< |d �d�rP|d dd� |d< tj�|�}|� dd�}|S )z�Normalizes the path component of a URI. Args: uri: URI to normalize. Returns: Normalized URI. Examples: gs://foo//bar -> gs://foo/bar gs://foo/./bar -> gs://foo/bar zgs://zfile://� z//r N) �replacer, r �parse�urlparse� posixpath�normpath� startswith� urlunparse)�uri�parsedZunparsedr r r � _NormalizeURI� s r� c C s& | j s| js| jdkrdS | j p$| jS )z�Returns a the generation for a StorageUri. Args: uri: boto.storage_uri.StorageURI object to get the URI from. Returns: Generation string for the URI. �s3�null)� generation� version_id�scheme)r� r r r �GenerationFromURI� s r� c G s� t | �r,d�tj�tjj| jg|�R � ��S t| tj �rRd�tjj| g|�R � �S t� | j �}|r�dd� |D �}t d�|g| ��}|� d�r�|dd� }|S )a� Returns the storage URI string for a given StorageUri or file object. Args: obj: The object to get the URI from. Can be a file object, a subclass of boto.storage_uri.StorageURI, or a string. If a string, it is assumed to be a local on-disk path. *suffixes: Suffixes to append. For example, ObjectToUri(bucketuri, 'foo') would return the URI for a key name 'foo' inside the given bucket. Returns: Storage URI string. z file://{}c S s g | ]}t �|��qS r ro )r �suffixr r r r � r z ObjectToURI.<locals>.<listcomp>�/Nr< )�is_filerF r/ �path�abspath�join�namerv r@ � string_typesrp r� r� �endswith)r� �suffixesr� Z suffixes_listr r r r� � s � r� c s e Zd Z� fdd�Z� ZS )�GSMockConnectionc s( d|d< d| _ tt| �j|i |�� d S )N�gs�providerr )�debug�superr� �__init__)�self�args�kwargs�� __class__r r r� � s zGSMockConnection.__init__)r� r� r� r� � __classcell__r r r� r r� � s r� c @ s\ e Zd Zddd�Zddd�Zddd�Zddd �Zdd d �Zdd d �Zddd�Z ddd�Z dS )�GSMockBucketStorageUriNc C s t S r. )�mock_connection)r� � access_key_id�secret_access_keyr r r �connect� s z GSMockBucketStorageUri.connectc C s | � � S )z:Dummy implementation to allow parallel uploads with tests.)�new_key)r� � components�headersr r r �compose� s z GSMockBucketStorageUri.composec C s dS )N�USr �r� r� r r r � get_location� s z#GSMockBucketStorageUri.get_locationc C s t jj�� S r. )rC r� �cors�Corsr� r r r �get_cors� s z GSMockBucketStorageUri.get_corsc C s t jj�� S r. )rC r� �encryptionconfig�EncryptionConfigr� r r r �get_encryption_config� s z,GSMockBucketStorageUri.get_encryption_configc C s d S r. r r� r r r �get_lifecycle_config� s z+GSMockBucketStorageUri.get_lifecycle_configc C s d S r. r r� r r r �get_website_config� s z)GSMockBucketStorageUri.get_website_configc C s d S r. r r� r r r �get_versioning_config� s z,GSMockBucketStorageUri.get_versioning_config)NN)N)N)N)N)N)N)N) r� r� r� r� r� r� r� r� r� r� r� r r r r r� � s r� ZTestRemoveSectionc C st t j�| |d�}t j�| �s8|�| tdf� t j�| � |�| ||f� |du r`t j�| |� nt j�| ||� dS )a� Sets boto configuration temporarily for testing. SetBotoConfigForTest should be called by tests instead of this function. This will ensure that the configuration is reverted to its original setting using _RevertBotoConfig. Args: section: Boto config section to set name: Boto config name to set value: Value to set revert_list: List for tracking configs to revert. N) rC rD �get� has_section�append�TEST_BOTO_REMOVE_SECTION� add_section� remove_optionr3 )�sectionr� �value� revert_listZ prev_valuer r r �_SetBotoConfig� s r� c C sj g }| D ]F\}}}|du r>|t kr.|�|� qNtj�||� qtj�|||� q|D ]}tj�|� qTdS )z�Reverts boto config modifications made by _SetBotoConfig. Args: revert_list: List of boto config modifications created by calls to _SetBotoConfig. N)r� r� rC rD r� r3 �remove_section)r� Zsections_to_remover� r� r� r r r �_RevertBotoConfig s r� c s t �� �� fdd��}|S )a` Decorator for tests that perform file to object transfers, or vice versa. This forces the test to run once normally, and again with special boto config settings that will ensure that the test follows the parallel composite upload and/or sliced object download code paths. Args: func: Function to wrap. Returns: Wrapped function. c sV � | i |�� t sRt� rRtg d��� � | i |�� W d � n1 sH0 Y d S )N))r> �#parallel_composite_upload_threshold�1)r> � sliced_object_download_thresholdr� )r> �%sliced_object_download_max_components�3)r> � check_hashes�always)� RUN_S3_TESTSr �SetBotoConfigForTest)r� r� r� r r �Wrapper1 s z.SequentialAndParallelTransfer.<locals>.Wrapper)� functools�wraps)r� r r r� r � SequentialAndParallelTransfer# s r c C sR i }| D ]D}|d |d |d }}}||vr8i ||< |dur||| |< q|S )a� Converts the input config list to a dict that is easy to write to a file. This is used to reset the boto config contents for a test instead of preserving the existing values. Args: boto_config_list: list of tuples of: (boto config section to set, boto config name to set, value to set) If value to set is None, no entry is created. Returns: Dictionary of {section: {keys: values}} for writing to the file. r r r� Nr )�boto_config_list�sectionsZ config_entryr� �keyr� r r r �_SectionDictFromConfigListC s r c C sv t |d��X}t�| �D ]<\}}|�d| � t�|�D ]\}}|�d||f � q6qW d � n1 sh0 Y dS )zFWrites a section dict from _SectionDictFromConfigList to tmp_filename.�wz[%s] z%s = %s N)�openr@ rY �write)Z section_dict� tmp_filename�tmp_filer� Zkey_value_pairsr r� r r r �_WriteSectionDictToFile\ s r c c s4 t dg�� dV W d � n1 s&0 Y dS )zHSets a dummy project in boto config for the duration of a 'with' clause.)r> �default_project_idZ dummy_projN)r r r r r �SetDummyProjectForUnitTeste s r c c sN g }d}�z t jdd�\}}t�|� |r�| D ]<}|d }tjrTt|t�rT|�t �}t |d |d ||� q.t |d�� }t j �|� W d � q�1 s�0 Y ntt| �|� t|�� dV W d � n1 s�0 Y W t|� |�rJzt�|� W n t�y Y n0 n4t|� |�rHzt�|� W n t�yF Y n0 0 dS )aA Sets the input list of boto configs for the duration of a 'with' clause. This preserves any existing boto configuration unless it is overwritten in the provided boto_config_list. Args: boto_config_list: list of tuples of: (boto config section to set, boto config name to set, value to set) use_existing_config: If True, apply boto_config_list to the existing configuration, preserving any original values unless they are overwritten. Otherwise, apply boto_config_list to a blank configuration. Yields: Once after config is set. Nzgsutil-temp-cfg)�prefixr� r r r )�tempfile�mkstempr/ �closer@ ru rv rw �decoder r� r rC rD r r r �_SetBotoConfigFileForTestr� �remove�OSError)r Zuse_existing_configZrevert_configsr Ztmp_fd� boto_configZ boto_valuer r r r r n sB � ,� &�r c # sT � fdd�}i � | D ]}t j�|�� |< qz|| � dV W |� � n |� � 0 dS )z0Sets OS environment variables for a single test.c sN t �| �D ]>\}}tj�|�� |< |d ur6|tj|< q |tjv r tj|= q d S r. )r@ rY r/ rU r� )Z dict_to_applyra rb �Z old_valuesr r �_ApplyDictToEnvironment� s z6SetEnvironmentForTest.<locals>._ApplyDictToEnvironmentN)r/ rU r� )Zenv_variable_dictr! ra r r r �SetEnvironmentForTest� s r" c c s� zt jd }d}W n ty( d}Y n0 | t jd< z(dV W |rN|t jd< q~t j�dd� n |rn|t jd< nt j�dd� 0 dS )a� Sets a given file as the boto config file for a single test. This function applies only the configuration in boto_config_path and will ignore existing configuration. It should not be called directly by tests; instead, use SetBotoConfigForTest. Args: boto_config_path: Path to config file to use. Yields: When configuration has been applied, and again when reverted. � BOTO_CONFIGTFN)r/ rU r! �pop)Zboto_config_pathZ old_boto_config_env_variableZboto_config_was_setr r r r � s � r c C sH t �d�} g }t�tj�D ](\}}}| �|�}|r|�|�d�� q|S )z?Returns a list of the names of the test modules in gslib.tests.z^test_(?P<name>.*)$r� ) r� r� �pkgutil� iter_modules� gslib_tests�__path__�matchr� r )�matcher�names�_�modname�mr r r � GetTestNames� s r/ c C s t jrt| t�S t| tj�S r. )r@ �PY2rv �file�io�IOBase)r� r r r r� � s r� c C s8 t | tjtjf�r"| �dd��� S td�t| ����dS )a Returns a copy of the given name with any invalid characters replaced. Args: name Union[str, unicode, bytes]: The bucket name to transform into a valid name. Returns: Union[str, unicode, bytes] The version of the bucket name containing only valid characters. r, �-z*Unable to format name. Incorrect Type: {0}N) rv r@ � text_type� binary_typer� �lower� TypeErrorrF �type)r� r r r �MakeBucketNameValid� s �r: c c sh d}z t �� }W n ty" Y n0 | r2t �| � z dV W | rd|rdt �|� n| rb|rbt �|� 0 dS )a Changes the working directory for the duration of a 'with' call. Args: new_working_directory: The directory to switch to before executing wrapped code. A None value indicates that no switching is necessary. Yields: Once after working directory has been changed. N)r/ �getcwdr �chdir)Znew_working_directoryZprev_working_directoryr r r �WorkingDirectory� s �r= c C s" t � �dd �} | sdS d| jjv S )N� __loader__F� zipimport)�globalsr� r� r� )�loaderr r r rI s rI c C s t � rt�| �S dd� S d S )Nc S s | S r. r r� r r r r9 r z SkipForParFile.<locals>.<lambda>)rI r� r� r� r r r �SkipForParFile s rB c @ s e Zd ZdZdd� Zdd� ZdS )�HaltingCopyCallbackHandlerzFTest callback handler for intentionally stopping a resumable transfer.c C s || _ || _d S r. )� _is_upload� _halt_at_byte)r� � is_upload� halt_at_byter r r r� & s z#HaltingCopyCallbackHandler.__init__c C sF || j krBtj�d| j t|�t|�f � | jr:td��ntd��dS )aD Forcibly exits if the transfer has passed the halting point. Note that this function is only called when the conditions in gslib.progress_callback.ProgressCallbackWithTimeout.Progress are met, so self._halt_at_byte is only precise if it's divisible by gslib.progress_callback._START_BYTES_PER_CALLBACK. z4Halting transfer after byte %s. %s/%s transferred. zArtifically halting upload.� Artifically halting download.N)rE rK rQ r r rD r r )r� �total_bytes_transferred� total_sizer r r �call+ s ��� z HaltingCopyCallbackHandler.callN�r� r� r� r� r� rK r r r r rC # s rC c @ s e Zd ZdZdd� Zdd� ZdS )�#HaltOneComponentCopyCallbackHandlerz=Test callback handler for stopping part of a sliced download.c C s d | _ || _d S r. )�_last_progress_byterE )r� rG r r r r� A s z,HaltOneComponentCopyCallbackHandler.__init__c C sD | j dur:| j | j k r"|k r:n ntj�d� td��|| _ dS )zCForcibly exits if the passed the halting point since the last call.NzHalting transfer. rH )rN rE rK rQ r r )r� Zcurrent_progress_byteZtotal_size_unusedr r r rK G s �� z(HaltOneComponentCopyCallbackHandler.callNrL r r r r rM > s rM c @ s e Zd ZdZddd�ZdS )� TestParamsak Allows easier organization of test parameters. This class allows grouping of test parameters, which include args and kwargs to be used, as well as the expected result based on those arguments. For example, to test an Add function, one might do: params = TestParams(args=(1, 2, 3), expected=6) self.assertEqual(Add(*(params.args)), params.expected) Nc C s\ |d u rt � n|| _|d u r"t� n|| _|| _t|t tf�sDtd��t| jt�sXtd��d S )Nz'TestParam args must be a tuple or list.z TestParam kwargs must be a dict.)�tupler� rX r� �expectedrv r, r8 )r� r� r� rQ r r r r� \ s zTestParams.__init__)NNN)r� r� r� r� r� r r r r rO P s rO c @ s e Zd ZdZdd� Zdd� ZdS )� CaptureStdoutz�Context manager. For example, this function has the lines printed by the function call stored as a list in output: with CaptureStdout() as output: function(input_to_function) c C s t j| _t� t _| _| S r. )rK rP �_stdoutr � _stringio)r� r r r � __enter__q s zCaptureStdout.__enter__c G s$ | � | j�� �� � | `| jt_d S r. )�extendrT �getvalue� splitlinesrS rK rP )r� r� r r r �__exit__v s zCaptureStdout.__exit__N)r� r� r� r� rU rY r r r r rR g s rR )F)N)N)T)�� __future__r r r r � contextlibr r rs r� r/ r% r� r� r2 rh r[ rK r ry r� r@ � six.movesr r rC �crcmodrG � gslib.kms_apir �gslib.project_idr Zmock_storage_service�gslib.cloud_apir r �gslib.lazy_wrapperr � gslib.tests�testsr' Z gslib.utilsr Zgslib.utils.boto_utilr r �gslib.utils.constantsr � gslib.utils.encryption_helperr �gslib.utils.system_utilr �gslib.utils.unit_utilr r r5 r% r- r) r* �InitializeDefaultMode�int�SYSTEM_POSIX_MODE� DEFAULT_MODE�getuidr7 r1 Z INVALID_UIDZ INVALID_GID� USER_GROUPSrN rc r� ZTEST_ENCRYPTION_KEY1Z TEST_ENCRYPTION_KEY1_SHA256_B64ZTEST_ENCRYPTION_KEY2Z TEST_ENCRYPTION_KEY2_SHA256_B64ZTEST_ENCRYPTION_KEY3Z TEST_ENCRYPTION_KEY3_SHA256_B64ZTEST_ENCRYPTION_KEY4Z TEST_ENCRYPTION_KEY4_SHA256_B64ZTEST_ENCRYPTION_KEY_S3ZTEST_ENCRYPTION_KEY_S3_MD5ZTEST_ENCRYPTION_CONTENT1Z TEST_ENCRYPTION_CONTENT1_MD5Z TEST_ENCRYPTION_CONTENT1_CRC32CZTEST_ENCRYPTION_CONTENT2Z TEST_ENCRYPTION_CONTENT2_MD5Z TEST_ENCRYPTION_CONTENT2_CRC32CZTEST_ENCRYPTION_CONTENT3Z TEST_ENCRYPTION_CONTENT3_MD5Z TEST_ENCRYPTION_CONTENT3_CRC32CZTEST_ENCRYPTION_CONTENT4Z TEST_ENCRYPTION_CONTENT4_MD5Z TEST_ENCRYPTION_CONTENT4_CRC32CZTEST_ENCRYPTION_CONTENT5Z TEST_ENCRYPTION_CONTENT5_MD5Z TEST_ENCRYPTION_CONTENT5_CRC32C�RUN_INTEGRATION_TESTSZRUN_UNIT_TESTSr �USE_MULTIREGIONAL_BUCKETSZ%PARALLEL_COMPOSITE_UPLOAD_TEST_CONFIGZ ORPHANED_FILEZPOSIX_MODE_ERRORZPOSIX_GID_ERRORZPOSIX_UID_ERRORZ POSIX_INSUFFICIENT_ACCESS_ERROR�objectr� r� r� r� r� rD r� � HAS_S3_CREDSr� r� ZHAS_NON_DEFAULT_GS_HOSTZ HAS_GS_HOSTZ HAS_GS_PORT�upperZUSING_JSON_APIr� r� ZARGCOMPLETE_AVAILABLEr� r� r� ZMockConnectionr� r� ZMockBucketStorageUrir� r� r� r� r r r r r r" r r/ r� r: r= rI rB rC rM rO r, rR r r r r �<module> s, � &����� � �� . tests/__pycache__/__init__.cpython-39.pyc 0000644 00000000513 15025013616 0014304 0 ustar 00 a +(Wg( � @ s8 d Z ddlmZ ddlmZ ddlmZ ddlmZ dS )z Contains test cases for gsutil.� )�absolute_import)�print_function)�division)�unicode_literalsN)�__doc__� __future__r r r r � r r �#/opt/gsutil/gslib/tests/__init__.py�<module> s tests/test_metrics.py 0000644 00000147233 15025013616 0010776 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint:mode=test """Unit tests for analytics data collection.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import pickle import re import socket import subprocess import sys import tempfile import pprint import six from apitools.base.py import exceptions as apitools_exceptions from apitools.base.py import http_wrapper from boto.storage_uri import BucketStorageUri from gslib import metrics from gslib import VERSION from gslib.cs_api_map import ApiSelector import gslib.exception from gslib.gcs_json_api import GcsJsonApi from gslib.metrics import MetricsCollector from gslib.metrics_tuple import Metric from gslib.tests.mock_logging_handler import MockLoggingHandler import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import HAS_S3_CREDS from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SkipForParFile from gslib.tests.util import unittest from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import FileMessage from gslib.thread_message import RetryableErrorMessage from gslib.utils.constants import START_CALLBACK_PER_BYTES from gslib.utils.retry_util import LogAndHandleRetries from gslib.utils.system_util import IS_LINUX from gslib.utils.system_util import IS_WINDOWS from gslib.utils.unit_util import ONE_KIB from gslib.utils.unit_util import ONE_MIB from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock # A piece of the URL logged for all of the tests. GLOBAL_DIMENSIONS_URL_PARAMS = ( 'a=b&c=d&cd1=cmd1+action1&cd10=0&cd2=x%2Cy%2Cz&cd3=opta%2Coptb&' 'cd6=CommandException&cm1=0') GLOBAL_PARAMETERS = [ 'a=b', 'c=d', 'cd1=cmd1 action1', 'cd2=x,y,z', 'cd3=opta,optb', 'cd6=CommandException', 'cm1=0', 'ev=0', 'el={0}'.format(VERSION) ] COMMAND_AND_ERROR_TEST_METRICS = set([ Metric( 'https://example.com', 'POST', '{0}&cm2=3&ea=cmd1+action1&ec={1}&el={2}&ev=0'.format( GLOBAL_DIMENSIONS_URL_PARAMS, metrics._GA_COMMANDS_CATEGORY, VERSION), 'user-agent-007'), Metric( 'https://example.com', 'POST', '{0}&cm2=2&ea=Exception&ec={1}&el={2}&ev=0'.format( GLOBAL_DIMENSIONS_URL_PARAMS, metrics._GA_ERRORRETRY_CATEGORY, VERSION), 'user-agent-007'), Metric( 'https://example.com', 'POST', '{0}&cm2=1&ea=ValueError&ec={1}&el={2}&ev=0'.format( GLOBAL_DIMENSIONS_URL_PARAMS, metrics._GA_ERRORRETRY_CATEGORY, VERSION), 'user-agent-007'), Metric( 'https://example.com', 'POST', '{0}&ea=CommandException&ec={1}&el={2}&ev=0'.format( GLOBAL_DIMENSIONS_URL_PARAMS, metrics._GA_ERRORFATAL_CATEGORY, VERSION), 'user-agent-007') ]) # A regex to find the list of metrics in log output. METRICS_LOG_RE = re.compile(r'(\[Metric.*\])') def _TryExceptAndPass(func, *args, **kwargs): """Calls the given function with the arguments and ignores exceptions. In these tests, we often force a failure that doesn't matter in order to check that a metric was collected. Args: func: The function to call. *args: Any arguments to call the function with. **kwargs: Any named arguments to call the function with. """ try: func(*args, **kwargs) except: # pylint: disable=bare-except pass def _LogAllTestMetrics(): """Logs all the common metrics for a test.""" metrics.LogCommandParams(command_name='cmd1', subcommands=['action1'], global_opts=[('-y', 'value'), ('-z', ''), ('-x', '')], sub_opts=[('optb', ''), ('opta', '')]) retry_msg_1 = RetryableErrorMessage(Exception(), 0) retry_msg_2 = RetryableErrorMessage(ValueError(), 0) metrics.LogRetryableError(retry_msg_1) metrics.LogRetryableError(retry_msg_1) metrics.LogRetryableError(retry_msg_2) metrics.LogFatalError(gslib.exception.CommandException('test')) class RetryableErrorsQueue(object): """Emulates Cloud API status queue, processes only RetryableErrorMessages.""" def put(self, status_item): # pylint: disable=invalid-name if isinstance(status_item, RetryableErrorMessage): metrics.LogRetryableError(status_item) @SkipForParFile('Do not try spawning the interpreter nested in the archive.') @mock.patch('time.time', new=mock.MagicMock(return_value=0)) class TestMetricsUnitTests(testcase.GsUtilUnitTestCase): """Unit tests for analytics data collection.""" def setUp(self): super(TestMetricsUnitTests, self).setUp() # Save the original state of the collector. self.original_collector_instance = MetricsCollector.GetCollector() # Set dummy attributes for the collector. MetricsCollector.StartTestCollector('https://example.com', 'user-agent-007', { 'a': 'b', 'c': 'd' }) self.collector = MetricsCollector.GetCollector() self.log_handler = MockLoggingHandler() # Use metrics logger to avoid impacting the root logger which may # interfere with other tests. logging.getLogger('metrics').setLevel(logging.DEBUG) logging.getLogger('metrics').addHandler(self.log_handler) def tearDown(self): super(TestMetricsUnitTests, self).tearDown() # Reset to default collection settings. MetricsCollector.StopTestCollector( original_instance=self.original_collector_instance) def testDisabling(self): """Tests enabling/disabling of metrics collection.""" self.assertEqual(self.collector, MetricsCollector.GetCollector()) # Test when gsutil is part of the Cloud SDK and the user opted in there. with mock.patch.dict(os.environ, values={ 'CLOUDSDK_WRAPPER': '1', 'GA_CID': '555' }): MetricsCollector._CheckAndSetDisabledCache() self.assertFalse(MetricsCollector._disabled_cache) self.assertEqual(self.collector, MetricsCollector.GetCollector()) # Test that when using the shim analytics are disabled. with mock.patch('boto.config.getbool', return_value=True): MetricsCollector._CheckAndSetDisabledCache() self.assertTrue(MetricsCollector._disabled_cache) self.assertEqual(None, MetricsCollector.GetCollector()) # Test when gsutil is part of the Cloud SDK and the user did not opt in # there. with mock.patch.dict(os.environ, values={ 'CLOUDSDK_WRAPPER': '1', 'GA_CID': '' }): MetricsCollector._CheckAndSetDisabledCache() self.assertTrue(MetricsCollector._disabled_cache) self.assertEqual(None, MetricsCollector.GetCollector()) # Test when gsutil is not part of the Cloud SDK and there is no UUID file. with mock.patch.dict(os.environ, values={'CLOUDSDK_WRAPPER': ''}): with mock.patch('os.path.exists', return_value=False): MetricsCollector._CheckAndSetDisabledCache() self.assertTrue(MetricsCollector._disabled_cache) self.assertEqual(None, MetricsCollector.GetCollector()) # Test when gsutil is not part of the Cloud SDK and there is a UUID file. with mock.patch.dict(os.environ, values={'CLOUDSDK_WRAPPER': ''}): with mock.patch('os.path.exists', return_value=True): # Mock the contents of the file. if six.PY2: builtin_open = '__builtin__.open' else: builtin_open = 'builtins.open' with mock.patch(builtin_open) as mock_open: mock_open.return_value.__enter__ = lambda s: s # Set the file.read() method to return the disabled text. mock_open.return_value.read.return_value = metrics._DISABLED_TEXT MetricsCollector._CheckAndSetDisabledCache() self.assertTrue(MetricsCollector._disabled_cache) self.assertEqual(None, MetricsCollector.GetCollector()) # Set the file.read() method to return a mock cid (analytics enabled). mock_open.return_value.read.return_value = 'mock_cid' MetricsCollector._CheckAndSetDisabledCache() self.assertFalse(MetricsCollector._disabled_cache) self.assertEqual(self.collector, MetricsCollector.GetCollector()) # Check that open/read was called twice. self.assertEqual(2, len(mock_open.call_args_list)) self.assertEqual(2, len(mock_open.return_value.read.call_args_list)) def testConfigValueValidation(self): """Tests the validation of potentially PII config values.""" string_and_bool_categories = [ 'check_hashes', 'content_language', 'disable_analytics_prompt', 'https_validate_certificates', 'json_api_version', 'parallel_composite_upload_component_size', 'parallel_composite_upload_threshold', 'prefer_api', 'sliced_object_download_component_size', 'sliced_object_download_threshold', 'tab_completion_time_logs', 'token_cache', 'use_magicfile' ] int_categories = [ 'debug', 'default_api_version', 'http_socket_timeout', 'max_retry_delay', 'num_retries', 'oauth2_refresh_retries', 'parallel_process_count', 'parallel_thread_count', 'resumable_threshold', 'rsync_buffer_lines', 'sliced_object_download_max_components', 'software_update_check_period', 'tab_completion_timeout', 'task_estimation_threshold' ] all_categories = sorted(string_and_bool_categories + int_categories) # Test general invalid values. with mock.patch('boto.config.get_value', return_value=None): self.assertEqual('', self.collector._ValidateAndGetConfigValues()) with mock.patch('boto.config.get_value', return_value='invalid string'): self.assertEqual( ','.join([category + ':INVALID' for category in all_categories]), self.collector._ValidateAndGetConfigValues()) # Test that non-ASCII characters are invalid. with mock.patch('boto.config.get_value', return_value='£'): self.assertEqual( ','.join([category + ':INVALID' for category in all_categories]), self.collector._ValidateAndGetConfigValues()) # Mock valid return values for specific string validations. def MockValidStrings(section, category): if section == 'GSUtil': if category == 'check_hashes': return 'if_fast_else_skip' if category == 'content_language': return 'chi' if category == 'json_api_version': return 'v3' if category == 'prefer_api': return 'xml' if category in ('disable_analytics_prompt', 'use_magicfile', 'tab_completion_time_logs'): return 'True' if section == 'OAuth2' and category == 'token_cache': return 'file_system' if section == 'Boto' and category == 'https_validate_certificates': return 'True' return '' with mock.patch('boto.config.get_value', side_effect=MockValidStrings): self.assertEqual( 'check_hashes:if_fast_else_skip,content_language:chi,' 'disable_analytics_prompt:True,https_validate_certificates:True,' 'json_api_version:v3,prefer_api:xml,tab_completion_time_logs:True,' 'token_cache:file_system,use_magicfile:True', self.collector._ValidateAndGetConfigValues()) # Test that "small" and "large" integers are appropriately validated. def MockValidSmallInts(_, category): if category in int_categories: return '1999' return '' with mock.patch('boto.config.get_value', side_effect=MockValidSmallInts): self.assertEqual( 'debug:1999,default_api_version:1999,http_socket_timeout:1999,' 'max_retry_delay:1999,num_retries:1999,oauth2_refresh_retries:1999,' 'parallel_process_count:1999,parallel_thread_count:1999,' 'resumable_threshold:1999,rsync_buffer_lines:1999,' 'sliced_object_download_max_components:1999,' 'software_update_check_period:1999,tab_completion_timeout:1999,' 'task_estimation_threshold:1999', self.collector._ValidateAndGetConfigValues()) def MockValidLargeInts(_, category): if category in int_categories: return '2001' return '' with mock.patch('boto.config.get_value', side_effect=MockValidLargeInts): self.assertEqual( 'debug:INVALID,default_api_version:INVALID,' 'http_socket_timeout:INVALID,max_retry_delay:INVALID,' 'num_retries:INVALID,oauth2_refresh_retries:INVALID,' 'parallel_process_count:INVALID,parallel_thread_count:INVALID,' 'resumable_threshold:2001,rsync_buffer_lines:2001,' 'sliced_object_download_max_components:INVALID,' 'software_update_check_period:INVALID,' 'tab_completion_timeout:INVALID,task_estimation_threshold:2001', self.collector._ValidateAndGetConfigValues()) # Test that a non-integer return value is invalid. def MockNonIntegerValue(_, category): if category in int_categories: return '10.28' return '' with mock.patch('boto.config.get_value', side_effect=MockNonIntegerValue): self.assertEqual( ','.join([category + ':INVALID' for category in int_categories]), self.collector._ValidateAndGetConfigValues()) # Test data size validation. def MockDataSizeValue(_, category): if category in ('parallel_composite_upload_component_size', 'parallel_composite_upload_threshold', 'sliced_object_download_component_size', 'sliced_object_download_threshold'): return '10MiB' return '' with mock.patch('boto.config.get_value', side_effect=MockDataSizeValue): self.assertEqual( 'parallel_composite_upload_component_size:10485760,' 'parallel_composite_upload_threshold:10485760,' 'sliced_object_download_component_size:10485760,' 'sliced_object_download_threshold:10485760', self.collector._ValidateAndGetConfigValues()) def testCommandAndErrorEventsCollection(self): """Tests the collection of command and error GA events.""" self.assertEqual([], self.collector._metrics) _LogAllTestMetrics() # Only the first command should be logged. metrics.LogCommandParams(command_name='cmd2') # Commands and errors should not be collected until we explicitly collect # them. self.assertEqual([], self.collector._metrics) self.collector._CollectCommandAndErrorMetrics() self.assertEqual(COMMAND_AND_ERROR_TEST_METRICS, set(self.collector._metrics)) def testPerformanceSummaryEventCollection(self): """Test the collection of PerformanceSummary GA events.""" # PerformanceSummaries are only collected for cp and rsync. self.collector.ga_params[metrics._GA_LABEL_MAP['Command Name']] = 'cp' # GetDiskCounters is called at initialization of _PerformanceSummaryParams, # which occurs during the first call to LogPerformanceSummaryParams. with mock.patch('gslib.metrics.system_util.GetDiskCounters', return_value={'fake-disk': (0, 0, 0, 0, 0, 0)}): metrics.LogPerformanceSummaryParams(uses_fan=True, uses_slice=True, avg_throughput=10, is_daisy_chain=True, has_file_dst=False, has_cloud_dst=True, has_file_src=False, has_cloud_src=True, total_bytes_transferred=100, total_elapsed_time=10, thread_idle_time=40, thread_execution_time=10, num_processes=2, num_threads=3, num_objects_transferred=3, provider_types=['gs']) # Log a retryable service error and two retryable network errors. service_retry_msg = RetryableErrorMessage( apitools_exceptions.CommunicationError(), 0) network_retry_msg = RetryableErrorMessage(socket.error(), 0) metrics.LogRetryableError(service_retry_msg) metrics.LogRetryableError(network_retry_msg) metrics.LogRetryableError(network_retry_msg) # Log some thread throughput. start_file_msg = FileMessage('src', 'dst', 0, size=100) end_file_msg = FileMessage('src', 'dst', 10, finished=True) start_file_msg.thread_id = end_file_msg.thread_id = 1 start_file_msg.process_id = end_file_msg.process_id = 1 metrics.LogPerformanceSummaryParams(file_message=start_file_msg) metrics.LogPerformanceSummaryParams(file_message=end_file_msg) self.assertEqual( self.collector.perf_sum_params.thread_throughputs[(1, 1)].GetThroughput(), 10) # GetDiskCounters is called a second time during collection. with mock.patch('gslib.metrics.system_util.GetDiskCounters', return_value={'fake-disk': (0, 0, 0, 0, 10, 10)}): self.collector._CollectPerformanceSummaryMetric() # Check for all the expected parameters. metric_body = self.collector._metrics[0].body label_and_value_pairs = [ ('Event Category', metrics._GA_PERFSUM_CATEGORY), ('Event Action', 'CloudToCloud%2CDaisyChain'), ('Execution Time', '10'), ('Parallelism Strategy', 'both'), ('Source URL Type', 'cloud'), ('Provider Types', 'gs'), ('Num Processes', '2'), ('Num Threads', '3'), ('Number of Files/Objects Transferred', '3'), ('Size of Files/Objects Transferred', '100'), ('Average Overall Throughput', '10'), ('Num Retryable Service Errors', '1'), ('Num Retryable Network Errors', '2'), ('Thread Idle Time Percent', '0.8'), ('Slowest Thread Throughput', '10'), ('Fastest Thread Throughput', '10'), ] if IS_LINUX: # Disk I/O time is only available on Linux. label_and_value_pairs.append(('Disk I/O Time', '20')) for label, exp_value in label_and_value_pairs: self.assertIn('{0}={1}'.format(metrics._GA_LABEL_MAP[label], exp_value), metric_body) def testCommandCollection(self): """Tests the collection of command parameters.""" _TryExceptAndPass(self.command_runner.RunNamedCommand, 'acl', ['set', '-a'], collect_analytics=True) self.assertEqual( 'acl set', self.collector.ga_params.get(metrics._GA_LABEL_MAP['Command Name'])) self.assertEqual( 'a', self.collector.ga_params.get( metrics._GA_LABEL_MAP['Command-Level Options'])) # Reset the ga_params, which store the command info. self.collector.ga_params.clear() self.command_runner.RunNamedCommand('list', collect_analytics=True) self.assertEqual( 'ls', self.collector.ga_params.get(metrics._GA_LABEL_MAP['Command Name'])) self.assertEqual( 'list', self.collector.ga_params.get(metrics._GA_LABEL_MAP['Command Alias'])) self.collector.ga_params.clear() _TryExceptAndPass(self.command_runner.RunNamedCommand, 'iam', ['get', 'dummy_bucket'], collect_analytics=True) self.assertEqual( 'iam get', self.collector.ga_params.get(metrics._GA_LABEL_MAP['Command Name'])) # We only care about the error logging, not the actual exceptions handling. @mock.patch.object(http_wrapper, 'HandleExceptionsAndRebuildHttpConnections') def testRetryableErrorCollection(self, mock_default_retry): """Tests the collection of a retryable error in the retry function.""" # A DiscardMessagesQueue has the same retryable error-logging code as the # UIThread and the MainThreadUIQueue. mock_queue = RetryableErrorsQueue() value_error_retry_args = http_wrapper.ExceptionRetryArgs( None, None, ValueError(), None, None, None) socket_error_retry_args = http_wrapper.ExceptionRetryArgs( None, None, socket.error(), None, None, None) metadata_retry_func = LogAndHandleRetries(is_data_transfer=False, status_queue=mock_queue) media_retry_func = LogAndHandleRetries(is_data_transfer=True, status_queue=mock_queue) metadata_retry_func(value_error_retry_args) self.assertEqual(self.collector.retryable_errors['ValueError'], 1) metadata_retry_func(value_error_retry_args) self.assertEqual(self.collector.retryable_errors['ValueError'], 2) metadata_retry_func(socket_error_retry_args) if six.PY2: self.assertEqual(self.collector.retryable_errors['SocketError'], 1) else: self.assertEqual(self.collector.retryable_errors['OSError'], 1) # The media retry function raises an exception after logging because # the GcsJsonApi handles retryable errors for media transfers itself. _TryExceptAndPass(media_retry_func, value_error_retry_args) _TryExceptAndPass(media_retry_func, socket_error_retry_args) self.assertEqual(self.collector.retryable_errors['ValueError'], 3) if six.PY2: self.assertEqual(self.collector.retryable_errors['SocketError'], 2) else: self.assertEqual(self.collector.retryable_errors['OSError'], 2) def testExceptionCatchingDecorator(self): """Tests the exception catching decorator CaptureAndLogException.""" # A wrapped function with an exception should not stop the process. mock_exc_fn = mock.MagicMock(__name__=str('mock_exc_fn'), side_effect=Exception()) wrapped_fn = metrics.CaptureAndLogException(mock_exc_fn) wrapped_fn() debug_messages = self.log_handler.messages['debug'] self.assertIn('Exception captured in mock_exc_fn during metrics collection', debug_messages[0]) self.log_handler.reset() self.assertEqual(1, mock_exc_fn.call_count) mock_err_fn = mock.MagicMock(__name__=str('mock_err_fn'), side_effect=TypeError()) wrapped_fn = metrics.CaptureAndLogException(mock_err_fn) wrapped_fn() self.assertEqual(1, mock_err_fn.call_count) debug_messages = self.log_handler.messages['debug'] self.assertIn('Exception captured in mock_err_fn during metrics collection', debug_messages[0]) self.log_handler.reset() # Test that exceptions in the unprotected metrics functions are caught. with mock.patch.object(MetricsCollector, 'GetCollector', return_value='not a collector'): # These calls should all fail, but the exceptions shouldn't propagate up. metrics.Shutdown() metrics.LogCommandParams() metrics.LogRetryableError() metrics.LogFatalError() metrics.LogPerformanceSummaryParams() metrics.CheckAndMaybePromptForAnalyticsEnabling('invalid argument') debug_messages = self.log_handler.messages['debug'] message_index = 0 for func_name in ('Shutdown', 'LogCommandParams', 'LogRetryableError', 'LogFatalError', 'LogPerformanceSummaryParams', 'CheckAndMaybePromptForAnalyticsEnabling'): self.assertIn( 'Exception captured in %s during metrics collection' % func_name, debug_messages[message_index]) message_index += 1 self.log_handler.reset() # Mock callback handlers to throw errors in integration tests, based on handlers # from test_cp.py. class _JSONForceHTTPErrorCopyCallbackHandler(object): """Test callback handler that raises an arbitrary HTTP error exception.""" def __init__(self, startover_at_byte, http_error_num): self._startover_at_byte = startover_at_byte self._http_error_num = http_error_num self.started_over_once = False # pylint: disable=invalid-name def call(self, total_bytes_transferred, unused_total_size): """Forcibly exits if the transfer has passed the halting point.""" if (total_bytes_transferred >= self._startover_at_byte and not self.started_over_once): self.started_over_once = True raise apitools_exceptions.HttpError({'status': self._http_error_num}, None, None) class _ResumableUploadRetryHandler(object): """Test callback handler for causing retries during a resumable transfer.""" def __init__(self, retry_at_byte, exception_to_raise, exc_args, num_retries=1): self._retry_at_byte = retry_at_byte self._exception_to_raise = exception_to_raise self._exception_args = exc_args self._num_retries = num_retries self._retries_made = 0 # pylint: disable=invalid-name def call(self, total_bytes_transferred, unused_total_size): """Cause a single retry at the retry point.""" if (total_bytes_transferred >= self._retry_at_byte and self._retries_made < self._num_retries): self._retries_made += 1 raise self._exception_to_raise(*self._exception_args) @SkipForParFile('Do not try spawning the interpreter nested in the archive.') class TestMetricsIntegrationTests(testcase.GsUtilIntegrationTestCase): """Integration tests for analytics data collection.""" def setUp(self): super(TestMetricsIntegrationTests, self).setUp() # Save the original state of the collector. self.original_collector_instance = MetricsCollector.GetCollector() # Set dummy attributes for the collector. MetricsCollector.StartTestCollector('https://example.com', 'user-agent-007', { 'a': 'b', 'c': 'd' }) self.collector = MetricsCollector.GetCollector() def tearDown(self): super(TestMetricsIntegrationTests, self).tearDown() # Reset to default collection settings. MetricsCollector.StopTestCollector( original_instance=self.original_collector_instance) def _RunGsUtilWithAnalyticsOutput(self, cmd, expected_status=0): """Runs the gsutil command to check for metrics log output. The env value is set so that the metrics collector in the subprocess will use testing parameters and output the metrics collected to the debugging log, which lets us check for proper collection in the stderr. Args: cmd: The command to run, as a list. expected_status: The expected return code. Returns: The string of metrics output. """ stderr = self.RunGsUtil(['-d'] + cmd, return_stderr=True, expected_status=expected_status, env_vars={'GSUTIL_TEST_ANALYTICS': '2'}) return METRICS_LOG_RE.search(stderr).group() def _StartObjectPatch(self, *args, **kwargs): """Runs mock.patch.object with the given args, and returns the mock object. This starts the patcher, returns the mock object, and registers the patcher to stop on test teardown. Args: *args: The args to pass to mock.patch.object() **kwargs: The kwargs to pass to mock.patch.object() Returns: Mock, The result of starting the patcher. """ patcher = mock.patch.object(*args, **kwargs) self.addCleanup(patcher.stop) return patcher.start() def _CheckParameterValue(self, param_name, exp_value, metrics_to_search): """Checks for a correct key=value pair in a log output string.""" self.assertIn( '{0}={1}'.format(metrics._GA_LABEL_MAP[param_name], exp_value), metrics_to_search) @mock.patch('time.time', new=mock.MagicMock(return_value=0)) def testMetricsReporting(self): """Tests the subprocess creation by Popen in metrics.py.""" popen_mock = self._StartObjectPatch(subprocess, 'Popen') # Set up the temp file for pickle dumping metrics into. metrics_file = tempfile.NamedTemporaryFile() metrics_file.close() temp_file_mock = self._StartObjectPatch(tempfile, 'NamedTemporaryFile') temp_file_mock.return_value = open(metrics_file.name, 'wb') # If there are no metrics, Popen should not be called. self.collector.ReportMetrics() self.assertEqual(0, popen_mock.call_count) _LogAllTestMetrics() # Report the metrics and check Popen calls. metrics.Shutdown() call_list = popen_mock.call_args_list self.assertEqual(1, len(call_list)) # Check to make sure that we have the proper PYTHONPATH in the subprocess. args = call_list[0] self.assertIn('PYTHONPATH', args[1]['env']) # Ensure that we can access the same modules as the main process from # PYTHONPATH. missing_paths = (set(sys.path) - set(args[1]['env']['PYTHONPATH'].split(os.pathsep))) self.assertEqual(set(), missing_paths) # Check that the metrics were correctly dumped into the temp file. with open(metrics_file.name, 'rb') as metrics_file: reported_metrics = pickle.load(metrics_file) self.assertEqual(COMMAND_AND_ERROR_TEST_METRICS, set(reported_metrics)) @unittest.skip("example.com is currently not accepting post requests, tracking issue: https://github.com/GoogleCloudPlatform/gsutil/issues/1800") @mock.patch('time.time', new=mock.MagicMock(return_value=0)) def testMetricsPosting(self): """Tests the metrics posting process as performed in metrics_reporter.py.""" # Windows has odd restrictions about attempting to open a named tempfile # while it's open. Regardless of platform, we don't need the file to be open # or even exist; we only need a valid file path to create a log file at. metrics_file = tempfile.NamedTemporaryFile() metrics_file_name = metrics_file.name metrics_file.close() # Logging statements will create a file at the path we just fetched. Make # sure we clean up the file afterward. def MetricsTempFileCleanup(file_path): try: os.unlink(file_path) except OSError: # Don't fail if the file was already cleaned up. pass self.addCleanup(MetricsTempFileCleanup, metrics_file_name) # Collect a metric and set log level for the metrics_reporter subprocess. def CollectMetricAndSetLogLevel(log_level, log_file_path): metrics.LogCommandParams(command_name='cmd1', subcommands=['action1'], sub_opts=[('optb', ''), ('opta', '')]) metrics.LogFatalError(gslib.exception.CommandException('test')) # Wait for report to make sure the log is written before we check it. self.collector.ReportMetrics(wait_for_report=True, log_level=log_level, log_file_path=log_file_path) self.assertEqual([], self.collector._metrics) metrics.LogCommandParams(global_opts=[('-y', 'value'), ('-z', ''), ('-x', '')]) # The log file should be empty unless the debug option is specified. CollectMetricAndSetLogLevel(logging.DEBUG, metrics_file.name) with open(metrics_file.name, 'rb') as metrics_log: log_text = metrics_log.read() if six.PY2: expected_response = ( b'Metric(endpoint=u\'https://example.com\', method=u\'POST\', ' b'body=\'{0}&cm2=0&ea=cmd1+action1&ec={1}&el={2}&ev=0\', ' b'user_agent=u\'user-agent-007\')'.format( GLOBAL_DIMENSIONS_URL_PARAMS, metrics._GA_COMMANDS_CATEGORY, VERSION)) else: expected_response = ( 'Metric(endpoint=\'https://example.com\', method=\'POST\', ' 'body=\'{0}&cm2=0&ea=cmd1+action1&ec={1}&el={2}&ev=0\', ' 'user_agent=\'user-agent-007\')'.format(GLOBAL_DIMENSIONS_URL_PARAMS, metrics._GA_COMMANDS_CATEGORY, VERSION)).encode('utf_8') self.assertIn(expected_response, log_text) self.assertIn(b'RESPONSE: 200', log_text) CollectMetricAndSetLogLevel(logging.INFO, metrics_file.name) with open(metrics_file.name, 'rb') as metrics_log: log_text = metrics_log.read() self.assertEqual(log_text, b'') CollectMetricAndSetLogLevel(logging.WARN, metrics_file.name) with open(metrics_file.name, 'rb') as metrics_log: log_text = metrics_log.read() self.assertEqual(log_text, b'') def testMetricsReportingWithFail(self): """Tests that metrics reporting error does not throw an exception.""" popen_mock = self._StartObjectPatch(subprocess, 'Popen') popen_mock.side_effect = OSError() self.collector._metrics.append('dummy metric') # Shouldn't raise an exception. self.collector.ReportMetrics() self.assertTrue(popen_mock.called) def testCommandCollection(self): """Tests the collection of commands.""" metrics_list = self._RunGsUtilWithAnalyticsOutput( ['-m', 'acl', 'set', '-a'], expected_status=1) self._CheckParameterValue('Event Category', metrics._GA_COMMANDS_CATEGORY, metrics_list) self._CheckParameterValue('Event Action', 'acl+set', metrics_list) # Check that the options were collected. self._CheckParameterValue('Global Options', 'd%2Cm', metrics_list) self._CheckParameterValue('Command-Level Options', 'a', metrics_list) metrics_list = self._RunGsUtilWithAnalyticsOutput(['ver']) self._CheckParameterValue('Event Category', metrics._GA_COMMANDS_CATEGORY, metrics_list) self._CheckParameterValue('Event Action', 'version', metrics_list) # Check the recording of the command alias. self._CheckParameterValue('Command Alias', 'ver', metrics_list) def testRetryableErrorMetadataCollection(self): """Tests that retryable errors are collected on JSON metadata operations.""" # Retryable errors will only be collected with the JSON API. if self.test_api != ApiSelector.JSON: return unittest.skip('Retryable errors are only collected in JSON') bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'bar') # Set the command name to rsync in order to collect PerformanceSummary info. self.collector.ga_params[metrics._GA_LABEL_MAP['Command Name']] = 'rsync' # Generate a JSON API instance to test with, because the RunGsUtil method # may use the XML API. gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(), RetryableErrorsQueue(), self.default_provider) # Don't wait for too many retries or for long periods between retries to # avoid long tests. gsutil_api.api_client.num_retries = 2 gsutil_api.api_client.max_retry_wait = 1 # Throw an error when transferring metadata. key = object_uri.get_key() src_obj_metadata = apitools_messages.Object(name=key.name, bucket=key.bucket.name, contentType=key.content_type) dst_obj_metadata = apitools_messages.Object( bucket=src_obj_metadata.bucket, name=self.MakeTempName('object'), contentType=src_obj_metadata.contentType) with mock.patch.object(http_wrapper, '_MakeRequestNoRetry', side_effect=socket.error()): _TryExceptAndPass(gsutil_api.CopyObject, src_obj_metadata, dst_obj_metadata) if six.PY2: self.assertEqual(self.collector.retryable_errors['SocketError'], 1) else: # In PY3, socket.* errors are deprecated aliases for OSError self.assertEqual(self.collector.retryable_errors['OSError'], 1) # Throw an error when removing a bucket. with mock.patch.object(http_wrapper, '_MakeRequestNoRetry', side_effect=apitools_exceptions.HttpError( 'unused', 'unused', 'unused')): _TryExceptAndPass(gsutil_api.DeleteObject, bucket_uri.bucket_name, object_uri.object_name) self.assertEqual(self.collector.retryable_errors['HttpError'], 1) # Check that the number of each kind of retryable error was logged. self.assertEqual( self.collector.perf_sum_params.num_retryable_network_errors, 1) self.assertEqual( self.collector.perf_sum_params.num_retryable_service_errors, 1) def testRetryableErrorMediaCollection(self): """Tests that retryable errors are collected on JSON media operations.""" # Retryable errors will only be collected with the JSON API. if self.test_api != ApiSelector.JSON: return unittest.skip('Retryable errors are only collected in JSON') boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB))] bucket_uri = self.CreateBucket() # For the resumable upload exception, we need to ensure at least one # callback occurs. halt_size = START_CALLBACK_PER_BYTES * 2 fpath = self.CreateTempFile(contents=b'a' * halt_size) # Test that the retry function for data transfers catches and logs an error. test_callback_file = self.CreateTempFile(contents=pickle.dumps( _ResumableUploadRetryHandler(5, apitools_exceptions.BadStatusCodeError, ('unused', 'unused', 'unused')))) with SetBotoConfigForTest(boto_config_for_test): metrics_list = self._RunGsUtilWithAnalyticsOutput([ 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ]) self._CheckParameterValue('Event Category', metrics._GA_ERRORRETRY_CATEGORY, metrics_list) self._CheckParameterValue('Event Action', 'BadStatusCodeError', metrics_list) self._CheckParameterValue('Retryable Errors', '1', metrics_list) self._CheckParameterValue('Num Retryable Service Errors', '1', metrics_list) # Test that the ResumableUploadStartOverException in copy_helper is caught. test_callback_file = self.CreateTempFile( contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404))) with SetBotoConfigForTest(boto_config_for_test): metrics_list = self._RunGsUtilWithAnalyticsOutput([ 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ]) self._CheckParameterValue('Event Category', metrics._GA_ERRORRETRY_CATEGORY, metrics_list) self._CheckParameterValue('Event Action', 'ResumableUploadStartOverException', metrics_list) self._CheckParameterValue('Retryable Errors', '1', metrics_list) self._CheckParameterValue('Num Retryable Service Errors', '1', metrics_list) # Test retryable error collection in a multithread/multiprocess situation. test_callback_file = self.CreateTempFile( contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404))) with SetBotoConfigForTest(boto_config_for_test): metrics_list = self._RunGsUtilWithAnalyticsOutput([ '-m', 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ]) self._CheckParameterValue('Event Category', metrics._GA_ERRORRETRY_CATEGORY, metrics_list) self._CheckParameterValue('Event Action', 'ResumableUploadStartOverException', metrics_list) self._CheckParameterValue('Retryable Errors', '1', metrics_list) self._CheckParameterValue('Num Retryable Service Errors', '1', metrics_list) def testFatalErrorCollection(self): """Tests that fatal errors are collected.""" def CheckForCommandException(log_output): self._CheckParameterValue('Event Category', metrics._GA_ERRORFATAL_CATEGORY, log_output) self._CheckParameterValue('Event Action', 'CommandException', log_output) metrics_list = self._RunGsUtilWithAnalyticsOutput(['invalid-command'], expected_status=1) CheckForCommandException(metrics_list) metrics_list = self._RunGsUtilWithAnalyticsOutput(['mb', '-invalid-option'], expected_status=1) CheckForCommandException(metrics_list) bucket_uri = self.CreateBucket() metrics_list = self._RunGsUtilWithAnalyticsOutput( ['cp', suri(bucket_uri), suri(bucket_uri)], expected_status=1) CheckForCommandException(metrics_list) def _GetAndCheckAllNumberMetrics(self, metrics_to_search, multithread=True): """Checks number metrics for PerformanceSummary tests. Args: metrics_to_search: The string of metrics to search. multithread: False if the the metrics were collected in a non-multithread situation. Returns: (slowest_throughput, fastest_throughput, io_time) as floats. """ def _ExtractNumberMetric(param_name): extracted_match = re.search( metrics._GA_LABEL_MAP[param_name] + r'=(\d+\.?\d*)&', metrics_to_search) if not extracted_match: self.fail( 'Could not find %s (%s) in metrics string %s' % (metrics._GA_LABEL_MAP[param_name], param_name, metrics_to_search)) return float(extracted_match.group(1)) # Thread idle time will only be recorded in a multithread situation. if multithread: thread_idle_time = _ExtractNumberMetric('Thread Idle Time Percent') # This should be a decimal between 0 and 1. self.assertGreaterEqual(thread_idle_time, 0) self.assertLessEqual(thread_idle_time, 1) throughput = _ExtractNumberMetric('Average Overall Throughput') self.assertGreater(throughput, 0) slowest_throughput = _ExtractNumberMetric('Slowest Thread Throughput') fastest_throughput = _ExtractNumberMetric('Fastest Thread Throughput') self.assertGreaterEqual(fastest_throughput, slowest_throughput) self.assertGreater(slowest_throughput, 0) self.assertGreater(fastest_throughput, 0) io_time = None if IS_LINUX: io_time = _ExtractNumberMetric('Disk I/O Time') self.assertGreaterEqual(io_time, 0) # Return some metrics to run tests in more specific scenarios. return (slowest_throughput, fastest_throughput, io_time) def testPerformanceSummaryFileToFile(self): """Tests PerformanceSummary collection in a file-to-file transfer.""" tmpdir1 = self.CreateTempDir() tmpdir2 = self.CreateTempDir() file_size = ONE_MIB self.CreateTempFile(tmpdir=tmpdir1, contents=b'a' * file_size) # Run an rsync file-to-file command with fan parallelism, without slice # parallelism. process_count = 1 if IS_WINDOWS else 6 with SetBotoConfigForTest([('GSUtil', 'parallel_process_count', str(process_count)), ('GSUtil', 'parallel_thread_count', '7')]): metrics_list = self._RunGsUtilWithAnalyticsOutput( ['-m', 'rsync', tmpdir1, tmpdir2]) self._CheckParameterValue('Event Category', metrics._GA_PERFSUM_CATEGORY, metrics_list) self._CheckParameterValue('Event Action', 'FileToFile', metrics_list) self._CheckParameterValue('Parallelism Strategy', 'fan', metrics_list) self._CheckParameterValue('Source URL Type', 'file', metrics_list) self._CheckParameterValue('Num Processes', str(process_count), metrics_list) self._CheckParameterValue('Num Threads', '7', metrics_list) self._CheckParameterValue('Provider Types', 'file', metrics_list) self._CheckParameterValue('Size of Files/Objects Transferred', file_size, metrics_list) self._CheckParameterValue('Number of Files/Objects Transferred', 1, metrics_list) (_, _, io_time) = self._GetAndCheckAllNumberMetrics(metrics_list) if IS_LINUX: # io_time will be None on other platforms. # We can't guarantee that the file read/write will consume a # reportable amount of disk I/O, but it should be reported as >= 0. self.assertGreaterEqual(io_time, 0) @SkipForS3('No slice parallelism support for S3.') def testPerformanceSummaryFileToCloud(self): """Tests PerformanceSummary collection in a file-to-cloud transfer.""" bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() file_size = 6 self.CreateTempFile(tmpdir=tmpdir, contents=b'a' * file_size) self.CreateTempFile(tmpdir=tmpdir, contents=b'b' * file_size) process_count = 1 if IS_WINDOWS else 2 # Run a parallel composite upload without fan parallelism. with SetBotoConfigForTest([ ('GSUtil', 'parallel_process_count', str(process_count)), ('GSUtil', 'parallel_thread_count', '3'), ('GSUtil', 'parallel_composite_upload_threshold', '1') ]): metrics_list = self._RunGsUtilWithAnalyticsOutput( ['rsync', tmpdir, suri(bucket_uri)]) self._CheckParameterValue('Event Category', metrics._GA_PERFSUM_CATEGORY, metrics_list) self._CheckParameterValue('Event Action', 'FileToCloud', metrics_list) self._CheckParameterValue('Parallelism Strategy', 'slice', metrics_list) self._CheckParameterValue('Num Processes', str(process_count), metrics_list) self._CheckParameterValue('Num Threads', '3', metrics_list) self._CheckParameterValue('Provider Types', 'file%2C' + bucket_uri.scheme, metrics_list) self._CheckParameterValue('Size of Files/Objects Transferred', 2 * file_size, metrics_list) self._CheckParameterValue('Number of Files/Objects Transferred', 2, metrics_list) (_, _, io_time) = self._GetAndCheckAllNumberMetrics(metrics_list) if IS_LINUX: # io_time will be None on other platforms. # We can't guarantee that the file read will consume a # reportable amount of disk I/O, but it should be reported as >= 0. self.assertGreaterEqual(io_time, 0) @SkipForS3('No slice parallelism support for S3.') def testPerformanceSummaryCloudToFile(self): """Tests PerformanceSummary collection in a cloud-to-file transfer.""" bucket_uri = self.CreateBucket() file_size = 6 object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'a' * file_size) fpath = self.CreateTempFile() # Run a sliced object download with fan parallelism. process_count = 1 if IS_WINDOWS else 4 with SetBotoConfigForTest([ ('GSUtil', 'parallel_process_count', str(process_count)), ('GSUtil', 'parallel_thread_count', '5'), ('GSUtil', 'sliced_object_download_threshold', '1'), ('GSUtil', 'test_assume_fast_crcmod', 'True') ]): metrics_list = self._RunGsUtilWithAnalyticsOutput( ['-m', 'cp', suri(object_uri), fpath]) self._CheckParameterValue('Event Category', metrics._GA_PERFSUM_CATEGORY, metrics_list) self._CheckParameterValue('Event Action', 'CloudToFile', metrics_list) self._CheckParameterValue('Parallelism Strategy', 'both', metrics_list) self._CheckParameterValue('Num Processes', str(process_count), metrics_list) self._CheckParameterValue('Num Threads', '5', metrics_list) self._CheckParameterValue('Provider Types', 'file%2C' + bucket_uri.scheme, metrics_list) self._CheckParameterValue('Number of Files/Objects Transferred', '1', metrics_list) self._CheckParameterValue('Size of Files/Objects Transferred', file_size, metrics_list) (_, _, io_time) = self._GetAndCheckAllNumberMetrics(metrics_list) if IS_LINUX: # io_time will be None on other platforms. # We can't guarantee that the file write will consume a # reportable amount of disk I/O, but it should be reported as >= 0. self.assertGreaterEqual(io_time, 0) def testPerformanceSummaryCloudToCloud(self): """Tests PerformanceSummary collection in a cloud-to-cloud transfer.""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() file_size = 6 key_uri = self.CreateObject(bucket_uri=bucket1_uri, contents=b'a' * file_size) # Run a daisy-chain cloud-to-cloud copy without parallelism. metrics_list = self._RunGsUtilWithAnalyticsOutput( ['cp', '-D', suri(key_uri), suri(bucket2_uri)]) (slowest_throughput, fastest_throughput, _) = self._GetAndCheckAllNumberMetrics(metrics_list, multithread=False) # Since there's a single thread, this must be the case. self.assertEqual(slowest_throughput, fastest_throughput) self._CheckParameterValue('Event Category', metrics._GA_PERFSUM_CATEGORY, metrics_list) self._CheckParameterValue('Event Action', 'CloudToCloud%2CDaisyChain', metrics_list) self._CheckParameterValue('Parallelism Strategy', 'none', metrics_list) self._CheckParameterValue('Source URL Type', 'cloud', metrics_list) self._CheckParameterValue('Num Processes', '1', metrics_list) self._CheckParameterValue('Num Threads', '1', metrics_list) self._CheckParameterValue('Provider Types', bucket1_uri.scheme, metrics_list) self._CheckParameterValue('Number of Files/Objects Transferred', '1', metrics_list) self._CheckParameterValue('Size of Files/Objects Transferred', file_size, metrics_list) @unittest.skipUnless(HAS_S3_CREDS, 'Test requires both S3 and GS credentials') def testCrossProviderDaisyChainCollection(self): """Tests the collection of daisy-chain operations.""" s3_bucket = self.CreateBucket(provider='s3') gs_bucket = self.CreateBucket(provider='gs') unused_s3_key = self.CreateObject(bucket_uri=s3_bucket, contents=b'foo') gs_key = self.CreateObject(bucket_uri=gs_bucket, contents=b'bar') metrics_list = self._RunGsUtilWithAnalyticsOutput( ['rsync', suri(s3_bucket), suri(gs_bucket)]) self._CheckParameterValue('Event Action', 'CloudToCloud%2CDaisyChain', metrics_list) self._CheckParameterValue('Provider Types', 'gs%2Cs3', metrics_list) metrics_list = self._RunGsUtilWithAnalyticsOutput( ['cp', suri(gs_key), suri(s3_bucket)]) self._CheckParameterValue('Event Action', 'CloudToCloud%2CDaisyChain', metrics_list) self._CheckParameterValue('Provider Types', 'gs%2Cs3', metrics_list) tests/test_resumable_streaming.py 0000644 00000030560 15025013616 0013352 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for resumable streaming upload functions and classes.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import pkgutil from six.moves import range from gslib.exception import CommandException from gslib.resumable_streaming_upload import ResumableStreamingJsonUploadWrapper import gslib.tests.testcase as testcase from gslib.utils.boto_util import GetJsonResumableChunkSize from gslib.utils.constants import TRANSFER_BUFFER_SIZE from gslib.utils.hashing_helper import CalculateHashesFromContents from gslib.utils.hashing_helper import CalculateMd5FromContents from gslib.utils.hashing_helper import GetMd5 _TEST_FILE = 'test.txt' class TestResumableStreamingJsonUploadWrapper(testcase.GsUtilUnitTestCase): """Unit tests for the TestResumableStreamingJsonUploadWrapper class.""" _temp_test_file = None _temp_test_file_contents = None _temp_test_file_len = None def _GetTestFile(self): if not self._temp_test_file: self._temp_test_file_contents = pkgutil.get_data( 'gslib', 'tests/test_data/%s' % _TEST_FILE) self._temp_test_file = self.CreateTempFile( file_name=_TEST_FILE, contents=self._temp_test_file_contents) self._temp_test_file_len = len(self._temp_test_file_contents) return self._temp_test_file def testReadInChunks(self): tmp_file = self._GetTestFile() with open(tmp_file, 'rb') as stream: wrapper = ResumableStreamingJsonUploadWrapper(stream, TRANSFER_BUFFER_SIZE, test_small_buffer=True) hash_dict = {'md5': GetMd5()} # CalculateHashesFromContents reads in chunks, but does not seek. CalculateHashesFromContents(wrapper, hash_dict) with open(tmp_file, 'rb') as stream: actual = CalculateMd5FromContents(stream) self.assertEqual(actual, hash_dict['md5'].hexdigest()) def testReadInChunksWithSeekToBeginning(self): """Reads one buffer, then seeks to 0 and reads chunks until the end.""" tmp_file = self._GetTestFile() for initial_read in (TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE + 1, TRANSFER_BUFFER_SIZE * 2 - 1, TRANSFER_BUFFER_SIZE * 2, TRANSFER_BUFFER_SIZE * 2 + 1, TRANSFER_BUFFER_SIZE * 3 - 1, TRANSFER_BUFFER_SIZE * 3, TRANSFER_BUFFER_SIZE * 3 + 1): for buffer_size in (TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE + 1, self._temp_test_file_len - 1, self._temp_test_file_len, self._temp_test_file_len + 1): # Can't seek to 0 if the buffer is too small, so we expect an # exception. expect_exception = buffer_size < self._temp_test_file_len with open(tmp_file, 'rb') as stream: wrapper = ResumableStreamingJsonUploadWrapper(stream, buffer_size, test_small_buffer=True) wrapper.read(initial_read) # CalculateMd5FromContents seeks to 0, reads in chunks, then seeks # to 0 again. try: hex_digest = CalculateMd5FromContents(wrapper) if expect_exception: self.fail('Did not get expected CommandException for ' 'initial read size %s, buffer size %s' % (initial_read, buffer_size)) except CommandException as e: if not expect_exception: self.fail('Got unexpected CommandException "%s" for ' 'initial read size %s, buffer size %s' % (str(e), initial_read, buffer_size)) if not expect_exception: with open(tmp_file, 'rb') as stream: actual = CalculateMd5FromContents(stream) self.assertEqual( actual, hex_digest, 'Digests not equal for initial read size %s, buffer size %s' % (initial_read, buffer_size)) def _testSeekBack(self, initial_reads, buffer_size, seek_back_amount): """Tests reading then seeking backwards. This function simulates an upload that is resumed after a connection break. It reads one transfer buffer at a time until it reaches initial_position, then seeks backwards (as if the server did not receive some of the bytes) and reads to the end of the file, ensuring the data read after the seek matches the original file. Args: initial_reads: List of integers containing read sizes to perform before seek. buffer_size: Maximum buffer size for the wrapper. seek_back_amount: Number of bytes to seek backward. Raises: AssertionError on wrong data returned by the wrapper. """ tmp_file = self._GetTestFile() initial_position = 0 for read_size in initial_reads: initial_position += read_size self.assertGreaterEqual( buffer_size, seek_back_amount, 'seek_back_amount must be less than initial position %s ' '(but was actually: %s)' % (buffer_size, seek_back_amount)) self.assertLess( initial_position, self._temp_test_file_len, 'initial_position must be less than test file size %s ' '(but was actually: %s)' % (self._temp_test_file_len, initial_position)) with open(tmp_file, 'rb') as stream: wrapper = ResumableStreamingJsonUploadWrapper(stream, buffer_size, test_small_buffer=True) position = 0 for read_size in initial_reads: data = wrapper.read(read_size) self.assertEqual( self._temp_test_file_contents[position:position + read_size], data, 'Data from position %s to %s did not match file contents.' % (position, position + read_size)) position += len(data) wrapper.seek(initial_position - seek_back_amount) self.assertEqual(wrapper.tell(), initial_position - seek_back_amount) data = wrapper.read() self.assertEqual( self._temp_test_file_len - (initial_position - seek_back_amount), len(data), 'Unexpected data length with initial pos %s seek_back_amount %s. ' 'Expected: %s, actual: %s.' % (initial_position, seek_back_amount, self._temp_test_file_len - (initial_position - seek_back_amount), len(data))) self.assertEqual( self._temp_test_file_contents[-len(data):], data, 'Data from position %s to EOF did not match file contents.' % position) def testReadSeekAndReadToEOF(self): """Tests performing reads on the wrapper, seeking, then reading to EOF.""" for initial_reads in ([1], [TRANSFER_BUFFER_SIZE - 1], [ TRANSFER_BUFFER_SIZE ], [TRANSFER_BUFFER_SIZE + 1], [1, TRANSFER_BUFFER_SIZE - 1], [ 1, TRANSFER_BUFFER_SIZE ], [1, TRANSFER_BUFFER_SIZE + 1 ], [TRANSFER_BUFFER_SIZE - 1, 1], [TRANSFER_BUFFER_SIZE, 1], [TRANSFER_BUFFER_SIZE + 1, 1], [TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE - 1 ], [TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE], [TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE + 1 ], [TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE - 1 ], [TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE], [TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE + 1], [TRANSFER_BUFFER_SIZE + 1, TRANSFER_BUFFER_SIZE - 1 ], [TRANSFER_BUFFER_SIZE + 1, TRANSFER_BUFFER_SIZE], [TRANSFER_BUFFER_SIZE + 1, TRANSFER_BUFFER_SIZE + 1], [ TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE ]): initial_position = 0 for read_size in initial_reads: initial_position += read_size for buffer_size in (initial_position, initial_position + 1, initial_position * 2 - 1, initial_position * 2): for seek_back_amount in (min(TRANSFER_BUFFER_SIZE - 1, initial_position), min(TRANSFER_BUFFER_SIZE, initial_position), min(TRANSFER_BUFFER_SIZE + 1, initial_position), min(TRANSFER_BUFFER_SIZE * 2 - 1, initial_position), min(TRANSFER_BUFFER_SIZE * 2, initial_position), min(TRANSFER_BUFFER_SIZE * 2 + 1, initial_position)): self._testSeekBack(initial_reads, buffer_size, seek_back_amount) def testBufferSizeLessThanChunkSize(self): ResumableStreamingJsonUploadWrapper(None, GetJsonResumableChunkSize()) try: ResumableStreamingJsonUploadWrapper(None, GetJsonResumableChunkSize() - 1) self.fail('Did not get expected CommandException') except CommandException as e: self.assertIn('Buffer size must be >= JSON resumable upload', str(e)) def testSeekPartialBuffer(self): """Tests seeking back partially within the buffer.""" tmp_file = self._GetTestFile() read_size = TRANSFER_BUFFER_SIZE with open(tmp_file, 'rb') as stream: wrapper = ResumableStreamingJsonUploadWrapper(stream, TRANSFER_BUFFER_SIZE * 3, test_small_buffer=True) position = 0 for _ in range(3): data = wrapper.read(read_size) self.assertEqual( self._temp_test_file_contents[position:position + read_size], data, 'Data from position %s to %s did not match file contents.' % (position, position + read_size)) position += len(data) data = wrapper.read(read_size // 2) # Buffer contents should now be have contents from: # read_size/2 through 7*read_size/2. position = read_size // 2 wrapper.seek(position) data = wrapper.read() self.assertEqual( self._temp_test_file_contents[-len(data):], data, 'Data from position %s to EOF did not match file contents.' % position) def testSeekEnd(self): tmp_file = self._GetTestFile() for buffer_size in (TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE + 1): for seek_back in (TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE + 1): expect_exception = seek_back > buffer_size with open(tmp_file, 'rb') as stream: wrapper = ResumableStreamingJsonUploadWrapper(stream, buffer_size, test_small_buffer=True) # Read to the end. while wrapper.read(TRANSFER_BUFFER_SIZE): pass try: wrapper.seek(seek_back, whence=os.SEEK_END) if expect_exception: self.fail('Did not get expected CommandException for ' 'seek_back size %s, buffer size %s' % (seek_back, buffer_size)) except CommandException as e: if not expect_exception: self.fail('Got unexpected CommandException "%s" for ' 'seek_back size %s, buffer size %s' % (str(e), seek_back, buffer_size)) tests/test_wildcard_iterator.py 0000644 00000054265 15025013616 0013034 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2010 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Unit tests for gsutil wildcard_iterator.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import six import tempfile from gslib import wildcard_iterator from gslib.exception import InvalidUrlError from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString import gslib.tests.testcase as testcase from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetDummyProjectForUnitTest class CloudWildcardIteratorTests(testcase.GsUtilUnitTestCase): """Unit tests for CloudWildcardIterator.""" def setUp(self): """Creates 2 mock buckets, each containing 4 objects, including 1 nested.""" super(CloudWildcardIteratorTests, self).setUp() self.immed_child_obj_names = ['abcd', 'abdd', 'ade$'] self.all_obj_names = [ 'abcd', 'abdd', 'ade$', 'nested1/nested2/xyz1', 'nested1/nested2/xyz2', 'nested1/nested2xyz1', 'nested1/nfile_abc' ] self.base_bucket_uri = self.CreateBucket() self.prefix_bucket_name = '%s_' % self.base_bucket_uri.bucket_name[:61] self.base_uri_str = suri(self.base_bucket_uri) self.base_uri_str = self.base_uri_str.replace( self.base_bucket_uri.bucket_name, self.prefix_bucket_name) self.test_bucket0_uri = self.CreateBucket(bucket_name='%s0' % self.prefix_bucket_name) self.test_bucket0_obj_uri_strs = set() for obj_name in self.all_obj_names: obj_uri = self.CreateObject(bucket_uri=self.test_bucket0_uri, object_name=obj_name, contents='') self.test_bucket0_obj_uri_strs.add(suri(obj_uri)) self.test_bucket1_uri = self.CreateBucket(bucket_name='%s1' % self.prefix_bucket_name) self.test_bucket1_obj_uri_strs = set() for obj_name in self.all_obj_names: obj_uri = self.CreateObject(bucket_uri=self.test_bucket1_uri, object_name=obj_name, contents='') self.test_bucket1_obj_uri_strs.add(suri(obj_uri)) self.test_bucket2_uri = self.CreateBucket(bucket_name='%s2' % self.prefix_bucket_name) self.test_bucket2_obj_uri_strs = set() object_list = [ # For testing ** patterns. # zf.txt has been added in every folder here to test that it does not # get listed for **/f.txt requests. 'f.txt', 'double/f.txt', 'double/zf.txt', 'double/foo/f.txt', 'double/foo/zf.txt', 'double/bar/f.txt', 'double/bar/zf.txt', ] for obj_name in object_list: obj_uri = self.CreateObject(bucket_uri=self.test_bucket2_uri, object_name=obj_name, contents='') self.test_bucket2_obj_uri_strs.add(suri(obj_uri)) def testNoOpObjectIterator(self): """Tests that bucket-only URI iterates just that one URI.""" results = list( self._test_wildcard_iterator( self.test_bucket0_uri).IterBuckets(bucket_fields=['id'])) self.assertEqual(1, len(results)) self.assertEqual(str(self.test_bucket0_uri), str(results[0])) def testMatchingAllObjects(self): """Tests matching all objects, based on wildcard.""" actual_obj_uri_strs = set( six.ensure_text(str(u)) for u in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('**')).IterAll( expand_top_level_buckets=True)) self.assertEqual(self.test_bucket0_obj_uri_strs, actual_obj_uri_strs) def testMatchingObjectSubset(self): """Tests matching a subset of objects, based on wildcard.""" exp_obj_uri_strs = set([ str(self.test_bucket0_uri.clone_replace_name('abcd')), str(self.test_bucket0_uri.clone_replace_name('abdd')) ]) actual_obj_uri_strs = set( str(u) for u in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('ab??')).IterAll( expand_top_level_buckets=True)) self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) def testMatchingNonWildcardedUri(self): """Tests matching a single named object.""" exp_obj_uri_strs = set( [str(self.test_bucket0_uri.clone_replace_name('abcd'))]) actual_obj_uri_strs = set( str(u) for u in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('abcd')).IterAll( expand_top_level_buckets=True)) self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) def testWildcardedObjectUriWithVsWithoutPrefix(self): """Tests that wildcarding w/ and w/o server prefix get same result.""" # (It's just more efficient to query w/o a prefix; wildcard # iterator will filter the matches either way.) with_prefix_uri_strs = set( str(u) for u in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('abcd')).IterAll( expand_top_level_buckets=True)) # By including a wildcard at the start of the string no prefix can be # used in server request. no_prefix_uri_strs = set( str(u) for u in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('?bcd')).IterAll( expand_top_level_buckets=True)) self.assertEqual(with_prefix_uri_strs, no_prefix_uri_strs) def testWildcardedObjectUriNestedSubdirMatch(self): """Tests wildcarding with a nested subdir.""" uri_strs = set() prefixes = set() for blr in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('*')): if blr.IsPrefix(): prefixes.add(blr.root_object) else: uri_strs.add(blr.url_string) exp_obj_uri_strs = set( [suri(self.test_bucket0_uri, x) for x in self.immed_child_obj_names]) self.assertEqual(exp_obj_uri_strs, uri_strs) self.assertEqual(1, len(prefixes)) self.assertTrue('nested1/' in prefixes) def testWildcardPlusSubdirMatch(self): """Tests gs://bucket/*/subdir matching.""" actual_uri_strs = set() actual_prefixes = set() for blr in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('*/nested1')): if blr.IsPrefix(): actual_prefixes.add(blr.root_object) else: actual_uri_strs.add(blr.url_string) expected_uri_strs = set() expected_prefixes = set(['nested1/']) self.assertEqual(expected_prefixes, actual_prefixes) self.assertEqual(expected_uri_strs, actual_uri_strs) def testWildcardPlusSubdirSubdirMatch(self): """Tests gs://bucket/*/subdir/* matching.""" actual_uri_strs = set() actual_prefixes = set() for blr in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('*/nested2/*')): if blr.IsPrefix(): actual_prefixes.add(blr.root_object) else: actual_uri_strs.add(blr.url_string) expected_uri_strs = set([ self.test_bucket0_uri.clone_replace_name('nested1/nested2/xyz1').uri, self.test_bucket0_uri.clone_replace_name('nested1/nested2/xyz2').uri ]) expected_prefixes = set() self.assertEqual(expected_prefixes, actual_prefixes) self.assertEqual(expected_uri_strs, actual_uri_strs) def testNoMatchingWildcardedObjectUri(self): """Tests that get back an empty iterator for non-matching wildcarded URI.""" res = list( self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('*x0')).IterAll( expand_top_level_buckets=True)) self.assertEqual(0, len(res)) def testWildcardedInvalidObjectUri(self): """Tests that we raise an exception for wildcarded invalid URI.""" try: for unused_ in self._test_wildcard_iterator('badscheme://asdf').IterAll( expand_top_level_buckets=True): self.assertFalse('Expected InvalidUrlError not raised.') except InvalidUrlError as e: # Expected behavior. self.assertTrue(e.message.find('Unrecognized scheme') != -1) def testSingleMatchWildcardedBucketUri(self): """Tests matching a single bucket based on a wildcarded bucket URI.""" exp_obj_uri_strs = set( [suri(self.test_bucket1_uri) + self.test_bucket1_uri.delim]) with SetDummyProjectForUnitTest(): actual_obj_uri_strs = set( str(u) for u in self._test_wildcard_iterator('%s*1' % self.base_uri_str).IterBuckets( bucket_fields=['id'])) self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) def testMultiMatchWildcardedBucketUri(self): """Tests matching a multiple buckets based on a wildcarded bucket URI.""" exp_obj_uri_strs = set([ suri(self.test_bucket0_uri) + self.test_bucket0_uri.delim, suri(self.test_bucket1_uri) + self.test_bucket1_uri.delim, suri(self.test_bucket2_uri) + self.test_bucket2_uri.delim, ]) with SetDummyProjectForUnitTest(): actual_obj_uri_strs = set( str(u) for u in self._test_wildcard_iterator('%s*' % self.base_uri_str).IterBuckets( bucket_fields=['id'])) self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) def testWildcardBucketAndObjectUri(self): """Tests matching with both bucket and object wildcards.""" exp_obj_uri_strs = set( [str(self.test_bucket0_uri.clone_replace_name('abcd'))]) with SetDummyProjectForUnitTest(): actual_obj_uri_strs = set( str(u) for u in self._test_wildcard_iterator( '%s0*/abc*' % self.base_uri_str).IterAll(expand_top_level_buckets=True)) self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) def testWildcardUpToFinalCharSubdirPlusObjectName(self): """Tests wildcard subd*r/obj name.""" exp_obj_uri_strs = set( [str(self.test_bucket0_uri.clone_replace_name('nested1/nested2/xyz1'))]) actual_obj_uri_strs = set( str(u) for u in self._test_wildcard_iterator( '%snested1/nest*2/xyz1' % self.test_bucket0_uri.uri).IterAll(expand_top_level_buckets=True)) self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) def testPostRecursiveWildcard(self): """Tests wildcard containing ** followed by an additional wildcard.""" exp_obj_uri_strs = set( [str(self.test_bucket0_uri.clone_replace_name('nested1/nested2/xyz2'))]) actual_obj_uri_strs = set( str(u) for u in self._test_wildcard_iterator( '%s**/*y*2' % self.test_bucket0_uri.uri).IterAll(expand_top_level_buckets=True)) self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) def testWildcardFields(self): """Tests that wildcard w/fields specification returns correct fields.""" blrs = set(u for u in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('**')).IterAll( bucket_listing_fields=['timeCreated'])) self.assertTrue(len(blrs)) for blr in blrs: self.assertTrue(blr.root_object and blr.root_object.timeCreated) blrs = set(u for u in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('**')).IterAll( bucket_listing_fields=['generation'])) self.assertTrue(len(blrs)) for blr in blrs: self.assertTrue(blr.root_object and not blr.root_object.timeCreated) def testDoesNotStripDelimiterForDoubleWildcard(self): """Tests gs://bucket/*/subdir matching.""" actual_uri_strs = set() actual_prefixes = set() for blr in self._test_wildcard_iterator( self.test_bucket0_uri.clone_replace_name('**/xyz*')): if blr.IsPrefix(): actual_prefixes.add(blr.root_object) else: actual_uri_strs.add(blr.url_string) expected_uri_strs = set([ self.test_bucket0_uri.clone_replace_name('nested1/nested2/xyz1').uri, self.test_bucket0_uri.clone_replace_name('nested1/nested2/xyz2').uri ]) expected_prefixes = set() self.assertEqual(expected_prefixes, actual_prefixes) self.assertEqual(expected_uri_strs, actual_uri_strs) def testDoubleWildcardAfterBucket(self): """Tests gs://bucket/**/object matching.""" actual_uri_strs = set() actual_prefixes = set() for blr in self._test_wildcard_iterator( self.test_bucket2_uri.clone_replace_name('**/f.txt')): if blr.IsPrefix(): actual_prefixes.add(blr.root_object) else: actual_uri_strs.add(blr.url_string) expected_uri_strs = set([ self.test_bucket2_uri.clone_replace_name('f.txt').uri, self.test_bucket2_uri.clone_replace_name('double/f.txt').uri, self.test_bucket2_uri.clone_replace_name('double/foo/f.txt').uri, self.test_bucket2_uri.clone_replace_name('double/bar/f.txt').uri, ]) expected_prefixes = set() self.assertEqual(expected_prefixes, actual_prefixes) self.assertEqual(expected_uri_strs, actual_uri_strs) def testDoubleWildcardAfterPrefix(self): """Tests gs://bucket/dir/**/object matching.""" actual_uri_strs = set() actual_prefixes = set() for blr in self._test_wildcard_iterator( self.test_bucket2_uri.clone_replace_name('double/**/f.txt')): if blr.IsPrefix(): actual_prefixes.add(blr.root_object) else: actual_uri_strs.add(blr.url_string) expected_uri_strs = set([ self.test_bucket2_uri.clone_replace_name('double/f.txt').uri, self.test_bucket2_uri.clone_replace_name('double/foo/f.txt').uri, self.test_bucket2_uri.clone_replace_name('double/bar/f.txt').uri, ]) expected_prefixes = set() self.assertEqual(expected_prefixes, actual_prefixes) self.assertEqual(expected_uri_strs, actual_uri_strs) def testDoubleWildcardBeforeAndAfterPrefix(self): """Tests gs://bucket/**/dir/**/object matching.""" actual_uri_strs = set() actual_prefixes = set() for blr in self._test_wildcard_iterator( self.test_bucket2_uri.clone_replace_name('**/double/**/f.txt')): if blr.IsPrefix(): actual_prefixes.add(blr.root_object) else: actual_uri_strs.add(blr.url_string) expected_uri_strs = set([ self.test_bucket2_uri.clone_replace_name('double/f.txt').uri, self.test_bucket2_uri.clone_replace_name('double/foo/f.txt').uri, self.test_bucket2_uri.clone_replace_name('double/bar/f.txt').uri, ]) expected_prefixes = set() self.assertEqual(expected_prefixes, actual_prefixes) self.assertEqual(expected_uri_strs, actual_uri_strs) class FileIteratorTests(testcase.GsUtilUnitTestCase): """Unit tests for FileWildcardIterator.""" def setUp(self): """Creates a test dir with 3 files and one nested subdirectory + file.""" super(FileIteratorTests, self).setUp() self.test_dir = self.CreateTempDir( test_files=['abcd', 'abdd', 'ade$', ('dir1', 'dir2', 'zzz')]) self.root_files_uri_strs = set([ suri(self.test_dir, 'abcd'), suri(self.test_dir, 'abdd'), suri(self.test_dir, 'ade$') ]) self.subdirs_uri_strs = set([suri(self.test_dir, 'dir1')]) self.nested_files_uri_strs = set( [suri(self.test_dir, 'dir1', 'dir2', 'zzz')]) self.immed_child_uri_strs = self.root_files_uri_strs | self.subdirs_uri_strs self.all_file_uri_strs = (self.root_files_uri_strs | self.nested_files_uri_strs) def testContainsWildcard(self): """Tests ContainsWildcard call.""" self.assertTrue(ContainsWildcard('a*.txt')) self.assertTrue(ContainsWildcard('a[0-9].txt')) self.assertFalse(ContainsWildcard('0-9.txt')) self.assertTrue(ContainsWildcard('?.txt')) def testNoOpDirectoryIterator(self): """Tests that directory-only URI iterates just that one URI.""" results = list( self._test_wildcard_iterator(suri( tempfile.tempdir)).IterAll(expand_top_level_buckets=True)) self.assertEqual(1, len(results)) self.assertEqual(suri(tempfile.tempdir), str(results[0])) def testMatchingAllFiles(self): """Tests matching all files, based on wildcard.""" uri = self._test_storage_uri(suri(self.test_dir, '*')) actual_uri_strs = set( str(u) for u in self._test_wildcard_iterator(uri).IterAll( expand_top_level_buckets=True)) self.assertEqual(self.immed_child_uri_strs, actual_uri_strs) def testMatchingAllFilesWithSize(self): """Tests matching all files, based on wildcard.""" uri = self._test_storage_uri(suri(self.test_dir, '*')) blrs = self._test_wildcard_iterator(uri).IterAll( expand_top_level_buckets=True, bucket_listing_fields=['size']) num_expected_objects = 3 num_actual_objects = 0 for blr in blrs: self.assertTrue(str(blr) in self.immed_child_uri_strs) if blr.IsObject(): num_actual_objects += 1 # Size is based on contents "Test N" as created by CreateTempDir. self.assertEqual(blr.root_object.size, 6) self.assertEqual(num_expected_objects, num_actual_objects) def testMatchingFileSubset(self): """Tests matching a subset of files, based on wildcard.""" exp_uri_strs = set( [suri(self.test_dir, 'abcd'), suri(self.test_dir, 'abdd')]) uri = self._test_storage_uri(suri(self.test_dir, 'ab??')) actual_uri_strs = set( str(u) for u in self._test_wildcard_iterator(uri).IterAll( expand_top_level_buckets=True)) self.assertEqual(exp_uri_strs, actual_uri_strs) def testMatchingNonWildcardedUri(self): """Tests matching a single named file.""" exp_uri_strs = set([suri(self.test_dir, 'abcd')]) uri = self._test_storage_uri(suri(self.test_dir, 'abcd')) actual_uri_strs = set( str(u) for u in self._test_wildcard_iterator(uri).IterAll( expand_top_level_buckets=True)) self.assertEqual(exp_uri_strs, actual_uri_strs) def testMatchingFilesIgnoringOtherRegexChars(self): """Tests ignoring non-wildcard regex chars (e.g., ^ and $).""" exp_uri_strs = set([suri(self.test_dir, 'ade$')]) uri = self._test_storage_uri(suri(self.test_dir, 'ad*$')) actual_uri_strs = set( str(u) for u in self._test_wildcard_iterator(uri).IterAll( expand_top_level_buckets=True)) self.assertEqual(exp_uri_strs, actual_uri_strs) def testRecursiveDirectoryOnlyWildcarding(self): """Tests recursive expansion of directory-only '**' wildcard.""" uri = self._test_storage_uri(suri(self.test_dir, '**')) actual_uri_strs = set( str(u) for u in self._test_wildcard_iterator(uri).IterAll( expand_top_level_buckets=True)) self.assertEqual(self.all_file_uri_strs, actual_uri_strs) def testRecursiveDirectoryPlusFileWildcarding(self): """Tests recursive expansion of '**' directory plus '*' wildcard.""" uri = self._test_storage_uri(suri(self.test_dir, '**', '*')) actual_uri_strs = set( str(u) for u in self._test_wildcard_iterator(uri).IterAll( expand_top_level_buckets=True)) self.assertEqual(self.all_file_uri_strs, actual_uri_strs) def testInvalidRecursiveDirectoryWildcard(self): """Tests that wildcard containing '***' raises exception.""" try: uri = self._test_storage_uri(suri(self.test_dir, '***', 'abcd')) for unused_ in self._test_wildcard_iterator(uri).IterAll( expand_top_level_buckets=True): self.fail('Expected WildcardException not raised.') except wildcard_iterator.WildcardException as e: # Expected behavior. self.assertTrue(str(e).find('more than 2 consecutive') != -1) def testMissingDir(self): """Tests that wildcard gets empty iterator when directory doesn't exist.""" res = list( self._test_wildcard_iterator(suri( 'no_such_dir', '*')).IterAll(expand_top_level_buckets=True)) self.assertEqual(0, len(res)) def testExistingDirNoFileMatch(self): """Tests that wildcard returns empty iterator when there's no match.""" uri = self._test_storage_uri(suri(self.test_dir, 'non_existent*')) res = list( self._test_wildcard_iterator(uri).IterAll( expand_top_level_buckets=True)) self.assertEqual(0, len(res)) def testExcludeDir(self): """Tests that the exclude regex will omit a nested directory.""" exp_uri_strs = self.root_files_uri_strs uri = self._test_storage_uri(suri(self.test_dir, '**')) exclude_tuple = (StorageUrlFromString(self.test_dir), True, re.compile('dir1')) actual_uri_strs = set( str(u) for u in self._test_wildcard_iterator(uri, exclude_tuple=exclude_tuple). IterAll(expand_top_level_buckets=True)) self.assertEqual(exp_uri_strs, actual_uri_strs) def testExcludeTupleButExcludeDirFalse(self): """Tests that the exclude regex will be disabled by exlude_dirs False.""" exp_uri_strs = self.all_file_uri_strs uri = self._test_storage_uri(suri(self.test_dir, '**')) exclude_tuple = (StorageUrlFromString(self.test_dir), False, re.compile('dir1')) actual_uri_strs = set( str(u) for u in self._test_wildcard_iterator(uri, exclude_tuple=exclude_tuple). IterAll(expand_top_level_buckets=True)) self.assertEqual(exp_uri_strs, actual_uri_strs) tests/test_shim_util.py 0000644 00000200306 15025013616 0011314 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for shim_util.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections from contextlib import contextmanager import os import re import subprocess from unittest import mock from boto import config from gslib import command from gslib import command_argument from gslib import exception from gslib.commands import rsync from gslib.commands import version from gslib.commands import test from gslib.cs_api_map import ApiSelector from gslib.tests import testcase from gslib.utils import boto_util from gslib.utils import constants from gslib.utils import shim_util from gslib.utils import system_util from gslib.tests import util @contextmanager def _mock_boto_config(boto_config_dict): """"Mock boto config replacing any exiting config. The util.SetBotoConfigForTest has a use_existing_config flag that can be set to False, but it does not work if the config has been already loaded, which is the case for all unit tests that do not use RunCommand method. Args: boto_config_dict. A dict with key=<boto section name> and value=<a dict of boto field name and the value for that field>. """ def _config_get_side_effect(section, key, default_value=None): return boto_config_dict.get(section, {}).get(key, default_value) with mock.patch.object(config, 'get', autospec=True) as mock_get: with mock.patch.object(config, 'getbool', autospec=True) as mock_getbool: with mock.patch.object(config, 'items', autospec=True) as mock_items: mock_get.side_effect = _config_get_side_effect mock_getbool.side_effect = _config_get_side_effect mock_items.return_value = boto_config_dict.items() yield class FakeCommandWithGcloudStorageMap(command.Command): """Implementation of a fake gsutil command.""" command_spec = command.Command.CreateCommandSpec('fake_shim', min_args=1, max_args=constants.NO_MAX, supported_sub_args='deilrz:', file_url_ok=True) gcloud_storage_map = shim_util.GcloudStorageMap( gcloud_command=['objects', 'fake'], flag_map={ '-r': shim_util.GcloudStorageFlag(gcloud_flag='-x'), '-z': shim_util.GcloudStorageFlag(gcloud_flag='--zip'), '-l': shim_util.GcloudStorageFlag( gcloud_flag='--ludicrous-list', repeat_type=shim_util.RepeatFlagType.LIST), '-d': shim_util.GcloudStorageFlag( gcloud_flag='--delightful-dict', repeat_type=shim_util.RepeatFlagType.DICT), '-e': shim_util.GcloudStorageFlag(gcloud_flag={ 'on': '--e-on', 'off': '--e-off' }), '-f': None, }) help_spec = command.Command.HelpSpec( help_name='fake_shim', help_name_aliases=[], help_type='command_help', help_one_line_summary='Fake one line summary for the command.', help_text='Help text for fake command.', subcommand_help_text={}, ) def RunCommand(self): print('FakeCommandWithGcloudStorageMap called') class FakeCommandWithSubCommandWithGcloudStorageMap(command.Command): """Implementation of a fake gsutil command.""" command_spec = command.Command.CreateCommandSpec( 'fake_with_sub', min_args=1, max_args=constants.NO_MAX, supported_sub_args='ay:', file_url_ok=True, argparse_arguments={ 'set': [ command_argument.CommandArgument. MakeZeroOrMoreCloudBucketURLsArgument() ], 'get': [ command_argument.CommandArgument.MakeNCloudBucketURLsArgument(1) ], }) gcloud_storage_map = shim_util.GcloudStorageMap(gcloud_command={ 'set': shim_util.GcloudStorageMap( gcloud_command=['buckets', 'update'], flag_map={ '-a': shim_util.GcloudStorageFlag(gcloud_flag='-x'), '-y': shim_util.GcloudStorageFlag(gcloud_flag='--yyy'), }), 'get': shim_util.GcloudStorageMap(gcloud_command=['buckets', 'describe'], flag_map={}) }, flag_map={}) help_spec = command.Command.HelpSpec( help_name='fake_with_sub', help_name_aliases=[], help_type='command_help', help_one_line_summary='Fake one line summary for the command.', help_text='Help text for fake command with sub commands.', subcommand_help_text={}, ) class TestGetGCloudStorageArgs(testcase.GsUtilUnitTestCase): """Test Command.get_gcloud_storage_args method.""" def setUp(self): super().setUp() self._fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['-z', 'opt1', '-r', 'arg1', 'arg2'], headers={}, debug=1, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) def test_get_gcloud_storage_args_parses_command_and_flags(self): gcloud_args = self._fake_command.get_gcloud_storage_args() self.assertEqual(gcloud_args, ['objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2']) def test_get_gcloud_storage_args_parses_custom_command_map(self): gcloud_args = self._fake_command.get_gcloud_storage_args( shim_util.GcloudStorageMap( gcloud_command=['objects', 'custom-fake'], flag_map={ '-z': shim_util.GcloudStorageFlag(gcloud_flag='-a'), '-r': shim_util.GcloudStorageFlag(gcloud_flag='-b'), })) self.assertEqual( gcloud_args, ['objects', 'custom-fake', '-a', 'opt1', '-b', 'arg1', 'arg2']) def test_get_gcloud_storage_args_parses_command_in_list_format(self): self._fake_command.gcloud_command = ['objects', 'fake'] gcloud_args = self._fake_command.get_gcloud_storage_args() self.assertEqual(gcloud_args, ['objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2']) def test_get_gcloud_storage_args_parses_subcommands(self): fake_with_subcommand = FakeCommandWithSubCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['set', '-y', 'opt1', '-a', 'arg1', 'arg2'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) gcloud_args = fake_with_subcommand.get_gcloud_storage_args() self.assertEqual( gcloud_args, ['buckets', 'update', '--yyy', 'opt1', '-x', 'arg1', 'arg2']) def test_get_gcloud_storage_args_with_flags_to_ignore(self): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['positional_arg', '-f', '-r', 'opt2', '-f'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) gcloud_args = fake_command.get_gcloud_storage_args() self.assertEqual(gcloud_args, ['objects', 'fake', 'positional_arg', '-x', 'opt2']) def test_get_gcloud_storage_args_with_positional_arg_at_beginning(self): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['positional_arg', '-z', 'opt1', '-r', 'opt2'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) gcloud_args = fake_command.get_gcloud_storage_args() self.assertEqual( gcloud_args, ['objects', 'fake', 'positional_arg', '--zip', 'opt1', '-x', 'opt2']) def test_get_gcloud_storage_args_with_positional_arg_in_middle(self): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['-z', 'opt1', 'positional_arg', '-r', 'opt2'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) gcloud_args = fake_command.get_gcloud_storage_args() self.assertEqual( gcloud_args, ['objects', 'fake', '--zip', 'opt1', 'positional_arg', '-x', 'opt2']) def test_get_gcloud_storage_args_with_repeat_flag_list(self): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['-l', 'flag_value1', '-l', 'flag_value2', 'positional_arg'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) gcloud_args = fake_command.get_gcloud_storage_args() self.assertEqual(gcloud_args, [ 'objects', 'fake', 'positional_arg', '--ludicrous-list=flag_value1,flag_value2' ]) def test_get_gcloud_storage_args_with_repeat_flag_dict(self): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=[ '-d', 'flag_key1:flag_value1', '-d', 'flag_key2:flag_value2', 'positional_arg' ], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) gcloud_args = fake_command.get_gcloud_storage_args() self.assertEqual(gcloud_args, [ 'objects', 'fake', 'positional_arg', '--delightful-dict=flag_key1=flag_value1,flag_key2=flag_value2' ]) def test_get_gcloud_storage_args_with_value_translated_to_flag(self): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['-e', 'on', 'positional_arg'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) gcloud_args = fake_command.get_gcloud_storage_args() self.assertEqual(gcloud_args, ['objects', 'fake', '--e-on', 'positional_arg']) fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=[ 'positional_arg', '-e', 'off', ], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) gcloud_args = fake_command.get_gcloud_storage_args() self.assertEqual(gcloud_args, ['objects', 'fake', 'positional_arg', '--e-off']) def test_raises_error_for_invalid_value_translated_to_flag(self): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['-e', 'incorrect', 'positional_arg'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) with self.assertRaisesRegex( ValueError, 'Flag value not in translation map for "-e": incorrect'): gcloud_args = fake_command.get_gcloud_storage_args() def test_raises_error_if_gcloud_storage_map_is_missing(self): self._fake_command.gcloud_storage_map = None with self.assertRaisesRegex( exception.GcloudStorageTranslationError, 'Command "fake_shim" cannot be translated to gcloud storage' ' because the translation mapping is missing'): self._fake_command.get_gcloud_storage_args() def test_raises_error_if_gcloud_command_is_of_incorrect_type(self): self._fake_command.gcloud_storage_map = shim_util.GcloudStorageMap( gcloud_command='some fake command as a string', flag_map={}) with self.assertRaisesRegex( ValueError, 'Incorrect mapping found for "fake_shim" command'): self._fake_command.get_gcloud_storage_args() def test_raises_error_if_command_option_mapping_is_missing(self): self._fake_command.gcloud_storage_map = shim_util.GcloudStorageMap( gcloud_command=['fake'], flag_map={ '-z': shim_util.GcloudStorageFlag('-a') # Mapping for -r is missing. }) with self.assertRaisesRegex( exception.GcloudStorageTranslationError, 'Command option "-r" cannot be translated to gcloud storage'): self._fake_command.get_gcloud_storage_args() def test_raises_error_if_sub_command_mapping_is_missing(self): fake_with_subcommand = FakeCommandWithSubCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['set', '-y', 'opt1', '-a', 'arg1', 'arg2'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) fake_with_subcommand.gcloud_storage_map = shim_util.GcloudStorageMap( gcloud_command={}, # Missing mapping for set. flag_map={}) with self.assertRaisesRegex( exception.GcloudStorageTranslationError, 'Command "fake_with_sub" cannot be translated to gcloud storage' ' because the translation mapping is missing.'): fake_with_subcommand.get_gcloud_storage_args() def test_raises_error_if_flags_mapping_at_top_level_for_subcommand(self): fake_with_subcommand = FakeCommandWithSubCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['set', '-y', 'opt1', '-a', 'arg1', 'arg2'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) fake_with_subcommand.gcloud_storage_map.flag_map = {'a': 'b'} with self.assertRaisesRegex( ValueError, 'Flags mapping should not be present at the top-level command' ' if a sub-command is used. Command: fake_with_sub'): fake_with_subcommand.get_gcloud_storage_args() class TestTranslateToGcloudStorageIfRequested(testcase.ShimUnitTestBase): """Test Command.translate_to_gcloud_storage_if_requested method.""" def setUp(self): super().setUp() self._fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['-z', 'opt1', '-r', 'arg1', 'arg2'], headers={}, debug=0, trace_token=None, parallel_operations=True, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) def test_gets_gcloud_binary_path_on_non_windows(self): with mock.patch.object(system_util, 'IS_WINDOWS', new=False): self.assertEqual(shim_util._get_gcloud_binary_path('fake_root'), os.path.join('fake_root', 'bin', 'gcloud')) def test_gets_gcloud_binary_path_on_windows(self): with mock.patch.object(system_util, 'IS_WINDOWS', new=True): self.assertEqual(shim_util._get_gcloud_binary_path('fake_root'), os.path.join('fake_root', 'bin', 'gcloud.cmd')) def test_returns_false_with_use_gcloud_storage_never(self): """Should not attempt translation.""" with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'False')]): with mock.patch.object(self._fake_command, 'get_gcloud_storage_args', autospec=True) as mock_get_gcloud_storage_args: self.assertFalse( self._fake_command.translate_to_gcloud_storage_if_requested()) self.assertFalse(mock_get_gcloud_storage_args.called) def test_returns_true_with_valid_gcloud_storage_map(self): """Should return True and perform the translation.""" with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self.assertTrue( self._fake_command.translate_to_gcloud_storage_if_requested()) # Verify translation. expected_gcloud_path = shim_util._get_gcloud_binary_path('fake_dir') self.assertEqual(self._fake_command._translated_gcloud_storage_command, [ expected_gcloud_path, 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2' ]) def test_with_cloudsdk_root_dir_unset_and_gcloud_binary_path_set(self): """Should return True and perform the translation.""" gcloud_path = shim_util._get_gcloud_binary_path('fake_dir') with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': None, 'GCLOUD_BINARY_PATH': gcloud_path, }): self.assertTrue( self._fake_command.translate_to_gcloud_storage_if_requested()) # Verify translation. self.assertEqual(self._fake_command._translated_gcloud_storage_command, [ gcloud_path, 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2' ]) def test_returns_false_if_invalid_use_gcloud_storage_value(self): with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'invalid') ]): self.assertFalse( self._fake_command.translate_to_gcloud_storage_if_requested()) def test_raises_error_if_invalid_hidden_shim_mode_value(self): with util.SetBotoConfigForTest([('GSUtil', 'hidden_shim_mode', 'invalid')]): with self.assertRaisesRegex( exception.CommandException, 'CommandException: Invalid option specified for' ' GSUtil:use_gcloud_storage config setting. Should be one of:' ' no_fallback | dry_run | none'): self._fake_command.translate_to_gcloud_storage_if_requested() def test_raises_error_if_cloudsdk_root_dir_is_none(self): with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_ROOT_DIR': None, }): with self.assertRaisesRegex( exception.CommandException, 'CommandException: Requested to use "gcloud storage" but the ' 'gcloud binary path cannot be found. This might happen if you' ' attempt to use gsutil that was not installed via Cloud SDK.' ' You can manually set the `CLOUDSDK_ROOT_DIR` environment variable' ' to point to the google-cloud-sdk installation directory to' ' resolve the issue. Alternatively, you can set' ' `use_gcloud_storage=False` to disable running the command' ' using gcloud storage.'): self._fake_command.translate_to_gcloud_storage_if_requested() def test_raises_error_if_pass_credentials_to_gsutil_is_missing(self): error_regex = ( r'CommandException: Requested to use "gcloud storage" but gsutil' r' is not using the same credentials as' r' gcloud. You can make gsutil use the same credentials' r' by running:\n' r'{} config set pass_credentials_to_gsutil True').format( re.escape(shim_util._get_gcloud_binary_path('fake_dir'))) with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_ROOT_DIR': 'fake_dir', 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': None }): with self.assertRaisesRegex(exception.CommandException, error_regex): self._fake_command.translate_to_gcloud_storage_if_requested() @mock.patch.object(boto_util, 'UsingGsHmac', return_value=True) def test_raises_error_if_using_gs_hmac_without_xml_support(self, _): with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_ROOT_DIR': 'fake_dir', 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', }): self._fake_command.command_spec = command.Command.CreateCommandSpec( 'fake_shim', gs_api_support=[ApiSelector.JSON]) with self.assertRaisesRegex( exception.CommandException, 'CommandException: Requested to use "gcloud storage" with Cloud' ' Storage XML API HMAC credentials but the "fake_shim" command can' ' only be used with the Cloud Storage JSON API.'): self._fake_command.translate_to_gcloud_storage_if_requested() def test_raises_error_if_gcloud_storage_map_missing(self): self._fake_command.gcloud_storage_map = None with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with self.assertRaisesRegex( exception.CommandException, 'CommandException: Command "fake_shim" cannot be translated to' ' gcloud storage because the translation mapping is missing.'): self._fake_command.translate_to_gcloud_storage_if_requested() def test_use_gcloud_storage_true_with_hidden_shim_mode_not_set(self): """Should not raise error.""" with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', None)]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): # return_stderr does not work here. Probably because we have # defined the FakeCommand in the same module. stdout, mock_log_handler = self.RunCommand('fake_shim', args=['-i', 'arg1'], return_stdout=True, return_log_handler=True) self.assertIn( 'Cannot translate gsutil command to gcloud storage.' ' Going to run gsutil command. Error: Command option "-i"' ' cannot be translated to gcloud storage', mock_log_handler.messages['error']) self.assertIn('FakeCommandWithGcloudStorageMap called', stdout) def test_dry_run_mode_prints_translated_command(self): """Should print the gcloud command and run gsutil.""" with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'True', 'hidden_shim_mode': 'dry_run' } }): with util.SetEnvironmentForTest({'CLOUDSDK_ROOT_DIR': 'fake_dir'}): stdout, mock_log_handler = self.RunCommand('fake_shim', args=['arg1'], return_stdout=True, return_log_handler=True) self.assertIn( 'Gcloud Storage Command: {} objects fake arg1'.format( shim_util._get_gcloud_binary_path('fake_dir')), mock_log_handler.messages['info']) self.assertIn( 'FakeCommandWithGcloudStorageMap called'.format( shim_util._get_gcloud_binary_path('fake_dir')), stdout) def test_non_dry_mode_logs_translated_command_to_debug_logs(self): with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with mock.patch.object(self._fake_command, 'logger', autospec=True) as mock_logger: self._fake_command.translate_to_gcloud_storage_if_requested() # Verify translation. mock_logger.debug.assert_has_calls([ mock.call('Gcloud Storage Command: {} objects' ' fake --zip opt1 -x arg1 arg2'.format( shim_util._get_gcloud_binary_path('fake_dir'))), mock.call('Environment variables for Gcloud Storage:'), mock.call('%s=%s', 'CLOUDSDK_METRICS_ENVIRONMENT', 'gsutil_shim'), mock.call('%s=%s', 'CLOUDSDK_STORAGE_RUN_BY_GSUTIL_SHIM', 'True') ], any_order=True) def test_print_gcloud_storage_env_vars_in_dry_run_mode(self): """Should log the command and env vars to logger.info""" with mock.patch.object(self._fake_command, 'logger', autospec=True) as mock_logger: self._fake_command._print_gcloud_storage_command_info( ['fake', 'gcloud', 'command'], {'fake_env_var': 'val'}, dry_run=True) expected_calls = [ mock.call('Gcloud Storage Command: fake gcloud command'), mock.call('Environment variables for Gcloud Storage:'), mock.call('%s=%s', 'fake_env_var', 'val'), ] self.assertEqual(mock_logger.info.mock_calls, expected_calls) def test_top_level_flags_get_translated(self): """Should return True and perform the translation.""" boto_config = { 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } } with _mock_boto_config(boto_config): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['arg1', 'arg2'], headers={}, # Headers will be tested separately. debug=3, # -D option trace_token='fake_trace_token', user_project='fake_user_project', parallel_operations=False, # Without the -m option. bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) self.assertTrue(fake_command.translate_to_gcloud_storage_if_requested()) # Verify translation. expected_gcloud_path = shim_util._get_gcloud_binary_path('fake_dir') self.assertEqual(fake_command._translated_gcloud_storage_command, [ expected_gcloud_path, 'objects', 'fake', 'arg1', 'arg2', '--verbosity', 'debug', '--billing-project=fake_user_project', '--trace-token=fake_trace_token' ]) self.assertCountEqual( fake_command._translated_env_variables, { 'CLOUDSDK_STORAGE_PROCESS_COUNT': '1', 'CLOUDSDK_STORAGE_THREAD_COUNT': '1', 'CLOUDSDK_METRICS_ENVIRONMENT': 'gsutil_shim', 'CLOUDSDK_STORAGE_RUN_BY_GSUTIL_SHIM': 'True', }) def test_anonymous_credentials_sets_disable_credentials_env_variable(self): self._mock_subprocess_run.return_value.stdout = '' boto_config = { 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } } with _mock_boto_config(boto_config): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self.assertTrue( self._fake_command.translate_to_gcloud_storage_if_requested()) # Verify translation. expected_gcloud_path = shim_util._get_gcloud_binary_path('fake_dir') self.assertCountEqual( self._fake_command._translated_env_variables, { 'CLOUDSDK_METRICS_ENVIRONMENT': 'gsutil_shim', 'CLOUDSDK_STORAGE_RUN_BY_GSUTIL_SHIM': 'True', 'CLOUDSDK_AUTH_DISABLE_CREDENTIALS': 'True', }) self._mock_subprocess_run.assert_called_once_with( [expected_gcloud_path, 'config', 'get', 'account'], stdout=-1, stderr=-1, encoding='utf-8') def test_gcloud_config_get_account_nonzero_returncode_raises_error(self): self._mock_subprocess_run.return_value.stdout = b'' self._mock_subprocess_run.return_value.stderr = b'fake error message' self._mock_subprocess_run.return_value.returncode = 1 expected_gcloud_path = shim_util._get_gcloud_binary_path('fake_dir') boto_config = { 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } } with _mock_boto_config(boto_config): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with self.assertRaisesRegex( exception.CommandException, "Error occurred while trying to retrieve gcloud's active account." " Error: b'fake error message"): self._fake_command.translate_to_gcloud_storage_if_requested() self._mock_subprocess_run.assert_called_once_with( [expected_gcloud_path, 'config', 'get', 'account'], stdout=-1, stderr=-1, encoding='utf-8', ) def test_parallel_operations_true_does_not_add_process_count_env_vars(self): with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self._fake_command.parallel_operations = True self._fake_command.translate_to_gcloud_storage_if_requested() self.assertNotIn('CLOUDSDK_STORAGE_PROCESS_COUNT', self._fake_command._translated_env_variables) self.assertNotIn('CLOUDSDK_STORAGE_THREAD_COUNT', self._fake_command._translated_env_variables) def test_parallel_operations_false_but_parallelism_turned_on_for_rsync(self): command = rsync.RsyncCommand(command_runner=mock.ANY, args=['arg1', 'arg2'], headers={}, debug=0, trace_token=None, parallel_operations=False, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): command.translate_to_gcloud_storage_if_requested() self.assertNotIn('CLOUDSDK_STORAGE_PROCESS_COUNT', command._translated_env_variables) self.assertNotIn('CLOUDSDK_STORAGE_THREAD_COUNT', command._translated_env_variables) def test_parallelism_turned_on_for_rsync_unless_boto_set_sequential(self): with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback'), ('GSUtil', 'parallel_process_count', '1'), ('GSUtil', 'thread_process_count', '1')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): command = rsync.RsyncCommand( command_runner=mock.ANY, args=['arg1', 'arg2'], headers={}, debug=0, trace_token=None, parallel_operations=False, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) command.translate_to_gcloud_storage_if_requested() self.assertEqual( command._translated_env_variables['CLOUDSDK_STORAGE_PROCESS_COUNT'], '1') self.assertEqual( command._translated_env_variables['CLOUDSDK_STORAGE_THREAD_COUNT'], '1') def test_debug_value_4_adds_log_http_flag(self): # Debug level 4 represents the -DD option. with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self._fake_command.debug = 4 self._fake_command.translate_to_gcloud_storage_if_requested() self.assertEqual(self._fake_command._translated_gcloud_storage_command, [ shim_util._get_gcloud_binary_path('fake_dir'), 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2', '--verbosity', 'debug', '--log-http' ]) @mock.patch.object(constants, 'IMPERSONATE_SERVICE_ACCOUNT', new='fake_service_account') def test_impersonate_service_account_translation(self): """Should add the --impersonate-service-account flag.""" with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self._fake_command.translate_to_gcloud_storage_if_requested() self.assertEqual( self._fake_command._translated_gcloud_storage_command, [ shim_util._get_gcloud_binary_path('fake_dir'), 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2', '--impersonate-service-account=fake_service_account' ]) def test_quiet_mode_translation_adds_no_user_output_enabled_flag(self): with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self._fake_command.quiet_mode = True self._fake_command.translate_to_gcloud_storage_if_requested() self.assertEqual(self._fake_command._translated_gcloud_storage_command, [ shim_util._get_gcloud_binary_path('fake_dir'), 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2', '--no-user-output-enabled' ]) def test_returns_false_for_version_command(self): command = version.VersionCommand( command_runner=mock.ANY, args=[], headers={}, debug=0, trace_token=None, parallel_operations=True, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with mock.patch.object(command, 'get_gcloud_storage_args', autospec=True) as mock_get_gcloud_storage_args: self.assertFalse(command.translate_to_gcloud_storage_if_requested()) self.assertFalse(mock_get_gcloud_storage_args.called) def test_returns_false_for_test_command(self): command = test.TestCommand(command_runner=mock.ANY, args=[], headers={}, debug=0, trace_token=None, parallel_operations=True, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with mock.patch.object(command, 'get_gcloud_storage_args', autospec=True) as mock_get_gcloud_storage_args: self.assertFalse(command.translate_to_gcloud_storage_if_requested()) self.assertFalse(mock_get_gcloud_storage_args.called) class TestHeaderTranslation(testcase.GsUtilUnitTestCase): """Test gsutil header translation.""" def setUp(self): super().setUp() self._fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['-z', 'opt1', '-r', 'arg1', 'arg2'], headers={}, debug=0, trace_token=None, parallel_operations=True, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) @mock.patch.object(shim_util, 'COMMANDS_SUPPORTING_ALL_HEADERS', new={'fake_shim'}) @mock.patch.object(subprocess, 'run', autospec=True) def test_translated_headers_get_added_to_final_command( self, mock_subprocess_run): mock_subprocess_run.return_value.returncode = 0 with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'always', 'hidden_shim_mode': 'no_fallback' } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['arg1', 'arg2'], headers={'Content-Type': 'fake_val'}, debug=1, trace_token=None, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) self.assertTrue(fake_command.translate_to_gcloud_storage_if_requested()) self.assertEqual(fake_command._translated_gcloud_storage_command, [ shim_util._get_gcloud_binary_path('fake_dir'), 'objects', 'fake', 'arg1', 'arg2', '--content-type=fake_val' ]) @mock.patch.object(shim_util, 'COMMANDS_SUPPORTING_ALL_HEADERS', new={'fake_shim'}) def test_translate_headers_returns_correct_flags_for_data_transfer_command( self): self._fake_command.headers = { # Data tranfer related headers. 'Cache-Control': 'fake_Cache_Control', 'Content-Disposition': 'fake_Content-Disposition', 'Content-Encoding': 'fake_Content-Encoding', 'Content-Language': 'fake_Content-Language', 'Content-Type': 'fake_Content-Type', 'Content-MD5': 'fake_Content-MD5', 'custom-time': 'fake_time', # Precondition headers. 'x-goog-if-generation-match': 'fake_gen_match', 'x-goog-if-metageneration-match': 'fake_metagen_match', # Custom metadata. 'x-goog-meta-cAsE': 'sEnSeTiVe', 'x-goog-meta-gfoo': 'fake_goog_meta', 'x-amz-meta-afoo': 'fake_amz_meta', 'x-amz-afoo': 'fake_amz_custom_header', } flags = self._fake_command._translate_headers() self.assertCountEqual(flags, [ '--cache-control=fake_Cache_Control', '--content-disposition=fake_Content-Disposition', '--content-encoding=fake_Content-Encoding', '--content-language=fake_Content-Language', '--content-type=fake_Content-Type', '--content-md5=fake_Content-MD5', '--custom-time=fake_time', '--if-generation-match=fake_gen_match', '--if-metageneration-match=fake_metagen_match', '--update-custom-metadata=cAsE=sEnSeTiVe', '--update-custom-metadata=gfoo=fake_goog_meta', '--update-custom-metadata=afoo=fake_amz_meta', '--additional-headers=x-amz-afoo=fake_amz_custom_header', ]) @mock.patch.object(shim_util, 'COMMANDS_SUPPORTING_ALL_HEADERS', new={'fake_shim'}) def test_translate_custom_headers_returns_correct_flags(self): flags = self._fake_command._translate_headers( {'Cache-Control': 'fake_Cache_Control'}) self.assertCountEqual(flags, ['--cache-control=fake_Cache_Control']) @mock.patch.object(shim_util, 'COMMANDS_SUPPORTING_ALL_HEADERS', new={'fake_shim'}) def test_translate_custom_headers_handles_multiple_additional_headers(self): flags = self._fake_command._translate_headers( collections.OrderedDict([('header1', 'value1'), ('header2', 'value2')])) self.assertCountEqual( flags, ['--additional-headers=header1=value1,header2=value2']) @mock.patch.object(shim_util, 'COMMANDS_SUPPORTING_ALL_HEADERS', new={'fake_shim'}) def test_translate_clear_headers_returns_correct_flags(self): flags = self._fake_command._translate_headers( {'Cache-Control': 'fake_Cache_Control'}, unset=True) self.assertCountEqual(flags, ['--clear-cache-control']) @mock.patch.object(shim_util, 'COMMANDS_SUPPORTING_ALL_HEADERS', new={'fake_shim'}) def test_translate_headers_for_data_transfer_command_with_additional_header( self): """Should log a warning.""" self._fake_command.headers = {'additional': 'header'} with mock.patch.object(self._fake_command.logger, 'warn', autospec=True) as mock_warning: self.assertEqual(self._fake_command._translate_headers(), ['--additional-headers=additional=header']) mock_warning.assert_called_once_with( 'Header additional:header cannot be translated to a gcloud' ' storage equivalent flag. It is being treated as an arbitrary' ' request header.') @mock.patch.object(shim_util, 'PRECONDITONS_ONLY_SUPPORTED_COMMANDS', new={'fake_shim'}) def test_translate_valid_headers_for_precondition_supported_command(self): self._fake_command.headers = { 'x-goog-if-generation-match': 'fake_gen_match', 'x-goog-if-metageneration-match': 'fake_metagen_match', # Custom metadata. These should be ignored. 'x-goog-meta-foo': 'fake_goog_meta', } flags = self._fake_command._translate_headers() self.assertCountEqual(flags, [ '--if-generation-match=fake_gen_match', '--if-metageneration-match=fake_metagen_match', ]) @mock.patch.object(shim_util, 'PRECONDITONS_ONLY_SUPPORTED_COMMANDS', new={'fake_shim'}) def test_translate_short_headers_for_precondition_supported_command(self): self._fake_command.headers = { 'x-goog-generation-match': 'fake_gen_match', 'x-goog-metageneration-match': 'fake_metagen_match', # Custom metadata. These should be ignored. 'x-goog-meta-foo': 'fake_goog_meta', } flags = self._fake_command._translate_headers() self.assertCountEqual(flags, [ '--if-generation-match=fake_gen_match', '--if-metageneration-match=fake_metagen_match', ]) @mock.patch.object(shim_util, 'PRECONDITONS_ONLY_SUPPORTED_COMMANDS', new={'fake_shim'}) def test_translate_headers_for_precondition_supported_command_with_additional_header( self): """Should be ignored and not raise any error.""" self._fake_command.headers = {'additional': 'header'} with mock.patch.object(self._fake_command.logger, 'warn', autospec=True) as mock_warning: self.assertEqual(self._fake_command._translate_headers(), ['--additional-headers=additional=header']) mock_warning.assert_called_once_with( 'Header additional:header cannot be translated to a gcloud' ' storage equivalent flag. It is being treated as an arbitrary' ' request header.') def test_translate_headers_only_uses_additional_headers_for_commands_not_in_allowlist( self): # Allowlist is defined by the shim_util.COMMANDS_SUPPORTING_ALL_HEADERS and # the shim_util.PRECONDITONS_ONLY_SUPPORTED_COMMANDS list. # The fake_shim command defined by self._fake_command is not part of # either of the list, and hence the headers must be ignored. self._fake_command.headers = { # Header from the DATA_TRANSFER_HEADERS list. 'Cache-Control': 'fake_Cache_Control', # Header from the PRECONDITIONS_HEADERS list. 'x-goog-if-generation-match': 'fake_gen_match', # Custom metadata. These should be ignored. 'x-goog-meta-foo': 'fake_goog_meta', # Additional header. Should be added. 'additional': 'header' } self.assertEqual(self._fake_command._translate_headers(), ['--additional-headers=additional=header']) @mock.patch.object(shim_util, 'PRECONDITONS_ONLY_SUPPORTED_COMMANDS', new={'fake_shim'}) def test_translate_headers_ignores_x_goog_api_version_header(self): self._fake_command.headers = { 'x-goog-if-generation-match': 'fake_gen_match', 'x-goog-api-version': '2', # Should be ignored. } self.assertEqual(self._fake_command._translate_headers(), ['--if-generation-match=fake_gen_match']) class TestGetFlagFromHeader(testcase.GsUtilUnitTestCase): """Test Command.get_flag_from_header function. We only test the unset functionality because rest of the workflows have been already tested indirectly in TestHeaderTranslation. """ def test_get_flag_from_header_with_unset_true_for_data_transfer_headers(self): # Ideally we should use parameterized, but avoiding adding a new dependency # just for the shim. headers_to_expected_flag_map = { 'Cache-Control': '--clear-cache-control', 'Content-Disposition': '--clear-content-disposition', 'Content-Encoding': '--clear-content-encoding', 'Content-Language': '--clear-content-language', 'Content-Type': '--clear-content-type', 'custom-time': '--clear-custom-time', } for header, expected_flag in headers_to_expected_flag_map.items(): result = shim_util.get_flag_from_header(header, 'fake_val', unset=True) self.assertEqual(result, expected_flag) def test_get_flag_from_header_with_unset_true_for_precondition_headers(self): """Should return None.""" # Ideally we should use parameterized, but avoiding adding a new dependency # just for the shim. for header in [ 'x-goog-if-generation-match', 'x-goog-if-metageneration-match', ]: result = shim_util.get_flag_from_header(header, 'fake_val', unset=True) self.assertIsNone(result) def test_get_flag_from_header_with_unset_true_for_content_md5(self): """Should return None.""" result = shim_util.get_flag_from_header('Content-MD5', 'fake_val', unset=True) self.assertIsNone(result) def test_get_flag_from_header_with_unset_true_for_invalid_header(self): """Should return None.""" result = shim_util.get_flag_from_header('invalid_header', 'fake_val', unset=True) self.assertIsNone(result) def test_get_flag_from_header_with_unset_true_for_metadata_headers(self): """Should return --remove-custom-metadata flag.""" headers_to_expected_flag_map = { 'x-goog-meta-foo': '--remove-custom-metadata=foo', 'x-amz-meta-foo': '--remove-custom-metadata=foo', } for header, expected_flag in headers_to_expected_flag_map.items(): result = shim_util.get_flag_from_header(header, 'fake_val', unset=True) self.assertEqual(result, expected_flag) class TestFormatFlagUtils(testcase.GsUtilUnitTestCase): """Test utils used for generating gcloud --format flags.""" def test_gets_format_flag_caret_on_non_windows(self): with mock.patch.object(system_util, 'IS_WINDOWS', new=False): self.assertEqual(shim_util.get_format_flag_caret(), '^') def test_gets_format_flag_escaped_caret_on_windows(self): with mock.patch.object(system_util, 'IS_WINDOWS', new=True): self.assertEqual(shim_util.get_format_flag_caret(), '^^^^') def test_gets_format_flag_newline_on_non_windows(self): with mock.patch.object(system_util, 'IS_WINDOWS', new=False): self.assertEqual(shim_util.get_format_flag_newline(), '\n') def test_gets_format_flag_escaped_newline_on_windows(self): with mock.patch.object(system_util, 'IS_WINDOWS', new=True): self.assertEqual(shim_util.get_format_flag_newline(), r'^\^n') class TestBotoTranslation(testcase.GsUtilUnitTestCase): """Test gsutil header translation.""" def setUp(self): super().setUp() self._fake_command = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['-z', 'opt1', '-r', 'arg1', 'arg2'], headers={}, debug=0, trace_token=None, parallel_operations=True, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) @mock.patch.object(shim_util, 'COMMANDS_SUPPORTING_ALL_HEADERS', new={'fake_shim'}) @mock.patch.object(subprocess, 'run', autospec=True) def test_translated_boto_config_gets_added(self, mock_subprocess_run): """Should add translated env vars as well flags.""" mock_subprocess_run.return_value.returncode = 0 with _mock_boto_config({ 'GSUtil': { 'use_gcloud_storage': 'True', 'hidden_shim_mode': 'no_fallback', 'content_language': 'foo', 'default_project_id': 'fake_project', } }): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self.assertTrue( self._fake_command.translate_to_gcloud_storage_if_requested()) # Verify translation. expected_gcloud_path = shim_util._get_gcloud_binary_path('fake_dir') self.assertEqual( self._fake_command._translated_gcloud_storage_command, [ expected_gcloud_path, 'objects', 'fake', '--zip', 'opt1', '-x', 'arg1', 'arg2', '--content-language=foo' ]) self.assertEqual( self._fake_command._translated_env_variables, { 'CLOUDSDK_CORE_PROJECT': 'fake_project', 'CLOUDSDK_METRICS_ENVIRONMENT': 'gsutil_shim', 'CLOUDSDK_STORAGE_RUN_BY_GSUTIL_SHIM': 'True' }) def test_gcs_json_endpoint_translation(self): with _mock_boto_config({ 'Credentials': { 'gs_json_host': 'foo_host', 'gs_json_port': '1234', 'json_api_version': 'v2', } }): flags, env_vars = self._fake_command._translate_boto_config() self.assertEqual(flags, []) self.assertEqual( env_vars, { 'CLOUDSDK_API_ENDPOINT_OVERRIDES_STORAGE': 'https://foo_host:1234/storage/v2', }) def test_gcs_json_endpoint_translation_with_missing_port(self): with _mock_boto_config({ 'Credentials': { 'gs_json_host': 'foo_host', 'json_api_version': 'v2', } }): flags, env_vars = self._fake_command._translate_boto_config() self.assertEqual(flags, []) self.assertEqual(env_vars, { 'CLOUDSDK_API_ENDPOINT_OVERRIDES_STORAGE': 'https://foo_host/storage/v2', }) def test_gcs_json_endpoint_translation_usees_default_version_v1(self): with _mock_boto_config( {'Credentials': { 'gs_json_host': 'foo_host', 'gs_json_port': '1234', }}): flags, env_vars = self._fake_command._translate_boto_config() self.assertEqual(flags, []) self.assertEqual( env_vars, { 'CLOUDSDK_API_ENDPOINT_OVERRIDES_STORAGE': 'https://foo_host:1234/storage/v1' }) def test_s3_endpoint_translation(self): with _mock_boto_config( {'Credentials': { 's3_host': 's3_host', 's3_port': '1234', }}): flags, env_vars = self._fake_command._translate_boto_config() self.assertEqual(flags, []) self.assertEqual( env_vars, {'CLOUDSDK_STORAGE_S3_ENDPOINT_URL': 'https://s3_host:1234'}) def test_s3_endpoint_translation_with_missing_port(self): with _mock_boto_config({'Credentials': {'s3_host': 's3_host',}}): flags, env_vars = self._fake_command._translate_boto_config() self.assertEqual(flags, []) self.assertEqual(env_vars, {'CLOUDSDK_STORAGE_S3_ENDPOINT_URL': 'https://s3_host'}) @mock.patch.object(shim_util, 'ENCRYPTION_SUPPORTED_COMMANDS', new={'fake_shim'}) def test_encryption_key_gets_converted_to_flag(self): with _mock_boto_config({'GSUtil': {'encryption_key': 'fake_key'}}): flags, _ = self._fake_command._translate_boto_config() self.assertEqual(flags, ['--encryption-key=fake_key']) @mock.patch.object(shim_util, 'ENCRYPTION_SUPPORTED_COMMANDS', new={'fake_shim'}) def test_decryption_key_gets_converted_to_flag(self): with _mock_boto_config({ 'GSUtil': collections.OrderedDict([ ('decryption_key1', 'key1'), ('decryption_key12', 'key12'), ('decryption_key100', 'key100'), ]) }): flags, _ = self._fake_command._translate_boto_config() self.assertEqual(flags, ['--decryption-keys=key1,key12,key100']) @mock.patch.object(boto_util, 'UsingGsHmac', return_value=False) def test_gs_hmac_auth_env_when_not_using_gs_hmac(self, mock_using_gs_hmac): with _mock_boto_config({ 'Credentials': { 'gs_access_key_id': 'foo', 'gs_secret_access_key': 'bar', } }): flags, env_vars = self._fake_command._translate_boto_config() self.assertEqual(mock_using_gs_hmac.call_count, 2) self.assertEqual(flags, []) self.assertEqual(env_vars, {}) @mock.patch.object(boto_util, 'UsingGsHmac', return_value=True) def test_boto_config_translation_for_supported_fields(self, _): with _mock_boto_config({ 'Credentials': { 'aws_access_key_id': 'AWS_ACCESS_KEY_ID_value', 'aws_secret_access_key': 'AWS_SECRET_ACCESS_KEY_value', 'gs_access_key_id': 'CLOUDSDK_STORAGE_GS_XML_ACCESS_KEY_ID_value', 'gs_secret_access_key': 'CLOUDSDK_STORAGE_GS_XML_SECRET_ACCESS_KEY_value', 'use_client_certificate': True, }, 'Boto': { 'proxy': 'CLOUDSDK_PROXY_ADDRESS_value', 'proxy_type': 'CLOUDSDK_PROXY_TYPE_value', 'proxy_port': 'CLOUDSDK_PROXY_PORT_value', 'proxy_user': 'CLOUDSDK_PROXY_USERNAME_value', 'proxy_pass': 'CLOUDSDK_PROXY_PASSWORD_value', 'proxy_rdns': 'CLOUDSDK_PROXY_RDNS_value', 'http_socket_timeout': 'HTTP_TIMEOUT_value', 'ca_certificates_file': 'CA_CERTS_FILE_value', 'https_validate_certificates': False, 'max_retry_delay': 'BASE_RETRY_DELAY_value', 'num_retries': 'MAX_RETRIES_value', }, 'GSUtil': { 'check_hashes': 'CHECK_HASHES_value', 'default_project_id': 'CLOUDSDK_CORE_PROJECT_value', 'disable_analytics_prompt': 'USAGE_REPORTING_value', 'use_magicfile': 'USE_MAGICFILE_value', 'parallel_composite_upload_threshold': '100M', 'resumable_threshold': '256K', 'rsync_buffer_lines': '32000', }, 'OAuth2': { 'client_id': 'CLOUDSDK_AUTH_CLIENT_ID_value', 'client_secret': 'AUTH_CLIENT_SECRET_value', 'provider_authorization_uri': 'CLOUDSDK_AUTH_AUTH_HOST_value', 'provider_token_uri': 'CLOUDSDK_AUTH_TOKEN_HOST_value', }, }): flags, env_vars = self._fake_command._translate_boto_config() self.assertEqual(flags, []) self.maxDiff = None self.assertDictEqual( env_vars, { 'AWS_ACCESS_KEY_ID': 'AWS_ACCESS_KEY_ID_value', 'AWS_SECRET_ACCESS_KEY': 'AWS_SECRET_ACCESS_KEY_value', 'CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE': True, 'CLOUDSDK_STORAGE_GS_XML_ACCESS_KEY_ID': 'CLOUDSDK_STORAGE_GS_XML_ACCESS_KEY_ID_value', 'CLOUDSDK_STORAGE_GS_XML_SECRET_ACCESS_KEY': 'CLOUDSDK_STORAGE_GS_XML_SECRET_ACCESS_KEY_value', 'CLOUDSDK_PROXY_ADDRESS': 'CLOUDSDK_PROXY_ADDRESS_value', 'CLOUDSDK_PROXY_ADDRESS': 'CLOUDSDK_PROXY_ADDRESS_value', 'CLOUDSDK_PROXY_TYPE': 'CLOUDSDK_PROXY_TYPE_value', 'CLOUDSDK_PROXY_PORT': 'CLOUDSDK_PROXY_PORT_value', 'CLOUDSDK_PROXY_USERNAME': 'CLOUDSDK_PROXY_USERNAME_value', 'CLOUDSDK_PROXY_PASSWORD': 'CLOUDSDK_PROXY_PASSWORD_value', 'CLOUDSDK_PROXY_RDNS': 'CLOUDSDK_PROXY_RDNS_value', 'CLOUDSDK_CORE_HTTP_TIMEOUT': 'HTTP_TIMEOUT_value', 'CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILE': 'CA_CERTS_FILE_value', 'CLOUDSDK_AUTH_DISABLE_SSL_VALIDATION': True, 'CLOUDSDK_STORAGE_BASE_RETRY_DELAY': 'BASE_RETRY_DELAY_value', 'CLOUDSDK_STORAGE_MAX_RETRIES': 'MAX_RETRIES_value', 'CLOUDSDK_STORAGE_CHECK_HASHES': 'CHECK_HASHES_value', 'CLOUDSDK_CORE_PROJECT': 'CLOUDSDK_CORE_PROJECT_value', 'CLOUDSDK_CORE_DISABLE_USAGE_REPORTING': 'USAGE_REPORTING_value', 'CLOUDSDK_STORAGE_USE_MAGICFILE': 'USE_MAGICFILE_value', 'CLOUDSDK_STORAGE_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD': '100M', 'CLOUDSDK_STORAGE_RESUMABLE_THRESHOLD': '256K', 'CLOUDSDK_STORAGE_RSYNC_LIST_CHUNK_SIZE': '32000', 'CLOUDSDK_AUTH_CLIENT_ID': 'CLOUDSDK_AUTH_CLIENT_ID_value', 'CLOUDSDK_AUTH_CLIENT_SECRET': 'AUTH_CLIENT_SECRET_value', 'CLOUDSDK_AUTH_AUTH_HOST': 'CLOUDSDK_AUTH_AUTH_HOST_value', 'CLOUDSDK_AUTH_TOKEN_HOST': 'CLOUDSDK_AUTH_TOKEN_HOST_value', }) def test_missing_mappging_gets_ignored(self): with _mock_boto_config({'GSUtil': {'unsupported_field': 'foo'}}): flags, env_vars = self._fake_command._translate_boto_config() self.assertEqual(flags, []) self.assertEqual(env_vars, {}) def test_truthy_https_validate_certificates(self): """Should not set CLOUDSDK_AUTH_DISABLE_SSL_VALIDATION.""" with _mock_boto_config({'GSUtil': {'https_validate_certificates': True}}): flags, env_vars = self._fake_command._translate_boto_config() self.assertEqual(flags, []) self.assertEqual(env_vars, {}) def test_missing_required_fields_log_error(self): with _mock_boto_config({ 'GSUtil': { 'stet_binary_path': 'foo_bin', 'stet_config_path': 'foo_config' } }): with mock.patch.object(self._fake_command.logger, 'error', autospec=True) as mock_logger: flags, env_vars = self._fake_command._translate_boto_config() self.assertEqual(flags, []) self.assertEqual(env_vars, {}) self.assertCountEqual(mock_logger.mock_calls, [ mock.call('The boto config field GSUtil:stet_binary_path cannot be' ' translated to gcloud storage equivalent.'), mock.call('The boto config field GSUtil:stet_config_path cannot be' ' translated to gcloud storage equivalent.') ]) class TestRunGcloudStorage(testcase.GsUtilUnitTestCase): """Test Command.run_gcloud_storage method.""" @mock.patch.object(os.environ, 'copy', return_value={'old_key': 'old_value'}) @mock.patch.object(subprocess, 'run', autospec=True) def test_calls_subprocess_with_translated_command_and_env_vars( self, mock_run, mock_environ_copy): command_instance = FakeCommandWithGcloudStorageMap( command_runner=mock.ANY, args=['-z', 'opt1', '-r', 'arg1', 'arg2'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): command_instance._translated_env_variables = { 'new_key': 'new_value', } command_instance._translated_gcloud_storage_command = ['gcloud', 'foo'] actual_return_code = command_instance.run_gcloud_storage() mock_run.assert_called_once_with(['gcloud', 'foo'], env={ 'old_key': 'old_value', 'new_key': 'new_value' }) mock_environ_copy.assert_called_once_with() self.assertEqual(actual_return_code, mock_run.return_value.returncode) class TestShimE2E(testcase.GsUtilIntegrationTestCase): def test_runs_gcloud_storage_if_use_gcloud_storage_true(self): with util.SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'no_fallback')]): with util.SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': None, }): stderr = self.RunGsUtil(['-D', 'ls'], return_stderr=True, expected_status=1) # This is a proxy to ensure that the test attempted to call # gcloud binary. self.assertIn('gcloud binary path cannot be found', stderr) tests/rewrite_helper.py 0000644 00000005463 15025013616 0011307 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for testing calls to the Rewrite API.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class RewriteHaltException(Exception): pass class HaltingRewriteCallbackHandler(object): """Test callback handler for intentionally stopping a rewrite operation.""" def __init__(self, halt_at_byte): self._halt_at_byte = halt_at_byte # pylint: disable=invalid-name def call(self, total_bytes_rewritten, unused_total_size): """Forcibly exits if the operation has passed the halting point.""" if total_bytes_rewritten >= self._halt_at_byte: raise RewriteHaltException('Artificially halting rewrite') class EnsureRewriteResumeCallbackHandler(object): """Test callback handler for ensuring a rewrite operation resumed.""" def __init__(self, required_byte): self._required_byte = required_byte # pylint: disable=invalid-name def call(self, total_bytes_rewritten, unused_total_size): """Exits if the total bytes rewritten is less than expected.""" if total_bytes_rewritten <= self._required_byte: raise RewriteHaltException( 'Rewrite did not resume; %s bytes written, but %s bytes should ' 'have already been written.' % (total_bytes_rewritten, self._required_byte)) class EnsureRewriteRestartCallbackHandler(object): """Test callback handler for ensuring a rewrite operation restarted.""" def __init__(self, required_byte): self._required_byte = required_byte self._got_restart_bytes = False # pylint: disable=invalid-name def call(self, total_bytes_rewritten, unused_total_size): """Exits if the total bytes rewritten is greater than expected.""" if not self._got_restart_bytes: if total_bytes_rewritten <= self._required_byte: # Restarted successfully, so future calls are allowed to rewrite the # rest of the bytes. self._got_restart_bytes = True else: raise RewriteHaltException( 'Rewrite did not restart; %s bytes written, but no more than %s ' 'bytes should have already been written.' % (total_bytes_rewritten, self._required_byte)) tests/test_help.py 0000644 00000007005 15025013616 0010250 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Unit tests for help command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.command import Command import gslib.tests.testcase as testcase class HelpUnitTests(testcase.GsUtilUnitTestCase): """Help command unit test suite.""" def test_help_noargs(self): stdout = self.RunCommand('help', return_stdout=True) self.assertIn('Available commands', stdout) def test_help_subcommand_arg(self): stdout = self.RunCommand('help', ['web', 'set'], return_stdout=True) self.assertIn('gsutil web set', stdout) self.assertNotIn('gsutil web get', stdout) def test_help_invalid_subcommand_arg(self): stdout = self.RunCommand('help', ['web', 'asdf'], return_stdout=True) self.assertIn('help about one of the subcommands', stdout) def test_help_with_subcommand_for_command_without_subcommands(self): stdout = self.RunCommand('help', ['ls', 'asdf'], return_stdout=True) self.assertIn('has no subcommands', stdout) def test_help_command_arg(self): stdout = self.RunCommand('help', ['ls'], return_stdout=True) self.assertIn('ls - List providers, buckets', stdout) def test_command_help_arg(self): stdout = self.RunCommand('ls', ['--help'], return_stdout=True) self.assertIn('ls - List providers, buckets', stdout) def test_subcommand_help_arg(self): stdout = self.RunCommand('web', ['set', '--help'], return_stdout=True) self.assertIn('gsutil web set', stdout) self.assertNotIn('gsutil web get', stdout) def test_command_args_with_help(self): stdout = self.RunCommand('cp', ['foo', 'bar', '--help'], return_stdout=True) self.assertIn('cp - Copy files and objects', stdout) class HelpIntegrationTests(testcase.GsUtilIntegrationTestCase): """Help command integration test suite.""" def test_help_wrong_num_args(self): stderr = self.RunGsUtil(['cp'], return_stderr=True, expected_status=1) self.assertIn('Usage:', stderr) def test_help_runs_for_all_commands(self): # This test is particularly helpful because the `help` command can fail # under unusual circumstances (e.g. someone adds a new command and they make # the "one-line" summary longer than the defined character limit). for command in Command.__subclasses__(): # Raises exception if the exit code is non-zero. self.RunGsUtil(['help', command.command_spec.command_name]) tests/test_defacl.py 0000644 00000034554 15025013616 0010547 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for the defacl command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import re import six from gslib.commands import defacl from gslib.cs_api_map import ApiSelector import gslib.tests.testcase as case from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import unittest from gslib.utils.constants import UTF8 from gslib.utils import shim_util from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock PUBLIC_READ_JSON_ACL_TEXT = '"entity":"allUsers","role":"READER"' @SkipForS3('S3 does not support default object ACLs.') class TestDefacl(case.GsUtilIntegrationTestCase): """Integration tests for the defacl command.""" _defacl_ch_prefix = ['defacl', 'ch'] _defacl_get_prefix = ['defacl', 'get'] _defacl_set_prefix = ['defacl', 'set'] def _MakeScopeRegex(self, role, entity_type, email_address): template_regex = (r'\{.*"entity":\s*"%s-%s".*"role":\s*"%s".*\}' % (entity_type, email_address, role)) return re.compile(template_regex, flags=re.DOTALL) def testChangeDefaultAcl(self): """Tests defacl ch.""" bucket = self.CreateBucket() test_regex = self._MakeScopeRegex('OWNER', 'group', self.GROUP_TEST_ADDRESS) test_regex2 = self._MakeScopeRegex('READER', 'group', self.GROUP_TEST_ADDRESS) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil(self._defacl_ch_prefix + ['-g', self.GROUP_TEST_ADDRESS + ':FC', suri(bucket)]) json_text2 = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertRegex(json_text2, test_regex) self.RunGsUtil(self._defacl_ch_prefix + ['-g', self.GROUP_TEST_ADDRESS + ':READ', suri(bucket)]) json_text3 = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertRegex(json_text3, test_regex2) stderr = self.RunGsUtil( self._defacl_ch_prefix + ['-g', self.GROUP_TEST_ADDRESS + ':WRITE', suri(bucket)], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('WRITER is not a valid value', stderr) else: self.assertIn('WRITER cannot be set as a default object ACL', stderr) def testChangeDefaultAclEmpty(self): """Tests adding and removing an entry from an empty default object ACL.""" bucket = self.CreateBucket() # First, clear out the default object ACL on the bucket. self.RunGsUtil(self._defacl_set_prefix + ['private', suri(bucket)]) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) empty_regex = r'\[\]\s*' self.assertRegex(json_text, empty_regex) group_regex = self._MakeScopeRegex('READER', 'group', self.GROUP_TEST_ADDRESS) self.RunGsUtil(self._defacl_ch_prefix + ['-g', self.GROUP_TEST_ADDRESS + ':READ', suri(bucket)]) json_text2 = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertRegex(json_text2, group_regex) if self.test_api == ApiSelector.JSON: # TODO: Enable when JSON service respects creating a private (no entries) # default object ACL via PATCH. For now, only supported in XML. return # After adding and removing a group, the default object ACL should be empty. self.RunGsUtil(self._defacl_ch_prefix + ['-d', self.GROUP_TEST_ADDRESS, suri(bucket)]) json_text3 = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertRegex(json_text3, empty_regex) def testChangeMultipleBuckets(self): """Tests defacl ch on multiple buckets.""" bucket1 = self.CreateBucket() bucket2 = self.CreateBucket() test_regex = self._MakeScopeRegex('READER', 'group', self.GROUP_TEST_ADDRESS) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket1)], return_stdout=True) self.assertNotRegex(json_text, test_regex) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket2)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil( self._defacl_ch_prefix + ['-g', self.GROUP_TEST_ADDRESS + ':READ', suri(bucket1), suri(bucket2)]) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket1)], return_stdout=True) self.assertRegex(json_text, test_regex) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket2)], return_stdout=True) self.assertRegex(json_text, test_regex) def testChangeMultipleAcls(self): """Tests defacl ch with multiple ACL entries.""" bucket = self.CreateBucket() test_regex_group = self._MakeScopeRegex('READER', 'group', self.GROUP_TEST_ADDRESS) test_regex_user = self._MakeScopeRegex('OWNER', 'user', self.USER_TEST_ADDRESS) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertNotRegex(json_text, test_regex_group) self.assertNotRegex(json_text, test_regex_user) self.RunGsUtil(self._defacl_ch_prefix + [ '-g', self.GROUP_TEST_ADDRESS + ':READ', '-u', self.USER_TEST_ADDRESS + ':fc', suri(bucket) ]) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertRegex(json_text, test_regex_group) self.assertRegex(json_text, test_regex_user) def testEmptyDefAcl(self): bucket = self.CreateBucket() self.RunGsUtil(self._defacl_set_prefix + ['private', suri(bucket)]) stdout = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertEqual(stdout.rstrip(), '[]') self.RunGsUtil(self._defacl_ch_prefix + ['-u', self.USER_TEST_ADDRESS + ':fc', suri(bucket)]) def testDeletePermissionsWithCh(self): """Tests removing permissions with defacl ch.""" bucket = self.CreateBucket() test_regex = self._MakeScopeRegex('OWNER', 'user', self.USER_TEST_ADDRESS) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil(self._defacl_ch_prefix + ['-u', self.USER_TEST_ADDRESS + ':fc', suri(bucket)]) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertRegex(json_text, test_regex) self.RunGsUtil(self._defacl_ch_prefix + ['-d', self.USER_TEST_ADDRESS, suri(bucket)]) json_text = self.RunGsUtil(self._defacl_get_prefix + [suri(bucket)], return_stdout=True) self.assertNotRegex(json_text, test_regex) def testTooFewArgumentsFails(self): """Tests calling defacl with insufficient number of arguments.""" # No arguments for get, but valid subcommand. stderr = self.RunGsUtil(self._defacl_get_prefix, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for set, but valid subcommand. stderr = self.RunGsUtil(self._defacl_set_prefix, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for ch, but valid subcommand. stderr = self.RunGsUtil(self._defacl_ch_prefix, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Neither arguments nor subcommand. stderr = self.RunGsUtil(['defacl'], return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) class TestDefaclShim(case.ShimUnitTestBase): @mock.patch.object(defacl.DefAclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_defacl_get(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('defacl', ['get', 'gs://bucket'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage buckets describe' ' --format=multi(defaultObjectAcl:format=json)' ' --raw gs://bucket').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(defacl.DefAclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_set_defacl_file(self): acl_string = 'acl_string' inpath = self.CreateTempFile(contents=acl_string.encode(UTF8)) with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'defacl', ['set', inpath, 'gs://b1', 'gs://b2'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage buckets update' ' --default-object-acl-file={} gs://b1 gs://b2').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(defacl.DefAclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_set_predefined_defacl(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'defacl', ['set', 'bucket-owner-read', 'gs://b1', 'gs://b2'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage buckets update' ' --predefined-default-object-acl={} gs://b1 gs://b2').format( shim_util._get_gcloud_binary_path('fake_dir'), 'bucketOwnerRead'), info_lines) @mock.patch.object(defacl.DefAclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_xml_predefined_defacl_for_set(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'defacl', ['set', 'authenticated-read', 'gs://b1', 'gs://b2'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage buckets update' ' --predefined-default-object-acl={} gs://b1 gs://b2').format( shim_util._get_gcloud_binary_path('fake_dir'), 'authenticatedRead'), info_lines) @mock.patch.object(defacl.DefAclCommand, 'RunCommand', new=mock.Mock()) def test_shim_changes_defacls_for_user(self): # The helper function this behavior relies on is tested more # thoroughly in test_acl.py. inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'defacl', ['ch', '-f', '-u', 'user@example.com:R', 'gs://bucket1'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(( 'Gcloud Storage Command: {} storage buckets update' ' --continue-on-error' ' --add-default-object-acl-grant entity=user-user@example.com,role=READER' ' gs://bucket1').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) class TestDefaclOldAlias(TestDefacl): _defacl_ch_prefix = ['chdefacl'] _defacl_get_prefix = ['getdefacl'] _defacl_set_prefix = ['setdefacl'] tests/test_encryption_helper.py 0000644 00000011174 15025013616 0013053 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for encryption_helper.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import base64 import os import six import boto from gslib.exception import CommandException from gslib.tests.testcase.unit_testcase import GsUtilUnitTestCase from gslib.tests.util import SetBotoConfigForTest from gslib.utils.encryption_helper import Base64Sha256FromBase64EncryptionKey from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.encryption_helper import FindMatchingCSEKInBotoConfig class TestEncryptionHelper(GsUtilUnitTestCase): """Unit tests for encryption helper functions.""" def testMaxDecryptionKeys(self): """Tests a config file with the maximum number of decryption keys.""" keys = [] boto_101_key_config = [] # Generate 101 keys. for i in range(1, 102): try: keys.append(base64.encodebytes(os.urandom(32)).rstrip(b'\n')) except AttributeError: # For Python 2 compatability. keys.append(base64.encodestring(os.urandom(32)).rstrip(b'\n')) boto_101_key_config.append( ('GSUtil', 'decryption_key%s' % i, keys[i - 1])) with SetBotoConfigForTest(boto_101_key_config): self.assertIsNotNone( FindMatchingCSEKInBotoConfig( Base64Sha256FromBase64EncryptionKey(keys[0]), boto.config)) self.assertIsNotNone( FindMatchingCSEKInBotoConfig( Base64Sha256FromBase64EncryptionKey(keys[99]), boto.config)) # Only 100 keys are supported. self.assertIsNone( FindMatchingCSEKInBotoConfig( Base64Sha256FromBase64EncryptionKey(keys[100]), boto.config)) boto_100_key_config = list(boto_101_key_config) boto_100_key_config.pop() with SetBotoConfigForTest(boto_100_key_config): self.assertIsNotNone( FindMatchingCSEKInBotoConfig( Base64Sha256FromBase64EncryptionKey(keys[0]), boto.config)) self.assertIsNotNone( FindMatchingCSEKInBotoConfig( Base64Sha256FromBase64EncryptionKey(keys[99]), boto.config)) def testNonSequentialDecryptionKeys(self): """Tests a config file with non-sequential decryption key numbering.""" keys = [] for _ in range(3): try: keys.append(base64.encodebytes(os.urandom(32)).rstrip(b'\n')) except AttributeError: # For Python 2 compatability. keys.append(base64.encodestring(os.urandom(32)).rstrip(b'\n')) boto_config = [('GSUtil', 'decryption_key4', keys[2]), ('GSUtil', 'decryption_key1', keys[0]), ('GSUtil', 'decryption_key2', keys[1])] with SetBotoConfigForTest(boto_config): # Because decryption_key3 does not exist in boto_config, decryption_key4 # should be ignored. self.assertIsNone( FindMatchingCSEKInBotoConfig( Base64Sha256FromBase64EncryptionKey(keys[2]), boto.config)) # decryption_key1 and decryption_key2 should work, though. self.assertIsNotNone( FindMatchingCSEKInBotoConfig( Base64Sha256FromBase64EncryptionKey(keys[0]), boto.config)) self.assertIsNotNone( FindMatchingCSEKInBotoConfig( Base64Sha256FromBase64EncryptionKey(keys[1]), boto.config)) def testInvalidCSEKConfigurationRaises(self): invalid_key = 'aP7KbmxLqDw1SWHeKvlfKOVgNRNNZc8L2sFz8ybLN===' with self.assertRaises(CommandException) as cm: CryptoKeyWrapperFromKey(invalid_key) self.assertIn( 'Configured encryption_key or decryption_key looked like a CSEK', cm.exception.reason) def testInvalidCMEKConfigurationRaises(self): invalid_key = ( 'projects/my-project/locations/some-location/keyRings/keyring/' 'cryptoKeyWHOOPS-INVALID-RESOURCE-PORTION/somekey') with self.assertRaises(CommandException) as cm: CryptoKeyWrapperFromKey(invalid_key) self.assertIn( 'Configured encryption_key or decryption_key looked like a CMEK', cm.exception.reason) tests/test_rewrite.py 0000644 00000077177 15025013616 0011022 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for rewrite command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import re import unittest from boto.storage_uri import BucketStorageUri from gslib.cs_api_map import ApiSelector from gslib.discard_messages_queue import DiscardMessagesQueue from gslib.gcs_json_api import GcsJsonApi from gslib.project_id import PopulateProjectId from gslib.tests.rewrite_helper import EnsureRewriteRestartCallbackHandler from gslib.tests.rewrite_helper import EnsureRewriteResumeCallbackHandler from gslib.tests.rewrite_helper import HaltingRewriteCallbackHandler from gslib.tests.rewrite_helper import RewriteHaltException import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import AuthorizeProjectToUseTestingKmsKey from gslib.tests.util import GenerationFromURI as urigen from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import TEST_ENCRYPTION_KEY1 from gslib.tests.util import TEST_ENCRYPTION_KEY2 from gslib.tests.util import TEST_ENCRYPTION_KEY3 from gslib.tests.util import TEST_ENCRYPTION_KEY4 from gslib.tests.util import unittest from gslib.tracker_file import DeleteTrackerFile from gslib.tracker_file import GetRewriteTrackerFilePath from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.unit_util import ONE_MIB @SkipForS3('gsutil doesn\'t support S3 customer-supplied encryption keys.') class TestRewrite(testcase.GsUtilIntegrationTestCase): """Integration tests for rewrite command.""" def setUp(self): super(TestRewrite, self).setUp() if self._use_gcloud_storage: self.rotating_message = 'Rewriting' self.skipping_message = 'Patching' self.encrypting_message = 'Rewriting' self.decrypting_message = 'Rewriting' else: self.rotating_message = 'Rotating' self.skipping_message = 'Skipping' self.encrypting_message = 'Encrypting' self.decrypting_message = 'Decrypting' def test_rewrite_missing_flag(self): """Tests rewrite with no transformation flag.""" stderr = self.RunGsUtil( ['rewrite', '%s://some_url' % self.default_provider], return_stderr=True, expected_status=1) self.assertIn('command requires at least one transformation flag', stderr) def test_rewrite_generation_url(self): """Tests that rewrite fails on a URL that includes a generation.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) generation = object_uri.generation stderr = self.RunGsUtil( ['rewrite', '-k', '%s#%s' % (suri(object_uri), generation)], return_stderr=True, expected_status=1) self.assertIn('"rewrite" called on URL with generation', stderr) def test_rewrite_missing_decryption_key(self): """Tests that rewrite fails when no decryption key matches.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(object_name='foo', contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True, expected_status=1) self.assertIn('No decryption key matches object %s' % suri(object_uri), stderr) def test_rewrite_stdin_args(self): """Tests rewrite with arguments supplied on stdin.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) stdin_arg = suri(object_uri) boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): self.RunGsUtil(['rewrite', '-k', '-I'], stdin=stdin_arg) self.AssertObjectUsesCSEK(stdin_arg, TEST_ENCRYPTION_KEY2) def test_rewrite_overwrite_acl(self): """Tests rewrite with the -O flag.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) self.RunGsUtil(['acl', 'ch', '-u', 'AllUsers:R', suri(object_uri)]) stdout = self.RunGsUtil(['acl', 'get', suri(object_uri)], return_stdout=True) self.assertIn('allUsers', stdout) boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): self.RunGsUtil(['rewrite', '-k', '-O', suri(object_uri)]) self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2) stdout = self.RunGsUtil(['acl', 'get', suri(object_uri)], return_stdout=True) self.assertNotIn('allUsers', stdout) def test_rewrite_bucket_recursive(self): """Tests rewrite command recursively on a bucket.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() self._test_rewrite_key_rotation_bucket( bucket_uri, ['rewrite', '-k', '-r', suri(bucket_uri)]) def test_parallel_rewrite_bucket_flat_wildcard(self): """Tests parallel rewrite command with a flat wildcard on a bucket.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() self._test_rewrite_key_rotation_bucket( bucket_uri, ['-d', '-m', 'rewrite', '-k', suri(bucket_uri, '**')]) def _test_rewrite_key_rotation_bucket(self, bucket_uri, command_args): """Helper function for testing key rotation on a bucket. Args: bucket_uri: bucket StorageUri to use for the test. command_args: list of args to gsutil command. """ object_contents = b'bar' object_uri1 = self.CreateObject(bucket_uri=bucket_uri, object_name='foo/foo', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY1) object_uri2 = self.CreateObject(bucket_uri=bucket_uri, object_name='foo/bar', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY2) object_uri3 = self.CreateObject(bucket_uri=bucket_uri, object_name='foo/baz', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY3) object_uri4 = self.CreateObject(bucket_uri=bucket_uri, object_name='foo/qux', contents=object_contents) # Rotate all keys to TEST_ENCRYPTION_KEY1. boto_config_for_test = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1.decode('utf-8')), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2.decode('utf-8')), ('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY3.decode('utf-8')) ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(command_args, return_stderr=True) # Object one already has the correct key. self.assertIn('{} {}'.format(self.skipping_message, suri(object_uri1)), stderr) # Other objects should be rotated. self.assertIn(self.rotating_message, stderr) for object_uri_str in (suri(object_uri1), suri(object_uri2), suri(object_uri3), suri(object_uri4)): self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY1) # Remove all encryption. boto_config_for_test2 = [('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1) ] with SetBotoConfigForTest(boto_config_for_test2): stderr = self.RunGsUtil(command_args, return_stderr=True) self.assertIn(self.decrypting_message, stderr) for object_uri_str in (suri(object_uri1), suri(object_uri2), suri(object_uri3), suri(object_uri4)): self.AssertObjectUnencrypted(object_uri_str) def test_rewrite_seek_ahead(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) # Remove encryption boto_config_for_test = [('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1), ('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True')] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['-m', 'rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn( 'Estimated work for this command: objects: 1, total size: 3', stderr) def test_rewrite_unintentional_key_rotation_fails(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') encrypted_obj_uri = self.CreateObject(contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) unencrypted_obj_uri = self.CreateObject(contents=b'bar') boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): # Executing rewrite without the -k flag should fail if your boto file has # a different encryption_key than was last used to encrypt the object. stderr = self.RunGsUtil(['rewrite', '-s', 'dra', suri(encrypted_obj_uri)], return_stderr=True, expected_status=1) self.assertIn('EncryptionException', stderr) # Should also fail for a previously unencrypted object. stderr = self.RunGsUtil( ['rewrite', '-s', 'dra', suri(unencrypted_obj_uri)], return_stderr=True, expected_status=1) self.assertIn('EncryptionException', stderr) def test_rewrite_key_rotation_single_object(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) # Rotate key. boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn(self.rotating_message, stderr) self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2) # Remove encryption. boto_config_for_test2 = [('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2) ] with SetBotoConfigForTest(boto_config_for_test2): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn(self.decrypting_message, stderr) self.AssertObjectUnencrypted(suri(object_uri)) def test_rewrite_key_rotation_bucket_subdir(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() object_contents = b'bar' rotate_subdir = suri(bucket_uri, 'bar') object_uri1 = self.CreateObject(bucket_uri=bucket_uri, object_name='foo/bar', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY1) object_uri2 = self.CreateObject(bucket_uri=bucket_uri, object_name='bar/foo', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY2) object_uri3 = self.CreateObject(bucket_uri=bucket_uri, object_name='bar/baz', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY3) object_uri4 = self.CreateObject(bucket_uri=bucket_uri, object_name='bar/qux', contents=object_contents) # Rotate subdir keys to TEST_ENCRYPTION_KEY3. boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY3), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY1)] self.AssertNObjectsInBucket(bucket_uri, 4) with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir], return_stderr=True) # Cannot check for "Rotating [object URL]" because output gets corrupt: # "\nRotating ...ewrite-k" self.assertIn(self.rotating_message, stderr) self.assertIn('{} {}'.format(self.skipping_message, suri(object_uri3)), stderr) self.assertIn(self.encrypting_message, stderr) # First subdir should be unaffected. self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1) for object_uri_str in (suri(object_uri2), suri(object_uri3), suri(object_uri4)): self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY3) # Remove encryption in subdir. boto_config_for_test2 = [('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3) ] with SetBotoConfigForTest(boto_config_for_test2): stderr = self.RunGsUtil(['rewrite', '-r', '-k', rotate_subdir], return_stderr=True) self.assertIn(self.decrypting_message, stderr) # First subdir should be unaffected. self.AssertObjectUsesCSEK(suri(object_uri1), TEST_ENCRYPTION_KEY1) for object_uri_str in (suri(object_uri2), suri(object_uri3), suri(object_uri4)): self.AssertObjectUnencrypted(object_uri_str) def test_rewrite_with_nonkey_transform_works_when_key_is_unchanged(self): # Tests that when a valid transformation flag aside from "-k" is supplied, # the "-k" flag is not supplied, and the encryption key previously used to # encrypt the target object matches the encryption_key in the user's boto # config file (via hash comparison), that the rewrite command properly # passes the same tuple for decryption and encryption, in addition to # performing the other desired transformations. if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['rewrite', '-s', 'nearline', suri(object_uri)], return_stderr=True) self.assertIn('Rewriting', stderr) def test_rewrite_key_rotation_with_storage_class_change(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) # Rotate key and change storage class to nearline. boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-s', 'nearline', '-k', suri(object_uri)], return_stderr=True) self.assertIn(self.rotating_message, stderr) self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY2) stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True) self.assertRegexpMatchesWithFlags( stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE, msg=('Storage class appears not to have been changed.')) def test_rewrite_with_only_storage_class_change(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar') # Change storage class to nearline. stderr = self.RunGsUtil(['rewrite', '-s', 'nearline', suri(object_uri)], return_stderr=True) self.assertIn('Rewriting', stderr) stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True) self.assertRegexpMatchesWithFlags( stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE, msg=('Storage class appears not to have been changed.')) def test_rewrite_to_same_storage_class_is_skipped(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar') stderr = self.RunGsUtil(['rewrite', '-s', 'standard', suri(object_uri)], return_stderr=True) self.assertIn('{} {}'.format(self.skipping_message, suri(object_uri)), stderr) def test_rewrite_with_same_key_and_storage_class_is_skipped(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'foo', encryption_key=TEST_ENCRYPTION_KEY1, storage_class='standard') boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', '-s', 'standard', suri(object_uri)], return_stderr=True) self.assertIn('{} {}'.format(self.skipping_message, suri(object_uri)), stderr) def test_rewrite_with_no_value_for_minus_s(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') stderr = self.RunGsUtil(['rewrite', '-s', 'gs://some-random-name'], return_stderr=True, expected_status=1) self.assertIn('CommandException', stderr) self.assertIn('expects at least one URL', stderr) def test_rewrite_resume(self): self._test_rewrite_resume_or_restart(TEST_ENCRYPTION_KEY1, TEST_ENCRYPTION_KEY2) def test_rewrite_resume_restart_source_encryption_changed(self): self._test_rewrite_resume_or_restart(TEST_ENCRYPTION_KEY1, TEST_ENCRYPTION_KEY2, new_dec_key=TEST_ENCRYPTION_KEY3) def test_rewrite_resume_restart_dest_encryption_changed(self): self._test_rewrite_resume_or_restart(TEST_ENCRYPTION_KEY1, TEST_ENCRYPTION_KEY2, new_enc_key=TEST_ENCRYPTION_KEY3) def test_rewrite_resume_restart_both_encryption_changed(self): self._test_rewrite_resume_or_restart(TEST_ENCRYPTION_KEY1, TEST_ENCRYPTION_KEY2, new_dec_key=TEST_ENCRYPTION_KEY3, new_enc_key=TEST_ENCRYPTION_KEY4) def test_rewrite_to_kms_then_unencrypted(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') key_fqn = AuthorizeProjectToUseTestingKmsKey() object_uri = self.CreateObject(contents=b'foo') boto_config_for_test = [('GSUtil', 'encryption_key', key_fqn)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn(self.encrypting_message, stderr) self.AssertObjectUsesCMEK(suri(object_uri), key_fqn) # Rewrite back to unencrypted and make sure no KMS key was used. boto_config_for_test = [('GSUtil', 'encryption_key', None)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn(self.decrypting_message, stderr) self.AssertObjectUnencrypted(suri(object_uri)) def test_rewrite_to_kms_then_csek(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') key_fqn = AuthorizeProjectToUseTestingKmsKey() object_uri = self.CreateObject(contents=b'foo') boto_config_for_test = [('GSUtil', 'encryption_key', key_fqn)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn(self.encrypting_message, stderr) self.AssertObjectUsesCMEK(suri(object_uri), key_fqn) # Rewrite from CMEK to CSEK encryption. boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn(self.rotating_message, stderr) self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY1) def test_rewrite_to_csek_then_kms(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') key_fqn = AuthorizeProjectToUseTestingKmsKey() object_uri = self.CreateObject(contents=b'foo') boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn(self.encrypting_message, stderr) self.AssertObjectUsesCSEK(suri(object_uri), TEST_ENCRYPTION_KEY1) # Rewrite from CSEK to CMEK encryption. boto_config_for_test = [ ('GSUtil', 'encryption_key', key_fqn), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1), ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn(self.rotating_message, stderr) self.AssertObjectUsesCMEK(suri(object_uri), key_fqn) def test_rewrite_with_no_encryption_key_operates_on_unencrypted_objects(self): if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') # Since the introduction of default KMS keys for GCS buckets, rewriting # with no explicitly specified CSEK/CMEK can still result in the rewritten # objects being encrypted. Before KMS support, this would always result in # decrypted objects. With this new possibility, we want to always rewrite # every specified object when no encryption_key was set in the boto config, # since we don't know if the operation will end up decrypting the object or # implicitly encrypting it with the bucket's default KMS key. key_fqn = AuthorizeProjectToUseTestingKmsKey() # Create an unencrypted object. bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'foo') # Set the bucket's default KMS key. self.RunGsUtil(['kms', 'encryption', '-k', key_fqn, suri(bucket_uri)]) # Rewriting with no encryption_key should rewrite the object, resulting in # the bucket's default KMS key being used to encrypt it. with SetBotoConfigForTest([('GSUtil', 'encryption_key', None)]): stderr = self.RunGsUtil( ['rewrite', '-k', suri(object_uri)], return_stderr=True) self.assertIn('Rewriting', stderr) self.AssertObjectUsesCMEK(suri(object_uri), key_fqn) def _test_rewrite_resume_or_restart(self, initial_dec_key, initial_enc_key, new_dec_key=None, new_enc_key=None): """Tests that the rewrite command restarts if the object's key changed. Args: initial_dec_key: Initial key the object is encrypted with, used as decryption key in the first rewrite call. initial_enc_key: Initial encryption key to rewrite the object with, used as encryption key in the first rewrite call. new_dec_key: Decryption key for the second rewrite call; if specified, object will be overwritten with a new encryption key in between the first and second rewrite calls, and this key will be used for the second rewrite call. new_enc_key: Encryption key for the second rewrite call; if specified, this key will be used for the second rewrite call, otherwise the initial key will be used. Returns: None """ if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() # If the source and destination are in the same location and have the same # storage class the rewrite completes in a single request. Using a different # storage class for destination so that maxBytesPerCall gets used. destination_bucket_uri = self.CreateBucket(storage_class='NEARLINE') # maxBytesPerCall must be >= 1 MiB, so create an object > 2 MiB because we # need 2 response from the service: 1 success, 1 failure prior to # completion. object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=(b'12' * ONE_MIB) + b'bar', prefer_json_api=True, encryption_key=initial_dec_key) destination_object_uri = self.CreateObject( bucket_uri=destination_bucket_uri, object_name='foo', contents='test', prefer_json_api=True, encryption_key=initial_dec_key) gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(), DiscardMessagesQueue(), self.default_provider) with SetBotoConfigForTest([('GSUtil', 'decryption_key1', initial_dec_key)]): src_obj_metadata = gsutil_api.GetObjectMetadata( object_uri.bucket_name, object_uri.object_name, provider=self.default_provider, fields=['bucket', 'contentType', 'etag', 'name']) dst_obj_metadata = gsutil_api.GetObjectMetadata( destination_object_uri.bucket_name, destination_object_uri.object_name, provider=self.default_provider, fields=['bucket', 'contentType', 'etag', 'name']) tracker_file_name = GetRewriteTrackerFilePath(src_obj_metadata.bucket, src_obj_metadata.name, dst_obj_metadata.bucket, dst_obj_metadata.name, self.test_api) decryption_tuple = CryptoKeyWrapperFromKey(initial_dec_key) decryption_tuple2 = CryptoKeyWrapperFromKey(new_dec_key or initial_dec_key) encryption_tuple = CryptoKeyWrapperFromKey(initial_enc_key) encryption_tuple2 = CryptoKeyWrapperFromKey(new_enc_key or initial_enc_key) try: try: gsutil_api.CopyObject(src_obj_metadata, dst_obj_metadata, progress_callback=HaltingRewriteCallbackHandler( ONE_MIB * 2).call, max_bytes_per_call=ONE_MIB, decryption_tuple=decryption_tuple, encryption_tuple=encryption_tuple) self.fail('Expected RewriteHaltException.') except RewriteHaltException: pass # Tracker file should be left over. self.assertTrue(os.path.exists(tracker_file_name)) if new_dec_key: # Recreate the object with a different encryption key. self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=(b'12' * ONE_MIB) + b'bar', prefer_json_api=True, encryption_key=new_dec_key, gs_idempotent_generation=urigen(object_uri)) with SetBotoConfigForTest([('GSUtil', 'decryption_key1', new_dec_key or initial_dec_key)]): original_md5 = gsutil_api.GetObjectMetadata( src_obj_metadata.bucket, src_obj_metadata.name, fields=['customerEncryption', 'md5Hash']).md5Hash if new_dec_key or new_enc_key: # Keys changed, rewrite should be restarted. progress_callback = EnsureRewriteRestartCallbackHandler(ONE_MIB).call else: # Keys are the same, rewrite should be resumed. progress_callback = EnsureRewriteResumeCallbackHandler(ONE_MIB * 2).call # Now resume. Callback ensures the appropriate resume/restart behavior. gsutil_api.CopyObject(src_obj_metadata, dst_obj_metadata, progress_callback=progress_callback, max_bytes_per_call=ONE_MIB, decryption_tuple=decryption_tuple2, encryption_tuple=encryption_tuple2) # Copy completed; tracker file should be deleted. self.assertFalse(os.path.exists(tracker_file_name)) final_enc_key = new_enc_key or initial_enc_key with SetBotoConfigForTest([('GSUtil', 'encryption_key', final_enc_key)]): self.assertEqual( original_md5, gsutil_api.GetObjectMetadata( dst_obj_metadata.bucket, dst_obj_metadata.name, fields=['customerEncryption', 'md5Hash']).md5Hash, 'Error: Rewritten object\'s hash doesn\'t match source object.') finally: # Clean up if something went wrong. DeleteTrackerFile(tracker_file_name) tests/test_rsync_funcs.py 0000644 00000006561 15025013616 0011662 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for functions in rsync command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os from gslib.commands.rsync import _ComputeNeededFileChecksums from gslib.commands.rsync import _NA from gslib.tests.testcase.unit_testcase import GsUtilUnitTestCase from gslib.utils.hashing_helper import CalculateB64EncodedCrc32cFromContents from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents class TestRsyncFuncs(GsUtilUnitTestCase): def test_compute_needed_file_checksums(self): """Tests that we compute all/only needed file checksums.""" size = 4 logger = logging.getLogger() tmpdir = self.CreateTempDir() file_url_str = 'file://%s' % os.path.join(tmpdir, 'obj1') self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1') cloud_url_str = 'gs://whatever' with open(os.path.join(tmpdir, 'obj1'), 'rb') as fp: crc32c = CalculateB64EncodedCrc32cFromContents(fp) fp.seek(0) md5 = CalculateB64EncodedMd5FromContents(fp) # Test case where source is a file and dest has CRC32C. (src_crc32c, src_md5, dst_crc32c, dst_md5) = _ComputeNeededFileChecksums(logger, file_url_str, size, _NA, _NA, cloud_url_str, size, crc32c, _NA) self.assertEqual(crc32c, src_crc32c) self.assertEqual(_NA, src_md5) self.assertEqual(crc32c, dst_crc32c) self.assertEqual(_NA, dst_md5) # Test case where source is a file and dest has MD5 but not CRC32C. (src_crc32c, src_md5, dst_crc32c, dst_md5) = _ComputeNeededFileChecksums(logger, file_url_str, size, _NA, _NA, cloud_url_str, size, _NA, md5) self.assertEqual(_NA, src_crc32c) self.assertEqual(md5, src_md5) self.assertEqual(_NA, dst_crc32c) self.assertEqual(md5, dst_md5) # Test case where dest is a file and src has CRC32C. (src_crc32c, src_md5, dst_crc32c, dst_md5) = _ComputeNeededFileChecksums(logger, cloud_url_str, size, crc32c, _NA, file_url_str, size, _NA, _NA) self.assertEqual(crc32c, dst_crc32c) self.assertEqual(_NA, src_md5) self.assertEqual(crc32c, src_crc32c) self.assertEqual(_NA, src_md5) # Test case where dest is a file and src has MD5 but not CRC32C. (src_crc32c, src_md5, dst_crc32c, dst_md5) = _ComputeNeededFileChecksums(logger, cloud_url_str, size, _NA, md5, file_url_str, size, _NA, _NA) self.assertEqual(_NA, dst_crc32c) self.assertEqual(md5, src_md5) self.assertEqual(_NA, src_crc32c) self.assertEqual(md5, src_md5) tests/test_daisy_chain_wrapper.py 0000644 00000034203 15025013616 0013333 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for daisy chain wrapper class.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import math import os import pkgutil import six import gslib.cloud_api from gslib.daisy_chain_wrapper import DaisyChainWrapper from gslib.storage_url import StorageUrlFromString import gslib.tests.testcase as testcase from gslib.utils.constants import TRANSFER_BUFFER_SIZE _TEST_FILE = 'test.txt' class TestDaisyChainWrapper(testcase.GsUtilUnitTestCase): """Unit tests for the DaisyChainWrapper class.""" _temp_test_file = None _dummy_url = StorageUrlFromString('gs://bucket/object') def setUp(self): super(TestDaisyChainWrapper, self).setUp() self.test_data_file = self._GetTestFile() self.test_data_file_len = os.path.getsize(self.test_data_file) def _GetTestFile(self): contents = pkgutil.get_data('gslib', 'tests/test_data/%s' % _TEST_FILE) if not self._temp_test_file: # Write to a temp file because pkgutil doesn't expose a stream interface. self._temp_test_file = self.CreateTempFile(file_name=_TEST_FILE, contents=contents) return self._temp_test_file class MockDownloadCloudApi(gslib.cloud_api.CloudApi): """Mock CloudApi that implements GetObjectMedia for testing.""" def __init__(self, write_values): """Initialize the mock that will be used by the download thread. Args: write_values: List of values that will be used for calls to write(), in order, by the download thread. An Exception class may be part of the list; if so, the Exception will be raised after previous values are consumed. """ self._write_values = write_values self.get_calls = 0 def GetObjectMedia(self, unused_bucket_name, unused_object_name, download_stream, start_byte=0, end_byte=None, **kwargs): """Writes self._write_values to the download_stream.""" # Writes from start_byte up to, but not including end_byte (if not None). # Does not slice values; # self._write_values must line up with start/end_byte. self.get_calls += 1 bytes_read = 0 for write_value in self._write_values: if bytes_read < start_byte: bytes_read += len(write_value) continue if end_byte and bytes_read >= end_byte: break if isinstance(write_value, Exception): raise write_value download_stream.write(write_value) bytes_read += len(write_value) def _WriteFromWrapperToFile(self, daisy_chain_wrapper, file_path): """Writes all contents from the DaisyChainWrapper to the named file.""" with open(file_path, 'wb') as upload_stream: while True: data = daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE) if not data: break upload_stream.write(data) def testDownloadSingleChunk(self): """Tests a single call to GetObjectMedia.""" write_values = [] with open(self.test_data_file, 'rb') as stream: while True: data = stream.read(TRANSFER_BUFFER_SIZE) if not data: break write_values.append(data) upload_file = self.CreateTempFile() # Test for a single call even if the chunk size is larger than the data. for chunk_size in (self.test_data_file_len, self.test_data_file_len + 1): mock_api = self.MockDownloadCloudApi(write_values) daisy_chain_wrapper = DaisyChainWrapper(self._dummy_url, self.test_data_file_len, mock_api, download_chunk_size=chunk_size) self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file) # Since the chunk size is >= the file size, only a single GetObjectMedia # call should be made. self.assertEqual(mock_api.get_calls, 1) with open(upload_file, 'rb') as upload_stream: with open(self.test_data_file, 'rb') as download_stream: self.assertEqual(upload_stream.read(), download_stream.read()) def testDownloadMultiChunk(self): """Tests multiple calls to GetObjectMedia.""" upload_file = self.CreateTempFile() write_values = [] with open(self.test_data_file, 'rb') as stream: while True: data = stream.read(TRANSFER_BUFFER_SIZE) if not data: break write_values.append(data) mock_api = self.MockDownloadCloudApi(write_values) daisy_chain_wrapper = DaisyChainWrapper( self._dummy_url, self.test_data_file_len, mock_api, download_chunk_size=TRANSFER_BUFFER_SIZE) self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file) num_expected_calls = self.test_data_file_len // TRANSFER_BUFFER_SIZE if self.test_data_file_len % TRANSFER_BUFFER_SIZE: num_expected_calls += 1 # Since the chunk size is < the file size, multiple calls to GetObjectMedia # should be made. self.assertEqual(mock_api.get_calls, num_expected_calls) with open(upload_file, 'rb') as upload_stream: with open(self.test_data_file, 'rb') as download_stream: self.assertEqual(upload_stream.read(), download_stream.read()) def testDownloadWithDifferentChunkSize(self): """Tests multiple calls to GetObjectMedia.""" upload_file = self.CreateTempFile() write_values = [] with open(self.test_data_file, 'rb') as stream: # Use an arbitrary size greater than TRANSFER_BUFFER_SIZE for writing # data to the buffer. For reading from the buffer # WriteFromWrapperToFile will use TRANSFER_BUFFER_SIZE. buffer_write_size = TRANSFER_BUFFER_SIZE * 2 + 10 while True: # Write data with size. data = stream.read(buffer_write_size) if not data: break write_values.append(data) mock_api = self.MockDownloadCloudApi(write_values) daisy_chain_wrapper = DaisyChainWrapper( self._dummy_url, self.test_data_file_len, mock_api, download_chunk_size=TRANSFER_BUFFER_SIZE) self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file) num_expected_calls = math.ceil(self.test_data_file_len / TRANSFER_BUFFER_SIZE) # Since the chunk size is < the file size, multiple calls to GetObjectMedia # should be made. self.assertEqual(mock_api.get_calls, num_expected_calls) with open(upload_file, 'rb') as upload_stream: with open(self.test_data_file, 'rb') as download_stream: self.assertEqual(upload_stream.read(), download_stream.read()) def testDownloadWithZeroWrites(self): """Tests 0-byte writes to the download stream from GetObjectMedia.""" write_values = [] with open(self.test_data_file, 'rb') as stream: while True: write_values.append(b'') data = stream.read(TRANSFER_BUFFER_SIZE) write_values.append(b'') if not data: break write_values.append(data) upload_file = self.CreateTempFile() mock_api = self.MockDownloadCloudApi(write_values) daisy_chain_wrapper = DaisyChainWrapper( self._dummy_url, self.test_data_file_len, mock_api, download_chunk_size=self.test_data_file_len) self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file) self.assertEqual(mock_api.get_calls, 1) with open(upload_file, 'rb') as upload_stream: with open(self.test_data_file, 'rb') as download_stream: self.assertEqual(upload_stream.read(), download_stream.read()) def testDownloadWithPartialWrite(self): """Tests unaligned writes to the download stream from GetObjectMedia.""" with open(self.test_data_file, 'rb') as stream: chunk = stream.read(TRANSFER_BUFFER_SIZE) # Though it may seem equivalent, the `:1` is actually necessary, without # it in python 3, `one_byte` would be int(77) and with it, `one_byte` is # the expected value of b'M' (using case where start of chunk is b'MJoTM...') one_byte = chunk[0:1] chunk_minus_one_byte = chunk[1:TRANSFER_BUFFER_SIZE] half_chunk = chunk[0:TRANSFER_BUFFER_SIZE // 2] write_values_dict = { 'First byte first chunk unaligned': (one_byte, chunk_minus_one_byte, chunk, chunk), 'Last byte first chunk unaligned': (chunk_minus_one_byte, chunk, chunk), 'First byte second chunk unaligned': (chunk, one_byte, chunk_minus_one_byte, chunk), 'Last byte second chunk unaligned': (chunk, chunk_minus_one_byte, one_byte, chunk), 'First byte final chunk unaligned': (chunk, chunk, one_byte, chunk_minus_one_byte), 'Last byte final chunk unaligned': (chunk, chunk, chunk_minus_one_byte, one_byte), 'Half chunks': (half_chunk, half_chunk, half_chunk), 'Many unaligned': (one_byte, half_chunk, one_byte, half_chunk, chunk, chunk_minus_one_byte, chunk, one_byte, half_chunk, one_byte) } upload_file = self.CreateTempFile() for case_name, write_values in six.iteritems(write_values_dict): expected_contents = b'' for write_value in write_values: expected_contents += write_value mock_api = self.MockDownloadCloudApi(write_values) daisy_chain_wrapper = DaisyChainWrapper( self._dummy_url, len(expected_contents), mock_api, download_chunk_size=self.test_data_file_len) self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file) with open(upload_file, 'rb') as upload_stream: self.assertEqual( upload_stream.read(), expected_contents, 'Uploaded file contents for case %s did not match' % case_name) def testSeekAndReturn(self): """Tests seeking to the end of the wrapper (simulates getting size).""" write_values = [] with open(self.test_data_file, 'rb') as stream: while True: data = stream.read(TRANSFER_BUFFER_SIZE) if not data: break write_values.append(data) upload_file = self.CreateTempFile() mock_api = self.MockDownloadCloudApi(write_values) daisy_chain_wrapper = DaisyChainWrapper( self._dummy_url, self.test_data_file_len, mock_api, download_chunk_size=self.test_data_file_len) with open(upload_file, 'wb') as upload_stream: current_position = 0 daisy_chain_wrapper.seek(0, whence=os.SEEK_END) daisy_chain_wrapper.seek(current_position) while True: data = daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE) current_position += len(data) daisy_chain_wrapper.seek(0, whence=os.SEEK_END) daisy_chain_wrapper.seek(current_position) if not data: break upload_stream.write(data) self.assertEqual(mock_api.get_calls, 1) with open(upload_file, 'rb') as upload_stream: with open(self.test_data_file, 'rb') as download_stream: self.assertEqual(upload_stream.read(), download_stream.read()) def testRestartDownloadThread(self): """Tests seek to non-stored position; this restarts the download thread.""" write_values = [] with open(self.test_data_file, 'rb') as stream: while True: data = stream.read(TRANSFER_BUFFER_SIZE) if not data: break write_values.append(data) upload_file = self.CreateTempFile() mock_api = self.MockDownloadCloudApi(write_values) daisy_chain_wrapper = DaisyChainWrapper( self._dummy_url, self.test_data_file_len, mock_api, download_chunk_size=self.test_data_file_len) daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE) daisy_chain_wrapper.read(TRANSFER_BUFFER_SIZE) daisy_chain_wrapper.seek(0) self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file) self.assertEqual(mock_api.get_calls, 2) with open(upload_file, 'rb') as upload_stream: with open(self.test_data_file, 'rb') as download_stream: self.assertEqual(upload_stream.read(), download_stream.read()) def testDownloadThreadException(self): """Tests that an exception is propagated via the upload thread.""" class DownloadException(Exception): pass write_values = [ b'a', b'b', DownloadException('Download thread forces failure') ] upload_file = self.CreateTempFile() mock_api = self.MockDownloadCloudApi(write_values) daisy_chain_wrapper = DaisyChainWrapper( self._dummy_url, self.test_data_file_len, mock_api, download_chunk_size=self.test_data_file_len) try: self._WriteFromWrapperToFile(daisy_chain_wrapper, upload_file) self.fail('Expected exception') except DownloadException as e: self.assertIn('Download thread forces failure', str(e)) def testInvalidSeek(self): """Tests that seeking fails for unsupported seek arguments.""" daisy_chain_wrapper = DaisyChainWrapper(self._dummy_url, self.test_data_file_len, self.MockDownloadCloudApi([])) try: # SEEK_CUR is invalid. daisy_chain_wrapper.seek(0, whence=os.SEEK_CUR) self.fail('Expected exception') except IOError as e: self.assertIn('does not support seek mode', str(e)) try: # Seeking from the end with an offset is invalid. daisy_chain_wrapper.seek(1, whence=os.SEEK_END) self.fail('Expected exception') except IOError as e: self.assertIn('Invalid seek during daisy chain', str(e)) tests/util.py 0000644 00000072011 15025013616 0007235 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from contextlib import contextmanager import functools import locale import logging import os import pkgutil import posixpath import re import io import signal import subprocess import sys import tempfile import threading import unittest import six from six.moves import urllib from six.moves import cStringIO import boto import crcmod import gslib from gslib.kms_api import KmsApi from gslib.project_id import PopulateProjectId import mock_storage_service # From boto/tests/integration/s3 from gslib.cloud_api import ResumableDownloadException from gslib.cloud_api import ResumableUploadException from gslib.lazy_wrapper import LazyWrapper import gslib.tests as gslib_tests from gslib.utils import posix_util from gslib.utils.boto_util import UsingCrcmodExtension, HasUserSpecifiedGsHost from gslib.utils.constants import UTF8 from gslib.utils.encryption_helper import Base64Sha256FromBase64EncryptionKey from gslib.utils.system_util import IS_WINDOWS from gslib.utils.unit_util import MakeHumanReadable # pylint: disable=g-import-not-at-top, g-long-lambda if not IS_WINDOWS: import grp import pwd def GetInvalidGid(): # Get a list of all GIDs on the system for quick reference. all_gid = sorted([group.gr_gid for group in grp.getgrall()]) # gid - Current GID being tested, 2k is close to a large empty span on most # unix systems and a good starting point. gid = 2000 # OverflowError should prevent loop from reaching 5b, but adding a number # to the loop ensures that infinite loop does not occur while gid < 5000000000: if gid in all_gid: # Shortcut check; if gid is in list then the group exists. gid += 1 continue try: # Testing for expected behaviour while testing POSIX permissions. # For more on grp see: # https://docs.python.org/3.7/library/grp.html grp.getgrgid(gid) gid += 1 except KeyError: # This is the target exception for invalid GID and the behaviour needed # for testing. return gid except OverflowError: # Limit reached without a usable GID found. break raise Exception("Unable to generate GID for ") def GetNonPrimaryGid(): # Select a group for the current user that is not the user's primary group. # If the length of the user's groups is 1, then we must use the primary # group. Otherwise put all of the user's groups (except the primary group) # in a list, and use the first element. This guarantees us a group that is # not the user's primary group (unless the user is only a member of one # group). primary_gid = GetPrimaryGid() user_groups = GetUserGroups() if len(user_groups) == 1: return primary_gid return [g for g in list(user_groups) if g != primary_gid][0] def GetPrimaryGid(): return os.getgid() def GetUserGroups(): return set([GetPrimaryGid()] + [g.gr_gid for g in grp.getgrall() if USER_NAME() in g.gr_mem]) posix_util.InitializeDefaultMode() DEFAULT_MODE = int(posix_util.SYSTEM_POSIX_MODE, 8) USER_ID = os.getuid() USER_NAME = LazyWrapper(lambda: pwd.getpwuid(USER_ID).pw_name) # Take the current user's UID and increment it by one, this counts as an # invalid UID, as the metric used is if the UID matches the current user's, # exactly. INVALID_UID = LazyWrapper( lambda: sorted([user.pw_uid for user in pwd.getpwall()])[-1] + 1) # Note that because the system's GID mapping can change mid-test, tests that # check for specific errors should always re-fetch these GID-related values, # rather than reusing these LazyWrapper values. INVALID_GID = LazyWrapper(lambda: GetInvalidGid()) # Get a list of all groups on the system where the current username is listed # as a member of the group in the gr_mem group attribute. Make this a list of # all group IDs and cast as a set for more efficient lookup times. USER_GROUPS = LazyWrapper(lambda: GetUserGroups()) def GetGsutilCommand(raw_command, force_gsutil=False): """Adds config options to a list of strings defining a gsutil subcommand.""" # TODO(b/203250512) Remove this once all the commands are supported # via gcloud storage. if force_gsutil: use_gcloud_storage = False else: use_gcloud_storage = boto.config.getbool('GSUtil', 'use_gcloud_storage', False) gcloud_storage_setting = [ '-o', 'GSUtil:use_gcloud_storage={}'.format(use_gcloud_storage), '-o', 'GSUtil:hidden_shim_mode=no_fallback', ] gsutil_command = [ gslib.GSUTIL_PATH, '--testexceptiontraces', '-o', 'GSUtil:default_project_id=' + PopulateProjectId() ] + gcloud_storage_setting + raw_command # Checks to see if the test was invoked from a par file (bundled archive). # If not, adds the Python executable path to ensure that the correct version # of Python is used for testing. if not InvokedFromParFile(): gsutil_command_with_executable_path = [str(sys.executable)] + gsutil_command else: gsutil_command_with_executable_path = gsutil_command return [six.ensure_str(part) for part in gsutil_command_with_executable_path] def GetGsutilSubprocess(cmd, env_vars=None): """Returns a subprocess.Popen object for for running a gsutil command.""" env = os.environ.copy() if env_vars: env.update(env_vars) envstr = dict() for k, v in six.iteritems(env): envstr[six.ensure_str(k)] = six.ensure_str(v) # The os.setsid call allows us to kill the process group below # if execution times out. With Python 2.7, there's no other way to # stop execution (p.kill() doesn't work). Since os.setsid is not available # on Windows, we just deal with the occasional timeouts on Windows. preexec_fn = os.setsid if hasattr(os, 'setsid') else None return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, env=envstr, preexec_fn=preexec_fn) def CommunicateWithTimeout(process, stdin=None): if stdin is not None: if six.PY3: if not isinstance(stdin, bytes): stdin = stdin.encode(UTF8) else: stdin = stdin.encode(UTF8) comm_kwargs = {'input': stdin} def Kill(): os.killpg(os.getpgid(process.pid), signal.SIGKILL) if six.PY3: # TODO(b/135936279): Make this number configurable in .boto comm_kwargs['timeout'] = 180 else: timer = threading.Timer(180, Kill) timer.start() c_out = process.communicate(**comm_kwargs) if not six.PY3: timer.cancel() try: c_out = [six.ensure_text(output) for output in c_out] except UnicodeDecodeError: c_out = [ six.ensure_text(output, locale.getpreferredencoding(False)) for output in c_out ] return c_out # 256-bit base64 encryption keys used for testing AES256 customer-supplied # encryption. These are public and open-source, so don't ever use them for # real data. TEST_ENCRYPTION_KEY1 = b'iMSM9eeXliDZHSBJZO71R98tfeW/+87VXTpk5chGd6Y=' TEST_ENCRYPTION_KEY1_SHA256_B64 = Base64Sha256FromBase64EncryptionKey( TEST_ENCRYPTION_KEY1) TEST_ENCRYPTION_KEY2 = b'4TSaQ3S4U+5oxAbByA7HgIigD51zfzGed/c03Ts2TXc=' TEST_ENCRYPTION_KEY2_SHA256_B64 = Base64Sha256FromBase64EncryptionKey( TEST_ENCRYPTION_KEY2) TEST_ENCRYPTION_KEY3 = b'HO4Q2X28N/6SmuAJ1v1CTuJjf5emQcXf7YriKzT1gj0=' TEST_ENCRYPTION_KEY3_SHA256_B64 = Base64Sha256FromBase64EncryptionKey( TEST_ENCRYPTION_KEY3) TEST_ENCRYPTION_KEY4 = b'U6zIErjZCK/IpIeDS0pJrDayqlZurY8M9dvPJU0SXI8=' TEST_ENCRYPTION_KEY4_SHA256_B64 = Base64Sha256FromBase64EncryptionKey( TEST_ENCRYPTION_KEY4) TEST_ENCRYPTION_KEY_S3 = b'MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=' TEST_ENCRYPTION_KEY_S3_MD5 = b'dnF5x6K/8ZZRzpfSlMMM+w==' TEST_ENCRYPTION_CONTENT1 = b'bar' TEST_ENCRYPTION_CONTENT1_MD5 = 'N7UdGUp1E+RbVvZSTy1R8g==' TEST_ENCRYPTION_CONTENT1_CRC32C = 'CrcTMQ==' TEST_ENCRYPTION_CONTENT2 = b'bar2' TEST_ENCRYPTION_CONTENT2_MD5 = 'Ik4lOfUiA+szcorNIotEMg==' TEST_ENCRYPTION_CONTENT2_CRC32C = 'QScXtg==' TEST_ENCRYPTION_CONTENT3 = b'bar3' TEST_ENCRYPTION_CONTENT3_MD5 = '9iW6smjfu9hm0A//VQTQfw==' TEST_ENCRYPTION_CONTENT3_CRC32C = 's0yUtQ==' TEST_ENCRYPTION_CONTENT4 = b'bar4' TEST_ENCRYPTION_CONTENT4_MD5 = 'kPCx6uZiUOU7W6E+cDCZFg==' TEST_ENCRYPTION_CONTENT4_CRC32C = 'Z4bwXg==' TEST_ENCRYPTION_CONTENT5 = b'bar5' TEST_ENCRYPTION_CONTENT5_MD5 = '758XbXQOVkp8fTKMm83NXA==' TEST_ENCRYPTION_CONTENT5_CRC32C = 'le1zXQ==' # Flags for running different types of tests. RUN_INTEGRATION_TESTS = True RUN_UNIT_TESTS = True RUN_S3_TESTS = False USE_MULTIREGIONAL_BUCKETS = False PARALLEL_COMPOSITE_UPLOAD_TEST_CONFIG = '/tmp/.boto.parallel_upload_test_config' ORPHANED_FILE = ('This sync will orphan file(s), please fix their permissions ' 'before trying again.') POSIX_MODE_ERROR = 'Mode for %s won\'t allow read access.' POSIX_GID_ERROR = 'GID for %s doesn\'t exist on current system.' POSIX_UID_ERROR = 'UID for %s doesn\'t exist on current system.' POSIX_INSUFFICIENT_ACCESS_ERROR = 'Insufficient access with uid/gid/mode for %s' class KmsTestingResources(object): """Constants for KMS resource names to be used in integration testing.""" KEYRING_LOCATION = 'us-central1' # Since KeyRings and their child resources cannot be deleted, we minimize the # number of resources created by using a hard-coded keyRing name. KEYRING_NAME = 'keyring-for-gsutil-integration-tests' # Used by tests where we don't need to alter the state of a cryptoKey and/or # its IAM policy bindings once it's initialized the first time. CONSTANT_KEY_NAME = 'key-for-gsutil-integration-tests' CONSTANT_KEY_NAME2 = 'key-for-gsutil-integration-tests2' # This key should not be authorized so it can be used for failure cases. CONSTANT_KEY_NAME_DO_NOT_AUTHORIZE = 'key-for-gsutil-no-auth' # Pattern used for keys that should only be operated on by one tester at a # time. Because multiple integration test invocations can run at the same # time, we want to minimize the risk of them operating on each other's key, # while also not creating too many one-time-use keys (as they cannot be # deleted). Tests should fill in the %d entries with a digit between 0 and 9. MUTABLE_KEY_NAME_TEMPLATE = 'cryptokey-for-gsutil-integration-tests-%d%d%d' def AuthorizeProjectToUseTestingKmsKey( key_name=KmsTestingResources.CONSTANT_KEY_NAME): """Ensures test keys exist and that the service agent is authorized.""" kms_api = KmsApi(logging.getLogger()) keyring_fully_qualified_name = kms_api.CreateKeyRing( PopulateProjectId(None), KmsTestingResources.KEYRING_NAME, location=KmsTestingResources.KEYRING_LOCATION) key_fully_qualified_name = kms_api.CreateCryptoKey( keyring_fully_qualified_name, key_name) cmd = GetGsutilCommand(['kms', 'authorize', '-k', key_fully_qualified_name], force_gsutil=True) process = GetGsutilSubprocess(cmd) CommunicateWithTimeout(process) return key_fully_qualified_name def BuildErrorRegex(obj, err_str): """Builds a regex to match a file name for a file that would be orphaned. Args: obj: Object uri. err_str: The error string to search for. Returns: A regex that will match the file name and with the error text for a file that would be orphaned. """ return re.compile(err_str % ObjectToURI(obj)) def TailSet(start_point, listing): """Returns set of object name tails. Tails can be compared between source and dest, past the point at which the command was done. For example if test ran {cp,mv,rsync} gs://bucket1/dir gs://bucket2/dir2, the tails for listings from bucket1 would start after "dir", while the tails for listings from bucket2 would start after "dir2". Args: start_point: The target of the cp command, e.g., for the above command it would be gs://bucket1/dir for the bucket1 listing results and gs://bucket2/dir2 for the bucket2 listing results. listing: The listing over which to compute tail. Returns: Object name tails. """ return set(l[len(start_point):] for l in listing.strip().split('\n')) HAS_S3_CREDS = (boto.config.get('Credentials', 'aws_access_key_id', None) and boto.config.get('Credentials', 'aws_secret_access_key', None)) HAS_P12_CREDS = boto.config.get('Credentials', 'gs_service_key_file', '').endswith('.p12') HAS_NON_DEFAULT_GS_HOST = HasUserSpecifiedGsHost() HAS_GS_HOST = boto.config.get('Credentials', 'gs_host', None) is not None HAS_GS_PORT = boto.config.get('Credentials', 'gs_port', None) is not None USING_JSON_API = boto.config.get('GSUtil', 'prefer_api', 'json').upper() != 'XML' def SkipForP12Creds(reason): if HAS_P12_CREDS: return unittest.skip(reason) else: return lambda func: func def _ArgcompleteAvailable(): argcomplete = None if not IS_WINDOWS: try: # pylint: disable=g-import-not-at-top import argcomplete except ImportError: pass return argcomplete is not None ARGCOMPLETE_AVAILABLE = _ArgcompleteAvailable() def _NormalizeURI(uri): """Normalizes the path component of a URI. Args: uri: URI to normalize. Returns: Normalized URI. Examples: gs://foo//bar -> gs://foo/bar gs://foo/./bar -> gs://foo/bar """ # Note: we have to do this dance of changing gs:// to file:// because on # Windows, the urlparse function won't work with URL schemes that are not # known. urlparse('gs://foo/bar') on Windows turns into: # scheme='gs', netloc='', path='//foo/bar' # while on non-Windows platforms, it turns into: # scheme='gs', netloc='foo', path='/bar' uri = uri.replace('gs://', 'file://') parsed = list(urllib.parse.urlparse(uri)) parsed[2] = posixpath.normpath(parsed[2]) if parsed[2].startswith('//'): # The normpath function doesn't change '//foo' -> '/foo' by design. parsed[2] = parsed[2][1:] unparsed = urllib.parse.urlunparse(parsed) unparsed = unparsed.replace('file://', 'gs://') return unparsed def GenerationFromURI(uri): """Returns a the generation for a StorageUri. Args: uri: boto.storage_uri.StorageURI object to get the URI from. Returns: Generation string for the URI. """ if not (uri.generation or uri.version_id): if uri.scheme == 's3': return 'null' return uri.generation or uri.version_id def ObjectToURI(obj, *suffixes): """Returns the storage URI string for a given StorageUri or file object. Args: obj: The object to get the URI from. Can be a file object, a subclass of boto.storage_uri.StorageURI, or a string. If a string, it is assumed to be a local on-disk path. *suffixes: Suffixes to append. For example, ObjectToUri(bucketuri, 'foo') would return the URI for a key name 'foo' inside the given bucket. Returns: Storage URI string. """ if is_file(obj): return 'file://{}'.format(os.path.abspath(os.path.join(obj.name, *suffixes))) if isinstance(obj, six.string_types): return 'file://{}'.format(os.path.join(obj, *suffixes)) uri = six.ensure_text(obj.uri) if suffixes: suffixes_list = [six.ensure_text(suffix) for suffix in suffixes] uri = _NormalizeURI('/'.join([uri] + suffixes_list)) # Storage URIs shouldn't contain a trailing slash. if uri.endswith('/'): uri = uri[:-1] return uri class GSMockConnection(mock_storage_service.MockConnection): def __init__(self, *args, **kwargs): kwargs['provider'] = 'gs' self.debug = 0 super(GSMockConnection, self).__init__(*args, **kwargs) mock_connection = GSMockConnection() class GSMockBucketStorageUri(mock_storage_service.MockBucketStorageUri): def connect(self, access_key_id=None, secret_access_key=None): return mock_connection def compose(self, components, headers=None): """Dummy implementation to allow parallel uploads with tests.""" return self.new_key() def get_location(self, headers=None): return 'US' def get_cors(self, headers=None): return boto.gs.cors.Cors() def get_encryption_config(self, headers=None): return boto.gs.encryptionconfig.EncryptionConfig() def get_lifecycle_config(self, headers=None): return None def get_website_config(self, headers=None): return None def get_versioning_config(self, headers=None): return None TEST_BOTO_REMOVE_SECTION = 'TestRemoveSection' def _SetBotoConfig(section, name, value, revert_list): """Sets boto configuration temporarily for testing. SetBotoConfigForTest should be called by tests instead of this function. This will ensure that the configuration is reverted to its original setting using _RevertBotoConfig. Args: section: Boto config section to set name: Boto config name to set value: Value to set revert_list: List for tracking configs to revert. """ prev_value = boto.config.get(section, name, None) if not boto.config.has_section(section): revert_list.append((section, TEST_BOTO_REMOVE_SECTION, None)) boto.config.add_section(section) revert_list.append((section, name, prev_value)) if value is None: boto.config.remove_option(section, name) else: boto.config.set(section, name, value) def _RevertBotoConfig(revert_list): """Reverts boto config modifications made by _SetBotoConfig. Args: revert_list: List of boto config modifications created by calls to _SetBotoConfig. """ sections_to_remove = [] for section, name, value in revert_list: if value is None: if name == TEST_BOTO_REMOVE_SECTION: sections_to_remove.append(section) else: boto.config.remove_option(section, name) else: boto.config.set(section, name, value) for section in sections_to_remove: boto.config.remove_section(section) def SequentialAndParallelTransfer(func): """Decorator for tests that perform file to object transfers, or vice versa. This forces the test to run once normally, and again with special boto config settings that will ensure that the test follows the parallel composite upload and/or sliced object download code paths. Args: func: Function to wrap. Returns: Wrapped function. """ @functools.wraps(func) def Wrapper(*args, **kwargs): # Run the test normally once. func(*args, **kwargs) if not RUN_S3_TESTS and UsingCrcmodExtension(): # Try again, forcing parallel upload and sliced download. with SetBotoConfigForTest([ ('GSUtil', 'parallel_composite_upload_threshold', '1'), ('GSUtil', 'sliced_object_download_threshold', '1'), ('GSUtil', 'sliced_object_download_max_components', '3'), ('GSUtil', 'check_hashes', 'always') ]): func(*args, **kwargs) return Wrapper def _SectionDictFromConfigList(boto_config_list): """Converts the input config list to a dict that is easy to write to a file. This is used to reset the boto config contents for a test instead of preserving the existing values. Args: boto_config_list: list of tuples of: (boto config section to set, boto config name to set, value to set) If value to set is None, no entry is created. Returns: Dictionary of {section: {keys: values}} for writing to the file. """ sections = {} for config_entry in boto_config_list: section, key, value = (config_entry[0], config_entry[1], config_entry[2]) if section not in sections: sections[section] = {} if value is not None: sections[section][key] = value return sections def _WriteSectionDictToFile(section_dict, tmp_filename): """Writes a section dict from _SectionDictFromConfigList to tmp_filename.""" with open(tmp_filename, 'w') as tmp_file: for section, key_value_pairs in six.iteritems(section_dict): tmp_file.write('[%s]\n' % section) for key, value in six.iteritems(key_value_pairs): tmp_file.write('%s = %s\n' % (key, value)) @contextmanager def SetDummyProjectForUnitTest(): """Sets a dummy project in boto config for the duration of a 'with' clause.""" # Listing buckets requires a project ID, but unit tests should run # regardless of whether one is specified in config. with SetBotoConfigForTest([('GSUtil', 'default_project_id', 'dummy_proj')]): yield @contextmanager def SetBotoConfigForTest(boto_config_list, use_existing_config=True): """Sets the input list of boto configs for the duration of a 'with' clause. This preserves any existing boto configuration unless it is overwritten in the provided boto_config_list. Args: boto_config_list: list of tuples of: (boto config section to set, boto config name to set, value to set) use_existing_config: If True, apply boto_config_list to the existing configuration, preserving any original values unless they are overwritten. Otherwise, apply boto_config_list to a blank configuration. Yields: Once after config is set. """ revert_configs = [] tmp_filename = None try: tmp_fd, tmp_filename = tempfile.mkstemp(prefix='gsutil-temp-cfg') os.close(tmp_fd) if use_existing_config: for boto_config in boto_config_list: boto_value = boto_config[2] if six.PY3: if isinstance(boto_value, bytes): boto_value = boto_value.decode(UTF8) _SetBotoConfig(boto_config[0], boto_config[1], boto_value, revert_configs) with open(tmp_filename, 'w') as tmp_file: boto.config.write(tmp_file) else: _WriteSectionDictToFile(_SectionDictFromConfigList(boto_config_list), tmp_filename) with _SetBotoConfigFileForTest(tmp_filename): yield finally: _RevertBotoConfig(revert_configs) if tmp_filename: try: os.remove(tmp_filename) except OSError: pass @contextmanager def SetEnvironmentForTest(env_variable_dict): """Sets OS environment variables for a single test.""" def _ApplyDictToEnvironment(dict_to_apply): for k, v in six.iteritems(dict_to_apply): old_values[k] = os.environ.get(k) if v is not None: os.environ[k] = v elif k in os.environ: del os.environ[k] old_values = {} for k in env_variable_dict: old_values[k] = os.environ.get(k) try: _ApplyDictToEnvironment(env_variable_dict) yield finally: _ApplyDictToEnvironment(old_values) @contextmanager def _SetBotoConfigFileForTest(boto_config_path): """Sets a given file as the boto config file for a single test. This function applies only the configuration in boto_config_path and will ignore existing configuration. It should not be called directly by tests; instead, use SetBotoConfigForTest. Args: boto_config_path: Path to config file to use. Yields: When configuration has been applied, and again when reverted. """ # Setup for entering "with" block. try: old_boto_config_env_variable = os.environ['BOTO_CONFIG'] boto_config_was_set = True except KeyError: boto_config_was_set = False os.environ['BOTO_CONFIG'] = boto_config_path try: yield finally: # Teardown for exiting "with" block. if boto_config_was_set: os.environ['BOTO_CONFIG'] = old_boto_config_env_variable else: os.environ.pop('BOTO_CONFIG', None) def GetTestNames(): """Returns a list of the names of the test modules in gslib.tests.""" matcher = re.compile(r'^test_(?P<name>.*)$') names = [] for _, modname, _ in pkgutil.iter_modules(gslib_tests.__path__): m = matcher.match(modname) if m: names.append(m.group('name')) return names def is_file(obj): if six.PY2: return isinstance(obj, file) # pylint: disable=undefined-variable return isinstance(obj, io.IOBase) def MakeBucketNameValid(name): """Returns a copy of the given name with any invalid characters replaced. Args: name Union[str, unicode, bytes]: The bucket name to transform into a valid name. Returns: Union[str, unicode, bytes] The version of the bucket name containing only valid characters. """ # Neither underscores nor uppercase letters are valid characters for a # bucket name. Replace those with hyphens and lowercase characters. if isinstance(name, (six.text_type, six.binary_type)): return name.replace('_', '-').lower() else: raise TypeError('Unable to format name. Incorrect Type: {0}'.format( type(name))) @contextmanager def WorkingDirectory(new_working_directory): """Changes the working directory for the duration of a 'with' call. Args: new_working_directory: The directory to switch to before executing wrapped code. A None value indicates that no switching is necessary. Yields: Once after working directory has been changed. """ prev_working_directory = None try: prev_working_directory = os.getcwd() except OSError: # This can happen if the current working directory no longer exists. pass if new_working_directory: os.chdir(new_working_directory) try: yield finally: if new_working_directory and prev_working_directory: os.chdir(prev_working_directory) def InvokedFromParFile(): loader = globals().get('__loader__', None) if not loader: return False return 'zipimport' in loader.__class__.__module__ def SkipForParFile(reason): if InvokedFromParFile(): return unittest.skip(reason) else: return lambda func: func # Custom test callbacks must be pickleable, and therefore at global scope. class HaltingCopyCallbackHandler(object): """Test callback handler for intentionally stopping a resumable transfer.""" def __init__(self, is_upload, halt_at_byte): self._is_upload = is_upload self._halt_at_byte = halt_at_byte # pylint: disable=invalid-name def call(self, total_bytes_transferred, total_size): """Forcibly exits if the transfer has passed the halting point. Note that this function is only called when the conditions in gslib.progress_callback.ProgressCallbackWithTimeout.Progress are met, so self._halt_at_byte is only precise if it's divisible by gslib.progress_callback._START_BYTES_PER_CALLBACK. """ if total_bytes_transferred >= self._halt_at_byte: sys.stderr.write( 'Halting transfer after byte %s. %s/%s transferred.\r\n' % (self._halt_at_byte, MakeHumanReadable(total_bytes_transferred), MakeHumanReadable(total_size))) if self._is_upload: raise ResumableUploadException('Artifically halting upload.') else: raise ResumableDownloadException('Artifically halting download.') class HaltOneComponentCopyCallbackHandler(object): """Test callback handler for stopping part of a sliced download.""" def __init__(self, halt_at_byte): self._last_progress_byte = None self._halt_at_byte = halt_at_byte # pylint: disable=invalid-name # pylint: disable=unused-argument def call(self, current_progress_byte, total_size_unused): """Forcibly exits if the passed the halting point since the last call.""" if (self._last_progress_byte is not None and self._last_progress_byte < self._halt_at_byte < current_progress_byte): sys.stderr.write('Halting transfer.\r\n') raise ResumableDownloadException('Artifically halting download.') self._last_progress_byte = current_progress_byte class TestParams(object): """Allows easier organization of test parameters. This class allows grouping of test parameters, which include args and kwargs to be used, as well as the expected result based on those arguments. For example, to test an Add function, one might do: params = TestParams(args=(1, 2, 3), expected=6) self.assertEqual(Add(*(params.args)), params.expected) """ def __init__(self, args=None, kwargs=None, expected=None): self.args = tuple() if args is None else args self.kwargs = dict() if kwargs is None else kwargs self.expected = expected if not isinstance(args, (tuple, list)): raise TypeError('TestParam args must be a tuple or list.') if not isinstance(self.kwargs, dict): raise TypeError('TestParam kwargs must be a dict.') class CaptureStdout(list): """Context manager. For example, this function has the lines printed by the function call stored as a list in output: with CaptureStdout() as output: function(input_to_function) """ def __enter__(self): self._stdout = sys.stdout sys.stdout = self._stringio = cStringIO() return self def __exit__(self, *args): self.extend(self._stringio.getvalue().splitlines()) del self._stringio sys.stdout = self._stdout tests/test_retention_util.py 0000644 00000012767 15025013616 0012377 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for retention_util module.""" from __future__ import absolute_import import gslib.tests.testcase as testcase from gslib.utils.retention_util import _RetentionPeriodToString from gslib.utils.retention_util import DaysToSeconds from gslib.utils.retention_util import MonthsToSeconds from gslib.utils.retention_util import RetentionInDaysMatch from gslib.utils.retention_util import RetentionInMonthsMatch from gslib.utils.retention_util import RetentionInSeconds from gslib.utils.retention_util import RetentionInSecondsMatch from gslib.utils.retention_util import RetentionInYearsMatch from gslib.utils.retention_util import SECONDS_IN_DAY from gslib.utils.retention_util import SECONDS_IN_MONTH from gslib.utils.retention_util import SECONDS_IN_YEAR from gslib.utils.retention_util import YearsToSeconds class TestRetentionUtil(testcase.GsUtilUnitTestCase): """Unit tests for gsutil retention_util module.""" def testDaysToSeconds(self): secs = DaysToSeconds(1) self.assertEqual(secs, 1 * SECONDS_IN_DAY) secs = DaysToSeconds(3) self.assertEqual(secs, 3 * SECONDS_IN_DAY) def testMonthsToSeconds(self): secs = MonthsToSeconds(1) self.assertEqual(secs, 1 * SECONDS_IN_MONTH) secs = MonthsToSeconds(3) self.assertEqual(secs, 3 * SECONDS_IN_MONTH) def testYearsToSeconds(self): secs = YearsToSeconds(1) self.assertEqual(secs, 1 * SECONDS_IN_YEAR) secs = YearsToSeconds(3) self.assertEqual(secs, 3 * SECONDS_IN_YEAR) def testRetentionInSecondsMatch(self): secs = '30s' secs_match = RetentionInSecondsMatch(secs) self.assertEqual('30', secs_match.group('number')) secs = '1s' secs_match = RetentionInSecondsMatch(secs) self.assertEqual('1', secs_match.group('number')) secs = '1second' secs_match = RetentionInSecondsMatch(secs) self.assertEqual(None, secs_match) def testRetentionInMonthsMatch(self): months = '30m' months_match = RetentionInMonthsMatch(months) self.assertEqual('30', months_match.group('number')) months = '1m' months_match = RetentionInMonthsMatch(months) self.assertEqual('1', months_match.group('number')) months = '1month' months_match = RetentionInMonthsMatch(months) self.assertEqual(None, months_match) def testRetentionInDaysMatch(self): days = '30d' days_match = RetentionInDaysMatch(days) self.assertEqual('30', days_match.group('number')) days = '1d' days_match = RetentionInDaysMatch(days) self.assertEqual('1', days_match.group('number')) days = '1day' days_match = RetentionInDaysMatch(days) self.assertEqual(None, days_match) def testRetentionInYearsMatch(self): years = '30y' years_match = RetentionInYearsMatch(years) self.assertEqual('30', years_match.group('number')) years = '1y' years_match = RetentionInYearsMatch(years) self.assertEqual('1', years_match.group('number')) years = '1year' years_match = RetentionInYearsMatch(years) self.assertEqual(None, years_match) def testRetentionInSeconds(self): one_year = '1y' one_year_in_seconds = RetentionInSeconds(one_year) self.assertEqual(SECONDS_IN_YEAR, one_year_in_seconds) one_month = '1m' one_month_in_seconds = RetentionInSeconds(one_month) self.assertEqual(SECONDS_IN_MONTH, one_month_in_seconds) one_day = '1d' one_day_in_seconds = RetentionInSeconds(one_day) self.assertEqual(SECONDS_IN_DAY, one_day_in_seconds) one_second = '1s' one_second_in_seconds = RetentionInSeconds(one_second) self.assertEqual(1, one_second_in_seconds) def testRetentionPeriodToString(self): retention_str = _RetentionPeriodToString(SECONDS_IN_DAY) self.assertRegex(retention_str, r'Duration: 1 Day\(s\)') retention_str = _RetentionPeriodToString(SECONDS_IN_DAY - 1) self.assertRegex(retention_str, r'Duration: 86399 Second\(s\)') retention_str = _RetentionPeriodToString(SECONDS_IN_DAY + 1) self.assertRegex(retention_str, r'Duration: 86401 Seconds \(~1 Day\(s\)\)') retention_str = _RetentionPeriodToString(SECONDS_IN_MONTH) self.assertRegex(retention_str, r'Duration: 1 Month\(s\)') retention_str = _RetentionPeriodToString(SECONDS_IN_MONTH - 1) self.assertRegex(retention_str, r'Duration: 2678399 Seconds \(~30 Day\(s\)\)') retention_str = _RetentionPeriodToString(SECONDS_IN_MONTH + 1) self.assertRegex(retention_str, r'Duration: 2678401 Seconds \(~31 Day\(s\)\)') retention_str = _RetentionPeriodToString(SECONDS_IN_YEAR) self.assertRegex(retention_str, r'Duration: 1 Year\(s\)') retention_str = _RetentionPeriodToString(SECONDS_IN_YEAR - 1) self.assertRegex(retention_str, r'Duration: 31557599 Seconds \(~365 Day\(s\)\)') retention_str = _RetentionPeriodToString(SECONDS_IN_YEAR + 1) self.assertRegex(retention_str, r'Duration: 31557601 Seconds \(~365 Day\(s\)\)') tests/test_Doption.py 0000644 00000023344 15025013616 0010740 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for gsutil -D option.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import platform import re import six import gslib from gslib.cs_api_map import ApiSelector import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.utils.unit_util import ONE_KIB @SkipForS3('-D output is implementation-specific.') class TestDOption(testcase.GsUtilIntegrationTestCase): """Integration tests for gsutil -D option.""" def assert_header_in_output(self, name, value, output): """Asserts that httplib2's debug logger printed out a specified header. This method is fairly primitive and uses assertIn statements, and thus is case-sensitive. Values should be normalized (e.g. to lowercase) if capitalization of the expected characters may vary. Args: name: (str) The header name, e.g. "Content-Length". value: (Union[str, None]) The header value, e.g. "4096". If no value is expected for the header or the value is unknown, this argument should be `None`. output: (str) The string in which to search for the specified header. """ expected = 'header: %s:' % name if value: # Only append a space and then the header value if a value was expected. expected += ' %s' % value if expected in output: return # Try the other format - when sending requests via the XML API, headers are # printed in a list of 2-tuples (by Boto), so we test for that output style # as well. The above style is generally preferred, but Python's http client # doesn't print all values in scenarios where a header is sent multiple # times with different values, e.g. in this case: # x-goog-hash: md5=blah2 # x-goog-hash: crc32c=blah1 # the debug logger would just print the last one to occur (the crc32c hash). alt_expected = "('%s'" % name if value: # Only check for the second part of the tuple if a value was expected. alt_expected += ", '%s')" % value if not alt_expected in output: self.fail('Neither of these two header formats were found in the output:' '\n1) %s\n2) %s\nOutput string: %s' % (expected, alt_expected, output)) def test_minus_D_multipart_upload(self): """Tests that debug option does not output upload media body.""" # We want to ensure it works with and without a trailing newline. for file_contents in (b'a1b2c3d4', b'a1b2c3d4\n'): fpath = self.CreateTempFile(contents=file_contents) bucket_uri = self.CreateBucket() with SetBotoConfigForTest([('GSUtil', 'resumable_threshold', str(ONE_KIB)) ]): stderr = self.RunGsUtil( ['-D', 'cp', fpath, suri(bucket_uri)], return_stderr=True) print('command line:' + ' '.join(['-D', 'cp', fpath, suri(bucket_uri)])) if self.test_api == ApiSelector.JSON: self.assertIn('media body', stderr) self.assertNotIn('a1b2c3d4', stderr) self.assertIn('Comparing local vs cloud md5-checksum for', stderr) self.assertIn('total_bytes_transferred: %d' % len(file_contents), stderr) def test_minus_D_perf_trace_cp(self): """Test upload and download with a sample perf trace token.""" file_name = 'bar' fpath = self.CreateTempFile(file_name=file_name, contents=b'foo') bucket_uri = self.CreateBucket() stderr = self.RunGsUtil( ['-D', '--perf-trace-token=123', 'cp', fpath, suri(bucket_uri)], return_stderr=True) self.assertIn('\'cookie\': \'123\'', stderr) stderr2 = self.RunGsUtil([ '-D', '--perf-trace-token=123', 'cp', suri(bucket_uri, file_name), fpath ], return_stderr=True) self.assertIn('\'cookie\': \'123\'', stderr2) def test_minus_D_arbitrary_header_cp(self): """Test upload and download with a sample perf trace token.""" file_name = 'bar' fpath = self.CreateTempFile(file_name=file_name, contents=b'foo') bucket_uri = self.CreateBucket() stderr = self.RunGsUtil( ['-D', '-h', 'custom-header:asdf', 'cp', fpath, suri(bucket_uri)], return_stderr=True) self.assertRegex(stderr, r"Headers: \{[\s\S]*'custom-header': 'asdf'[\s\S]*\}") stderr2 = self.RunGsUtil([ '-D', '-h', 'custom-header:asdf', 'cp', suri(bucket_uri, file_name), fpath ], return_stderr=True) self.assertRegex(stderr2, r"Headers: \{[\s\S]*'custom-header': 'asdf'[\s\S]*\}") # Ensure the header wasn't set in metadata somehow: stdout = self.RunGsUtil( ['ls', '-L', suri(bucket_uri, file_name)], return_stdout=True) self.assertNotIn('custom', stdout) self.assertNotIn('asdf', stdout) def test_minus_D_resumable_upload(self): fpath = self.CreateTempFile(contents=b'a1b2c3d4') bucket_uri = self.CreateBucket() with SetBotoConfigForTest([('GSUtil', 'resumable_threshold', '4')]): stderr = self.RunGsUtil( ['-D', 'cp', fpath, suri(bucket_uri)], return_stderr=True) self.assertNotIn('a1b2c3d4', stderr) self.assertIn('Comparing local vs cloud md5-checksum for', stderr) self.assertIn('total_bytes_transferred: 8', stderr) def test_minus_D_cat(self): """Tests cat command with debug option.""" key_uri = self.CreateObject(contents=b'0123456789') with SetBotoConfigForTest([('Boto', 'proxy_pass', 'secret')]): (stdout, stderr) = self.RunGsUtil(['-D', 'cat', suri(key_uri)], return_stdout=True, return_stderr=True) # Check for log messages we output. self.assertIn('You are running gsutil with debug output enabled.', stderr) self.assertIn('config:', stderr) if six.PY2: self.assertIn("('proxy_pass', u'REDACTED')", stderr) else: self.assertIn("('proxy_pass', 'REDACTED')", stderr) # Check for log messages from httplib2 / http_client. self.assertIn("reply: 'HTTP/1.1 200 OK", stderr) self.assert_header_in_output('Expires', None, stderr) self.assert_header_in_output('Date', None, stderr) self.assert_header_in_output('Content-Type', 'application/octet-stream', stderr) self.assert_header_in_output('Content-Length', '10', stderr) if self.test_api == ApiSelector.XML: self.assert_header_in_output('Cache-Control', None, stderr) self.assert_header_in_output('ETag', '"781e5e245d69b566979b86e28d23f2c7"', stderr) self.assert_header_in_output('Last-Modified', None, stderr) self.assert_header_in_output('x-goog-generation', None, stderr) self.assert_header_in_output('x-goog-metageneration', '1', stderr) self.assert_header_in_output('x-goog-hash', 'crc32c=KAwGng==', stderr) self.assert_header_in_output('x-goog-hash', 'md5=eB5eJF1ptWaXm4bijSPyxw==', stderr) # Check request fields show correct segments. regex_str = r'''send:\s+([b|u]')?HEAD /%s/%s HTTP/[^\\]*\\r\\n(.*)''' % ( key_uri.bucket_name, key_uri.object_name) regex = re.compile(regex_str) match = regex.search(stderr) if not match: self.fail('Did not find this regex in stderr:\nRegex: %s\nStderr: %s' % (regex_str, stderr)) request_fields_str = match.group(2) self.assertIn('Content-Length: 0', request_fields_str) self.assertRegex( request_fields_str, 'User-Agent: .*gsutil/%s.*interactive/False command/cat' % gslib.VERSION) elif self.test_api == ApiSelector.JSON: if six.PY2: self.assertIn("md5Hash: u'eB5eJF1ptWaXm4bijSPyxw=='", stderr) else: self.assertIn("md5Hash: 'eB5eJF1ptWaXm4bijSPyxw=='", stderr) self.assert_header_in_output( 'Cache-Control', 'no-cache, no-store, max-age=0, must-revalidate', stderr) self.assertRegex( stderr, '.*GET.*b/%s/o/%s' % (key_uri.bucket_name, key_uri.object_name)) self.assertRegex( stderr, 'Python/%s.gsutil/%s.*interactive/False command/cat' % (platform.python_version(), gslib.VERSION)) if gslib.IS_PACKAGE_INSTALL: self.assertIn('PACKAGED_GSUTIL_INSTALLS_DO_NOT_HAVE_CHECKSUMS', stdout) else: self.assertRegex(stdout, r'.*checksum: [0-9a-f]{32}.*') self.assertIn('gsutil version: %s' % gslib.VERSION, stdout) self.assertIn('boto version: ', stdout) self.assertIn('python version: ', stdout) self.assertIn('OS: ', stdout) self.assertIn('multiprocessing available: ', stdout) self.assertIn('using cloud sdk: ', stdout) self.assertIn('pass cloud sdk credentials to gsutil: ', stdout) self.assertIn('config path(s): ', stdout) self.assertIn('gsutil path: ', stdout) self.assertIn('compiled crcmod: ', stdout) self.assertIn('installed via package manager: ', stdout) self.assertIn('editable install: ', stdout) tests/test_rm.py 0000644 00000103346 15025013616 0007743 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for rm command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import re import sys from unittest import mock from gslib.exception import NO_URLS_MATCHED_PREFIX from gslib.exception import NO_URLS_MATCHED_TARGET import gslib.tests.testcase as testcase from gslib.tests.testcase.base import MAX_BUCKET_LENGTH from gslib.tests.testcase.integration_testcase import SkipForS3 import gslib.tests.util as util from gslib.tests.util import GenerationFromURI as urigen from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.utils import shim_util from gslib.utils.retry_util import Retry MACOS_WARNING = ( 'If you experience problems with multiprocessing on MacOS, they might be ' 'related to https://bugs.python.org/issue33725. You can disable ' 'multiprocessing by editing your .boto config or by adding the following ' 'flag to your command: `-o "GSUtil:parallel_process_count=1"`. Note that ' 'multithreading is still available even if you disable multiprocessing.') class TestRm(testcase.GsUtilIntegrationTestCase): """Integration tests for rm command.""" def _CleanRmUiOutputBeforeChecking(self, stderr): """Excludes everything coming from the UI to avoid assert errors. Args: stderr: The cumulative stderr output. Returns: The cumulative stderr output without the expected UI output. """ if self._use_gcloud_storage: return self._CleanOutputLinesForGcloudStorage(stderr) ui_output_pattern = '[^\n\r]*objects][^\n\r]*[\n\r]' final_message_pattern = 'Operation completed over[^\n]*' ui_spinner_list = ['\\\r', '|\r', '/\r', '-\r'] ui_lines_list = (re.findall(ui_output_pattern, stderr) + re.findall(final_message_pattern, stderr) + ui_spinner_list) for ui_line in ui_lines_list: stderr = stderr.replace(ui_line, '') return stderr def _CleanOutputLinesForGcloudStorage(self, stderr): """Remove irrelevant lines from the output lines.""" stderr_lines = stderr.splitlines() valid_lines = [] strings_to_remove = set(['Removing objects:', 'Removing Buckets:', ' ']) for line in stderr_lines: if line in strings_to_remove: continue # Gcloud storage logs a '.' for each resource in non-interactive mode. # This might get added to the start of a valid line. # e.g. ".Removing gs://bucket/obj" cleaned_line = line.lstrip('.') if cleaned_line: valid_lines.append(cleaned_line) return os.linesep.join(valid_lines) def _RunRemoveCommandAndCheck(self, command_and_args, objects_to_remove=None, buckets_to_remove=None, stdin=None): """Tests a remove command in the presence of eventual listing consistency. Eventual listing consistency means that a remove command may not see all of the objects to be removed at once. When removing multiple objects (or buckets via -r), some calls may return no matches and multiple calls to the rm command may be necessary to reach the desired state. This function retries the rm command, incrementally tracking what has been removed and ensuring that the exact set of objects/buckets are removed across all retried calls. The caller is responsible for confirming the existence of buckets/objects prior to calling this function. Args: command_and_args: List of strings representing the rm command+args to run. objects_to_remove: List of object URL strings (optionally including generation) that should be removed by the command, if any. buckets_to_remove: List of bucket URL strings that should be removed by the command, if any. stdin: String of data to pipe to the process as standard input (for testing -I option). """ bucket_strings = [] for bucket_to_remove in buckets_to_remove or []: bucket_strings.append('Removing %s/...' % bucket_to_remove) object_strings = [] for object_to_remove in objects_to_remove or []: object_strings.append('Removing %s...' % object_to_remove) expected_stderr_lines = set(object_strings + bucket_strings) if not self.multiregional_buckets and self.default_provider == 'gs': stderr = self.RunGsUtil(command_and_args, return_stderr=True, expected_status=None, stdin=stdin) num_objects = len(object_strings) # Asserting for operation completion if '-q' not in command_and_args and not self._use_gcloud_storage: if '-m' in command_and_args: self.assertIn('[%d/%d objects]' % (num_objects, num_objects), stderr) else: self.assertIn('[%d objects]' % num_objects, stderr) stderr = self._CleanRmUiOutputBeforeChecking(stderr) stderr_set = set(stderr.splitlines()) if '' in stderr_set: stderr_set.remove('') # Avoid groups represented by an empty string. if MACOS_WARNING in stderr_set: stderr_set.remove(MACOS_WARNING) if self._use_gcloud_storage: for to_check in expected_stderr_lines: self.assertIn(to_check, stderr) else: self.assertEqual(stderr_set, expected_stderr_lines) else: cumulative_stderr_lines = set() @Retry(AssertionError, tries=5, timeout_secs=1) def _RunRmCommandAndCheck(): """Runs/retries the command updating+checking cumulative output.""" stderr = self.RunGsUtil(command_and_args, return_stderr=True, expected_status=None, stdin=stdin) stderr = self._CleanRmUiOutputBeforeChecking(stderr) update_lines = True # Retry 404's and 409's due to eventual listing consistency, but don't # add the output to the set. if (NO_URLS_MATCHED_PREFIX in stderr or '409 BucketNotEmpty' in stderr or '409 VersionedBucketNotEmpty' in stderr): update_lines = False # For recursive deletes of buckets, it is possible that the bucket is # deleted before the objects are all present in the listing, in which # case we will never see all of the expected "Removing object..." # messages. Since this is still a successful outcome, just return # successfully. if self._use_gcloud_storage: bucket_not_found_string = 'not found' else: bucket_not_found_string = 'bucket does not exist' if '-r' in command_and_args and 'bucket does not exist' in stderr: for bucket_to_remove in buckets_to_remove: matching_bucket = re.match( r'.*404\s+%s\s+bucket does not exist' % re.escape(bucket_to_remove), stderr) if matching_bucket: for line in cumulative_stderr_lines: if 'Removing %s/...' % bucket_to_remove in line: return if 'Removing %s/...' % bucket_to_remove in stderr: return if update_lines: cumulative_stderr_lines.update( set([s for s in stderr.splitlines() if s])) # Ensure all of the expected strings are present. self.assertEqual(cumulative_stderr_lines, expected_stderr_lines) _RunRmCommandAndCheck() def test_all_versions_current(self): """Test that 'rm -a' for an object with a current version works.""" bucket_uri = self.CreateVersionedBucket() key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo') self.StorageUriSetContentsFromString(key_uri, 'bar') g1 = urigen(key_uri) self.StorageUriSetContentsFromString(key_uri, 'baz') g2 = urigen(key_uri) def _Check1(stderr_lines): stderr = self.RunGsUtil( ['-m', 'rm', '-a', suri(key_uri)], return_stderr=True) stderr_lines.update(set(stderr.splitlines())) stderr = '\n'.join(stderr_lines) self.assertEqual(stderr.count('Removing %s://' % self.default_provider), 2) self.assertIn('Removing %s#%s...' % (suri(key_uri), g1), stderr) self.assertIn('Removing %s#%s...' % (suri(key_uri), g2), stderr) all_stderr_lines = set() if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True) @Retry(AssertionError, tries=3, timeout_secs=1) # Use @Retry as hedge against bucket listing eventual consistency. def _CheckWithRetries(stderr_lines): _Check1(stderr_lines) _CheckWithRetries(all_stderr_lines) else: _Check1(all_stderr_lines) self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) def test_all_versions_no_current(self): """Test that 'rm -a' for an object without a current version works.""" bucket_uri = self.CreateVersionedBucket() key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo') self.StorageUriSetContentsFromString(key_uri, 'bar') g1 = urigen(key_uri) self.StorageUriSetContentsFromString(key_uri, 'baz') g2 = urigen(key_uri) self._RunRemoveCommandAndCheck( ['-m', 'rm', '-a', suri(key_uri)], objects_to_remove=[ '%s#%s' % (suri(key_uri), g1), '%s#%s' % (suri(key_uri), g2) ]) self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) def test_fails_for_missing_obj(self): bucket_uri = self.CreateVersionedBucket() stderr = self.RunGsUtil( ['rm', '-a', '%s' % suri(bucket_uri, 'foo')], return_stderr=True, expected_status=1) if self._use_gcloud_storage: no_url_matched_target = no_url_matched_target = ( 'The following URLs matched no objects or files:\n%s') else: no_url_matched_target = NO_URLS_MATCHED_TARGET self.assertIn(no_url_matched_target % suri(bucket_uri, 'foo'), stderr) def test_remove_recursive_prefix(self): bucket_uri = self.CreateBucket() obj_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='a/b/c', contents=b'foo') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 1) stderr = self.RunGsUtil(['rm', '-r', suri(bucket_uri, 'a')], return_stderr=True) self.assertIn('Removing %s' % suri(obj_uri), stderr) def test_remove_all_versions_recursive_on_bucket(self): """Test that 'rm -r' works on bucket.""" bucket_uri = self.CreateVersionedBucket() k1_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo') k2_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo2') self.StorageUriSetContentsFromString(k1_uri, 'bar') self.StorageUriSetContentsFromString(k2_uri, 'bar2') k1g1 = urigen(k1_uri) k2g1 = urigen(k2_uri) self.StorageUriSetContentsFromString(k1_uri, 'baz') self.StorageUriSetContentsFromString(k2_uri, 'baz2') k1g2 = urigen(k1_uri) k2g2 = urigen(k2_uri) if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True) self._RunRemoveCommandAndCheck(['rm', '-r', suri(bucket_uri)], objects_to_remove=[ '%s#%s' % (suri(k1_uri), k1g1), '%s#%s' % (suri(k1_uri), k1g2), '%s#%s' % (suri(k2_uri), k2g1), '%s#%s' % (suri(k2_uri), k2g2) ], buckets_to_remove=[suri(bucket_uri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): # Bucket should no longer exist. stderr = self.RunGsUtil(['ls', '-a', suri(bucket_uri)], return_stderr=True, expected_status=1) if self._use_gcloud_storage: if self._use_gcloud_storage: # GCS and S3 responses. self.assertTrue('not found: 404' in stderr or 'NoSuchBucket' in stderr) else: self.assertIn('bucket does not exist', stderr) _Check() def test_remove_all_versions_recursive_on_subdir(self): """Test that 'rm -r' works on subdir.""" bucket_uri = self.CreateVersionedBucket() k1_uri = self.StorageUriCloneReplaceName(bucket_uri, 'dir/foo') k2_uri = self.StorageUriCloneReplaceName(bucket_uri, 'dir/foo2') self.StorageUriSetContentsFromString(k1_uri, 'bar') self.StorageUriSetContentsFromString(k2_uri, 'bar2') k1g1 = urigen(k1_uri) k2g1 = urigen(k2_uri) self.StorageUriSetContentsFromString(k1_uri, 'baz') self.StorageUriSetContentsFromString(k2_uri, 'baz2') k1g2 = urigen(k1_uri) k2g2 = urigen(k2_uri) if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 4, versioned=True) self._RunRemoveCommandAndCheck( ['rm', '-r', '%s' % suri(bucket_uri, 'dir')], objects_to_remove=[ '%s#%s' % (suri(k1_uri), k1g1), '%s#%s' % (suri(k1_uri), k1g2), '%s#%s' % (suri(k2_uri), k2g1), '%s#%s' % (suri(k2_uri), k2g2) ]) self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) def test_rm_seek_ahead(self): object_uri = self.CreateObject(contents=b'foo') with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil(['-m', 'rm', suri(object_uri)], return_stderr=True) self.assertIn('Estimated work for this command: objects: 1\n', stderr) def test_rm_seek_ahead_stdin_args(self): object_uri = self.CreateObject(contents=b'foo') with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil(['-m', 'rm', '-I'], stdin=suri(object_uri), return_stderr=True) self.assertNotIn('Estimated work', stderr) def test_missing_first_force(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='present', contents=b'foo') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 1) if not self._use_gcloud_storage: # Gcloud storage continues on missing objects by default. # So we will skip running this for gcloud storage. self.RunGsUtil( ['rm', '%s' % suri(bucket_uri, 'missing'), suri(object_uri)], expected_status=1) stderr = self.RunGsUtil( ['rm', '-f', '%s' % suri(bucket_uri, 'missing'), suri(object_uri)], return_stderr=True, expected_status=1) self.assertEqual(stderr.count('Removing %s://' % self.default_provider), 1) self.RunGsUtil(['stat', suri(object_uri)], expected_status=1) def test_some_missing(self): """Test that 'rm -a' fails when some but not all uris don't exist.""" bucket_uri = self.CreateVersionedBucket() key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo') self.StorageUriSetContentsFromString(key_uri, 'bar') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 1, versioned=True) stderr = self.RunGsUtil( ['rm', '-a', suri(key_uri), '%s' % suri(bucket_uri, 'missing')], return_stderr=True, expected_status=1) self.assertEqual(stderr.count('Removing %s://' % self.default_provider), 1) if self._use_gcloud_storage: self.assertIn( 'The following URLs matched no objects or files:\n%s' % suri(bucket_uri, 'missing'), stderr) else: self.assertIn(NO_URLS_MATCHED_TARGET % suri(bucket_uri, 'missing'), stderr) def test_some_missing_force(self): """Test that 'rm -af' succeeds despite hidden first uri.""" bucket_uri = self.CreateVersionedBucket() key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo') self.StorageUriSetContentsFromString(key_uri, 'bar') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 1, versioned=True) stderr = self.RunGsUtil( ['rm', '-af', suri(key_uri), '%s' % suri(bucket_uri, 'missing')], return_stderr=True, expected_status=1) self.assertEqual(stderr.count('Removing %s://' % self.default_provider), 1) self.AssertNObjectsInBucket(bucket_uri, 0) def test_folder_objects_deleted(self): """Test for 'rm -r' of a folder with a dir_$folder$ marker.""" bucket_uri = self.CreateVersionedBucket() key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc/o1') self.StorageUriSetContentsFromString(key_uri, 'foobar') folder_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc_$folder$') self.StorageUriSetContentsFromString(folder_uri, '') def _RemoveAndCheck(): self.RunGsUtil(['rm', '-r', '%s' % suri(bucket_uri, 'abc')], expected_status=None) self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True) # This could fail due to eventual listing consistency, so use retry and # expected_status=None to guard against No URLs matched exceptions. @Retry(AssertionError, tries=3, timeout_secs=1) def _RemoveAndCheckWrapper(): _RemoveAndCheck() _RemoveAndCheckWrapper() else: _RemoveAndCheck() # Bucket should not be deleted (Should not get ServiceException). bucket_uri.get_location(validate=False) def test_folder_objects_deleted_with_wildcard(self): """Test for 'rm -r' of a folder with a dir_$folder$ marker.""" bucket_uri = self.CreateVersionedBucket() key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc/o1') self.StorageUriSetContentsFromString(key_uri, 'foobar') folder_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc_$folder$') self.StorageUriSetContentsFromString(folder_uri, '') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True) self._RunRemoveCommandAndCheck( ['rm', '-r', '%s' % suri(bucket_uri, '*')], objects_to_remove=[ '%s#%s' % (suri(key_uri), urigen(key_uri)), '%s#%s' % (suri(folder_uri), urigen(folder_uri)) ]) self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) # Bucket should not be deleted (Should not get ServiceException). bucket_uri.get_location(validate=False) def test_folder_objects_deleted_with_double_wildcard(self): """Test for 'rm -r' of a folder with a dir_$folder$ marker.""" bucket_uri = self.CreateVersionedBucket() key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc/o1') self.StorageUriSetContentsFromString(key_uri, 'foobar') folder_uri = self.StorageUriCloneReplaceName(bucket_uri, 'abc_$folder$') self.StorageUriSetContentsFromString(folder_uri, '') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True) self._RunRemoveCommandAndCheck( ['rm', '-r', '%s' % suri(bucket_uri, '**')], objects_to_remove=[ '%s#%s' % (suri(key_uri), urigen(key_uri)), '%s#%s' % (suri(folder_uri), urigen(folder_uri)) ]) self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) # Bucket should not be deleted (Should not get ServiceException). bucket_uri.get_location(validate=False) @SkipForS3('Listing/removing S3 DeleteMarkers is not supported') def test_recursive_bucket_rm(self): """Test for 'rm -r' of a bucket.""" bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri, contents=b'foo') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 1) self._RunRemoveCommandAndCheck( ['rm', '-r', suri(bucket_uri)], objects_to_remove=['%s#%s' % (suri(object_uri), urigen(object_uri))], buckets_to_remove=[suri(bucket_uri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): # Bucket should be deleted. stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stderr=True, expected_status=1, force_gsutil=True) self.assertIn('bucket does not exist', stderr) _Check1() # Now try same thing, but for a versioned bucket with multiple versions of # an object present. bucket_uri = self.CreateVersionedBucket() self.CreateObject(bucket_uri, 'obj', contents=b'z') self.CreateObject(bucket_uri, 'obj', contents=b'z') final_uri = self.CreateObject(bucket_uri, 'obj', contents=b'z') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True) self._RunRemoveCommandAndCheck(['rm', suri(bucket_uri, '**')], objects_to_remove=['%s' % final_uri]) stderr = self.RunGsUtil(['rb', suri(bucket_uri)], return_stderr=True, expected_status=1, force_gsutil=True) self.assertIn('not empty', stderr) # Now try with rm -r. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): self.RunGsUtil(['rm', '-r', suri(bucket_uri)]) # Bucket should be deleted. stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stderr=True, expected_status=1, force_gsutil=True) self.assertIn('bucket does not exist', stderr) _Check2() def test_recursive_bucket_rm_with_wildcarding(self): """Tests removing all objects and buckets matching a bucket wildcard.""" buri_base = 'gsutil-test-%s' % self.GetTestMethodName() buri_base = buri_base[:MAX_BUCKET_LENGTH - 20] buri_base = '%s-%s' % (buri_base, self.MakeRandomTestString()) buri_base = 'aaa-' + buri_base buri_base = util.MakeBucketNameValid(buri_base) buri1 = self.CreateBucket(bucket_name='%s-tbuck1' % buri_base) buri2 = self.CreateBucket(bucket_name='%s-tbuck2' % buri_base) buri3 = self.CreateBucket(bucket_name='%s-tb3' % buri_base) ouri1 = self.CreateObject(bucket_uri=buri1, object_name='o1', contents=b'z') ouri2 = self.CreateObject(bucket_uri=buri2, object_name='o2', contents=b'z') self.CreateObject(bucket_uri=buri3, object_name='o3', contents=b'z') if self.multiregional_buckets: self.AssertNObjectsInBucket(buri1, 1) self.AssertNObjectsInBucket(buri2, 1) self.AssertNObjectsInBucket(buri3, 1) self._RunRemoveCommandAndCheck( ['rm', '-r', '%s://%s-tbu*' % (self.default_provider, buri_base)], objects_to_remove=[ '%s#%s' % (suri(ouri1), urigen(ouri1)), '%s#%s' % (suri(ouri2), urigen(ouri2)) ], buckets_to_remove=[suri(buri1), suri(buri2)]) self.AssertNObjectsInBucket(buri3, 1) def test_rm_quiet(self): """Test that 'rm -q' outputs no progress indications.""" bucket_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 1) self._RunRemoveCommandAndCheck(['-q', 'rm', suri(key_uri)], []) self.AssertNObjectsInBucket(bucket_uri, 0) @SkipForS3('The boto lib used for S3 does not handle objects ' 'starting with slashes if we use V4 signature') def test_rm_object_with_prefix_slash(self): """Tests removing a bucket that has an object starting with slash. The boto lib used for S3 does not handle objects starting with slashes if we use V4 signature. Hence we are testing objects with prefix slashes separately. """ bucket_uri = self.CreateVersionedBucket() ouri1 = self.CreateObject(bucket_uri=bucket_uri, object_name='/dirwithslash/foo', contents=b'z') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 1, versioned=True) self._RunRemoveCommandAndCheck( ['rm', '-r', suri(bucket_uri)], objects_to_remove=['%s#%s' % (suri(ouri1), urigen(ouri1))], buckets_to_remove=[suri(bucket_uri)]) def test_rm_object_with_slashes(self): """Tests removing a bucket that has objects with slashes.""" bucket_uri = self.CreateVersionedBucket() ouri1 = self.CreateObject(bucket_uri=bucket_uri, object_name='h/e/l//lo', contents=b'z') ouri2 = self.CreateObject(bucket_uri=bucket_uri, object_name='dirnoslash/foo', contents=b'z') ouri3 = self.CreateObject(bucket_uri=bucket_uri, object_name='dirnoslash/foo2', contents=b'z') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 3, versioned=True) self._RunRemoveCommandAndCheck(['rm', '-r', suri(bucket_uri)], objects_to_remove=[ '%s#%s' % (suri(ouri1), urigen(ouri1)), '%s#%s' % (suri(ouri2), urigen(ouri2)), '%s#%s' % (suri(ouri3), urigen(ouri3)) ], buckets_to_remove=[suri(bucket_uri)]) @SkipForS3('The boto lib used for S3 does not handle objects ' 'starting with slashes if we use V4 signature') def test_slasher_horror_film(self): """Tests removing a bucket with objects that are filled with slashes.""" bucket_uri = self.CreateVersionedBucket() ouri1 = self.CreateObject(bucket_uri=bucket_uri, object_name='h/e/l//lo', contents=b'Halloween') ouri2 = self.CreateObject(bucket_uri=bucket_uri, object_name='/h/e/l/l/o', contents=b'A Nightmare on Elm Street') ouri3 = self.CreateObject(bucket_uri=bucket_uri, object_name='//h//e/l//l/o', contents=b'Friday the 13th') ouri4 = self.CreateObject(bucket_uri=bucket_uri, object_name='//h//e//l//l//o', contents=b'I Know What You Did Last Summer') ouri5 = self.CreateObject(bucket_uri=bucket_uri, object_name='/', contents=b'Scream') ouri6 = self.CreateObject(bucket_uri=bucket_uri, object_name='//', contents=b'Child\'s Play') ouri7 = self.CreateObject(bucket_uri=bucket_uri, object_name='///', contents=b'The Prowler') ouri8 = self.CreateObject(bucket_uri=bucket_uri, object_name='////', contents=b'Black Christmas') ouri9 = self.CreateObject( bucket_uri=bucket_uri, object_name='everything/is/better/with/slashes///////', contents=b'Maniac') if self.multiregional_buckets: self.AssertNObjectsInBucket(bucket_uri, 9, versioned=True) # We add a slash to URIs with a trailing slash, # because ObjectToURI (suri) removes one trailing slash. objects_to_remove = [ '%s#%s' % (suri(ouri1), urigen(ouri1)), '%s#%s' % (suri(ouri2), urigen(ouri2)), '%s#%s' % (suri(ouri3), urigen(ouri3)), '%s#%s' % (suri(ouri4), urigen(ouri4)), '%s#%s' % (suri(ouri5) + '/', urigen(ouri5)), '%s#%s' % (suri(ouri6) + '/', urigen(ouri6)), '%s#%s' % (suri(ouri7) + '/', urigen(ouri7)), '%s#%s' % (suri(ouri8) + '/', urigen(ouri8)), '%s#%s' % (suri(ouri9) + '/', urigen(ouri9)) ] self._RunRemoveCommandAndCheck( ['-m', 'rm', '-r', suri(bucket_uri)], objects_to_remove=objects_to_remove, buckets_to_remove=[suri(bucket_uri)]) @SkipForS3('GCS versioning headers not supported by S3') def test_rm_failing_precondition(self): """Test for '-h x-goog-if-generation-match:value rm' of an object.""" bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri, contents=b'foo') stderr = self.RunGsUtil( ['-h', 'x-goog-if-generation-match:12345', 'rm', suri(object_uri)], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertRegex(stderr, r'pre-conditions you specified did not hold') else: self.assertRegex(stderr, r'PreconditionException: 412') def test_stdin_args(self): """Tests rm with the -I option.""" buri1 = self.CreateVersionedBucket() ouri1 = self.CreateObject(bucket_uri=buri1, object_name='foo', contents=b'foocontents') self.CreateObject(bucket_uri=buri1, object_name='bar', contents=b'barcontents') ouri3 = self.CreateObject(bucket_uri=buri1, object_name='baz', contents=b'bazcontents') buri2 = self.CreateVersionedBucket() ouri4 = self.CreateObject(bucket_uri=buri2, object_name='moo', contents=b'moocontents') if self.multiregional_buckets: self.AssertNObjectsInBucket(buri1, 3, versioned=True) self.AssertNObjectsInBucket(buri2, 1, versioned=True) objects_to_remove = [ '%s#%s' % (suri(ouri1), urigen(ouri1)), '%s#%s' % (suri(ouri3), urigen(ouri3)), '%s#%s' % (suri(ouri4), urigen(ouri4)) ] stdin = '\n'.join(objects_to_remove) self._RunRemoveCommandAndCheck(['rm', '-I'], objects_to_remove=objects_to_remove, stdin=stdin) self.AssertNObjectsInBucket(buri1, 1, versioned=True) self.AssertNObjectsInBucket(buri2, 0, versioned=True) def test_rm_nonexistent_bucket_recursive(self): stderr = self.RunGsUtil([ 'rm', '-rf', '%s://%s' % (self.default_provider, self.nonexistent_bucket_name) ], return_stderr=True, expected_status=1) if self._use_gcloud_storage: # GCS and S3 responses. self.assertTrue('not found: 404' in stderr or 'NoSuchBucket' in stderr) else: self.assertIn('Encountered non-existent bucket', stderr) def test_rm_multiple_nonexistent_objects(self): bucket_uri = self.CreateBucket() nonexistent_object1 = suri(bucket_uri, 'nonexistent1') nonexistent_object2 = suri(bucket_uri, 'nonexistent2') stderr = self.RunGsUtil( ['rm', '-rf', nonexistent_object1, nonexistent_object2], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn( 'The following URLs matched no objects or files:\n{}\n{}'.format( nonexistent_object1, nonexistent_object2), stderr) else: self.assertIn('2 files/objects could not be removed.', stderr) class TestRmUnitTestsWithShim(testcase.ShimUnitTestBase): """Unit tests for gsutil rm with shim.""" def test_shim_translates_flags(self): bucket_uri = self.CreateBucket() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'rm', ['-r', '-R', '-a', '-f', suri(bucket_uri)], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage rm' ' -r -r -a --continue-on-error {}'.format( shim_util._get_gcloud_binary_path('fake_dir'), suri(bucket_uri)), info_lines) @mock.patch.object(sys, 'stdin') def test_shim_translates_stdin_flag(self, mock_stdin): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri, 'foo', 'abcd') mock_stdin.__iter__.return_value = [suri(object_uri)] with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('rm', ['-I'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage rm' ' -I'.format(shim_util._get_gcloud_binary_path('fake_dir')), info_lines) tests/test_lifecycle.py 0000644 00000033460 15025013616 0011263 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for lifecycle command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import os import posixpath from unittest import mock from xml.dom.minidom import parseString from gslib.cs_api_map import ApiSelector import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import unittest from gslib.utils.retry_util import Retry from gslib.utils.translation_helper import LifecycleTranslation from gslib.utils import shim_util @SkipForS3('Lifecycle command is only supported for gs:// URLs') class TestSetLifecycle(testcase.GsUtilIntegrationTestCase): """Integration tests for lifecycle command.""" empty_doc1 = '{}' xml_doc = parseString( '<LifecycleConfiguration><Rule>' '<Action><Delete/></Action>' '<Condition><Age>365</Age></Condition>' '</Rule></LifecycleConfiguration>').toprettyxml(indent=' ') bad_doc = ( '{"rule": [{"action": {"type": "Add"}, "condition": {"age": 365}}]}\n') lifecycle_doc = ( '{"rule": [' '{"action": {"type": "Delete"}, "condition": {"age": 365}}, ' '{"action": {"type": "SetStorageClass", "storageClass": "NEARLINE"},' ' "condition": {"matchesStorageClass": ["STANDARD"], "age": 366}}]}\n') lifecycle_json_obj = json.loads(lifecycle_doc) lifecycle_doc_bucket_style = ('{"lifecycle": ' + lifecycle_doc.rstrip() + '}\n') # TODO: Remove once Boto is updated to support new fields. lifecycle_doc_without_storage_class_fields = ( '{"rule": [{"action": {"type": "Delete"}, "condition": {"age": 365}}]}\n') lifecycle_created_before_doc = ( '{"rule": [{"action": {"type": "Delete"}, "condition": ' '{"createdBefore": "2014-10-01"}}]}\n') lifecycle_created_before_json_obj = json.loads(lifecycle_created_before_doc) lifecycle_with_falsy_condition_values = ( '{"rule": [{"action": {"type": "Delete"}, "condition": {' '"age": 0, "isLive": false, "numNewerVersions": 0}}]}') no_lifecycle_config = 'has no lifecycle configuration.' def test_lifecycle_translation(self): """Tests lifecycle translation for various formats.""" # TODO: Use lifecycle_doc again once Boto is updated to support new fields. # json_text = self.lifecycle_doc json_text = self.lifecycle_doc_without_storage_class_fields entries_list = LifecycleTranslation.JsonLifecycleToMessage(json_text) boto_lifecycle = LifecycleTranslation.BotoLifecycleFromMessage(entries_list) converted_entries_list = LifecycleTranslation.BotoLifecycleToMessage( boto_lifecycle) converted_json_text = LifecycleTranslation.JsonLifecycleFromMessage( converted_entries_list) self.assertEqual(json.loads(json_text), json.loads(converted_json_text)) def test_default_lifecycle(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) self.assertIn(self.no_lifecycle_config, stdout) def test_set_empty_lifecycle1(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.empty_doc1.encode('ascii')) self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)]) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) self.assertIn(self.no_lifecycle_config, stdout) def test_valid_lifecycle(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.lifecycle_doc.encode('ascii')) self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)]) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj) def test_valid_lifecycle_bucket_style(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile( contents=self.lifecycle_doc_bucket_style.encode('ascii')) self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)]) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj) def test_created_before_lifecycle(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile( contents=self.lifecycle_created_before_doc.encode('ascii')) self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)]) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_created_before_json_obj) def test_bad_lifecycle(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.bad_doc.encode('ascii')) stderr = self.RunGsUtil( ['lifecycle', 'set', fpath, suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertNotIn('XML lifecycle data provided', stderr) def test_bad_xml_lifecycle(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.xml_doc.encode('ascii')) stderr = self.RunGsUtil( ['lifecycle', 'set', fpath, suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn('XML lifecycle data provided', stderr) def test_translation_for_falsy_values_works_correctly(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile( contents=self.lifecycle_with_falsy_condition_values.encode('ascii')) self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)]) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) # The lifecycle policy we fetch should include all the False- and 0-valued # attributes that we just set. self.assertRegex(stdout, r'"age":\s+0') self.assertRegex(stdout, r'"isLive":\s+false') self.assertRegex(stdout, r'"numNewerVersions":\s+0') def test_set_lifecycle_and_reset(self): """Tests setting and turning off lifecycle configuration.""" bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() fpath = self.CreateTempFile(tmpdir=tmpdir, contents=self.lifecycle_doc.encode('ascii')) self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)]) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj) fpath = self.CreateTempFile(tmpdir=tmpdir, contents=self.empty_doc1.encode('ascii')) self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)]) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket_uri)], return_stdout=True) self.assertIn(self.no_lifecycle_config, stdout) def test_set_lifecycle_multi_buckets(self): """Tests setting lifecycle configuration on multiple buckets.""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.lifecycle_doc.encode('ascii')) self.RunGsUtil( ['lifecycle', 'set', fpath, suri(bucket1_uri), suri(bucket2_uri)]) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket1_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket2_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj) def test_set_lifecycle_wildcard(self): """Tests setting lifecycle with a wildcarded bucket URI.""" if self.test_api == ApiSelector.XML: # This test lists buckets with wildcards, but it is possible that another # test being run in parallel (in the same project) deletes a bucket after # it is listed in this test. This causes the subsequent XML metadata get # for the lifecycle configuration to fail on that just-deleted bucket, # even though that bucket is not used directly in this test. return unittest.skip('XML wildcard behavior can cause test to flake ' 'if a bucket in the same project is deleted ' 'during execution.') random_prefix = self.MakeRandomTestString() bucket1_name = self.MakeTempName('bucket', prefix=random_prefix) bucket2_name = self.MakeTempName('bucket', prefix=random_prefix) bucket1_uri = self.CreateBucket(bucket_name=bucket1_name) bucket2_uri = self.CreateBucket(bucket_name=bucket2_name) # This just double checks that the common prefix of the two buckets is what # we think it should be (based on implementation detail of CreateBucket). # We want to be careful when setting a wildcard on buckets to make sure we # don't step outside the test buckets to affect other buckets. common_prefix = posixpath.commonprefix( [suri(bucket1_uri), suri(bucket2_uri)]) self.assertTrue( common_prefix.startswith( 'gs://%sgsutil-test-test-set-lifecycle-wildcard-' % random_prefix)) wildcard = '%s*' % common_prefix fpath = self.CreateTempFile(contents=self.lifecycle_doc.encode('ascii')) actual_lines = set() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stderr = self.RunGsUtil(['lifecycle', 'set', fpath, wildcard], return_stderr=True) actual_lines.update(stderr.splitlines()) if self._use_gcloud_storage: # Gcloud may have unrelated characters on status line # as a result of output logic so can't do direct line string comparison. self.assertIn('Updating %s/...' % suri(bucket1_uri), stderr) self.assertIn('Updating %s/...' % suri(bucket2_uri), stderr) status_message = 'Updating' else: expected_lines = set([ 'Setting lifecycle configuration on %s/...' % suri(bucket1_uri), 'Setting lifecycle configuration on %s/...' % suri(bucket2_uri) ]) self.assertEqual(expected_lines, actual_lines) status_message = 'Setting lifecycle configuration' self.assertEqual(stderr.count(status_message), 2) _Check1() stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket1_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj) stdout = self.RunGsUtil( ['lifecycle', 'get', suri(bucket2_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.lifecycle_json_obj) class TestLifecycleUnitTestsWithShim(testcase.ShimUnitTestBase): """Unit tests for gsutil lifecycle with shim.""" def test_shim_translates_lifecycle_get_correctly(self): bucket_uri = self.CreateBucket() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('lifecycle', args=[ 'get', suri(bucket_uri), ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage buckets' ' describe --format=gsutiljson[key=lifecycle_config,empty=\' has' ' no lifecycle configuration.\',empty_prefix_key=storage_url]' ' {}').format(shim_util._get_gcloud_binary_path('fake_dir'), suri(bucket_uri)), info_lines) @mock.patch('gslib.commands.lifecycle.LifecycleCommand._SetLifecycleConfig', new=mock.Mock()) def test_shim_translates_lifecycle_set_correctly(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('lifecycle', args=[ 'set', 'fake-lifecycle-config.json', 'gs://fake-bucket1', 'gs://fake-bucket2', ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage buckets' ' update --lifecycle-file=fake-lifecycle-config.json' ' gs://fake-bucket1 gs://fake-bucket2').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) tests/test_creds_config.py 0000644 00000021224 15025013616 0011744 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for various combinations of configured credentials.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import boto from apitools.base.py import exceptions as apitools_exceptions from apitools.base.py import http_wrapper from gslib.cloud_api import AccessDeniedException from gslib.cred_types import CredTypes from gslib.discard_messages_queue import DiscardMessagesQueue from gslib.exception import CommandException from gslib.gcs_json_api import GcsJsonApi from gslib.tests.mock_logging_handler import MockLoggingHandler import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import unittest from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock from datetime import datetime def _Make403(message): return (apitools_exceptions.HttpError.FromResponse( http_wrapper.Response(info={'status': 403}, content='{"error": {"message": "%s"}}' % message, request_url='http://www.google.com'))) SERVICE_ACCOUNT = boto.config.get_value('GSUtil', 'test_impersonate_service_account') class TestCredsConfig(testcase.GsUtilUnitTestCase): """Tests for various combinations of configured credentials.""" def setUp(self): super(TestCredsConfig, self).setUp() self.log_handler = MockLoggingHandler() self.logger.addHandler(self.log_handler) def testMultipleConfiguredCreds(self): with SetBotoConfigForTest([ ('Credentials', 'gs_oauth2_refresh_token', 'foo'), ('Credentials', 'gs_service_client_id', 'bar'), ('Credentials', 'gs_service_key_file', 'baz'), ('Credentials', 'gs_impersonate_service_account', None) ]): try: GcsJsonApi(None, self.logger, DiscardMessagesQueue()) self.fail('Succeeded with multiple types of configured creds.') except CommandException as e: msg = str(e) self.assertIn('types of configured credentials', msg) self.assertIn(CredTypes.OAUTH2_USER_ACCOUNT, msg) self.assertIn(CredTypes.OAUTH2_SERVICE_ACCOUNT, msg) @SkipForS3('Tests only uses gs credentials.') @SkipForXML('Tests only run on JSON API.') @mock.patch('gslib.third_party.iamcredentials_apitools' '.iamcredentials_v1_client.IamcredentialsV1' '.ProjectsServiceAccountsService.GenerateAccessToken') def testImpersonationBlockedByIamCredentialsApiErrors( self, mock_iam_creds_generate_access_token): with SetBotoConfigForTest([ ('Credentials', 'gs_oauth2_refresh_token', 'foo'), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ('Credentials', 'gs_impersonate_service_account', 'bar') ]): mock_iam_creds_generate_access_token.side_effect = ( _Make403('The caller does not have permission')) try: GcsJsonApi(None, self.logger, DiscardMessagesQueue()) self.fail('Succeeded when IAM Credentials threw an error') except AccessDeniedException as e: self.assertIn('Service account impersonation failed.', str(e)) mock_iam_creds_generate_access_token.side_effect = ( _Make403('IAM Service Account Credentials API has not been used')) try: GcsJsonApi(None, self.logger, DiscardMessagesQueue()) self.fail('Succeeded when IAM Credentials threw an error') except AccessDeniedException as e: self.assertIn('IAM Service Account Credentials API has not', str(e)) @SkipForS3('Tests only uses gs credentials.') @SkipForXML('Tests only run on JSON API.') @mock.patch('gslib.third_party.iamcredentials_apitools' '.iamcredentials_v1_client.IamcredentialsV1' '.ProjectsServiceAccountsService.GenerateAccessToken') def testImpersonationSuccessfullyUsesToken( self, mock_iam_creds_generate_access_token): with SetBotoConfigForTest([ ('Credentials', 'gs_oauth2_refresh_token', 'foo'), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ('Credentials', 'gs_impersonate_service_account', 'bar') ]): fake_token = 'Mock token from IAM Credentials API' expire_time = datetime.now().strftime("%Y-%m-%dT23:59:59Z") mock_iam_creds_generate_access_token.return_value.accessToken = fake_token mock_iam_creds_generate_access_token.return_value.expireTime = expire_time api = GcsJsonApi(None, self.logger, DiscardMessagesQueue()) self.assertIn(fake_token, str(api.credentials.access_token)) class TestCredsConfigIntegration(testcase.GsUtilIntegrationTestCase): @SkipForS3('Tests only uses gs credentials.') def testExactlyOneInvalid(self): bucket_uri = self.CreateBucket() with SetBotoConfigForTest( [('Credentials', 'gs_oauth2_refresh_token', 'foo'), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ('Credentials', 'gs_impersonate_service_account', None)], use_existing_config=False): stderr = self.RunGsUtil(['ls', suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn('credentials are invalid', stderr) @unittest.skipUnless(SERVICE_ACCOUNT, 'Test requires test_impersonate_service_account.') @SkipForS3('Tests only uses gs credentials.') @SkipForXML('Tests only run on JSON API.') def testImpersonationCredentialsFromBotoConfig(self): with SetBotoConfigForTest([('Credentials', 'gs_impersonate_service_account', SERVICE_ACCOUNT)]): with SetEnvironmentForTest({}): stderr = self.RunGsUtil(['ls', 'gs://pub'], return_stderr=True) self.assertIn('using service account impersonation', stderr) @unittest.skipUnless(SERVICE_ACCOUNT, 'Test requires test_impersonate_service_account.') @SkipForS3('Tests only uses gs credentials.') @SkipForXML('Tests only run on JSON API.') def testImpersonationCredentialsFromGCloud(self): with SetBotoConfigForTest([('Credentials', 'gs_impersonate_service_account', None)]): with SetEnvironmentForTest( {'CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT': SERVICE_ACCOUNT}): stderr = self.RunGsUtil(['ls', 'gs://pub'], return_stderr=True) self.assertIn('using service account impersonation', stderr) @unittest.skipUnless(SERVICE_ACCOUNT, 'Test requires test_impersonate_service_account.') @SkipForS3('Tests only uses gs credentials.') @SkipForXML('Tests only run on JSON API.') def testImpersonationCredentialsFromOptionOverridesOthers(self): with SetBotoConfigForTest([('Credentials', 'gs_impersonate_service_account', 'foo@google.com')]): with SetEnvironmentForTest( {'CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT': 'bar@google.com'}): stderr = self.RunGsUtil(['-i', SERVICE_ACCOUNT, 'ls', 'gs://pub'], return_stderr=True) self.assertIn('API calls will be executed as [%s' % SERVICE_ACCOUNT, stderr) @unittest.skipUnless(SERVICE_ACCOUNT, 'Test requires test_impersonate_service_account.') @SkipForS3('Tests only uses gs credentials.') @SkipForXML('Tests only run on JSON API.') def testImpersonationSuccess(self): with SetBotoConfigForTest([('Credentials', 'gs_impersonate_service_account', SERVICE_ACCOUNT)]): stderr = self.RunGsUtil(['ls', 'gs://pub'], return_stderr=True) self.assertIn('API calls will be executed as [%s' % SERVICE_ACCOUNT, stderr) tests/test_posix_util.py 0000644 00000004042 15025013616 0011515 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2022 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for posix_util.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from gslib.tests import testcase from gslib.tests.util import unittest from gslib.utils import posix_util from gslib.utils.system_util import IS_WINDOWS from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock class TestPosixUtil(testcase.GsUtilUnitTestCase): """Unit tests for POSIX utils.""" @mock.patch.object(posix_util, 'InitializeUserGroups', autospec=True) @mock.patch.object(posix_util, 'InitializeDefaultMode', autospec=True) def test_initialize_preserve_posix_data_calls_correct_functions( self, mock_initialize_default_mode, mock_initialize_user_groups): posix_util.InitializePreservePosixData() mock_initialize_default_mode.assert_called_once_with() mock_initialize_user_groups.assert_called_once_with() @unittest.skipIf(IS_WINDOWS, 'os.umask always returns 0 on Windows.') @mock.patch.object(os, 'umask', autospec=True) def test_initialize_mode_sets_umask_to_correct_temporary_value_not_windows( self, mock_umask): # Abort before setting SYSTEM_POSIX_MODE to avoid side effects. mock_umask.side_effect = ValueError with self.assertRaises(ValueError): posix_util.InitializeDefaultMode() mock_umask.assert_called_once_with(0o177) tests/test_context_config.py 0000644 00000045435 15025013616 0012342 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2020 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for context_config.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import os import subprocess from unittest import mock import six from gslib import context_config from gslib import exception from gslib.tests import testcase from gslib.tests.testcase import base from gslib.tests.util import SetBotoConfigForTest DEFAULT_CERT_PROVIDER_FILE_CONTENTS = { 'cert_provider_command': [ os.path.join('some', 'helper'), '--print_certificate' ] } DEFAULT_CERT_PROVIDER_FILE_CONTENTS_WITH_SPACE = { 'cert_provider_command': ['some helper', '--print_certificate'] } DEFAULT_CERT_PROVIDER_FILE_NO_COMMAND = {'foo': 'foo'} CERT_SECTION = """-----BEGIN CERTIFICATE----- LKJHLSDJKFHLEUIORWUYERWEHJHL KLJHGFDLSJKH(@#*&$)@*#KJHFLKJDSFSD -----END CERTIFICATE----- """ ENCRYPTED_KEY_SECTION = """-----BEGIN ENCRYPTED PRIVATE KEY----- LKJWE:RUWEORIU)(#*&$@(#$KJHLKDJHF(I*F@YLFHSLDKJFS -----END ENCRYPTED PRIVATE KEY----- """ KEY_SECTION = """-----BEGIN PRIVATE KEY----- LKJWE:RUWEORIU)(#*&$@(#$KJHLKDJHF(I*F@YLFHSLDKJFS -----END PRIVATE KEY----- """ CERT_ENCRYPTED_KEY_SECTION = CERT_SECTION + ENCRYPTED_KEY_SECTION PASSWORD = '##invalid-password##' PASSWORD_SECTION = """ -----BEGIN PASSPHRASE----- %s -----END PASSPHRASE----- """ % PASSWORD FULL_ENCRYPTED_CERT = CERT_ENCRYPTED_KEY_SECTION + PASSWORD_SECTION FULL_CERT = CERT_SECTION + KEY_SECTION BAD_CERT_KEY_EMBEDDED_SECTION = """-----BEGIN CERTIFICATE----- LKJHLSDJKFHLEUIORWUYERWEHJHL KLJHGFDLSJKH(@#*&$)@*#KJHFLKJDSFSD -----BEGIN ENCRYPTED PRIVATE KEY----- LKJWE:RUWEORIU)(#*&$@(#$KJHLKDJHF(I*F@YLFHSLDKJFS -----END ENCRYPTED PRIVATE KEY----- -----END CERTIFICATE----- """ BAD_CERT_KEY_MISSING_END = """-----BEGIN CERTIFICATE----- LKJHLSDJKFHLEUIORWUYERWEHJHL KLJHGFDLSJKH(@#*&$)@*#KJHFLKJDSFSD -----END CERTIFICATE----- -----BEGIN ENCRYPTED PRIVATE KEY----- LKJWE:RUWEORIU)(#*&$@(#$KJHLKDJHF(I*F@YLFHSLDKJFS """ BAD_CERT_KEY_MISSING_BEGIN = """-----END CERTIFICATE----- -----BEGIN ENCRYPTED PRIVATE KEY----- LKJWE:RUWEORIU)(#*&$@(#$KJHLKDJHF(I*F@YLFHSLDKJFS -----END ENCRYPTED PRIVATE KEY----- """ BAD_CERT_KEY_MISMATCH = """-----BEGIN CERTIFICATE----- LKJHLSDJKFHLEUIORWUYERWEHJHL KLJHGFDLSJKH(@#*&$)@*#KJHFLKJDSFSD -----END ENCRYPTED PRIVATE KEY----- """ CERT_KEY_WITH_COMMENT_AT_BEGIN = """SOMECOMMENTS -----BEGIN CERTIFICATE----- LKJHLSDJKFHLEUIORWUYERWEHJHL KLJHGFDLSJKH(@#*&$)@*#KJHFLKJDSFSD -----END CERTIFICATE----- -----BEGIN ENCRYPTED PRIVATE KEY----- LKJWE:RUWEORIU)(#*&$@(#$KJHLKDJHF(I*F@YLFHSLDKJFS -----END ENCRYPTED PRIVATE KEY----- """ CERT_KEY_WITH_COMMENT_AT_END = """-----BEGIN CERTIFICATE----- LKJHLSDJKFHLEUIORWUYERWEHJHL KLJHGFDLSJKH(@#*&$)@*#KJHFLKJDSFSD -----END CERTIFICATE----- -----BEGIN ENCRYPTED PRIVATE KEY----- LKJWE:RUWEORIU)(#*&$@(#$KJHLKDJHF(I*F@YLFHSLDKJFS -----END ENCRYPTED PRIVATE KEY----- SOMECOMMENT """ CERT_KEY_WITH_COMMENT_IN_BETWEEN = """-----BEGIN CERTIFICATE----- LKJHLSDJKFHLEUIORWUYERWEHJHL KLJHGFDLSJKH(@#*&$)@*#KJHFLKJDSFSD -----END CERTIFICATE----- SOMECOMMENT -----BEGIN ENCRYPTED PRIVATE KEY----- LKJWE:RUWEORIU)(#*&$@(#$KJHLKDJHF(I*F@YLFHSLDKJFS -----END ENCRYPTED PRIVATE KEY----- """ OPEN_TO_PATCH = '__builtin__.open' if six.PY2 else 'builtins.open' @testcase.integration_testcase.SkipForS3('mTLS only runs on GCS JSON API.') @testcase.integration_testcase.SkipForXML('mTLS only runs on GCS JSON API.') class TestPemFileParser(testcase.GsUtilUnitTestCase): """Test PEM-format certificate parsing for mTLS.""" def test_pem_file_with_comment_at_beginning(self): sections = context_config._split_pem_into_sections( CERT_KEY_WITH_COMMENT_AT_BEGIN, self.logger) self.assertEqual(sections['CERTIFICATE'], CERT_SECTION) self.assertEqual(sections['ENCRYPTED PRIVATE KEY'], ENCRYPTED_KEY_SECTION) def test_pem_file_with_comment_at_end(self): sections = context_config._split_pem_into_sections( CERT_KEY_WITH_COMMENT_AT_END, self.logger) self.assertEqual(sections['CERTIFICATE'], CERT_SECTION) self.assertEqual(sections['ENCRYPTED PRIVATE KEY'], ENCRYPTED_KEY_SECTION) def test_pem_file_with_comment_in_between(self): sections = context_config._split_pem_into_sections( CERT_KEY_WITH_COMMENT_IN_BETWEEN, self.logger) self.assertEqual(sections['CERTIFICATE'], CERT_SECTION) self.assertEqual(sections['ENCRYPTED PRIVATE KEY'], ENCRYPTED_KEY_SECTION) def test_pem_file_with_bad_format_embedded_section(self): sections = context_config._split_pem_into_sections( BAD_CERT_KEY_EMBEDDED_SECTION, self.logger) self.assertIsNone(sections.get('CERTIFICATE')) self.assertEqual(sections.get('ENCRYPTED PRIVATE KEY'), ENCRYPTED_KEY_SECTION) def test_pem_file_with_bad_format_missing_ending(self): sections = context_config._split_pem_into_sections(BAD_CERT_KEY_MISSING_END, self.logger) self.assertEqual(sections.get('CERTIFICATE'), CERT_SECTION) self.assertIsNone(sections.get('ENCRYPTED PRIVATE KEY')) def test_pem_file_with_bad_format_missing_beginning(self): sections = context_config._split_pem_into_sections( BAD_CERT_KEY_MISSING_BEGIN, self.logger) self.assertIsNone(sections.get('CERTIFICATE')) self.assertEqual(sections.get('ENCRYPTED PRIVATE KEY'), ENCRYPTED_KEY_SECTION) def test_pem_file_with_bad_format_section_mismatch(self): sections = context_config._split_pem_into_sections(BAD_CERT_KEY_MISMATCH, self.logger) self.assertIsNone(sections.get('CERTIFICATE')) self.assertIsNone(sections.get('ENCRYPTED PRIVATE KEY')) # Setting global context_config singleton causes issues in parallel. @base.NotParallelizable @testcase.integration_testcase.SkipForS3('mTLS only runs on GCS JSON API.') @testcase.integration_testcase.SkipForXML('mTLS only runs on GCS JSON API.') class TestContextConfig(testcase.GsUtilUnitTestCase): """Test the global ContextConfig singleton.""" def setUp(self): super(TestContextConfig, self).setUp() self._old_context_config = context_config._singleton_config context_config._singleton_config = None self.mock_logger = mock.Mock() def tearDown(self): super(TestContextConfig, self).tearDown() context_config._singleton_config = self._old_context_config def test_context_config_is_a_singleton(self): first = context_config.create_context_config(self.mock_logger) with self.assertRaises( context_config.ContextConfigSingletonAlreadyExistsError): context_config.create_context_config(self.mock_logger) second = context_config.get_context_config() self.assertEqual(first, second) @mock.patch.object(subprocess, 'Popen') def test_does_not_provision_if_use_client_certificate_not_true( self, mock_Popen): context_config.create_context_config(self.mock_logger) mock_Popen.assert_not_called() @mock.patch('os.path.exists', new=mock.Mock(return_value=True)) @mock.patch.object(json, 'load', autospec=True) @mock.patch.object(subprocess, 'Popen', autospec=True) @mock.patch(OPEN_TO_PATCH, new_callable=mock.mock_open) def test_executes_provider_command_from_default_file(self, mock_open, mock_Popen, mock_json_load): mock_json_load.side_effect = [DEFAULT_CERT_PROVIDER_FILE_CONTENTS] with SetBotoConfigForTest([('Credentials', 'use_client_certificate', 'True') ]): # Purposely end execution here to avoid writing a file. with self.assertRaises(ValueError): context_config.create_context_config(self.mock_logger) mock_open.assert_called_with(context_config._DEFAULT_METADATA_PATH) mock_Popen.assert_called_once_with( os.path.realpath(os.path.join('some', 'helper')), '--print_certificate', '--with_passphrase') @mock.patch('os.path.exists', new=mock.Mock(return_value=True)) @mock.patch.object(json, 'load', autospec=True) @mock.patch.object(subprocess, 'Popen', autospec=True) @mock.patch(OPEN_TO_PATCH, new_callable=mock.mock_open) def test_executes_provider_command_with_space_from_default_file( self, mock_open, mock_Popen, mock_json_load): mock_json_load.side_effect = [ DEFAULT_CERT_PROVIDER_FILE_CONTENTS_WITH_SPACE ] with SetBotoConfigForTest([('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', None)]): # Purposely end execution here to avoid writing a file. with self.assertRaises(ValueError): context_config.create_context_config(self.mock_logger) mock_open.assert_called_with(context_config._DEFAULT_METADATA_PATH) mock_Popen.assert_called_once_with(os.path.realpath('cert helper'), '--print_certificate', '--with_passphrase') @mock.patch('os.path.exists', new=mock.Mock(return_value=True)) @mock.patch.object(json, 'load', autospec=True) @mock.patch(OPEN_TO_PATCH, new_callable=mock.mock_open) def test_default_provider_no_command_error(self, mock_open, mock_json_load): mock_json_load.return_value = DEFAULT_CERT_PROVIDER_FILE_NO_COMMAND with SetBotoConfigForTest([('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', None)]): context_config.create_context_config(self.mock_logger) mock_open.assert_called_with(context_config._DEFAULT_METADATA_PATH) self.mock_logger.error.assert_called_once_with( "Failed to provision client certificate: " "Client certificate provider command not found.") @mock.patch('os.path.exists', new=mock.Mock(return_value=False)) def test_default_provider_not_found_error(self): with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', None), # Avoids permissions error on Windows tests: ('GSUtil', 'state_dir', self.CreateTempDir()) ]): context_config.create_context_config(self.mock_logger) self.mock_logger.error.assert_called_once_with( "Failed to provision client certificate: " "Client certificate provider file not found.") @mock.patch.object(json, 'load', autospec=True) @mock.patch('os.path.exists', new=mock.Mock(return_value=True)) @mock.patch(OPEN_TO_PATCH, new_callable=mock.mock_open) def test_raises_cert_provision_error_on_json_load_error( self, mock_open, mock_json_load): mock_json_load.side_effect = ValueError('valueError') with SetBotoConfigForTest([('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', None)]): context_config.create_context_config(self.mock_logger) mock_open.assert_called_with(context_config._DEFAULT_METADATA_PATH) self.mock_logger.error.assert_called_once_with( 'Failed to provision client certificate: valueError') @mock.patch.object(subprocess, 'Popen', autospec=True) def test_executes_custom_provider_command_from_boto_config(self, mock_Popen): with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'some/path') ]): # Purposely end execution here to avoid writing a file. with self.assertRaises(ValueError): context_config.create_context_config(self.mock_logger) mock_Popen.assert_called_once_with(os.path.realpath('some/path'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) @mock.patch.object(subprocess, 'Popen') def test_converts_and_logs_provisioning_cert_provider_unexpected_exit_error( self, mock_Popen): mock_command_process = mock.Mock() mock_command_process.communicate.return_value = (None, 'oh no') mock_Popen.return_value = mock_command_process with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'some/path') ]): context_config.create_context_config(self.mock_logger) self.mock_logger.error.assert_called_once_with( 'Failed to provision client certificate: oh no') @mock.patch.object(subprocess, 'Popen') def test_converts_and_logs_provisioning_os_error(self, mock_Popen): mock_Popen.side_effect = OSError('foobar') with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'some/path') ]): context_config.create_context_config(self.mock_logger) self.mock_logger.error.assert_called_once_with( 'Failed to provision client certificate: foobar') @mock.patch.object(subprocess, 'Popen') def test_converts_and_logs_provisioning_external_binary_error( self, mock_Popen): mock_Popen.side_effect = exception.ExternalBinaryError('foobar') with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'some/path') ]): context_config.create_context_config(self.mock_logger) self.mock_logger.error.assert_called_once_with( 'Failed to provision client certificate: foobar') @mock.patch.object(subprocess, 'Popen') def test_converts_and_logs_provisioning_key_error(self, mock_Popen): # Mocking f.write would make more sense, but mocking Popen earlier in the # function results in much less code and tests the same error handling. mock_Popen.side_effect = KeyError('foobar') with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'some/path') ]): context_config.create_context_config(self.mock_logger) unicode_escaped_error_string = "'foobar'" if six.PY3 else "u'foobar'" self.mock_logger.error.assert_called_once_with( "Failed to provision client certificate:" " Invalid output format from certificate provider, no " + unicode_escaped_error_string) @mock.patch.object(os, 'remove') def test_does_not_unprovision_if_no_client_certificate(self, mock_remove): context_config.create_context_config(self.mock_logger) context_config._singleton_config._unprovision_client_cert() mock_remove.assert_not_called() @mock.patch.object(os, 'remove') def test_handles_and_logs_unprovisioning_os_error(self, mock_remove): mock_remove.side_effect = OSError('no') context_config.create_context_config(self.mock_logger) context_config._singleton_config.client_cert_path = 'some/path' context_config._singleton_config._unprovision_client_cert() self.mock_logger.error.assert_called_once_with( 'Failed to remove client certificate: no') @mock.patch(OPEN_TO_PATCH, new_callable=mock.mock_open) @mock.patch.object(os, 'remove') @mock.patch.object(subprocess, 'Popen') def test_writes_and_deletes_encrypted_certificate_file_storing_password_to_memory( self, mock_Popen, mock_remove, mock_open): mock_command_process = mock.Mock() mock_command_process.returncode = 0 mock_command_process.communicate.return_value = ( FULL_ENCRYPTED_CERT.encode(), None) mock_Popen.return_value = mock_command_process with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'path --print_certificate') ]): # Mock logger argument to avoid atexit hook writing to stderr. test_config = context_config.create_context_config(mock.Mock()) # Test writes certificate file. # Can't check whole mock_calls list because SetBotoConfigForTest also # uses the mock in Python 3. Should work with any_order=False based on # docs description but does not in current environment. mock_open.assert_has_calls([ mock.call(test_config.client_cert_path, 'w+'), mock.call().write(CERT_SECTION), mock.call().write(ENCRYPTED_KEY_SECTION), ], any_order=True) # Test saves certificate password to memory. self.assertEqual(context_config._singleton_config.client_cert_password, PASSWORD) # Test deletes certificate file. context_config._singleton_config._unprovision_client_cert() mock_remove.assert_called_once_with(test_config.client_cert_path) @mock.patch(OPEN_TO_PATCH, new_callable=mock.mock_open) @mock.patch.object(os, 'remove') @mock.patch.object(subprocess, 'Popen') def test_writes_and_deletes_unencrypted_certificate_file_without_storing_password( self, mock_Popen, mock_remove, mock_open): """This is the format used by gcloud by default.""" mock_command_process = mock.Mock() mock_command_process.returncode = 0 mock_command_process.communicate.return_value = (FULL_CERT.encode(), None) mock_Popen.return_value = mock_command_process with SetBotoConfigForTest([ ('Credentials', 'use_client_certificate', 'True'), ('Credentials', 'cert_provider_command', 'path --print_certificate') ]): # Mock logger argument to avoid atexit hook writing to stderr. test_config = context_config.create_context_config(mock.Mock()) # Test writes certificate file. # Can't check whole mock_calls list because SetBotoConfigForTest also # uses the mock in Python 3. Should work with any_order=False based on # docs description but does not in current environment. mock_open.assert_has_calls([ mock.call(test_config.client_cert_path, 'w+'), mock.call().write(CERT_SECTION), mock.call().write(KEY_SECTION), ], any_order=True) # Does not save unnecessary password. self.assertIsNone(context_config._singleton_config.client_cert_password, PASSWORD) # Test deletes certificate file. context_config._singleton_config._unprovision_client_cert() mock_remove.assert_called_once_with(test_config.client_cert_path) tests/test_copy_helper_funcs.py 0000644 00000117413 15025013616 0013034 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for parallel upload functions in copy_helper.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import datetime import logging import os import pyu2f from apitools.base.py import exceptions as apitools_exceptions from gslib.bucket_listing_ref import BucketListingObject from gslib.bucket_listing_ref import BucketListingPrefix from gslib.cloud_api import CloudApi from gslib.cloud_api import ResumableUploadAbortException from gslib.cloud_api import ResumableUploadException from gslib.cloud_api import ResumableUploadStartOverException from gslib.cloud_api import ServiceException from gslib.command import CreateOrGetGsutilLogger from gslib.discard_messages_queue import DiscardMessagesQueue from gslib.exception import CommandException from gslib.gcs_json_api import GcsJsonApi from gslib.parallel_tracker_file import ObjectFromTracker from gslib.storage_url import StorageUrlFromString from gslib.tests.mock_cloud_api import MockCloudApi from gslib.tests.testcase.unit_testcase import GsUtilUnitTestCase from gslib.tests.util import GSMockBucketStorageUri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import unittest from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import copy_helper from gslib.utils import parallelism_framework_util from gslib.utils import posix_util from gslib.utils import system_util from gslib.utils import hashing_helper from gslib.utils.copy_helper import _CheckCloudHashes from gslib.utils.copy_helper import _DelegateUploadFileToObject from gslib.utils.copy_helper import _GetPartitionInfo from gslib.utils.copy_helper import _SelectUploadCompressionStrategy from gslib.utils.copy_helper import _SetContentTypeFromFile from gslib.utils.copy_helper import ExpandUrlToSingleBlr from gslib.utils.copy_helper import FilterExistingComponents from gslib.utils.copy_helper import GZIP_ALL_FILES from gslib.utils.copy_helper import PerformParallelUploadFileToObjectArgs from gslib.utils.copy_helper import WarnIfMvEarlyDeletionChargeApplies from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock _CalculateB64EncodedMd5FromContents = ( hashing_helper.CalculateB64EncodedMd5FromContents) class TestCpFuncs(GsUtilUnitTestCase): """Unit tests for parallel upload functions in cp command.""" def testGetPartitionInfo(self): """Tests the _GetPartitionInfo function.""" # Simplest case - threshold divides file_size. (num_components, component_size) = _GetPartitionInfo(300, 200, 10) self.assertEqual(30, num_components) self.assertEqual(10, component_size) # Threshold = 1 (mod file_size). (num_components, component_size) = _GetPartitionInfo(301, 200, 10) self.assertEqual(31, num_components) self.assertEqual(10, component_size) # Threshold = -1 (mod file_size). (num_components, component_size) = _GetPartitionInfo(299, 200, 10) self.assertEqual(30, num_components) self.assertEqual(10, component_size) # Too many components needed. (num_components, component_size) = _GetPartitionInfo(301, 2, 10) self.assertEqual(2, num_components) self.assertEqual(151, component_size) # Test num_components with huge numbers. (num_components, component_size) = _GetPartitionInfo((10**150) + 1, 10**200, 10) self.assertEqual((10**149) + 1, num_components) self.assertEqual(10, component_size) # Test component_size with huge numbers. (num_components, component_size) = _GetPartitionInfo((10**150) + 1, 10, 10) self.assertEqual(10, num_components) self.assertEqual((10**149) + 1, component_size) # Test component_size > file_size (make sure we get at least two components. (num_components, component_size) = _GetPartitionInfo(100, 500, 51) self.assertEqual(2, num_components) self.assertEqual(50, component_size) def testFilterExistingComponentsNonVersioned(self): """Tests upload with a variety of component states.""" mock_api = MockCloudApi() bucket_name = self.MakeTempName('bucket') tracker_file = self.CreateTempFile(file_name='foo', contents=b'asdf') tracker_file_lock = parallelism_framework_util.CreateLock() # dst_obj_metadata used for passing content-type and storage class. content_type = 'ContentType' storage_class = 'StorageClass' # Already uploaded, contents still match, component still used. fpath_uploaded_correctly = self.CreateTempFile(file_name='foo1', contents=b'1') fpath_uploaded_correctly_url = StorageUrlFromString( str(fpath_uploaded_correctly)) object_uploaded_correctly_url = StorageUrlFromString( '%s://%s/%s' % (self.default_provider, bucket_name, fpath_uploaded_correctly)) with open(fpath_uploaded_correctly, 'rb') as f_in: fpath_uploaded_correctly_md5 = _CalculateB64EncodedMd5FromContents(f_in) mock_api.MockCreateObjectWithMetadata(apitools_messages.Object( bucket=bucket_name, name=fpath_uploaded_correctly, md5Hash=fpath_uploaded_correctly_md5), contents=b'1') args_uploaded_correctly = PerformParallelUploadFileToObjectArgs( fpath_uploaded_correctly, 0, 1, fpath_uploaded_correctly_url, object_uploaded_correctly_url, '', content_type, storage_class, tracker_file, tracker_file_lock, None, False) # Not yet uploaded, but needed. fpath_not_uploaded = self.CreateTempFile(file_name='foo2', contents=b'2') fpath_not_uploaded_url = StorageUrlFromString(str(fpath_not_uploaded)) object_not_uploaded_url = StorageUrlFromString( '%s://%s/%s' % (self.default_provider, bucket_name, fpath_not_uploaded)) args_not_uploaded = PerformParallelUploadFileToObjectArgs( fpath_not_uploaded, 0, 1, fpath_not_uploaded_url, object_not_uploaded_url, '', content_type, storage_class, tracker_file, tracker_file_lock, None, False) # Already uploaded, but contents no longer match. Even though the contents # differ, we don't delete this since the bucket is not versioned and it # will be overwritten anyway. fpath_wrong_contents = self.CreateTempFile(file_name='foo4', contents=b'4') fpath_wrong_contents_url = StorageUrlFromString(str(fpath_wrong_contents)) object_wrong_contents_url = StorageUrlFromString( '%s://%s/%s' % (self.default_provider, bucket_name, fpath_wrong_contents)) with open(self.CreateTempFile(contents=b'_'), 'rb') as f_in: fpath_wrong_contents_md5 = _CalculateB64EncodedMd5FromContents(f_in) mock_api.MockCreateObjectWithMetadata(apitools_messages.Object( bucket=bucket_name, name=fpath_wrong_contents, md5Hash=fpath_wrong_contents_md5), contents=b'1') args_wrong_contents = PerformParallelUploadFileToObjectArgs( fpath_wrong_contents, 0, 1, fpath_wrong_contents_url, object_wrong_contents_url, '', content_type, storage_class, tracker_file, tracker_file_lock, None, False) # Exists in tracker file, but component object no longer exists. fpath_remote_deleted = self.CreateTempFile(file_name='foo5', contents=b'5') fpath_remote_deleted_url = StorageUrlFromString(str(fpath_remote_deleted)) args_remote_deleted = PerformParallelUploadFileToObjectArgs( fpath_remote_deleted, 0, 1, fpath_remote_deleted_url, '', '', content_type, storage_class, tracker_file, tracker_file_lock, None, False) # Exists in tracker file and already uploaded, but no longer needed. fpath_no_longer_used = self.CreateTempFile(file_name='foo6', contents=b'6') with open(fpath_no_longer_used, 'rb') as f_in: file_md5 = _CalculateB64EncodedMd5FromContents(f_in) mock_api.MockCreateObjectWithMetadata(apitools_messages.Object( bucket=bucket_name, name='foo6', md5Hash=file_md5), contents=b'6') dst_args = { fpath_uploaded_correctly: args_uploaded_correctly, fpath_not_uploaded: args_not_uploaded, fpath_wrong_contents: args_wrong_contents, fpath_remote_deleted: args_remote_deleted } existing_components = [ ObjectFromTracker(fpath_uploaded_correctly, ''), ObjectFromTracker(fpath_wrong_contents, ''), ObjectFromTracker(fpath_remote_deleted, ''), ObjectFromTracker(fpath_no_longer_used, '') ] bucket_url = StorageUrlFromString('%s://%s' % (self.default_provider, bucket_name)) (components_to_upload, uploaded_components, existing_objects_to_delete) = (FilterExistingComponents( dst_args, existing_components, bucket_url, mock_api)) uploaded_components = [i[0] for i in uploaded_components] for arg in [args_not_uploaded, args_wrong_contents, args_remote_deleted]: self.assertTrue(arg in components_to_upload) self.assertEqual(1, len(uploaded_components)) self.assertEqual(args_uploaded_correctly.dst_url.url_string, uploaded_components[0].url_string) self.assertEqual(1, len(existing_objects_to_delete)) no_longer_used_url = StorageUrlFromString( '%s://%s/%s' % (self.default_provider, bucket_name, fpath_no_longer_used)) self.assertEqual(no_longer_used_url.url_string, existing_objects_to_delete[0].url_string) def testFilterExistingComponentsVersioned(self): """Tests upload with versionined parallel components.""" mock_api = MockCloudApi() bucket_name = self.MakeTempName('bucket') mock_api.MockCreateVersionedBucket(bucket_name) # dst_obj_metadata used for passing content-type and storage class. content_type = 'ContentType' storage_class = 'StorageClass' tracker_file = self.CreateTempFile(file_name='foo', contents=b'asdf') tracker_file_lock = parallelism_framework_util.CreateLock() # Already uploaded, contents still match, component still used. fpath_uploaded_correctly = self.CreateTempFile(file_name='foo1', contents=b'1') fpath_uploaded_correctly_url = StorageUrlFromString( str(fpath_uploaded_correctly)) with open(fpath_uploaded_correctly, 'rb') as f_in: fpath_uploaded_correctly_md5 = _CalculateB64EncodedMd5FromContents(f_in) object_uploaded_correctly = mock_api.MockCreateObjectWithMetadata( apitools_messages.Object(bucket=bucket_name, name=fpath_uploaded_correctly, md5Hash=fpath_uploaded_correctly_md5), contents=b'1') object_uploaded_correctly_url = StorageUrlFromString( '%s://%s/%s#%s' % (self.default_provider, bucket_name, fpath_uploaded_correctly, object_uploaded_correctly.generation)) args_uploaded_correctly = PerformParallelUploadFileToObjectArgs( fpath_uploaded_correctly, 0, 1, fpath_uploaded_correctly_url, object_uploaded_correctly_url, object_uploaded_correctly.generation, content_type, storage_class, tracker_file, tracker_file_lock, None, False) # Duplicate object name in tracker file, but uploaded correctly. fpath_duplicate = fpath_uploaded_correctly fpath_duplicate_url = StorageUrlFromString(str(fpath_duplicate)) duplicate_uploaded_correctly = mock_api.MockCreateObjectWithMetadata( apitools_messages.Object(bucket=bucket_name, name=fpath_duplicate, md5Hash=fpath_uploaded_correctly_md5), contents=b'1') duplicate_uploaded_correctly_url = StorageUrlFromString( '%s://%s/%s#%s' % (self.default_provider, bucket_name, fpath_uploaded_correctly, duplicate_uploaded_correctly.generation)) args_duplicate = PerformParallelUploadFileToObjectArgs( fpath_duplicate, 0, 1, fpath_duplicate_url, duplicate_uploaded_correctly_url, duplicate_uploaded_correctly.generation, content_type, storage_class, tracker_file, tracker_file_lock, None, False) # Already uploaded, but contents no longer match. fpath_wrong_contents = self.CreateTempFile(file_name='foo4', contents=b'4') fpath_wrong_contents_url = StorageUrlFromString(str(fpath_wrong_contents)) with open(self.CreateTempFile(contents=b'_'), 'rb') as f_in: fpath_wrong_contents_md5 = _CalculateB64EncodedMd5FromContents(f_in) object_wrong_contents = mock_api.MockCreateObjectWithMetadata( apitools_messages.Object(bucket=bucket_name, name=fpath_wrong_contents, md5Hash=fpath_wrong_contents_md5), contents=b'_') wrong_contents_url = StorageUrlFromString( '%s://%s/%s#%s' % (self.default_provider, bucket_name, fpath_wrong_contents, object_wrong_contents.generation)) args_wrong_contents = PerformParallelUploadFileToObjectArgs( fpath_wrong_contents, 0, 1, fpath_wrong_contents_url, wrong_contents_url, '', content_type, storage_class, tracker_file, tracker_file_lock, None, False) dst_args = { fpath_uploaded_correctly: args_uploaded_correctly, fpath_wrong_contents: args_wrong_contents } existing_components = [ ObjectFromTracker(fpath_uploaded_correctly, object_uploaded_correctly_url.generation), ObjectFromTracker(fpath_duplicate, duplicate_uploaded_correctly_url.generation), ObjectFromTracker(fpath_wrong_contents, wrong_contents_url.generation) ] bucket_url = StorageUrlFromString('%s://%s' % (self.default_provider, bucket_name)) (components_to_upload, uploaded_components, existing_objects_to_delete) = (FilterExistingComponents( dst_args, existing_components, bucket_url, mock_api)) uploaded_components = [i[0] for i in uploaded_components] self.assertEqual([args_wrong_contents], components_to_upload) self.assertEqual(args_uploaded_correctly.dst_url.url_string, uploaded_components[0].url_string) expected_to_delete = [(args_wrong_contents.dst_url.object_name, args_wrong_contents.dst_url.generation), (args_duplicate.dst_url.object_name, args_duplicate.dst_url.generation)] for uri in existing_objects_to_delete: self.assertTrue((uri.object_name, uri.generation) in expected_to_delete) self.assertEqual(len(expected_to_delete), len(existing_objects_to_delete)) def testReauthChallengeIsPerformed(self): mock_api = mock.Mock(spec=CloudApi) destination_url = StorageUrlFromString('gs://bucket') with SetBotoConfigForTest([ ('GSUtil', 'trigger_reauth_challenge_for_parallel_operations', 'True') ]): copy_helper.TriggerReauthForDestinationProviderIfNecessary( destination_url, mock_api, worker_count=2) mock_api.GetBucket.assert_called_once_with('bucket', fields=['location'], provider='gs') def testReauthChallengeIsNotPerformedByDefault(self): mock_api = mock.Mock(spec=CloudApi) destination_url = StorageUrlFromString('gs://bucket') copy_helper.TriggerReauthForDestinationProviderIfNecessary(destination_url, mock_api, worker_count=2) mock_api.GetBucket.assert_not_called() def testReauthChallengeNotPerformedWithFileDestination(self): mock_api = mock.Mock(spec=CloudApi) destination_url = StorageUrlFromString('dir/file') with SetBotoConfigForTest([ ('GSUtil', 'trigger_reauth_challenge_for_parallel_operations', 'True') ]): copy_helper.TriggerReauthForDestinationProviderIfNecessary( destination_url, mock_api, worker_count=2) mock_api.GetBucket.assert_not_called() def testReauthChallengeNotPerformedWhenDestinationContainsWildcard(self): mock_api = mock.Mock(spec=CloudApi) destination_url = StorageUrlFromString('gs://bucket*') with SetBotoConfigForTest([ ('GSUtil', 'trigger_reauth_challenge_for_parallel_operations', 'True') ]): copy_helper.TriggerReauthForDestinationProviderIfNecessary( destination_url, mock_api, worker_count=2) mock_api.GetBucket.assert_not_called() def testReauthChallengeNotPerformedWithSequentialExecution(self): mock_api = mock.Mock(spec=CloudApi) destination_url = StorageUrlFromString('gs://bucket') with SetBotoConfigForTest([ ('GSUtil', 'trigger_reauth_challenge_for_parallel_operations', 'True') ]): copy_helper.TriggerReauthForDestinationProviderIfNecessary( destination_url, mock_api, worker_count=1) mock_api.GetBucket.assert_not_called() def testReauthChallengeRaisesReauthError(self): mock_api = mock.Mock(spec=CloudApi) mock_api.GetBucket.side_effect = pyu2f.errors.PluginError('Reauth error') destination_url = StorageUrlFromString('gs://bucket') with SetBotoConfigForTest([ ('GSUtil', 'trigger_reauth_challenge_for_parallel_operations', 'True') ]): with self.assertRaisesRegex(pyu2f.errors.PluginError, 'Reauth error'): copy_helper.TriggerReauthForDestinationProviderIfNecessary( destination_url, mock_api, worker_count=2) def testReauthChallengeSilencesOtherErrors(self): mock_api = mock.Mock(spec=CloudApi) mock_api.GetBucket.side_effect = Exception destination_url = StorageUrlFromString('gs://bucket') with SetBotoConfigForTest([ ('GSUtil', 'trigger_reauth_challenge_for_parallel_operations', 'True') ]): copy_helper.TriggerReauthForDestinationProviderIfNecessary( destination_url, mock_api, worker_count=2) mock_api.GetBucket.assert_called_once_with('bucket', fields=['location'], provider='gs') # pylint: disable=protected-access def testTranslateApitoolsResumableUploadException(self): """Tests that _TranslateApitoolsResumableUploadException works correctly.""" gsutil_api = GcsJsonApi(GSMockBucketStorageUri, CreateOrGetGsutilLogger('copy_test'), DiscardMessagesQueue()) gsutil_api.http.disable_ssl_certificate_validation = True exc = apitools_exceptions.HttpError({'status': 503}, None, None) translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) self.assertTrue(isinstance(translated_exc, ServiceException)) gsutil_api.http.disable_ssl_certificate_validation = False exc = apitools_exceptions.HttpError({'status': 503}, None, None) translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) self.assertTrue(isinstance(translated_exc, ResumableUploadException)) gsutil_api.http.disable_ssl_certificate_validation = False exc = apitools_exceptions.HttpError({'status': 429}, None, None) translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) self.assertTrue(isinstance(translated_exc, ResumableUploadException)) exc = apitools_exceptions.HttpError({'status': 410}, None, None) translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) self.assertTrue( isinstance(translated_exc, ResumableUploadStartOverException)) exc = apitools_exceptions.HttpError({'status': 404}, None, None) translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) self.assertTrue( isinstance(translated_exc, ResumableUploadStartOverException)) exc = apitools_exceptions.HttpError({'status': 401}, None, None) translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) self.assertTrue(isinstance(translated_exc, ResumableUploadAbortException)) exc = apitools_exceptions.TransferError('Aborting transfer') translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) self.assertTrue(isinstance(translated_exc, ResumableUploadAbortException)) exc = apitools_exceptions.TransferError('additional bytes left in stream') translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) self.assertTrue(isinstance(translated_exc, ResumableUploadAbortException)) self.assertIn('this can happen if a file changes size', translated_exc.reason) def testTranslateApitoolsResumableUploadExceptionStreamExhausted(self): """Test that StreamExhausted error gets handled.""" gsutil_api = GcsJsonApi(GSMockBucketStorageUri, CreateOrGetGsutilLogger('copy_test'), DiscardMessagesQueue()) exc = apitools_exceptions.StreamExhausted('Not enough bytes') translated_exc = gsutil_api._TranslateApitoolsResumableUploadException(exc) self.assertTrue(isinstance(translated_exc, ResumableUploadAbortException)) self.assertIn( 'if this issue persists, try deleting the tracker files' ' present under ~/.gsutil/tracker-files/', translated_exc.reason) def testSetContentTypeFromFile(self): """Tests that content type is correctly determined for symlinks.""" if system_util.IS_WINDOWS: return unittest.skip('use_magicfile features not available on Windows') surprise_html = b'<html><body>And you thought I was just text!</body></html>' temp_dir_path = self.CreateTempDir() txt_file_path = self.CreateTempFile(tmpdir=temp_dir_path, contents=surprise_html, file_name='html_in_disguise.txt') link_name = 'link_to_realfile' # Notice no file extension was supplied. os.symlink(txt_file_path, temp_dir_path + os.path.sep + link_name) # Content-type of a symlink should be obtained from the link's target. dst_obj_metadata_mock = mock.MagicMock(contentType=None) src_url_stub = mock.MagicMock(object_name=temp_dir_path + os.path.sep + link_name) src_url_stub.IsFileUrl.return_value = True src_url_stub.IsStream.return_value = False src_url_stub.IsFifo.return_value = False # The file command should detect HTML in the real file. with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'True')]): _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock) self.assertEqual('text/html; charset=us-ascii', dst_obj_metadata_mock.contentType) dst_obj_metadata_mock = mock.MagicMock(contentType=None) # The mimetypes module should guess based on the real file's extension. with SetBotoConfigForTest([('GSUtil', 'use_magicfile', 'False')]): _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock) self.assertEqual('text/plain', dst_obj_metadata_mock.contentType) def testSetsContentTypesForCommonFileExtensionsCorrectly(self): extension_rules = copy_helper.COMMON_EXTENSION_RULES.items() for extension, expected_content_type in extension_rules: dst_obj_metadata_mock = mock.MagicMock(contentType=None) src_url_stub = mock.MagicMock(object_name='file.' + extension) src_url_stub.IsFileUrl.return_value = True src_url_stub.IsStream.return_value = False src_url_stub.IsFifo.return_value = False _SetContentTypeFromFile(src_url_stub, dst_obj_metadata_mock) self.assertEqual(expected_content_type, dst_obj_metadata_mock.contentType) _PI_DAY = datetime.datetime(2016, 3, 14, 15, 9, 26) @mock.patch('time.time', new=mock.MagicMock( return_value=posix_util.ConvertDatetimeToPOSIX(_PI_DAY))) def testWarnIfMvEarlyDeletionChargeApplies(self): """Tests that WarnIfEarlyDeletionChargeApplies warns when appropriate.""" test_logger = logging.Logger('test') src_url = StorageUrlFromString('gs://bucket/object') # Recent nearline objects should generate a warning. for object_time_created in (self._PI_DAY, self._PI_DAY - datetime.timedelta(days=29, hours=23)): recent_nearline_obj = apitools_messages.Object( storageClass='NEARLINE', timeCreated=object_time_created) with mock.patch.object(test_logger, 'warn') as mocked_warn: WarnIfMvEarlyDeletionChargeApplies(src_url, recent_nearline_obj, test_logger) mocked_warn.assert_called_with( 'Warning: moving %s object %s may incur an early deletion ' 'charge, because the original object is less than %s days old ' 'according to the local system time.', 'nearline', src_url.url_string, 30) # Recent coldine objects should generate a warning. for object_time_created in (self._PI_DAY, self._PI_DAY - datetime.timedelta(days=89, hours=23)): recent_nearline_obj = apitools_messages.Object( storageClass='COLDLINE', timeCreated=object_time_created) with mock.patch.object(test_logger, 'warn') as mocked_warn: WarnIfMvEarlyDeletionChargeApplies(src_url, recent_nearline_obj, test_logger) mocked_warn.assert_called_with( 'Warning: moving %s object %s may incur an early deletion ' 'charge, because the original object is less than %s days old ' 'according to the local system time.', 'coldline', src_url.url_string, 90) # Recent archive objects should generate a warning. for object_time_created in (self._PI_DAY, self._PI_DAY - datetime.timedelta(days=364, hours=23)): recent_archive_obj = apitools_messages.Object( storageClass='ARCHIVE', timeCreated=object_time_created) with mock.patch.object(test_logger, 'warn') as mocked_warn: WarnIfMvEarlyDeletionChargeApplies(src_url, recent_archive_obj, test_logger) mocked_warn.assert_called_with( 'Warning: moving %s object %s may incur an early deletion ' 'charge, because the original object is less than %s days old ' 'according to the local system time.', 'archive', src_url.url_string, 365) # Sufficiently old objects should not generate a warning. with mock.patch.object(test_logger, 'warn') as mocked_warn: old_nearline_obj = apitools_messages.Object( storageClass='NEARLINE', timeCreated=self._PI_DAY - datetime.timedelta(days=30, seconds=1)) WarnIfMvEarlyDeletionChargeApplies(src_url, old_nearline_obj, test_logger) mocked_warn.assert_not_called() with mock.patch.object(test_logger, 'warn') as mocked_warn: old_coldline_obj = apitools_messages.Object( storageClass='COLDLINE', timeCreated=self._PI_DAY - datetime.timedelta(days=90, seconds=1)) WarnIfMvEarlyDeletionChargeApplies(src_url, old_coldline_obj, test_logger) mocked_warn.assert_not_called() with mock.patch.object(test_logger, 'warn') as mocked_warn: old_archive_obj = apitools_messages.Object( storageClass='ARCHIVE', timeCreated=self._PI_DAY - datetime.timedelta(days=365, seconds=1)) WarnIfMvEarlyDeletionChargeApplies(src_url, old_archive_obj, test_logger) mocked_warn.assert_not_called() # Recent standard storage class object should not generate a warning. with mock.patch.object(test_logger, 'warn') as mocked_warn: not_old_enough_nearline_obj = apitools_messages.Object( storageClass='STANDARD', timeCreated=self._PI_DAY) WarnIfMvEarlyDeletionChargeApplies(src_url, not_old_enough_nearline_obj, test_logger) mocked_warn.assert_not_called() def testSelectUploadCompressionStrategyAll(self): paths = ('file://test', 'test.xml', 'test.py') exts = GZIP_ALL_FILES for path in paths: zipped, gzip_encoded = _SelectUploadCompressionStrategy( path, False, exts, False) self.assertTrue(zipped) self.assertFalse(gzip_encoded) zipped, gzip_encoded = _SelectUploadCompressionStrategy( path, False, exts, True) self.assertFalse(zipped) self.assertTrue(gzip_encoded) def testSelectUploadCompressionStrategyFilter(self): zipped, gzip_encoded = _SelectUploadCompressionStrategy( 'test.xml', False, ['xml'], False) self.assertTrue(zipped) self.assertFalse(gzip_encoded) zipped, gzip_encoded = _SelectUploadCompressionStrategy( 'test.xml', False, ['yaml'], False) self.assertFalse(zipped) self.assertFalse(gzip_encoded) def testSelectUploadCompressionStrategyComponent(self): zipped, gzip_encoded = _SelectUploadCompressionStrategy( 'test.xml', True, ['not_matching'], True) self.assertFalse(zipped) self.assertTrue(gzip_encoded) def testDelegateUploadFileToObjectNormal(self): mock_stream = mock.Mock() mock_stream.close = mock.Mock() def DelegateUpload(): return 'a', 'b' elapsed_time, uploaded_object = _DelegateUploadFileToObject( DelegateUpload, 'url', mock_stream, False, False, False, None) # Ensure results are passed through. self.assertEqual(elapsed_time, 'a') self.assertEqual(uploaded_object, 'b') # Ensure close was called. self.assertTrue(mock_stream.close.called) @mock.patch('os.unlink') def testDelegateUploadFileToObjectZipped(self, mock_unlink): mock_stream = mock.Mock() mock_stream.close = mock.Mock() mock_upload_url = mock.Mock() mock_upload_url.object_name = 'Sample' def DelegateUpload(): return 'a', 'b' elapsed_time, uploaded_object = _DelegateUploadFileToObject( DelegateUpload, mock_upload_url, mock_stream, True, False, False, None) # Ensure results are passed through. self.assertEqual(elapsed_time, 'a') self.assertEqual(uploaded_object, 'b') # Ensure the file was unlinked. self.assertTrue(mock_unlink.called) # Ensure close was called. self.assertTrue(mock_stream.close.called) @mock.patch('gslib.command.concurrent_compressed_upload_lock') def testDelegateUploadFileToObjectGzipEncoded(self, mock_lock): mock_stream = mock.Mock() mock_stream.close = mock.Mock() def DelegateUpload(): # Ensure the lock was aquired before the delegate was called. self.assertTrue(mock_lock.__enter__.called) return 'a', 'b' elapsed_time, uploaded_object = _DelegateUploadFileToObject( DelegateUpload, 'url', mock_stream, False, True, False, None) # Ensure results are passed through. self.assertEqual(elapsed_time, 'a') self.assertEqual(uploaded_object, 'b') # Ensure close was called. self.assertTrue(mock_stream.close.called) # Ensure the lock was released. self.assertTrue(mock_lock.__exit__.called) @mock.patch('gslib.command.concurrent_compressed_upload_lock') def testDelegateUploadFileToObjectGzipEncodedComposite(self, mock_lock): mock_stream = mock.Mock() mock_stream.close = mock.Mock() def DelegateUpload(): # Ensure the lock was not aquired before the delegate was called. self.assertFalse(mock_lock.__enter__.called) return 'a', 'b' elapsed_time, uploaded_object = _DelegateUploadFileToObject( DelegateUpload, 'url', mock_stream, False, True, True, None) # Ensure results are passed through. self.assertEqual(elapsed_time, 'a') self.assertEqual(uploaded_object, 'b') # Ensure close was called. self.assertTrue(mock_stream.close.called) # Ensure the lock was released. self.assertFalse(mock_lock.__exit__.called) def testDoesNotGetSizeSourceFieldIfFileSizeWillChange(self): fields = copy_helper.GetSourceFieldsNeededForCopy( True, True, False, file_size_will_change=True) self.assertNotIn('size', fields) class TestExpandUrlToSingleBlr(GsUtilUnitTestCase): @mock.patch('gslib.cloud_api.CloudApi') @mock.patch('gslib.utils.copy_helper.CreateWildcardIterator') def testContainsWildcardMatchesNotObject(self, mock_CreateWildcardIterator, mock_gsutil_api): storage_url = StorageUrlFromString('gs://test/helloworld') mock_CreateWildcardIterator.return_value = iter( [BucketListingPrefix(storage_url)]) (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 'gs://test/hello*/', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) self.assertTrue(have_existing_dst_container) self.assertEqual(exp_url, storage_url) @mock.patch('gslib.cloud_api.CloudApi') @mock.patch('gslib.utils.copy_helper.CreateWildcardIterator') def testContainsWildcardMatchesObject(self, mock_CreateWildcardIterator, mock_gsutil_api): storage_url = StorageUrlFromString('gs://test/helloworld') mock_CreateWildcardIterator.return_value = iter( [BucketListingObject(storage_url)]) (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 'gs://test/hello*/', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) self.assertFalse(have_existing_dst_container) self.assertEqual(exp_url, storage_url) @mock.patch('gslib.cloud_api.CloudApi') @mock.patch('gslib.utils.copy_helper.CreateWildcardIterator') def testContainsWildcardMultipleMatches(self, mock_CreateWildcardIterator, mock_gsutil_api): mock_CreateWildcardIterator.return_value = iter([ BucketListingObject(StorageUrlFromString('gs://test/helloworld')), BucketListingObject(StorageUrlFromString('gs://test/helloworld2')) ]) with self.assertRaises(CommandException): ExpandUrlToSingleBlr('gs://test/hello*/', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) @mock.patch('gslib.cloud_api.CloudApi') @mock.patch('gslib.utils.copy_helper.CreateWildcardIterator') def testContainsWildcardNoMatches(self, mock_CreateWildcardIterator, mock_gsutil_api): mock_CreateWildcardIterator.return_value = iter([]) with self.assertRaises(CommandException): ExpandUrlToSingleBlr('gs://test/hello*/', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) @mock.patch('gslib.cloud_api.CloudApi') @mock.patch('gslib.utils.copy_helper.StorageUrlFromString') def testLocalFileDirectory(self, mock_StorageUrlFromString, mock_gsutil_api): mock_storage_url = mock.Mock() mock_storage_url.isFileUrl.return_value = True mock_storage_url.IsDirectory.return_value = True mock_StorageUrlFromString.return_value = mock_storage_url (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( '/home/test', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) self.assertTrue(have_existing_dst_container) self.assertEqual(exp_url, mock_storage_url) @mock.patch('gslib.cloud_api.CloudApi') @mock.patch('gslib.utils.copy_helper.StorageUrlFromString') def testLocalFileNotDirectory(self, mock_StorageUrlFromString, mock_gsutil_api): mock_storage_url = mock.Mock() mock_storage_url.isFileUrl.return_value = True mock_storage_url.IsDirectory.return_value = False mock_StorageUrlFromString.return_value = mock_storage_url (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( '/home/test', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) self.assertFalse(have_existing_dst_container) self.assertEqual(exp_url, mock_storage_url) @mock.patch('gslib.cloud_api.CloudApi') def testNoSlashPrefixExactMatch(self, mock_gsutil_api): mock_gsutil_api.ListObjects.return_value = iter([ CloudApi.CsObjectOrPrefix('folder/', CloudApi.CsObjectOrPrefixType.PREFIX) ]) (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 'gs://test/folder', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) self.assertTrue(have_existing_dst_container) self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder')) @mock.patch('gslib.cloud_api.CloudApi') def testNoSlashPrefixSubstringMatch(self, mock_gsutil_api): mock_gsutil_api.ListObjects.return_value = iter([ CloudApi.CsObjectOrPrefix('folderone/', CloudApi.CsObjectOrPrefixType.PREFIX) ]) (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 'gs://test/folder', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) self.assertFalse(have_existing_dst_container) self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder')) @mock.patch('gslib.cloud_api.CloudApi') def testNoSlashFolderPlaceholder(self, mock_gsutil_api): mock_gsutil_api.ListObjects.return_value = iter([ CloudApi.CsObjectOrPrefix( apitools_messages.Object(name='folder_$folder$'), CloudApi.CsObjectOrPrefixType.OBJECT) ]) (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 'gs://test/folder', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) self.assertTrue(have_existing_dst_container) self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder')) @mock.patch('gslib.cloud_api.CloudApi') def testNoSlashNoMatch(self, mock_gsutil_api): mock_gsutil_api.ListObjects.return_value = iter([]) (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 'gs://test/folder', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) self.assertFalse(have_existing_dst_container) self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder')) @mock.patch('gslib.cloud_api.CloudApi') def testWithSlashPrefixExactMatch(self, mock_gsutil_api): mock_gsutil_api.ListObjects.return_value = iter([ CloudApi.CsObjectOrPrefix('folder/', CloudApi.CsObjectOrPrefixType.PREFIX) ]) (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 'gs://test/folder/', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) self.assertTrue(have_existing_dst_container) self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder/')) @mock.patch('gslib.cloud_api.CloudApi') def testWithSlashNoMatch(self, mock_gsutil_api): mock_gsutil_api.ListObjects.return_value = iter([]) (exp_url, have_existing_dst_container) = ExpandUrlToSingleBlr( 'gs://test/folder/', mock_gsutil_api, 'project_id', False, CreateOrGetGsutilLogger('copy_test')) self.assertTrue(have_existing_dst_container) self.assertEqual(exp_url, StorageUrlFromString('gs://test/folder/')) def testCheckCloudHashesIsSkippedCorrectly(self): FakeObject = collections.namedtuple('FakeObject', ['md5Hash']) with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): # Should not raise a hash mismatch error: _CheckCloudHashes(logger=None, src_url=None, dst_url=None, src_obj_metadata=FakeObject(md5Hash='a'), dst_obj_metadata=FakeObject(md5Hash='b')) tests/test_gcs_json_media.py 0000644 00000016707 15025013616 0012275 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Tests for media helper functions and classes for GCS JSON API.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import email from gslib.gcs_json_media import BytesTransferredContainer from gslib.gcs_json_media import HttpWithDownloadStream from gslib.gcs_json_media import UploadCallbackConnectionClassFactory import gslib.tests.testcase as testcase import httplib2 import io import six from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import http_client from six.moves import mock # Assigning string representations of the appropriate package, used for # '@mock.patch` methods that take a string in the following format: # "package.module.ClassName" if six.PY2: https_connection = 'httplib.HTTPSConnection' else: https_connection = 'http.client.HTTPSConnection' class TestUploadCallbackConnection(testcase.GsUtilUnitTestCase): """Tests for the upload callback connection.""" def setUp(self): super(TestUploadCallbackConnection, self).setUp() self.bytes_container = BytesTransferredContainer() self.class_factory = UploadCallbackConnectionClassFactory( self.bytes_container, buffer_size=50, total_size=100, progress_callback='Sample') self.instance = self.class_factory.GetConnectionClass()('host') @mock.patch(https_connection) def testHeaderDefaultBehavior(self, mock_conn): """Test the size modifier is correct under expected headers.""" mock_conn.putheader.return_value = None self.instance.putheader('content-encoding', 'gzip') self.instance.putheader('content-length', '10') self.instance.putheader('content-range', 'bytes 0-104/*') # Ensure the modifier is as expected. self.assertAlmostEqual(self.instance.size_modifier, 10.5) @mock.patch(https_connection) def testHeaderIgnoreWithoutGzip(self, mock_conn): """Test that the gzip content-encoding is required to modify size.""" mock_conn.putheader.return_value = None self.instance.putheader('content-length', '10') self.instance.putheader('content-range', 'bytes 0-99/*') # Ensure the modifier is unchanged. self.assertAlmostEqual(self.instance.size_modifier, 1.0) @mock.patch(https_connection) def testHeaderRangeFormatX_YSlashStar(self, mock_conn): """Test content-range header format X-Y/* """ self.instance.putheader('content-encoding', 'gzip') self.instance.putheader('content-length', '10') self.instance.putheader('content-range', 'bytes 0-99/*') # Ensure the modifier is as expected. self.assertAlmostEqual(self.instance.size_modifier, 10.0) @mock.patch(https_connection) def testHeaderRangeFormatStarSlash100(self, mock_conn): """Test content-range header format */100 """ self.instance.putheader('content-encoding', 'gzip') self.instance.putheader('content-length', '10') self.instance.putheader('content-range', 'bytes */100') # Ensure the modifier is as expected. self.assertAlmostEqual(self.instance.size_modifier, 1.0) @mock.patch(https_connection) def testHeaderRangeFormat0_99Slash100(self, mock_conn): """Test content-range header format 0-99/100 """ self.instance.putheader('content-encoding', 'gzip') self.instance.putheader('content-length', '10') self.instance.putheader('content-range', 'bytes 0-99/100') # Ensure the modifier is as expected. self.assertAlmostEqual(self.instance.size_modifier, 10.0) @mock.patch(https_connection) def testHeaderParseFailure(self, mock_conn): """Test incorrect header values do not raise exceptions.""" mock_conn.putheader.return_value = None self.instance.putheader('content-encoding', 'gzip') self.instance.putheader('content-length', 'bytes 10') self.instance.putheader('content-range', 'not a number') # Ensure the modifier is unchanged. self.assertAlmostEqual(self.instance.size_modifier, 1.0) @mock.patch('gslib.progress_callback.ProgressCallbackWithTimeout') @mock.patch('httplib2.HTTPSConnectionWithTimeout') def testSendDefaultBehavior(self, mock_conn, mock_callback): mock_conn.send.return_value = None self.instance.size_modifier = 2 self.instance.processed_initial_bytes = True self.instance.callback_processor = mock_callback # Send 10 bytes of data. sample_data = b'0123456789' self.instance.send(sample_data) # Ensure the data is fully sent since the buffer size is 50 bytes. self.assertTrue(mock_conn.send.called) (_, sent_data), _ = mock_conn.send.call_args_list[0] self.assertEqual(sent_data, sample_data) # Ensure the progress callback is correctly scaled. self.assertTrue(mock_callback.Progress.called) [sent_bytes], _ = mock_callback.Progress.call_args_list[0] self.assertEqual(sent_bytes, 20) class TestHttpWithDownloadStream(testcase.GsUtilUnitTestCase): def test_incomplete_download_sets_content_length_correctly(self): expected_content_length = 100 bytes_returned_by_server = 'byte count less than content length' http_response = mock.Mock(spec=http_client.HTTPResponse) http_response.reason = 'reason' http_response.version = 'version' http_response.status = http_client.OK http_response.read.side_effect = [bytes_returned_by_server, ''] http_response.getheaders.return_value = [('Content-Length', expected_content_length)] # This method is only called with content-length: http_response.getheader.return_value = expected_content_length headers_stream = io.BytesIO(b'Content-Length:%d' % expected_content_length) if six.PY2: http_response.msg = http_client.HTTPMessage(headers_stream) else: http_response.msg = http_client.parse_headers(headers_stream) mock_connection = mock.Mock(spec=http_client.HTTPConnection) mock_connection.getresponse.return_value = http_response http = HttpWithDownloadStream() http.stream = mock.Mock(spec=io.BufferedIOBase) http.stream.mode = 'wb' http._conn_request(mock_connection, 'uri', 'GET', 'body', 'headers') # To resume this download correctly, the response needs to have the number # of bytes we actually received, not the number of bytes in the original # content-length header. self.assertEqual(int(http_response.msg['content-length']), len(bytes_returned_by_server)) tests/test_trace.py 0000644 00000003406 15025013616 0010417 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for gsutil --trace-token option.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.cs_api_map import ApiSelector import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri @SkipForS3('--trace-token is supported only on GCS JSON API.') class TestTraceTokenOption(testcase.GsUtilIntegrationTestCase): """Integration tests for gsutil --trace-token option.""" def test_minus_tracetoken_cat(self): """Tests cat command with trace-token option.""" key_uri = self.CreateObject(contents=b'0123456789') (_, stderr) = self.RunGsUtil( ['-D', '--trace-token=THISISATOKEN', 'cat', suri(key_uri)], return_stdout=True, return_stderr=True) if self.test_api == ApiSelector.JSON: self.assertIn('You are running gsutil with trace output enabled.', stderr) self.assertRegex( stderr, r'.*GET.*b/%s/o/%s\?.*trace=token%%3ATHISISATOKEN' % (key_uri.bucket_name, key_uri.object_name)) tests/test_copy_objects_iterator.py 0000644 00000010763 15025013616 0013721 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. . """Unit tests for name_expansion.CopyObjectsIterator class.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.commands.cp import DestinationInfo from gslib.name_expansion import CopyObjectsIterator from gslib.name_expansion import NameExpansionIteratorDestinationTuple from gslib.name_expansion import NameExpansionResult from gslib.storage_url import StorageUrlFromString import gslib.tests.testcase as testcase def _ConstructNameExpansionIterator(src_url_strs): for src_url_str in src_url_strs: storage_url = StorageUrlFromString(src_url_str) yield NameExpansionResult(source_storage_url=storage_url, is_multi_source_request=False, is_multi_top_level_source_request=False, names_container=False, expanded_storage_url=storage_url, expanded_result=None) def _ConstrcutNameExpansionIteratorDestinationTupleIterator( src_url_strs_array, dst_url_strs): for src_url_strs, dst_url_str in zip(src_url_strs_array, dst_url_strs): name_expansion_iter_dst_tuple = NameExpansionIteratorDestinationTuple( _ConstructNameExpansionIterator(src_url_strs), DestinationInfo(StorageUrlFromString(dst_url_str), False)) yield name_expansion_iter_dst_tuple class TestCopyObjectsIterator(testcase.GsUtilUnitTestCase): """Unit tests for CopyObjectsIterator.""" def setUp(self): super(TestCopyObjectsIterator, self).setUp() def test_iterator(self): src_strings_array = [ ['src_{}_{}'.format(i, j) for j in range(4)] for i in range(3) ] dst_strings = ['dest_' + str(i) for i in range(3)] copy_objects_iterator = CopyObjectsIterator( _ConstrcutNameExpansionIteratorDestinationTupleIterator( src_strings_array, dst_strings), False) # Flatten the src dst arrays src_dst_strings = [ (src, dst) for src_strings, dst in zip(src_strings_array, dst_strings) for src in src_strings ] for (src_string, dst_string) in src_dst_strings: copy_object_info = next(copy_objects_iterator) self.assertEqual(src_string, copy_object_info.source_storage_url.object_name) self.assertEqual(dst_string, copy_object_info.exp_dst_url.object_name) iterator_ended = False try: next(copy_objects_iterator) except StopIteration: iterator_ended = True self.assertTrue(iterator_ended) def test_iterator_metadata(self): src_strings_array = [['gs://bucket1'], ['source'], ['s3://bucket1']] dst_strings = ['gs://bucket2', 'dest', 'gs://bucket2'] copy_objects_iterator = CopyObjectsIterator( _ConstrcutNameExpansionIteratorDestinationTupleIterator( src_strings_array, dst_strings), False) self.assertFalse(copy_objects_iterator.has_cloud_src) self.assertFalse(copy_objects_iterator.has_file_src) self.assertEqual(len(copy_objects_iterator.provider_types), 0) next(copy_objects_iterator) self.assertTrue(copy_objects_iterator.has_cloud_src) self.assertFalse(copy_objects_iterator.has_file_src) self.assertEqual(len(copy_objects_iterator.provider_types), 1) self.assertTrue('gs' in copy_objects_iterator.provider_types) next(copy_objects_iterator) self.assertTrue(copy_objects_iterator.has_cloud_src) self.assertTrue(copy_objects_iterator.has_file_src) self.assertEqual(len(copy_objects_iterator.provider_types), 2) self.assertTrue('file' in copy_objects_iterator.provider_types) self.assertFalse(copy_objects_iterator.is_daisy_chain) next(copy_objects_iterator) self.assertEqual(len(copy_objects_iterator.provider_types), 3) self.assertTrue('s3' in copy_objects_iterator.provider_types) self.assertTrue(copy_objects_iterator.is_daisy_chain) tests/test_naming.py 0000644 00000176163 15025013616 0010605 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2010 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Tests for gsutil naming logic. The test code in this file runs against an in-memory storage service mock, so runs very quickly. This is valuable for testing changes that impact the naming rules, since those rules are complex and it's useful to be able to make small incremental changes and rerun the tests frequently. Additional end-to-end tests (which send traffic to the production Google Cloud Storage service) are available via the gsutil test command. """ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals import gzip import os import six from gslib.cloud_api import NotFoundException from gslib.cloud_api import ServiceException from gslib.exception import CommandException from gslib.exception import InvalidUrlError from gslib.exception import NO_URLS_MATCHED_GENERIC from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.storage_url import StorageUrlFromString import gslib.tests.testcase as testcase from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetDummyProjectForUnitTest from gslib.tests.util import unittest from gslib.utils.constants import UTF8 from gslib.utils import copy_helper from gslib.utils import system_util def _Overwrite(fp): """Overwrite first byte in an open file and flush contents.""" fp.seek(0) fp.write('x') fp.flush() def _Append(fp): """Append a byte at end of an open file and flush contents.""" fp.seek(0, 2) fp.write('x') fp.flush() # TODO: Re-enable SequentialAndParallelTransfer decorator on tests in this file # once we refactor to a thread-safe mock storage service implementation. class GsutilNamingTests(testcase.GsUtilUnitTestCase): """Unit tests for gsutil naming logic.""" def testGetPathBeforeFinalDir(self): """Tests GetPathBeforeFinalDir() (unit test).""" self.assertEqual( 'gs://', copy_helper.GetPathBeforeFinalDir( StorageUrlFromString('gs://bucket/'), StorageUrlFromString('gs://bucket/obj'))) self.assertEqual( 'gs://bucket', copy_helper.GetPathBeforeFinalDir( StorageUrlFromString('gs://bucket/dir/'), StorageUrlFromString('gs://bucket/dir/obj'))) self.assertEqual( 'gs://bucket', copy_helper.GetPathBeforeFinalDir( StorageUrlFromString('gs://bucket/dir'), StorageUrlFromString('gs://bucket/dir/obj'))) self.assertEqual( 'gs://bucket/dir', copy_helper.GetPathBeforeFinalDir( StorageUrlFromString('gs://bucket/dir/obj'), StorageUrlFromString('gs://bucket/dir/obj'))) self.assertEqual( 'gs://bucket/dir1', copy_helper.GetPathBeforeFinalDir( StorageUrlFromString('gs://bucket/*/dir2'), StorageUrlFromString('gs://bucket/dir1/dir2/obj'))) self.assertEqual( 'gs://bucket/dir1/dir2/dir3', copy_helper.GetPathBeforeFinalDir( StorageUrlFromString('gs://bucket/*/dir2/*/dir4'), StorageUrlFromString('gs://bucket/dir1/dir2/dir3/dir4/obj'))) src_dir = self.CreateTempDir() subdir = os.path.join(src_dir, 'subdir') os.mkdir(subdir) self.assertEqual( suri(src_dir), copy_helper.GetPathBeforeFinalDir( StorageUrlFromString(suri(subdir)), StorageUrlFromString(suri(subdir, 'obj')))) # @SequentialAndParallelTransfer def testCopyingTopLevelFileToBucket(self): """Tests copying one top-level file to a bucket.""" src_file = self.CreateTempFile(file_name='f0') dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', [src_file, suri(dst_bucket_uri)]) actual = list( self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) self.assertEqual(1, len(actual)) self.assertEqual('f0', actual[0].root_object.name) # @SequentialAndParallelTransfer def testCopyingMultipleFilesToBucket(self): """Tests copying multiple files to a bucket.""" src_file0 = self.CreateTempFile(file_name='f0') src_file1 = self.CreateTempFile(file_name='f1') dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', [src_file0, src_file1, suri(dst_bucket_uri)]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'f0'), suri(dst_bucket_uri, 'f1'), ]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingNestedFileToBucketSubdir(self): r"""Tests copying a nested file to a bucket subdir. Tests that we correctly translate local FS-specific delimiters (\ on Windows) to bucket delimiter (/). """ tmpdir = self.CreateTempDir() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) src_file = self.CreateTempFile(tmpdir=tmpdir, file_name='obj', contents=b'') dst_bucket_uri = self.CreateBucket() # Make an object under subdir so next copy will treat subdir as a subdir. self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'subdir/a')]) self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'subdir')]) actual = set( str(u) for u in self._test_wildcard_iterator(suri(dst_bucket_uri, '**')).IterObjects()) expected = set([ suri(dst_bucket_uri, 'subdir', 'a'), suri(dst_bucket_uri, 'subdir', 'obj'), ]) self.assertEqual(expected, actual) def testCopyingBucketSubdirsToBucket(self): """Ensure wildcarded recursive cp in bucket subdirs behaves like Unix.""" src_bucket_uri = self.CreateBucket() dst_bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(file_name='foo', contents=b'bar') self.RunCommand('cp', [fpath, suri(src_bucket_uri, 'Test/sub-test/foo')]) self.RunCommand('cp', [fpath, suri(src_bucket_uri, 'Test2/sub-test/foo')]) self.RunCommand('cp', [fpath, suri(src_bucket_uri, 'Test3/sub-test/foo')]) self.RunCommand( 'cp', ['-R', suri(src_bucket_uri, '*', 'sub-test'), suri(dst_bucket_uri)]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'sub-test', 'foo')]) self.assertEqual(expected, actual) src_bucket_uri2 = self.CreateBucket() dst_bucket_uri2 = self.CreateBucket() self.RunCommand('cp', [fpath, suri(src_bucket_uri2, 'Test/dir1/dir2/foo')]) self.RunCommand('cp', [fpath, suri(src_bucket_uri2, 'Test2/dir1/dir2/foo')]) self.RunCommand('cp', [fpath, suri(src_bucket_uri2, 'Test3/dir1/dir2/bar')]) self.RunCommand('cp', [ '-R', suri(src_bucket_uri2, '*', 'dir1', 'dir2'), suri(dst_bucket_uri2) ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri2, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri2, 'dir2', 'foo'), suri(dst_bucket_uri2, 'dir2', 'bar') ]) self.assertEqual(expected, actual) dst_bucket_uri3 = self.CreateBucket() self.RunCommand('cp', [ '-R', suri(src_bucket_uri2, 'Test*', '*', 'dir2'), suri(dst_bucket_uri3) ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri3, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri3, 'dir2', 'foo'), suri(dst_bucket_uri3, 'dir2', 'bar') ]) self.assertEqual(expected, actual) src_bucket_uri3 = self.CreateBucket() dst_bucket_uri4 = self.CreateBucket() self.RunCommand( 'cp', [fpath, suri(src_bucket_uri3, 'dir1/test1/dir2/dir3/foo')]) self.RunCommand( 'cp', [fpath, suri(src_bucket_uri3, 'dir1/test2/dir2/dir3/foo')]) self.RunCommand( 'cp', [fpath, suri(src_bucket_uri3, 'dir1/test3/dir2/dir3/bar')]) self.RunCommand('cp', [ '-R', suri(src_bucket_uri3, 'dir1', '*', 'dir2', 'dir3'), suri(dst_bucket_uri4) ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri4, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri4, 'dir3', 'foo'), suri(dst_bucket_uri4, 'dir3', 'bar') ]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingAbsolutePathDirToBucket(self): """Tests recursively copying absolute path directory to a bucket.""" dst_bucket_uri = self.CreateBucket() src_dir_root = self.CreateTempDir( test_files=['f0', 'f1', 'f2.txt', ('dir0', 'dir1', 'nested')]) self.RunCommand('cp', ['-R', src_dir_root, suri(dst_bucket_uri)]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) src_tmpdir = os.path.split(src_dir_root)[1] expected = set([ suri(dst_bucket_uri, src_tmpdir, 'f0'), suri(dst_bucket_uri, src_tmpdir, 'f1'), suri(dst_bucket_uri, src_tmpdir, 'f2.txt'), suri(dst_bucket_uri, src_tmpdir, 'dir0', 'dir1', 'nested') ]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingRelativePathDirToBucket(self): """Tests recursively copying relative directory to a bucket.""" dst_bucket_uri = self.CreateBucket() src_dir = self.CreateTempDir(test_files=[('dir0', 'f1')]) self.RunCommand('cp', ['-R', 'dir0', suri(dst_bucket_uri)], cwd=src_dir) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'dir0', 'f1')]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingRelPathSubDirToBucketSubdirWithDollarFolderObj(self): """Tests recursively copying relative sub-directory to bucket subdir. Subdir is signified by a $folder$ object. """ # Create a $folder$ object to simulate a folder created by the legacy GCS # console (or various other tools), which gsutil understands to mean there # is a folder into which the object is being copied. dst_bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=dst_bucket_uri, object_name='abc_$folder$', contents='') src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')]) self.RunCommand( 'cp', ['-R', os.path.join('dir0', 'dir1'), suri(dst_bucket_uri, 'abc')], cwd=src_dir) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'abc_$folder$'), suri(dst_bucket_uri, 'abc', 'dir1', 'f1') ]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingRelativePathSubDirToBucketSubdirSignifiedBySlash(self): """Tests recursively copying relative sub-directory to bucket subdir. Subdir is signified by a / object. """ dst_bucket_uri = self.CreateBucket() src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')]) self.RunCommand( 'cp', ['-R', os.path.join('dir0', 'dir1'), suri(dst_bucket_uri, 'abc') + '/'], cwd=src_dir) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'abc', 'dir1', 'f1')]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingRelativePathSubDirToBucket(self): """Tests recursively copying relative sub-directory to a bucket.""" dst_bucket_uri = self.CreateBucket() src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')]) self.RunCommand( 'cp', ['-R', os.path.join('dir0', 'dir1'), suri(dst_bucket_uri)], cwd=src_dir) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'dir1', 'f1')]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingDotSlashToBucket(self): """Tests copying ./ to a bucket produces expected naming.""" # When running a command like gsutil cp -r . gs://dest we expect the dest # obj names to be of the form gs://dest/abc, not gs://dest/./abc. dst_bucket_uri = self.CreateBucket() src_dir = self.CreateTempDir(test_files=['foo']) for rel_src_dir in ['.', '.%s' % os.sep]: self.RunCommand( 'cp', ['-R', rel_src_dir, suri(dst_bucket_uri)], cwd=src_dir) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'foo')]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingDirContainingOneFileToBucket(self): """Tests copying a directory containing 1 file to a bucket. We test this case to ensure that correct bucket handling isn't dependent on the copy being treated as a multi-source copy. """ dst_bucket_uri = self.CreateBucket() src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'foo')]) self.RunCommand( 'cp', ['-R', os.path.join(src_dir, 'dir0', 'dir1'), suri(dst_bucket_uri)]) actual = list((str(u) for u in self._test_wildcard_iterator( suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))) self.assertEqual(1, len(actual)) self.assertEqual(suri(dst_bucket_uri, 'dir1', 'foo'), actual[0]) def testCopyDotFilesToBucket(self): dst_bucket_uri = self.CreateBucket() src_dir = self.CreateTempDir(test_files=[('foo')]) object_named_dot = suri(dst_bucket_uri) + '/.' object_named_dotdot = suri(dst_bucket_uri) + '/..' for object_name in (object_named_dot, object_named_dotdot): try: self.RunCommand('cp', [os.path.join(src_dir, 'foo'), object_name]) self.fail('Expected InvalidUrlError for %s' % object_name) except InvalidUrlError: pass def testCopyingBucketToDir(self): """Tests copying from a bucket to a directory.""" src_bucket_uri = self.CreateBucket(test_objects=['foo', 'dir/foo2']) dst_dir = self.CreateTempDir() # Mock objects don't support hash digestion. with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): self.RunCommand('cp', ['-R', suri(src_bucket_uri), dst_dir]) actual = set( str(u) for u in self._test_wildcard_iterator( '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_dir, src_bucket_uri.bucket_name, 'foo'), suri(dst_dir, src_bucket_uri.bucket_name, 'dir', 'foo2') ]) self.assertEqual(expected, actual) @unittest.skipIf(system_util.IS_WINDOWS, 'os.symlink() is not available on Windows.') def testCopyingSymlinkDirectory(self): """Tests that cp warns when copying a symlink directory.""" bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() tmpdir2 = self.CreateTempDir() # Create a valid file, since cp expects to copy at least one source URL # successfully. subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) fpath1 = self.CreateTempFile(tmpdir=subdir, contents=b'foo') self.CreateTempFile(tmpdir=tmpdir2, contents=b'foo') os.mkdir(os.path.join(tmpdir, 'symlinkdir')) # Create a symlink to a directory to ensure we warn when encountering it. os.symlink(tmpdir2, os.path.join(subdir, 'symlinkdir')) mock_log_handler = self.RunCommand( 'cp', ['-r', tmpdir, suri(bucket_uri)], debug=1, return_log_handler=True) actual = set( str(u) for u in self._test_wildcard_iterator(suri(bucket_uri, '**')).IterAll( expand_top_level_buckets=True)) expected_object_path = suri(bucket_uri, os.path.basename(tmpdir), 'subdir', os.path.basename(fpath1)) expected = set([expected_object_path]) self.assertEqual(expected, actual) desired_msg = ('Skipping symlink directory "%s"' % os.path.join(subdir, 'symlinkdir')) self.assertIn( desired_msg, mock_log_handler.messages['info'], '"%s" not found in mock_log_handler["info"]: %s' % (desired_msg, str(mock_log_handler.messages))) def testCopyingBucketToBucket(self): """Tests copying from a bucket-only URI to a bucket.""" src_bucket_uri = self.CreateBucket(test_objects=['foo', 'dir/foo2']) dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', ['-R', suri(src_bucket_uri), suri(dst_bucket_uri)]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, src_bucket_uri.bucket_name, 'foo'), suri(dst_bucket_uri, src_bucket_uri.bucket_name, 'dir', 'foo2') ]) self.assertEqual(expected, actual) def testCopyingDirectoryToDirectory(self): """Tests copying from a directory to a directory.""" src_dir = self.CreateTempDir(test_files=['foo', ('dir', 'foo2')]) dst_dir = self.CreateTempDir() self.RunCommand('cp', ['-R', src_dir, dst_dir]) actual = set( str(u) for u in self._test_wildcard_iterator( '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) src_dir_base = os.path.split(src_dir)[1] expected = set([ suri(dst_dir, src_dir_base, 'foo'), suri(dst_dir, src_dir_base, 'dir', 'foo2') ]) self.assertEqual(expected, actual) def testCopyingFilesAndDirNonRecursive(self): """Tests copying containing files and a directory without -R.""" src_dir = self.CreateTempDir(test_files=[ 'foo', 'bar', ('d1', 'f2'), ('d2', 'f3'), ('d3', 'd4', 'f4') ]) dst_dir = self.CreateTempDir() self.RunCommand('cp', ['%s%s*' % (src_dir, os.sep), dst_dir]) actual = set( str(u) for u in self._test_wildcard_iterator( '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_dir, 'foo'), suri(dst_dir, 'bar')]) self.assertEqual(expected, actual) def testCopyingFileToDir(self): """Tests copying one file to a directory.""" src_file = self.CreateTempFile(file_name='foo') dst_dir = self.CreateTempDir() self.RunCommand('cp', [src_file, dst_dir]) actual = list( self._test_wildcard_iterator( '%s%s*' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) self.assertEqual(1, len(actual)) self.assertEqual(suri(dst_dir, 'foo'), str(actual[0])) # @SequentialAndParallelTransfer def testCopyingFileToObjectWithConsecutiveSlashes(self): """Tests copying a file to an object containing consecutive slashes.""" src_file = self.CreateTempFile(file_name='f0') dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', [src_file, suri(dst_bucket_uri) + '//obj']) actual = list( self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) self.assertEqual(1, len(actual)) self.assertEqual('/obj', actual[0].root_object.name) def testCopyingCompressedFileToBucket(self): """Tests copying one file with compression to a bucket.""" src_file = self.CreateTempFile(contents=b'plaintext', file_name='f2.txt') dst_bucket_uri = self.CreateBucket() self.RunCommand( 'cp', ['-z', 'txt', src_file, suri(dst_bucket_uri)], ) actual = list( self._test_wildcard_iterator(suri( dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True)) self.assertEqual(1, len(actual)) actual_obj = actual[0].root_object self.assertEqual('f2.txt', actual_obj.name) self.assertEqual('gzip', actual_obj.contentEncoding) stdout = self.RunCommand('cat', [suri(dst_bucket_uri, 'f2.txt')], return_stdout=True) f = gzip.GzipFile(fileobj=six.BytesIO(six.ensure_binary(stdout)), mode='rb') try: self.assertEqual(f.read(), b'plaintext') finally: f.close() def testCopyingObjectToObject(self): """Tests copying an object to an object.""" src_bucket_uri = self.CreateBucket(test_objects=['obj']) dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', [suri(src_bucket_uri, 'obj'), suri(dst_bucket_uri)]) actual = list( self._test_wildcard_iterator(suri( dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True)) self.assertEqual(1, len(actual)) self.assertEqual('obj', actual[0].root_object.name) def testCopyingObjectToObjectUsingDestWildcard(self): """Tests copying an object to an object using a dest wildcard.""" src_bucket_uri = self.CreateBucket(test_objects=['obj']) dst_bucket_uri = self.CreateBucket(test_objects=['dstobj']) self.RunCommand('cp', [suri(src_bucket_uri, 'obj'), '%s*' % dst_bucket_uri.uri]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'dstobj')]) self.assertEqual(expected, actual) def testCopyingObjsAndFilesToDir(self): """Tests copying objects and files to a directory.""" src_bucket_uri = self.CreateBucket(test_objects=['f1']) src_dir = self.CreateTempDir(test_files=['f2']) dst_dir = self.CreateTempDir() # Mock objects don't support hash digestion. with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): self.RunCommand('cp', [ '-R', suri(src_bucket_uri, '**'), os.path.join(src_dir, '**'), dst_dir ]) actual = set( str(u) for u in self._test_wildcard_iterator(os.path.join( dst_dir, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_dir, 'f1'), suri(dst_dir, 'f2')]) self.assertEqual(expected, actual) def testCopyingObjToDot(self): """Tests that copying an object to . or ./ downloads to correct name.""" src_bucket_uri = self.CreateBucket(test_objects=['f1']) dst_dir = self.CreateTempDir() for final_char in ('/', ''): # Mock objects don't support hash digestion. with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): self.RunCommand('cp', [suri(src_bucket_uri, 'f1'), '.%s' % final_char], cwd=dst_dir) actual = set() for dirname, dirnames, filenames in os.walk(dst_dir): for subdirname in dirnames: actual.add(os.path.join(dirname, subdirname)) for filename in filenames: actual.add(os.path.join(dirname, filename)) expected = set([os.path.join(dst_dir, 'f1')]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingObjsAndFilesToBucket(self): """Tests copying objects and files to a bucket.""" src_bucket_uri = self.CreateBucket(test_objects=['f1']) src_dir = self.CreateTempDir(test_files=['f2']) dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', [ '-R', suri(src_bucket_uri, '**'), '%s%s**' % (src_dir, os.sep), suri(dst_bucket_uri) ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'f1'), suri(dst_bucket_uri, 'f2')]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingSubdirRecursiveToNonexistentSubdir(self): """Tests copying a directory with a single file recursively to a bucket. The file should end up in a new bucket subdirectory with the file's directory structure starting below the recursive copy point, as in Unix cp. Example: filepath: dir1/dir2/foo cp -r dir1 dir3 Results in dir3/dir2/foo being created. """ src_dir = self.CreateTempDir() self.CreateTempFile(tmpdir=src_dir + '/dir1/dir2', file_name='foo') dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', ['-R', src_dir + '/dir1', suri(dst_bucket_uri, 'dir3')]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'dir3/dir2/foo')]) self.assertEqual(expected, actual) def testCopyingFileToDirRecursive(self): """Tests copying a file with -R.""" src_file = self.CreateTempFile(file_name='foo') dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', ['-R', src_file, suri(dst_bucket_uri, 'dir/foo')]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'dir/foo')]) self.assertEqual(expected, actual) def testCopyingFileToNonExistentDir(self): """Tests copying a file to a non-existent directory. Should create the directory and add the file to it """ src_file = self.CreateTempFile(file_name='foo') dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'dir') + '/']) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'dir/foo')]) self.assertEqual(expected, actual) def testCopyingMultipleFilesToDirRecursive(self): """Tests copying multiple files with -R.""" src_dir = self.CreateTempDir() src_file1 = self.CreateTempFile(tmpdir=src_dir, file_name='foo') src_file2 = self.CreateTempFile(tmpdir=src_dir, file_name='bar') dst_bucket_uri = self.CreateBucket() self.RunCommand( 'cp', ['-R', src_file1, src_file2, suri(dst_bucket_uri, 'dir/foo')]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'dir/foo/foo'), suri(dst_bucket_uri, 'dir/foo/bar') ]) self.assertEqual(expected, actual) def testAttemptDirCopyWithoutRecursion(self): """Tests copying a directory without -R.""" src_dir = self.CreateTempDir(test_files=1) dst_dir = self.CreateTempDir() try: self.RunCommand('cp', [src_dir, dst_dir]) self.fail('Did not get expected CommandException') except CommandException as e: self.assertIn(NO_URLS_MATCHED_GENERIC, e.reason) def testNonRecursiveFileAndSameNameSubdir(self): """Tests copying a file and subdirectory of the same name without -R.""" src_bucket_uri = self.CreateBucket(test_objects=['f1', 'f1/f2']) dst_dir = self.CreateTempDir() # Mock objects don't support hash digestion. with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): self.RunCommand('cp', [suri(src_bucket_uri, 'f1'), dst_dir]) actual = list( self._test_wildcard_iterator( '%s%s*' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) self.assertEqual(1, len(actual)) self.assertEqual(suri(dst_dir, 'f1'), str(actual[0])) # TODO: Assert that we omit the prefix here when unit_testcase supports # returning stderr. def testAttemptCopyingProviderOnlySrc(self): """Attempts to copy a src specified as a provider-only URI.""" src_bucket_uri = self.CreateBucket() try: self.RunCommand('cp', ['gs://', suri(src_bucket_uri)]) self.fail('Did not get expected CommandException') except CommandException as e: self.assertIn('provider-only', e.reason) def testAttemptCopyingOverlappingSrcDstFile(self): """Attempts to an object atop itself.""" src_file = self.CreateTempFile() try: self.RunCommand('cp', [src_file, src_file]) self.fail('Did not get expected CommandException') except CommandException as e: self.assertIn('are the same file - abort', e.reason) def testAttemptCopyingToMultiMatchWildcard(self): """Attempts to copy where dst wildcard matches >1 obj.""" src_bucket_uri = self.CreateBucket(test_objects=2) try: self.RunCommand('cp', [suri(src_bucket_uri, 'obj0'), suri(src_bucket_uri, '*')]) self.fail('Did not get expected CommandException') except CommandException as e: self.assertNotEqual(e.reason.find('must match exactly 1 URL'), -1) def testAttemptCopyingMultiObjsToFile(self): """Attempts to copy multiple objects to a file.""" src_bucket_uri = self.CreateBucket(test_objects=2) dst_file = self.CreateTempFile() try: self.RunCommand('cp', ['-R', suri(src_bucket_uri, '*'), dst_file]) self.fail('Did not get expected CommandException') except CommandException as e: self.assertIn('must name a directory, bucket, or', e.reason) def testAttemptCopyingWithFileDirConflict(self): """Attempts to copy objects that cause a file/directory conflict.""" # Create objects with name conflicts (a/b and a). Use 'dst' bucket because # it gets cleared after each test. bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='a') self.CreateObject(bucket_uri=bucket_uri, object_name='b/a') dst_dir = self.CreateTempDir() try: self.RunCommand('cp', ['-R', suri(bucket_uri), dst_dir]) self.fail('Did not get expected CommandException') except CommandException as e: self.assertNotEqual('exists where a directory needs to be created', e.reason) def testAttemptCopyingWithDirFileConflict(self): """Attempts to copy an object that causes a directory/file conflict.""" # Create an object that conflicts with dest subdir. tmpdir = self.CreateTempDir() os.mkdir(os.path.join(tmpdir, 'abc')) src_uri = self.CreateObject(object_name='abc', contents='bar') try: self.RunCommand('cp', [suri(src_uri), tmpdir + '/']) self.fail('Did not get expected CommandException') except CommandException as e: self.assertNotEqual('where the file needs to be created', e.reason) def testWildcardMoveWithinBucket(self): """Attempts to move using src wildcard that overlaps dest object. We want to ensure that this doesn't stomp the result data. """ dst_bucket_uri = self.CreateBucket(test_objects=['old']) self.RunCommand('mv', [suri(dst_bucket_uri, 'old*'), suri(dst_bucket_uri, 'new')]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'new')]) self.assertEqual(expected, actual) def testLsNonExistentObjectWithPrefixName(self): """Test ls of non-existent obj that matches prefix of existing objs.""" # Use an object name that matches a prefix of other names at that level, to # ensure the ls subdir handling logic doesn't pick up anything extra. src_bucket_uri = self.CreateBucket(test_objects=['obj_with_suffix']) try: self.RunCommand('ls', [suri(src_bucket_uri, 'obj')]) except CommandException as e: self.assertIn('matched no objects', e.reason) def testLsBucketNonRecursive(self): """Test that ls of a bucket returns expected results.""" src_bucket_uri = self.CreateBucket( test_objects=['foo1', 'd0/foo2', 'd1/d2/foo3']) output = self.RunCommand('ls', [suri(src_bucket_uri, '*')], return_stdout=True) expected = set([ suri(src_bucket_uri, 'foo1'), suri(src_bucket_uri, 'd1', ':'), suri(src_bucket_uri, 'd1', 'd2') + src_bucket_uri.delim, suri(src_bucket_uri, 'd0', ':'), suri(src_bucket_uri, 'd0', 'foo2') ]) expected.add('') # Blank line between subdir listings. actual = set([line.strip() for line in output.split('\n')]) self.assertEqual(expected, actual) def testLsBucketRecursive(self): """Test that ls -R of a bucket returns expected results.""" src_bucket_uri = self.CreateBucket( test_objects=['foo1', 'd0/foo2', 'd1/d2/foo3']) output = self.RunCommand('ls', ['-R', suri(src_bucket_uri, '*')], return_stdout=True) expected = set([ suri(src_bucket_uri, 'foo1'), suri(src_bucket_uri, 'd1', ':'), suri(src_bucket_uri, 'd1', 'd2', ':'), suri(src_bucket_uri, 'd1', 'd2', 'foo3'), suri(src_bucket_uri, 'd0', ':'), suri(src_bucket_uri, 'd0', 'foo2') ]) expected.add('') # Blank line between subdir listings. actual = set([line.strip() for line in output.split('\n')]) self.assertEqual(expected, actual) def testLsBucketRecursiveWithLeadingSlashObjectName(self): """Test that ls -R of a bucket with an object that has leading slash.""" dst_bucket_uri = self.CreateBucket(test_objects=['f0']) output = self.RunCommand('ls', ['-R', suri(dst_bucket_uri, '*')], return_stdout=True) expected = set([suri(dst_bucket_uri, 'f0')]) expected.add('') # Blank line between subdir listings. actual = set([line.strip() for line in output.split('\n')]) self.assertEqual(expected, actual) def testLsBucketSubdirNonRecursive(self): """Test that ls of a bucket subdir returns expected results.""" src_bucket_uri = self.CreateBucket( test_objects=['src_subdir/foo', 'src_subdir/nested/foo2']) output = self.RunCommand('ls', [suri(src_bucket_uri, 'src_subdir')], return_stdout=True) expected = set([ suri(src_bucket_uri, 'src_subdir', 'foo'), (suri(src_bucket_uri, 'src_subdir', 'nested') + src_bucket_uri.delim) ]) expected.add('') # Blank line between subdir listings. actual = set([line.strip() for line in output.split('\n')]) self.assertEqual(expected, actual) def testLsBucketSubdirRecursive(self): """Test that ls -R of a bucket subdir returns expected results.""" src_bucket_uri = self.CreateBucket( test_objects=['src_subdir/foo', 'src_subdir/nested/foo2']) for final_char in ('/', ''): output = self.RunCommand( 'ls', ['-R', suri(src_bucket_uri, 'src_subdir') + final_char], return_stdout=True) expected = set([ suri(src_bucket_uri, 'src_subdir', ':'), suri(src_bucket_uri, 'src_subdir', 'foo'), suri(src_bucket_uri, 'src_subdir', 'nested', ':'), suri(src_bucket_uri, 'src_subdir', 'nested', 'foo2') ]) expected.add('') # Blank line between subdir listings. actual = set([line.strip() for line in output.split('\n')]) self.assertEqual(expected, actual) def testSetAclOnBucketRuns(self): """Test that the 'acl set' command basically runs.""" # We don't test reading back the acl (via 'acl get' command) because at # present MockStorageService doesn't translate canned ACLs into actual ACL # XML. src_bucket_uri = self.CreateBucket() self.RunCommand('acl', ['set', 'private', suri(src_bucket_uri)]) def testSetAclOnWildcardNamedBucketRuns(self): """Test that 'acl set' basically runs against wildcard-named bucket.""" # We don't test reading back the acl (via 'acl get' command) because at # present MockStorageService doesn't translate canned ACLs into actual ACL # XML. src_bucket_uri = self.CreateBucket(test_objects=['f0']) with SetDummyProjectForUnitTest(): self.RunCommand( 'acl', ['set', 'private', suri(src_bucket_uri)[:-2] + '*']) def testSetAclOnObjectRuns(self): """Test that the 'acl set' command basically runs.""" src_bucket_uri = self.CreateBucket(test_objects=['f0']) self.RunCommand('acl', ['set', 'private', suri(src_bucket_uri, '*')]) def testSetDefAclOnBucketRuns(self): """Test that the 'defacl set' command basically runs.""" src_bucket_uri = self.CreateBucket() self.RunCommand('defacl', ['set', 'private', suri(src_bucket_uri)]) def testSetDefAclOnObjectFails(self): """Test that the 'defacl set' command fails when run against an object.""" src_bucket_uri = self.CreateBucket() try: self.RunCommand('defacl', ['set', 'private', suri(src_bucket_uri, '*')]) self.fail('Did not get expected CommandException') except CommandException as e: self.assertIn('URL must name a bucket', e.reason) # @SequentialAndParallelTransfer def testMinusDOptionWorks(self): """Tests using gsutil -D option.""" src_file = self.CreateTempFile(file_name='f0') dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', [src_file, suri(dst_bucket_uri)], debug=3) actual = list( self._test_wildcard_iterator(suri( dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True)) self.assertEqual(1, len(actual)) self.assertEqual('f0', actual[0].root_object.name) # @SequentialAndParallelTransfer def testFlatCopyingObjsAndFilesToBucketSubDir(self): """Tests copying flatly listed objects and files to bucket subdir.""" src_bucket_uri = self.CreateBucket(test_objects=['f0', 'd0/f1', 'd1/d2/f2']) src_dir = self.CreateTempDir( test_files=['f3', ('d3', 'f4'), ('d4', 'd5', 'f5')]) dst_bucket_uri = self.CreateBucket( test_objects=['dst_subdir0/existing', 'dst_subdir1/existing']) # Test with and without final slash on dest subdir. for i, final_char in enumerate(('/', '')): self.RunCommand('cp', [ '-R', suri(src_bucket_uri, '**'), os.path.join(src_dir, '**'), suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_char ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set() for i in range(2): expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing')) for j in range(6): expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'f%d' % j)) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testRecursiveCopyObjsAndFilesToExistingBucketSubDir(self): """Tests recursive copy of objects and files to existing bucket subdir.""" src_bucket_uri = self.CreateBucket(test_objects=['f0', 'nested/f1']) dst_bucket_uri = self.CreateBucket( test_objects=['dst_subdir0/existing_obj', 'dst_subdir1/existing_obj']) src_dir = self.CreateTempDir(test_files=['f2', ('nested', 'f3')]) # Test with and without final slash on dest subdir. for i, final_char in enumerate(('/', '')): self.RunCommand('cp', [ '-R', suri(src_bucket_uri), src_dir, suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_char ]) actual = set( str(u) for u in self._test_wildcard_iterator( suri(dst_bucket_uri, 'dst_subdir%d' % i, '**')).IterAll(expand_top_level_buckets=True)) tmp_dirname = os.path.split(src_dir)[1] bucketname = src_bucket_uri.bucket_name expected = set([ suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing_obj'), suri(dst_bucket_uri, 'dst_subdir%d' % i, bucketname, 'f0'), suri(dst_bucket_uri, 'dst_subdir%d' % i, bucketname, 'nested', 'f1'), suri(dst_bucket_uri, 'dst_subdir%d' % i, tmp_dirname, 'f2'), suri(dst_bucket_uri, 'dst_subdir%d' % i, tmp_dirname, 'nested', 'f3') ]) self.assertEqual(expected, actual) def testRecursiveCopyFileToExistingBucketSubDirInvalidSourceParent(self): """Tests recursive copy of invalid path file to existing bucket subdir.""" src_dir = self.CreateTempDir(test_files=[('nested', 'f0')]) dst_bucket_uri = self.CreateBucket(test_objects=['dst_subdir/existing_obj']) for relative_path_string in ['.', '.' + os.sep]: self.RunCommand('cp', [ '-R', src_dir + os.sep + relative_path_string, suri(dst_bucket_uri, 'dst_subdir'), ]) actual = set( str(u) for u in self._test_wildcard_iterator( suri(dst_bucket_uri, 'dst_subdir', '**')).IterAll( expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'dst_subdir', 'nested', 'f0'), suri(dst_bucket_uri, 'dst_subdir', 'existing_obj'), ]) self.assertEqual(expected, actual) def testRecursiveCopyFilesToExistingBucketSubDirInvalidSourceParent(self): """Tests recursive copy of invalid paths files to existing bucket subdir.""" src_dir1 = self.CreateTempDir(test_files=['f1']) src_dir2 = os.path.join(src_dir1, 'nested') os.mkdir(src_dir2) self.CreateTempFile(tmpdir=src_dir2, file_name='f2') dst_bucket_uri = self.CreateBucket(test_objects=['dst_subdir/existing_obj']) for relative_path_string in ['.', '.' + os.sep, '..', '..' + os.sep]: with self.subTest(relative_path_string=relative_path_string): invalid_parent_dir = os.path.join(src_dir2, relative_path_string) with self.assertRaises(InvalidUrlError): self.RunCommand('cp', [ '-R', src_dir1, invalid_parent_dir, suri(dst_bucket_uri, 'dst_subdir'), ]) def testRecursiveCopyObjsAndFilesToNonExistentBucketSubDir(self): """Tests recursive copy of objs + files to non-existent bucket subdir.""" src_bucket_uri = self.CreateBucket(test_objects=['f0', 'nested/f1']) src_dir = self.CreateTempDir(test_files=[ ('parent', 'f2'), ('parent', 'nested', 'f3'), ]) dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', [ '-R', os.path.join(src_dir, 'parent'), suri(src_bucket_uri), suri(dst_bucket_uri, 'dst_subdir') ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'dst_subdir', src_bucket_uri.bucket_name, 'f0'), suri(dst_bucket_uri, 'dst_subdir', src_bucket_uri.bucket_name, 'nested', 'f1'), suri(dst_bucket_uri, 'dst_subdir', 'parent', 'f2'), suri(dst_bucket_uri, 'dst_subdir', 'parent', 'nested', 'f3') ]) self.assertEqual(expected, actual) def testRecursiveCopyNestedObjsToNonExistentBucketSubDir(self): """Tests recursive copy of objs + files to non-existent bucket subdir.""" src_bucket_uri = self.CreateBucket( test_objects=['parent/f0', 'parent/nested/f1']) dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', [ '-R', suri(src_bucket_uri, 'parent'), suri(dst_bucket_uri, 'dst_subdir') ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'dst_subdir', 'f0'), suri(dst_bucket_uri, 'dst_subdir', 'nested', 'f1'), ]) self.assertEqual(expected, actual) def testRecursiveCopyFilesToNonExistentBucketSubDir(self): """Tests recursive copy of objs + files to non-existent bucket subdir.""" src_dir = self.CreateTempDir(test_files=['f2', ('nested', 'f3')]) dst_bucket_uri = self.CreateBucket() self.RunCommand('cp', ['-R', src_dir, suri(dst_bucket_uri, 'dst_subdir')]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'dst_subdir', 'f2'), suri(dst_bucket_uri, 'dst_subdir', 'nested', 'f3') ]) self.assertEqual(expected, actual) def testCopyingBucketSubDirToDir(self): """Tests copying a bucket subdir to a directory.""" src_bucket_uri = self.CreateBucket(test_objects=['src_subdir/obj']) dst_dir = self.CreateTempDir() # Test with and without final slash on dest subdir. for (final_src_char, final_dst_char) in (('', ''), ('', '/'), ('/', ''), ('/', '/')): # Mock objects don't support hash digestion. with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): self.RunCommand('cp', [ '-R', suri(src_bucket_uri, 'src_subdir') + final_src_char, dst_dir + final_dst_char ]) actual = set( str(u) for u in self._test_wildcard_iterator( '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_dir, 'src_subdir', 'obj')]) self.assertEqual(expected, actual) def testCopyingWildcardSpecifiedBucketSubDirToExistingDir(self): """Tests copying a wildcard-specified bucket subdir to a directory.""" src_bucket_uri = self.CreateBucket(test_objects=[ 'src_sub0dir/foo', 'src_sub1dir/foo', 'src_sub2dir/foo', 'src_sub3dir/foo' ]) dst_dir = self.CreateTempDir() # Test with and without final slash on dest subdir. for i, (final_src_char, final_dst_char) in enumerate( (('', ''), ('', '/'), ('/', ''), ('/', '/'))): # Mock objects don't support hash digestion. with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): self.RunCommand('cp', [ '-R', suri(src_bucket_uri, 'src_sub%d*' % i) + final_src_char, dst_dir + final_dst_char ]) actual = set( str(u) for u in self._test_wildcard_iterator( os.path.join(dst_dir, 'src_sub%ddir' % i, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_dir, 'src_sub%ddir' % i, 'foo')]) self.assertEqual(expected, actual) def testCopyingBucketSubDirToDirFailsWithoutMinusR(self): """Tests for failure when attempting bucket subdir copy without -R.""" src_bucket_uri = self.CreateBucket(test_objects=['src_subdir/obj']) dst_dir = self.CreateTempDir() try: self.RunCommand('cp', [suri(src_bucket_uri, 'src_subdir'), dst_dir]) self.fail('Did not get expected CommandException') except CommandException as e: self.assertIn(NO_URLS_MATCHED_GENERIC, e.reason) def testCopyingBucketSubDirToBucketSubDir(self): """Tests copying a bucket subdir to another bucket subdir.""" src_bucket_uri = self.CreateBucket( test_objects=['src_subdir_%d/obj' % i for i in range(4)]) dst_bucket_uri = self.CreateBucket( test_objects=['dst_subdir_%d/obj2' % i for i in range(4)]) # Test with and without final slash on dest subdir. for i, (final_src_char, final_dst_char) in enumerate( (('', ''), ('', '/'), ('/', ''), ('/', '/'))): self.RunCommand('cp', [ '-R', suri(src_bucket_uri, 'src_subdir_%d' % i) + final_src_char, suri(dst_bucket_uri, 'dst_subdir_%d' % i) + final_dst_char ]) actual = set( str(u) for u in self._test_wildcard_iterator( suri(dst_bucket_uri, 'dst_subdir_%d' % i, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'dst_subdir_%d' % i, 'src_subdir_%d' % i, 'obj'), suri(dst_bucket_uri, 'dst_subdir_%d' % i, 'obj2') ]) self.assertEqual(expected, actual) def testCopyingBucketSubDirToBucketSubDirWithNested(self): """Tests copying a bucket subdir to another bucket subdir with nesting.""" src_bucket_uri = self.CreateBucket( test_objects=['src_subdir_%d/obj' % i for i in range(4)] + ['src_subdir_%d/nested/obj' % i for i in range(4)]) dst_bucket_uri = self.CreateBucket( test_objects=['dst_subdir_%d/obj2' % i for i in range(4)]) # Test with and without final slash on dest subdir. for i, (final_src_char, final_dst_char) in enumerate( (('', ''), ('', '/'), ('/', ''), ('/', '/'))): self.RunCommand('cp', [ '-R', suri(src_bucket_uri, 'src_subdir_%d' % i) + final_src_char, suri(dst_bucket_uri, 'dst_subdir_%d' % i) + final_dst_char ]) actual = set( str(u) for u in self._test_wildcard_iterator( suri(dst_bucket_uri, 'dst_subdir_%d' % i, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'dst_subdir_%d' % i, 'src_subdir_%d' % i, 'obj'), suri(dst_bucket_uri, 'dst_subdir_%d' % i, 'src_subdir_%d' % i, 'nested', 'obj'), suri(dst_bucket_uri, 'dst_subdir_%d' % i, 'obj2') ]) self.assertEqual(expected, actual) def testMovingBucketSubDirToExistingBucketSubDir(self): """Tests moving a bucket subdir to a existing bucket subdir.""" src_objs = ['foo'] for i in range(4): src_objs.extend(['src_subdir%d/foo2' % i, 'src_subdir%d/nested/foo3' % i]) src_bucket_uri = self.CreateBucket(test_objects=src_objs) dst_bucket_uri = self.CreateBucket( test_objects=['dst_subdir%d/existing' % i for i in range(4)]) # Test with and without final slash on dest subdir. for i, (final_src_char, final_dst_char) in enumerate( (('', ''), ('', '/'), ('/', ''), ('/', '/'))): self.RunCommand('mv', [ suri(src_bucket_uri, 'src_subdir%d' % i) + final_src_char, suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_dst_char ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set() for i in range(4): expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing')) expected.add( suri(dst_bucket_uri, 'dst_subdir%d' % i, 'src_subdir%d' % i, 'foo2')) expected.add( suri(dst_bucket_uri, 'dst_subdir%d' % i, 'src_subdir%d' % i, 'nested', 'foo3')) self.assertEqual(expected, actual) def testCopyingObjectToBucketSubDir(self): """Tests copying an object to a bucket subdir.""" src_bucket_uri = self.CreateBucket(test_objects=['obj0']) dst_bucket_uri = self.CreateBucket( test_objects=['dir0/existing', 'dir1/existing']) # Test with and without final slash on dest subdir. for i, final_dst_char in enumerate(('', '/')): self.RunCommand('cp', [ suri(src_bucket_uri, 'obj0'), suri(dst_bucket_uri, 'dir%d' % i) + final_dst_char ]) actual = set( str(u) for u in self._test_wildcard_iterator( suri(dst_bucket_uri, 'dir%d' % i, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'dir%d' % i, 'obj0'), suri(dst_bucket_uri, 'dir%d' % i, 'existing') ]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingWildcardedFilesToBucketSubDir(self): """Tests copying wildcarded files to a bucket subdir.""" dst_bucket_uri = self.CreateBucket( test_objects=['subdir0/existing', 'subdir1/existing']) src_dir = self.CreateTempDir(test_files=['f0', 'f1', 'f2']) # Test with and without final slash on dest subdir. for i, final_dst_char in enumerate(('', '/')): self.RunCommand('cp', [ os.path.join(src_dir, 'f?'), suri(dst_bucket_uri, 'subdir%d' % i) + final_dst_char ]) actual = set( str(u) for u in self._test_wildcard_iterator( suri(dst_bucket_uri, 'subdir%d' % i, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'subdir%d' % i, 'existing'), suri(dst_bucket_uri, 'subdir%d' % i, 'f0'), suri(dst_bucket_uri, 'subdir%d' % i, 'f1'), suri(dst_bucket_uri, 'subdir%d' % i, 'f2') ]) self.assertEqual(expected, actual) # @SequentialAndParallelTransfer def testCopyingOneNestedFileToBucketSubDir(self): """Tests copying one nested file to a bucket subdir.""" dst_bucket_uri = self.CreateBucket( test_objects=['d0/placeholder', 'd1/placeholder']) src_dir = self.CreateTempDir(test_files=[('d3', 'd4', 'nested', 'f1')]) # Test with and without final slash on dest subdir. for i, final_dst_char in enumerate(('', '/')): self.RunCommand('cp', [ '-r', suri(src_dir, 'd3'), suri(dst_bucket_uri, 'd%d' % i) + final_dst_char ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'd0', 'placeholder'), suri(dst_bucket_uri, 'd1', 'placeholder'), suri(dst_bucket_uri, 'd0', 'd3', 'd4', 'nested', 'f1'), suri(dst_bucket_uri, 'd1', 'd3', 'd4', 'nested', 'f1') ]) self.assertEqual(expected, actual) def testMovingWildcardedFilesToNonExistentBucketSubDir(self): """Tests moving files to a non-existent bucket subdir.""" # This tests for how we allow users to do something like: # gsutil cp *.txt gs://bucket/dir # where *.txt matches more than 1 file and gs://bucket/dir # doesn't exist as a subdir. # src_bucket_uri = self.CreateBucket( test_objects=['f0f0', 'f0f1', 'f1f0', 'f1f1']) dst_bucket_uri = self.CreateBucket( test_objects=['dst_subdir0/existing_obj', 'dst_subdir1/existing_obj']) # Test with and without final slash on dest subdir. for i, final_dst_char in enumerate(('', '/')): # Copy some files into place in dst bucket. self.RunCommand('cp', [ suri(src_bucket_uri, 'f%df*' % i), suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_dst_char ]) # Now do the move test. self.RunCommand('mv', [ suri(src_bucket_uri, 'f%d*' % i), suri(dst_bucket_uri, 'nonexisting%d' % i) + final_dst_char ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'dst_subdir0', 'existing_obj'), suri(dst_bucket_uri, 'dst_subdir0', 'f0f0'), suri(dst_bucket_uri, 'dst_subdir0', 'f0f1'), suri(dst_bucket_uri, 'nonexisting0', 'f0f0'), suri(dst_bucket_uri, 'nonexisting0', 'f0f1'), suri(dst_bucket_uri, 'dst_subdir1', 'existing_obj'), suri(dst_bucket_uri, 'dst_subdir1', 'f1f0'), suri(dst_bucket_uri, 'dst_subdir1', 'f1f1'), suri(dst_bucket_uri, 'nonexisting1', 'f1f0'), suri(dst_bucket_uri, 'nonexisting1', 'f1f1') ]) self.assertEqual(expected, actual) def testMovingObjectToBucketSubDir(self): """Tests moving an object to a bucket subdir.""" src_bucket_uri = self.CreateBucket(test_objects=['obj0', 'obj1']) dst_bucket_uri = self.CreateBucket( test_objects=['dst_subdir0/existing_obj', 'dst_subdir1/existing_obj']) # Test with and without final slash on dest subdir. for i, final_dst_char in enumerate(('', '/')): self.RunCommand('mv', [ suri(src_bucket_uri, 'obj%d' % i), suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_dst_char ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([ suri(dst_bucket_uri, 'dst_subdir0', 'existing_obj'), suri(dst_bucket_uri, 'dst_subdir0', 'obj0'), suri(dst_bucket_uri, 'dst_subdir1', 'existing_obj'), suri(dst_bucket_uri, 'dst_subdir1', 'obj1') ]) self.assertEqual(expected, actual) actual = set( str(u) for u in self._test_wildcard_iterator(suri( src_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) self.assertEqual(actual, set()) def testMovingBucketSubDirToNonExistentBucketSubDir(self): """Tests moving a bucket subdir to a non-existent bucket subdir.""" src_bucket = self.CreateBucket(test_objects=[ 'foo', 'src_subdir0/foo2', 'src_subdir0/nested/foo3', 'src_subdir1/foo2', 'src_subdir1/nested/foo3' ]) dst_bucket = self.CreateBucket() # Test with and without final slash on dest subdir. for i, final_src_char in enumerate(('', '/')): self.RunCommand('mv', [ suri(src_bucket, 'src_subdir%d' % i) + final_src_char, suri(dst_bucket, 'dst_subdir%d' % i) ]) actual = set( str(u) for u in self._test_wildcard_iterator(suri(dst_bucket, '**')).IterAll( expand_top_level_buckets=True)) # Unlike the case with copying, with mv we expect renaming to occur # at the level of the src subdir, vs appending that subdir beneath the # dst subdir like is done for copying. expected = set([ suri(dst_bucket, 'dst_subdir0', 'foo2'), suri(dst_bucket, 'dst_subdir1', 'foo2'), suri(dst_bucket, 'dst_subdir0', 'nested', 'foo3'), suri(dst_bucket, 'dst_subdir1', 'nested', 'foo3') ]) self.assertEqual(expected, actual) def testRemovingBucketSubDir(self): """Tests removing a bucket subdir.""" dst_bucket_uri = self.CreateBucket(test_objects=[ 'f0', 'dir0/f1', 'dir0/nested/f2', 'dir1/f1', 'dir1/nested/f2' ]) # Test with and without final slash on dest subdir. for i, final_src_char in enumerate(('', '/')): # Test removing bucket subdir. self.RunCommand( 'rm', ['-R', suri(dst_bucket_uri, 'dir%d' % i) + final_src_char]) actual = set( str(u) for u in self._test_wildcard_iterator(suri( dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)) expected = set([suri(dst_bucket_uri, 'f0')]) self.assertEqual(expected, actual) def testRecursiveRemoveObjsInBucket(self): """Tests removing all objects in bucket via rm -R gs://bucket.""" bucket_uris = [ self.CreateBucket(test_objects=['f0', 'dir/f1', 'dir/nested/f2']), self.CreateBucket(test_objects=['f0', 'dir/f1', 'dir/nested/f2']) ] # Test with and without final slash on dest subdir. for i, final_src_char in enumerate(('', '/')): # Test removing all objects via rm -R. self.RunCommand('rm', ['-R', suri(bucket_uris[i]) + final_src_char]) try: self.RunCommand('ls', [suri(bucket_uris[i])]) # Ensure exception is raised. self.assertTrue(False) except NotFoundException as e: self.assertEqual(e.status, 404) def testUnicodeArgs(self): """Tests that you can list an object with unicode characters.""" object_name = 'フォ' bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name=object_name, contents=b'foo') stdout = self.RunCommand('ls', [suri(bucket_uri, object_name)], return_stdout=True) self.assertIn(six.ensure_text(object_name), six.ensure_text(stdout)) def testRecursiveListTrailingSlash(self): bucket_uri = self.CreateBucket() obj_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='/', contents=b'foo') stdout = self.RunCommand('ls', ['-R', suri(bucket_uri)], return_stdout=True) # Note: The suri function normalizes the URI, so the double slash gets # removed. self.assertEqual(stdout.splitlines(), [(suri(obj_uri) + '/:'), (suri(obj_uri) + '/')]) def FinalObjNameComponent(self, uri): """For gs://bucket/abc/def/ghi returns ghi.""" return uri.uri.rpartition('/')[-1] def testFileContainingColon(self): url_str = 'abc:def' url = StorageUrlFromString(url_str) self.assertEqual('file', url.scheme) self.assertEqual('file://%s' % url_str, url.url_string) # TODO: These should all be moved to their own test_*.py testing files. class GsUtilCommandTests(testcase.GsUtilUnitTestCase): """Basic sanity check tests to make sure commands run.""" def testDisableLoggingCommandRuns(self): """Test that the 'logging set off' command basically runs.""" src_bucket_uri = self.CreateBucket() self.RunCommand('logging', ['set', 'off', suri(src_bucket_uri)]) def testEnableLoggingCommandRuns(self): """Test that the 'logging set on' command basically runs.""" src_bucket_uri = self.CreateBucket() self.RunCommand( 'logging', ['set', 'on', '-b', 'gs://log_bucket', suri(src_bucket_uri)]) def testHelpCommandDoesntRaise(self): """Test that the help command doesn't raise (sanity checks all help).""" # Unset PAGER if defined, so help output paginating into $PAGER doesn't # cause test to pause. if 'PAGER' in os.environ: del os.environ['PAGER'] self.RunCommand('help', []) def testCatCommandRuns(self): """Test that the cat command basically runs.""" src_uri = self.CreateObject(contents='foo') stdout = self.RunCommand('cat', [suri(src_uri)], return_stdout=True) self.assertEqual(stdout, 'foo') def testGetLoggingCommandRuns(self): """Test that the 'logging get' command basically runs.""" src_bucket_uri = self.CreateBucket() self.RunCommand('logging', ['get', suri(src_bucket_uri)]) def testMakeBucketsCommand(self): """Test mb on existing bucket.""" dst_bucket_uri = self.CreateBucket() try: with SetDummyProjectForUnitTest(): self.RunCommand('mb', [suri(dst_bucket_uri)]) self.fail('Did not get expected StorageCreateError') except ServiceException as e: self.assertEqual(e.status, 409) def testRemoveObjsCommand(self): """Test rm command on non-existent object.""" dst_bucket_uri = self.CreateBucket() try: self.RunCommand('rm', [suri(dst_bucket_uri, 'non_existent')]) self.fail('Did not get expected CommandException') except CommandException as e: self.assertIn(NO_URLS_MATCHED_TARGET % dst_bucket_uri, e.reason) # Now that gsutil ver computes a checksum it adds 1-3 seconds to test run # time (for in memory mocked tests that otherwise take ~ 0.1 seconds). Since # it provides very little test value, we're leaving this test commented out. # def testVerCommmandRuns(self): # """Test that the Ver command basically runs""" # self.RunCommand('ver', []) tests/test_ubla.py 0000644 00000007607 15025013616 0010253 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for ubla command.""" from __future__ import absolute_import import re from gslib.cs_api_map import ApiSelector import gslib.tests.testcase as testcase from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import unittest from gslib.utils.retry_util import Retry class TestUbla(testcase.GsUtilIntegrationTestCase): """Integration tests for ubla command.""" _set_ubla_cmd = ['ubla', 'set'] _get_ubla_cmd = ['ubla', 'get'] def _AssertEnabled(self, bucket_uri, value): stdout = self.RunGsUtil(self._get_ubla_cmd + [suri(bucket_uri)], return_stdout=True) uniform_bucket_level_access_re = re.compile( r'^\s*Enabled:\s+(?P<enabled_val>.+)$', re.MULTILINE) uniform_bucket_level_access_match = re.search( uniform_bucket_level_access_re, stdout) uniform_bucket_level_access_val = uniform_bucket_level_access_match.group( 'enabled_val') self.assertEqual(str(value), uniform_bucket_level_access_val) def test_off_on_default_buckets(self): if self.test_api == ApiSelector.XML: return unittest.skip( 'XML API has no concept of Uniform bucket-level access') bucket_uri = self.CreateBucket() self._AssertEnabled(bucket_uri, False) def test_turning_off_on_enabled_buckets(self): if self.test_api == ApiSelector.XML: return unittest.skip( 'XML API has no concept of Uniform bucket-level access') # TODO(mynameisrafe): Replace bucket_policy_only with uniform_bucket_level_access when the property is live. bucket_uri = self.CreateBucket(bucket_policy_only=True, prefer_json_api=True) self._AssertEnabled(bucket_uri, True) self.RunGsUtil(self._set_ubla_cmd + ['off', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, False) def test_turning_on(self): if self.test_api == ApiSelector.XML: return unittest.skip( 'XML API has no concept of Uniform bucket-level access') bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_ubla_cmd + ['on', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, True) def test_turning_on_and_off(self): if self.test_api == ApiSelector.XML: return unittest.skip( 'XML API has no concept of Uniform bucket-level access') bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_ubla_cmd + ['on', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, True) self.RunGsUtil(self._set_ubla_cmd + ['off', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, False) def testTooFewArgumentsFails(self): """Ensures ubla commands fail with too few arguments.""" # No arguments for set, but valid subcommand. stderr = self.RunGsUtil(self._set_ubla_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for get, but valid subcommand. stderr = self.RunGsUtil(self._get_ubla_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Neither arguments nor subcommand. stderr = self.RunGsUtil(['ubla'], return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) tests/test_storage_url.py 0000644 00000016027 15025013616 0011652 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. . """Unit tests for storage URLs.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import sys from gslib.exception import CommandException from gslib import storage_url from gslib.exception import InvalidUrlError from gslib.tests.testcase import base from unittest import mock _UNSUPPORTED_DOUBLE_WILDCARD_WARNING_TEXT = ( '** behavior is undefined if directly preceeded or followed by' ' with characters other than / in the cloud and {} locally.'.format(os.sep)) class TestStorageUrl(base.GsUtilTestCase): """Unit tests for storage URLs.""" def setUp(self): super(TestStorageUrl, self).setUp() def test_is_file_url_string(self): self.assertTrue(storage_url.IsFileUrlString('abc')) self.assertTrue(storage_url.IsFileUrlString('file://abc')) self.assertFalse(storage_url.IsFileUrlString('gs://abc')) self.assertFalse(storage_url.IsFileUrlString('s3://abc')) def test_storage_url_from_string(self): url = storage_url.StorageUrlFromString('abc') self.assertTrue(url.IsFileUrl()) self.assertEqual('abc', url.object_name) url = storage_url.StorageUrlFromString('file://abc/123') self.assertTrue(url.IsFileUrl()) self.assertEqual('abc%s123' % os.sep, url.object_name) url = storage_url.StorageUrlFromString('gs://abc/123/456') self.assertTrue(url.IsCloudUrl()) self.assertEqual('abc', url.bucket_name) self.assertEqual('123/456', url.object_name) url = storage_url.StorageUrlFromString('gs://abc///:/') self.assertTrue(url.IsCloudUrl()) self.assertEqual('abc', url.bucket_name) self.assertEqual('//:/', url.object_name) url = storage_url.StorageUrlFromString('s3://abc/123/456') self.assertTrue(url.IsCloudUrl()) self.assertEqual('abc', url.bucket_name) self.assertEqual('123/456', url.object_name) def test_raises_error_for_too_many_slashes_after_scheme(self): with self.assertRaises(InvalidUrlError): storage_url.StorageUrlFromString('gs:///') with self.assertRaises(InvalidUrlError): storage_url.StorageUrlFromString('gs://////') @mock.patch.object(sys.stderr, 'write', autospec=True) def test_does_not_warn_if_supported_double_wildcard(self, mock_stderr): storage_url.StorageUrlFromString('**') storage_url.StorageUrlFromString('gs://bucket/**') storage_url.StorageUrlFromString('**' + os.sep) storage_url.StorageUrlFromString('gs://bucket/**/') storage_url.StorageUrlFromString(os.sep + '**') storage_url.StorageUrlFromString('gs://bucket//**') storage_url.StorageUrlFromString(os.sep + '**' + os.sep) mock_stderr.assert_not_called() @mock.patch.object(sys.stderr, 'write', autospec=True) def test_warns_if_unsupported_double_wildcard(self, mock_stderr): storage_url.StorageUrlFromString('abc**') storage_url.StorageUrlFromString('gs://bucket/object**') storage_url.StorageUrlFromString('**abc') storage_url.StorageUrlFromString('gs://bucket/**object') storage_url.StorageUrlFromString('abc**' + os.sep) storage_url.StorageUrlFromString('gs://bucket/object**/') storage_url.StorageUrlFromString(os.sep + '**abc') storage_url.StorageUrlFromString('gs://bucket//**object') storage_url.StorageUrlFromString(os.sep + '**' + os.sep + 'abc**') storage_url.StorageUrlFromString('gs://bucket/**/abc**') storage_url.StorageUrlFromString('abc**' + os.sep + 'abc') storage_url.StorageUrlFromString('gs://bucket/abc**/abc') storage_url.StorageUrlFromString(os.sep + 'abc**' + os.sep + '**') storage_url.StorageUrlFromString('gs://bucket/abc**/**') storage_url.StorageUrlFromString('gs://b**') storage_url.StorageUrlFromString('gs://**b') mock_stderr.assert_has_calls( [mock.call(_UNSUPPORTED_DOUBLE_WILDCARD_WARNING_TEXT)] * 14) def test_urls_are_mix_of_objects_and_buckets_is_false_for_all_buckets(self): urls = list(map(storage_url.StorageUrlFromString, ['gs://b1', 'gs://b2'])) self.assertFalse(storage_url.UrlsAreMixOfBucketsAndObjects(urls)) def test_urls_are_mix_of_objects_and_buckets_is_false_for_all_objects(self): urls = list(map(storage_url.StorageUrlFromString, ['gs://b/o', 'gs://b/p'])) self.assertFalse(storage_url.UrlsAreMixOfBucketsAndObjects(urls)) def test_urls_are_mix_of_objects_and_buckets_is_true_for_a_mix(self): urls = list(map(storage_url.StorageUrlFromString, ['gs://b/o', 'gs://b'])) self.assertTrue(storage_url.UrlsAreMixOfBucketsAndObjects(urls)) def test_urls_are_mix_of_objects_and_buckets_is_null_for_invalid(self): urls = list(map(storage_url.StorageUrlFromString, ['gs://b', 'f:o@o:o'])) self.assertIsNone(storage_url.UrlsAreMixOfBucketsAndObjects(urls)) def test_urls_raise_error_if_bucket_followed_by_object(self): urls = list(map(storage_url.StorageUrlFromString, ['gs://b1', 'gs://b/o'])) with self.assertRaisesRegex( CommandException, 'Cannot operate on a mix of buckets and objects.'): storage_url.RaiseErrorIfUrlsAreMixOfBucketsAndObjects( urls, recursion_requested=False) def test_urls_raise_error_if_object_followed_by_bucket(self): urls = list(map(storage_url.StorageUrlFromString, ['gs://b/o', 'gs://b'])) with self.assertRaisesRegex( CommandException, 'Cannot operate on a mix of buckets and objects.'): storage_url.RaiseErrorIfUrlsAreMixOfBucketsAndObjects( urls, recursion_requested=False) def test_accepts_mix_of_objects_and_buckets_if_recursion_requested(self): # No error raised. urls = list(map(storage_url.StorageUrlFromString, ['gs://b1', 'gs://b/o'])) storage_url.RaiseErrorIfUrlsAreMixOfBucketsAndObjects( urls, recursion_requested=True) def test_not_raising_error_if_multiple_objects_without_recursion(self): urls = list(map(storage_url.StorageUrlFromString, ['gs://b/o', 'gs://b/p'])) storage_url.RaiseErrorIfUrlsAreMixOfBucketsAndObjects( urls, recursion_requested=False) def test_not_raising_error_if_multiple_buckets_with_recursion(self): urls = list(map(storage_url.StorageUrlFromString, ['gs://b/o', 'gs://b/p'])) storage_url.RaiseErrorIfUrlsAreMixOfBucketsAndObjects( urls, recursion_requested=True) def test_not_raising_error_if_multiple_objects_with_recursion(self): urls = list(map(storage_url.StorageUrlFromString, ['gs://b/o', 'gs://b/p'])) storage_url.RaiseErrorIfUrlsAreMixOfBucketsAndObjects( urls, recursion_requested=True) tests/test_ui.py 0000644 00000206656 15025013616 0007752 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Tests for gsutil UI functions.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import pickle import crcmod import six from six.moves import queue as Queue from gslib.cs_api_map import ApiSelector from gslib.parallel_tracker_file import ObjectFromTracker from gslib.parallel_tracker_file import WriteParallelUploadTrackerFile from gslib.storage_url import StorageUrlFromString import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import HaltingCopyCallbackHandler from gslib.tests.util import HaltOneComponentCopyCallbackHandler from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import TailSet from gslib.tests.util import TEST_ENCRYPTION_KEY1 from gslib.tests.util import TEST_ENCRYPTION_KEY2 from gslib.tests.util import unittest from gslib.thread_message import FileMessage from gslib.thread_message import FinalMessage from gslib.thread_message import MetadataMessage from gslib.thread_message import ProducerThreadMessage from gslib.thread_message import ProgressMessage from gslib.thread_message import SeekAheadMessage from gslib.tracker_file import DeleteTrackerFile from gslib.tracker_file import GetSlicedDownloadTrackerFilePaths from gslib.tracker_file import GetTrackerFilePath from gslib.tracker_file import TrackerFileType from gslib.ui_controller import BytesToFixedWidthString from gslib.ui_controller import DataManager from gslib.ui_controller import MainThreadUIQueue from gslib.ui_controller import MetadataManager from gslib.ui_controller import UIController from gslib.ui_controller import UIThread from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.constants import START_CALLBACK_PER_BYTES from gslib.utils.constants import UTF8 from gslib.utils.copy_helper import PARALLEL_UPLOAD_STATIC_SALT from gslib.utils.copy_helper import PARALLEL_UPLOAD_TEMP_NAMESPACE from gslib.utils.hashing_helper import GetMd5 from gslib.utils.parallelism_framework_util import PutToQueueWithTimeout from gslib.utils.parallelism_framework_util import ZERO_TASKS_TO_DO_ARGUMENT from gslib.utils.retry_util import Retry from gslib.utils.unit_util import HumanReadableWithDecimalPlaces from gslib.utils.unit_util import MakeHumanReadable from gslib.utils.unit_util import ONE_KIB DOWNLOAD_SIZE = 300 UPLOAD_SIZE = 400 # Ensures at least one progress callback is made HALT_SIZE = START_CALLBACK_PER_BYTES * 2 # After waiting this long, assume the UIThread is hung. THREAD_WAIT_TIME = 5 def JoinThreadAndRaiseOnTimeout(ui_thread, thread_wait_time=THREAD_WAIT_TIME): """Joins the ui_thread and ensures it has not timed out. Args: ui_thread: the UIThread to be joined. thread_wait_time: the time to wait to join Raises: Exception: Warns UIThread is still alive. """ ui_thread.join(thread_wait_time) if ui_thread.is_alive(): raise Exception('UIThread is still alive') def _FindAppropriateDescriptionString(metadata): """Returns the correspondent string (objects or files) for the operation type. Args: metadata: Describes whether this is a metadata operation. Returns: ' objects' if a metadata operation; ' files' otherwise. """ return ' objects' if metadata else ' files' # TODO: migrate CheckUiOutput functions to integration_testcase # and call them directly from the adapted tests so we do not have to duplicate # code. def CheckUiOutputWithMFlag(test_case, content, num_objects, total_size=0, metadata=False): """Checks if the UI output works as expected with the -m flag enabled. Args: test_case: Testcase used to maintain the same assert structure. content: The output provided by the UI. num_objects: The number of objects processed. total_size: The total size transferred in the operation. Used for data operations only. metadata: Indicates whether this is a metadata operation. """ description_string = _FindAppropriateDescriptionString(metadata) # We must have transferred 100% of our data. test_case.assertIn('100% Done', content) # All files should be completed. files_completed_string = str(num_objects) + '/' + str(num_objects) test_case.assertIn(files_completed_string + description_string, content) final_message = 'Operation completed over %s objects' % num_objects if not metadata: # The total_size must also been successfully transferred. total_size_string = BytesToFixedWidthString(total_size) test_case.assertIn(total_size_string + '/' + total_size_string, content) final_message += '/%s' % HumanReadableWithDecimalPlaces(total_size) test_case.assertIn(final_message, content) def CheckUiOutputWithNoMFlag(test_case, content, num_objects, total_size=0, metadata=False): """Checks if the UI output works as expected with the -m flag not enabled. Args: test_case: Testcase used to maintain the same assert structure. content: The output provided by the UI. num_objects: The number of objects processed. total_size: The total size transferred in the operation. Used for data operations only. metadata: Indicates whether this is a metadata operation. """ description_string = _FindAppropriateDescriptionString(metadata) # All files should be completed. files_completed_string = str(num_objects) test_case.assertIn(files_completed_string + description_string, content) final_message = 'Operation completed over %s objects' % num_objects if not metadata: # The total_size must also been successfully transferred. total_size_string = BytesToFixedWidthString(total_size) test_case.assertIn(total_size_string + '/' + total_size_string, content) final_message += '/%s' % HumanReadableWithDecimalPlaces(total_size) test_case.assertIn(final_message, content) def CheckBrokenUiOutputWithMFlag(test_case, content, num_objects, total_size=0, metadata=False): """Checks if the UI output fails as expected with the -m flag enabled. Args: test_case: Testcase used to maintain the same assert structure. content: The output provided by the UI. num_objects: The number of objects processed. total_size: The total size transferred in the operation. Used for data operations only. metadata: Indicates whether this is a metadata operation. """ description_string = _FindAppropriateDescriptionString(metadata) # We must not have transferred 100% of our data. test_case.assertNotIn('100% Done', content) # We cannot have completed a file. files_completed_string = str(num_objects) + '/' + str(num_objects) test_case.assertNotIn(files_completed_string + description_string, content) if not metadata: total_size_string = BytesToFixedWidthString(total_size) zero = BytesToFixedWidthString(0) # Zero bytes must have been transferred in the beginning. test_case.assertIn(zero + '/' + total_size_string, content) # The total_size must have not been successfully transferred. test_case.assertNotIn(total_size_string + '/' + total_size_string, content) final_message_prefix = 'Operation completed over' test_case.assertNotIn(final_message_prefix, content) def CheckBrokenUiOutputWithNoMFlag(test_case, content, num_objects, total_size=0, metadata=False): """Checks if the UI output fails as expected with the -m flag not enabled. Args: test_case: Testcase used to maintain the same assert structure. content: The output provided by the UI. num_objects: The number of objects processed. total_size: The total size transferred in the operation. Used for data operations only. metadata: Indicates whether this is a metadata operation. """ description_string = _FindAppropriateDescriptionString(metadata) # 0 files should be completed. no_files_string = str(0) test_case.assertIn(no_files_string + description_string, content) # We cannot have completed a file. files_completed_string = str(num_objects) test_case.assertNotIn(files_completed_string + description_string, content) if not metadata: total_size_string = BytesToFixedWidthString(total_size) zero = BytesToFixedWidthString(0) # Zero bytes must have been transferred in the beginning. test_case.assertIn(zero + '/' + total_size_string, content) # The total_size must have not been successfully transferred. test_case.assertNotIn(total_size_string + '/' + total_size_string, content) final_message_prefix = 'Operation completed over' test_case.assertNotIn(final_message_prefix, content) class TestUi(testcase.GsUtilIntegrationTestCase): """Integration tests for UI functions.""" def test_ui_download_single_objects_with_m_flag(self): """Tests UI for a single object download with the -m flag enabled. This test indirectly tests the correctness of ProducerThreadMessage in the UIController. """ bucket_uri = self.CreateBucket() file_contents = b'd' * DOWNLOAD_SIZE object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=file_contents) fpath = self.CreateTempFile() stderr = self.RunGsUtil(['-m', 'cp', suri(object_uri), fpath], return_stderr=True) CheckUiOutputWithMFlag(self, stderr, 1, total_size=DOWNLOAD_SIZE) def test_ui_download_single_objects_with_no_m_flag(self): """Tests UI for a single object download with the -m flag not enabled. The UI should behave differently from the -m flag option because in the latter we have a ProducerThreadMessage that allows us to know our progress percentage and total number of files. """ bucket_uri = self.CreateBucket() file_contents = b'd' * DOWNLOAD_SIZE object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=file_contents) fpath = self.CreateTempFile() stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) CheckUiOutputWithNoMFlag(self, stderr, 1, total_size=DOWNLOAD_SIZE) def test_ui_upload_single_object_with_m_flag(self): """Tests UI for a single object upload with -m flag enabled. This test indirectly tests the correctness of ProducerThreadMessage in the UIController. """ bucket_uri = self.CreateBucket() file_contents = b'u' * UPLOAD_SIZE fpath = self.CreateTempFile(file_name='sample-file.txt', contents=file_contents) stderr = self.RunGsUtil( ['-m', 'cp', suri(fpath), suri(bucket_uri)], return_stderr=True) CheckUiOutputWithMFlag(self, stderr, 1, total_size=UPLOAD_SIZE) def test_ui_upload_single_object_with_no_m_flag(self): """Tests UI for a single object upload with -m flag not enabled. The UI should behave differently from the -m flag option because in the latter we have a ProducerThreadMessage that allows us to know our progress percentage and total number of files. """ bucket_uri = self.CreateBucket() file_contents = b'u' * UPLOAD_SIZE fpath = self.CreateTempFile(file_name='sample-file.txt', contents=file_contents) stderr = self.RunGsUtil( ['cp', suri(fpath), suri(bucket_uri)], return_stderr=True) CheckUiOutputWithNoMFlag(self, stderr, 1, total_size=UPLOAD_SIZE) def test_ui_download_multiple_objects_with_m_flag(self): """Tests UI for a multiple object download with the -m flag enabled. This test indirectly tests the correctness of ProducerThreadMessage in the UIController. """ bucket_uri = self.CreateBucket() num_objects = 7 argument_list = ['-m', 'cp'] total_size = 0 for i in range(num_objects): file_size = DOWNLOAD_SIZE // 3 file_contents = b'd' * file_size object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo' + str(i), contents=file_contents) total_size += file_size argument_list.append(suri(object_uri)) fpath = self.CreateTempDir() argument_list.append(fpath) stderr = self.RunGsUtil(argument_list, return_stderr=True) CheckUiOutputWithMFlag(self, stderr, num_objects, total_size=total_size) def test_ui_download_multiple_objects_with_no_m_flag(self): """Tests UI for a multiple object download with the -m flag not enabled. The UI should behave differently from the -m flag option because in the latter we have a ProducerThreadMessage that allows us to know our progress percentage and total number of files. """ bucket_uri = self.CreateBucket() num_objects = 7 argument_list = ['cp'] total_size = 0 for i in range(num_objects): file_size = DOWNLOAD_SIZE // 3 file_contents = b'd' * file_size object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo' + str(i), contents=file_contents) total_size += file_size argument_list.append(suri(object_uri)) fpath = self.CreateTempDir() argument_list.append(fpath) stderr = self.RunGsUtil(argument_list, return_stderr=True) CheckUiOutputWithNoMFlag(self, stderr, num_objects, total_size=total_size) def test_ui_upload_mutliple_objects_with_m_flag(self): """Tests UI for a multiple object upload with -m flag enabled. This test indirectly tests the correctness of ProducerThreadMessage in the UIController. """ bucket_uri = self.CreateBucket() num_objects = 7 argument_list = ['-m', 'cp'] total_size = 0 for i in range(num_objects): file_size = UPLOAD_SIZE // 3 file_contents = b'u' * file_size fpath = self.CreateTempFile(file_name='foo' + str(i), contents=file_contents) total_size += file_size argument_list.append(suri(fpath)) argument_list.append(suri(bucket_uri)) stderr = self.RunGsUtil(argument_list, return_stderr=True) CheckUiOutputWithMFlag(self, stderr, num_objects, total_size=total_size) def test_ui_upload_mutliple_objects_with_no_m_flag(self): """Tests UI for a multiple object upload with -m flag not enabled. The UI should behave differently from the -m flag option because in the latter we have a ProducerThreadMessage that allows us to know our progress percentage and total number of files. """ bucket_uri = self.CreateBucket() num_objects = 7 argument_list = ['cp'] total_size = 0 for i in range(num_objects): file_size = UPLOAD_SIZE // 3 file_contents = b'u' * file_size fpath = self.CreateTempFile(file_name='foo' + str(i), contents=file_contents) total_size += file_size argument_list.append(suri(fpath)) argument_list.append(suri(bucket_uri)) stderr = self.RunGsUtil(argument_list, return_stderr=True) CheckUiOutputWithNoMFlag(self, stderr, num_objects, total_size=total_size) @SkipForS3('No resumable upload support for S3.') def test_ui_resumable_upload_break_with_m_flag(self): """Tests UI for upload resumed after a connection break with -m flag. This was adapted from test_cp_resumable_upload_break. """ bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'a' * HALT_SIZE) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'parallel_composite_upload_component_size', str(ONE_KIB)) ] test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5))) with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ '-m', 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting upload', stderr) CheckBrokenUiOutputWithMFlag(self, stderr, 1, total_size=HALT_SIZE) stderr = self.RunGsUtil( ['-m', 'cp', fpath, suri(bucket_uri)], return_stderr=True) self.assertIn('Resuming upload', stderr) CheckUiOutputWithMFlag(self, stderr, 1, total_size=HALT_SIZE) @SkipForS3('No resumable upload support for S3.') def test_ui_resumable_upload_break_with_no_m_flag(self): """Tests UI for upload resumed after a connection break with no -m flag. This was adapted from test_cp_resumable_upload_break. """ bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'a' * HALT_SIZE) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'parallel_composite_upload_component_size', str(ONE_KIB)) ] test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5))) with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting upload', stderr) CheckBrokenUiOutputWithNoMFlag(self, stderr, 1, total_size=HALT_SIZE) stderr = self.RunGsUtil(['cp', fpath, suri(bucket_uri)], return_stderr=True) self.assertIn('Resuming upload', stderr) CheckUiOutputWithNoMFlag(self, stderr, 1, total_size=HALT_SIZE) def _test_ui_resumable_download_break_helper(self, boto_config, gsutil_flags=None): """Helper function for testing UI on a resumable download break. This was adapted from _test_cp_resumable_download_break_helper. Args: boto_config: List of boto configuration tuples for use with SetBotoConfigForTest. gsutil_flags: List of flags to run gsutil with, or None. """ if not gsutil_flags: gsutil_flags = [] bucket_uri = self.CreateBucket() file_contents = b'a' * HALT_SIZE object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=file_contents) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) with SetBotoConfigForTest(boto_config): gsutil_args = (gsutil_flags + [ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), fpath ]) stderr = self.RunGsUtil(gsutil_args, expected_status=1, return_stderr=True) self.assertIn('Artifically halting download.', stderr) if '-q' not in gsutil_flags: if '-m' in gsutil_flags: CheckBrokenUiOutputWithMFlag(self, stderr, 1, total_size=HALT_SIZE) else: CheckBrokenUiOutputWithNoMFlag(self, stderr, 1, total_size=HALT_SIZE) tracker_filename = GetTrackerFilePath(StorageUrlFromString(fpath), TrackerFileType.DOWNLOAD, self.test_api) self.assertTrue(os.path.isfile(tracker_filename)) gsutil_args = gsutil_flags + ['cp', suri(object_uri), fpath] stderr = self.RunGsUtil(gsutil_args, return_stderr=True) if '-q' not in gsutil_args: self.assertIn('Resuming download', stderr) with open(fpath, 'rb') as f: self.assertEqual(f.read(), file_contents, 'File contents differ') if '-q' in gsutil_flags: self.assertEqual('', stderr) elif '-m' in gsutil_flags: CheckUiOutputWithMFlag(self, stderr, 1, total_size=HALT_SIZE) else: CheckUiOutputWithNoMFlag(self, stderr, 1, total_size=HALT_SIZE) def test_ui_resumable_download_break_with_m_flag(self): """Tests UI on a resumable download break with -m flag. This was adapted from test_cp_resumable_download_break. """ self._test_ui_resumable_download_break_helper( [('GSUtil', 'resumable_threshold', str(ONE_KIB))], gsutil_flags=['-m']) def test_ui_resumable_download_break_with_no_m_flag(self): """Tests UI on a resumable download break with no -m flag. This was adapted from test_cp_resumable_download_break. """ self._test_ui_resumable_download_break_helper([ ('GSUtil', 'resumable_threshold', str(ONE_KIB)) ]) def test_ui_resumable_download_break_with_q_flag(self): """Tests UI on a resumable download break with -q flag but no -m flag. This was adapted from test_cp_resumable_download_break, and the UI output should be empty. """ self._test_ui_resumable_download_break_helper( [('GSUtil', 'resumable_threshold', str(ONE_KIB))], gsutil_flags=['-q']) def test_ui_resumable_download_break_with_q_and_m_flags(self): """Tests UI on a resumable download break with -q and -m flags. This was adapted from test_cp_resumable_download_break, and the UI output should be empty. """ self._test_ui_resumable_download_break_helper( [('GSUtil', 'resumable_threshold', str(ONE_KIB))], gsutil_flags=['-m', '-q']) def _test_ui_composite_upload_resume_helper(self, gsutil_flags=None): """Helps testing UI on a resumable upload with finished components. Args: gsutil_flags: List of flags to run gsutil with, or None. """ if not gsutil_flags: gsutil_flags = [] bucket_uri = self.CreateBucket() dst_url = StorageUrlFromString(suri(bucket_uri, 'foo')) file_contents = b'foobar' file_name = 'foobar' source_file = self.CreateTempFile(contents=file_contents, file_name=file_name) src_url = StorageUrlFromString(source_file) # Simulate an upload that had occurred by writing a tracker file # that points to a previously uploaded component. tracker_file_name = GetTrackerFilePath(dst_url, TrackerFileType.PARALLEL_UPLOAD, self.test_api, src_url) tracker_prefix = '123' # Create component 0 to be used in the resume; it must match the name # that will be generated in copy_helper, so we use the same scheme. encoded_name = (PARALLEL_UPLOAD_STATIC_SALT + source_file).encode(UTF8) content_md5 = GetMd5() content_md5.update(encoded_name) digest = content_md5.hexdigest() component_object_name = (tracker_prefix + PARALLEL_UPLOAD_TEMP_NAMESPACE + digest + '_0') component_size = 3 object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name=component_object_name, contents=file_contents[:component_size]) existing_component = ObjectFromTracker(component_object_name, str(object_uri.generation)) existing_components = [existing_component] WriteParallelUploadTrackerFile(tracker_file_name, tracker_prefix, existing_components) try: # Now "resume" the upload. with SetBotoConfigForTest([ ('GSUtil', 'parallel_composite_upload_threshold', '1'), ('GSUtil', 'parallel_composite_upload_component_size', str(component_size)) ]): gsutil_args = ( gsutil_flags + ['cp', source_file, suri(bucket_uri, 'foo')]) stderr = self.RunGsUtil(gsutil_args, return_stderr=True) self.assertIn('Found 1 existing temporary components to reuse.', stderr) self.assertFalse( os.path.exists(tracker_file_name), 'Tracker file %s should have been deleted.' % tracker_file_name) read_contents = self.RunGsUtil(['cat', suri(bucket_uri, 'foo')], return_stdout=True) self.assertEqual(read_contents.encode(UTF8), file_contents) if '-m' in gsutil_flags: CheckUiOutputWithMFlag(self, stderr, 1, total_size=len(file_contents)) else: CheckUiOutputWithNoMFlag(self, stderr, 1, total_size=len(file_contents)) finally: # Clean up if something went wrong. DeleteTrackerFile(tracker_file_name) @SkipForS3('No resumable upload support for S3.') def test_ui_composite_upload_resume_with_m_flag(self): """Tests UI on a resumable upload with finished components and -m flag.""" self._test_ui_composite_upload_resume_helper(gsutil_flags=['-m']) @SkipForS3('No resumable upload support for S3.') def test_ui_composite_upload_resume_with_no_m_flag(self): """Tests UI on a resumable upload with finished components and no -m flag. """ self._test_ui_composite_upload_resume_helper() @unittest.skipUnless(UsingCrcmodExtension(), 'Sliced download requires fast crcmod.') @SkipForS3('No sliced download support for S3.') def _test_ui_sliced_download_partial_resume_helper(self, gsutil_flags=None): """Helps testing UI for sliced download with some finished components. This was adapted from test_sliced_download_partial_resume_helper. Args: gsutil_flags: List of flags to run gsutil with, or None. """ if not gsutil_flags: gsutil_flags = [] bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'abc' * HALT_SIZE) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltOneComponentCopyCallbackHandler(5))) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(HALT_SIZE)), ('GSUtil', 'sliced_object_download_threshold', str(HALT_SIZE)), ('GSUtil', 'sliced_object_download_max_components', '3') ] with SetBotoConfigForTest(boto_config_for_test): gsutil_args = gsutil_flags + [ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), suri(fpath) ] stderr = self.RunGsUtil(gsutil_args, return_stderr=True, expected_status=1) if '-m' in gsutil_args: CheckBrokenUiOutputWithMFlag(self, stderr, 1, total_size=(len('abc') * HALT_SIZE)) else: CheckBrokenUiOutputWithNoMFlag(self, stderr, 1, total_size=(len('abc') * HALT_SIZE)) # Each tracker file should exist. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertTrue(os.path.isfile(tracker_filename)) gsutil_args = gsutil_flags + ['cp', suri(object_uri), fpath] stderr = self.RunGsUtil(gsutil_args, return_stderr=True) self.assertIn('Resuming download', stderr) self.assertIn('Download already complete', stderr) # Each tracker file should have been deleted. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertFalse(os.path.isfile(tracker_filename)) with open(fpath, 'r') as f: self.assertEqual(f.read(), 'abc' * HALT_SIZE, 'File contents differ') if '-m' in gsutil_args: CheckUiOutputWithMFlag(self, stderr, 1, total_size=(len('abc') * HALT_SIZE)) else: CheckUiOutputWithNoMFlag(self, stderr, 1, total_size=(len('abc') * HALT_SIZE)) @SkipForS3('No resumable upload support for S3.') def test_ui_sliced_download_partial_resume_helper_with_m_flag(self): """Tests UI on a resumable download with finished components and -m flag. """ self._test_ui_sliced_download_partial_resume_helper(gsutil_flags=['-m']) @SkipForS3('No resumable upload support for S3.') def _test_ui_sliced_download_partial_resume_helper_with_no_m_flag(self): """Tests UI on a resumable upload with finished components and no -m flag. """ self._test_ui_sliced_download_partial_resume_helper() def test_ui_hash_mutliple_objects_with_no_m_flag(self): """Tests UI for a multiple object hashing with no -m flag enabled. This test indirectly tests the correctness of ProducerThreadMessage in the UIController. """ num_objects = 7 argument_list = ['hash'] total_size = 0 for i in range(num_objects): file_size = UPLOAD_SIZE // 3 file_contents = b'u' * file_size fpath = self.CreateTempFile(file_name='foo' + str(i), contents=file_contents) total_size += file_size argument_list.append(suri(fpath)) stderr = self.RunGsUtil(argument_list, return_stderr=True) CheckUiOutputWithNoMFlag(self, stderr, num_objects, total_size) def test_ui_rewrite_with_m_flag(self): """Tests UI output for rewrite and -m flag. Adapted from test_rewrite_stdin_args. """ if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) stdin_arg = suri(object_uri) boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['-m', 'rewrite', '-k', '-I'], stdin=stdin_arg, return_stderr=True) self.AssertObjectUsesCSEK(stdin_arg, TEST_ENCRYPTION_KEY2) num_objects = 1 total_size = len(b'bar') CheckUiOutputWithMFlag(self, stderr, num_objects, total_size) def test_ui_rewrite_with_no_m_flag(self): """Tests UI output for rewrite and -m flag not enabled. Adapted from test_rewrite_stdin_args. """ if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') object_uri = self.CreateObject(contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) stdin_arg = suri(object_uri) boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['rewrite', '-k', '-I'], stdin=stdin_arg, return_stderr=True) self.AssertObjectUsesCSEK(stdin_arg, TEST_ENCRYPTION_KEY2) num_objects = 1 total_size = len(b'bar') CheckUiOutputWithNoMFlag(self, stderr, num_objects, total_size) def test_ui_setmeta_with_m_flag(self): """Tests a recursive setmeta command with m flag has expected UI output. Adapted from test_recursion_works on test_setmeta. """ bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') object2_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') stderr = self.RunGsUtil([ '-m', 'setmeta', '-h', 'content-type:footype', suri(object1_uri), suri(object2_uri) ], return_stderr=True) for obj_uri in [object1_uri, object2_uri]: stdout = self.RunGsUtil(['stat', suri(obj_uri)], return_stdout=True) self.assertIn('footype', stdout) CheckUiOutputWithMFlag(self, stderr, 2, metadata=True) def test_ui_setmeta_with_no_m_flag(self): """Tests a recursive setmeta command with no m flag has expected UI output. Adapted from test_recursion_works on test_setmeta. """ bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') object2_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') stderr = self.RunGsUtil([ 'setmeta', '-h', 'content-type:footype', suri(object1_uri), suri(object2_uri) ], return_stderr=True) for obj_uri in [object1_uri, object2_uri]: stdout = self.RunGsUtil(['stat', suri(obj_uri)], return_stdout=True) self.assertIn('footype', stdout) CheckUiOutputWithNoMFlag(self, stderr, 2, metadata=True) def test_ui_acl_with_m_flag(self): """Tests UI output for an ACL command with m flag enabled. Adapted from test_set_valid_acl_object. """ get_acl_prefix = ['-m', 'acl', 'get'] set_acl_prefix = ['-m', 'acl', 'set'] obj_uri = suri(self.CreateObject(contents=b'foo')) acl_string = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True) inpath = self.CreateTempFile(contents=acl_string.encode(UTF8)) stderr = self.RunGsUtil(set_acl_prefix + ['public-read', obj_uri], return_stderr=True) CheckUiOutputWithMFlag(self, stderr, 1, metadata=True) acl_string2 = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True) stderr = self.RunGsUtil(set_acl_prefix + [inpath, obj_uri], return_stderr=True) CheckUiOutputWithMFlag(self, stderr, 1, metadata=True) acl_string3 = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True) self.assertNotEqual(acl_string, acl_string2) self.assertEqual(acl_string, acl_string3) def test_ui_acl_with_no_m_flag(self): """Tests UI output for an ACL command with m flag not enabled. Adapted from test_set_valid_acl_object. """ get_acl_prefix = ['acl', 'get'] set_acl_prefix = ['acl', 'set'] obj_uri = suri(self.CreateObject(contents=b'foo')) acl_string = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True) inpath = self.CreateTempFile(contents=acl_string.encode(UTF8)) stderr = self.RunGsUtil(set_acl_prefix + ['public-read', obj_uri], return_stderr=True) CheckUiOutputWithNoMFlag(self, stderr, 1, metadata=True) acl_string2 = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True) stderr = self.RunGsUtil(set_acl_prefix + [inpath, obj_uri], return_stderr=True) CheckUiOutputWithNoMFlag(self, stderr, 1, metadata=True) acl_string3 = self.RunGsUtil(get_acl_prefix + [obj_uri], return_stdout=True) self.assertNotEqual(acl_string, acl_string2) self.assertEqual(acl_string, acl_string3) def _test_ui_rsync_bucket_to_bucket_helper(self, gsutil_flags=None): """Helper class to test UI output for rsync command. Args: gsutil_flags: List of flags to run gsutil with, or None. Adapted from test_bucket_to_bucket in test_rsync. """ if not gsutil_flags: gsutil_flags = [] # Create 2 buckets with 1 overlapping object, 1 extra object at root level # in each, and 1 extra object 1 level down in each, where one of the objects # starts with "." to test that we don't skip those objects. Make the # overlapping objects named the same but with different content, to test # that we detect and properly copy in that case. bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents=b'obj1') self.CreateObject(bucket_uri=bucket1_uri, object_name='.obj2', contents=b'.obj2', mtime=10) self.CreateObject(bucket_uri=bucket1_uri, object_name='subdir/obj3', contents=b'subdir/obj3') self.CreateObject(bucket_uri=bucket1_uri, object_name='obj6', contents=b'obj6_', mtime=100) # .obj2 will be replaced and have mtime of 10 self.CreateObject(bucket_uri=bucket2_uri, object_name='.obj2', contents=b'.OBJ2') self.CreateObject(bucket_uri=bucket2_uri, object_name='obj4', contents=b'obj4') self.CreateObject(bucket_uri=bucket2_uri, object_name='subdir/obj5', contents=b'subdir/obj5') self.CreateObject(bucket_uri=bucket2_uri, object_name='obj6', contents=b'obj6', mtime=100) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" gsutil_args = (gsutil_flags + ['rsync', suri(bucket1_uri), suri(bucket2_uri)]) stderr = self.RunGsUtil(gsutil_args, return_stderr=True) num_objects = 3 total_size = len('obj1') + len('.obj2') + len('obj6_') CheckUiOutputWithNoMFlag(self, stderr, num_objects, total_size) listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri)) listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri)) # First bucket should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir/obj3', '/obj6'])) # Second bucket should have new objects added from source bucket (without # removing extraneeous object found in dest bucket), and without the # subdir objects synchronized. self.assertEqual( listing2, set(['/obj1', '/.obj2', '/obj4', '/subdir/obj5', '/obj6'])) # Assert that the src/dest objects that had same length but different # content were correctly synchronized (bucket to bucket rsync uses # checksums). self.assertEqual( '.obj2', self.RunGsUtil(['cat', suri(bucket1_uri, '.obj2')], return_stdout=True)) self.assertEqual( '.obj2', self.RunGsUtil(['cat', suri(bucket2_uri, '.obj2')], return_stdout=True)) self.assertEqual( 'obj6_', self.RunGsUtil(['cat', suri(bucket2_uri, 'obj6')], return_stdout=True)) _Check1() def test_ui_rsync_bucket_to_bucket_with_m_flag(self): """Tests UI output for rsync with -m flag enabled works as expected.""" self._test_ui_rsync_bucket_to_bucket_helper(gsutil_flags=['-m']) def test_ui_rsync_bucket_to_bucket_with_no_m_flag(self): """Tests UI output for rsync with -m flag not enabled works as expected.""" self._test_ui_rsync_bucket_to_bucket_helper() class TestUiUnitTests(testcase.GsUtilUnitTestCase): """Unit tests for UI functions.""" upload_size = UPLOAD_SIZE start_time = 10000 def test_ui_seek_ahead_message(self): """Tests if a seek ahead message is correctly printed.""" status_queue = Queue.Queue() stream = six.StringIO() # No time constraints for displaying messages. start_time = self.start_time ui_controller = UIController(0, 0, 0, 0, custom_time=start_time) ui_thread = UIThread(status_queue, stream, ui_controller) num_objects = 10 total_size = 1024**3 PutToQueueWithTimeout(status_queue, SeekAheadMessage(num_objects, total_size, start_time)) # Adds a file. Because this message was already theoretically processed # by the SeekAheadThread, the number of files reported by the UIController # should not change. fpath = self.CreateTempFile(file_name='sample-file.txt', contents=b'foo') PutToQueueWithTimeout( status_queue, FileMessage(StorageUrlFromString(suri(fpath)), None, start_time + 10, size=UPLOAD_SIZE, message_type=FileMessage.FILE_UPLOAD, finished=False)) PutToQueueWithTimeout( status_queue, FileMessage(StorageUrlFromString(suri(fpath)), None, start_time + 20, size=UPLOAD_SIZE, message_type=FileMessage.FILE_UPLOAD, finished=True)) PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT) JoinThreadAndRaiseOnTimeout(ui_thread) content = stream.getvalue() expected_message = ( 'Estimated work for this command: objects: %s, total size: %s\n' % (num_objects, MakeHumanReadable(total_size))) self.assertIn(expected_message, content) # This ensures the SeekAheadMessage did its job. self.assertIn('/' + str(num_objects), content) # This ensures a FileMessage did not affect the total number of files # obtained by the SeekAheadMessage. self.assertNotIn('/' + str(num_objects + 1), content) def test_ui_seek_ahead_zero_size(self): """Tests the case where the SeekAheadThread returns total size of 0.""" current_time_ms = self.start_time status_queue = Queue.Queue() stream = six.StringIO() ui_controller = UIController(custom_time=current_time_ms) ui_thread = UIThread(status_queue, stream, ui_controller) PutToQueueWithTimeout(status_queue, SeekAheadMessage(100, 0, current_time_ms)) for i in range(100): current_time_ms += 200 PutToQueueWithTimeout( status_queue, FileMessage(StorageUrlFromString('gs://foo%s' % i), StorageUrlFromString('bar%s' % i), current_time_ms, message_type=FileMessage.FILE_DOWNLOAD)) for i in range(100): current_time_ms += 200 PutToQueueWithTimeout( status_queue, FileMessage(StorageUrlFromString('gs://foo%s' % i), StorageUrlFromString('bar%s' % i), current_time_ms, finished=True, message_type=FileMessage.FILE_DOWNLOAD)) PutToQueueWithTimeout( status_queue, ProducerThreadMessage(100, 0, current_time_ms, finished=True)) PutToQueueWithTimeout(status_queue, FinalMessage(current_time_ms)) PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT) JoinThreadAndRaiseOnTimeout(ui_thread) self.assertIn('100/100', stream.getvalue()) def test_ui_empty_list(self): """Tests if status queue is empty after processed by UIThread.""" status_queue = Queue.Queue() stream = six.StringIO() ui_controller = UIController() ui_thread = UIThread(status_queue, stream, ui_controller) for i in range(10000): # pylint: disable=unused-variable PutToQueueWithTimeout(status_queue, 'foo') PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT) JoinThreadAndRaiseOnTimeout(ui_thread) self.assertEqual(0, status_queue.qsize()) def test_ui_controller_shared_states(self): """Tests that UIController correctly integrates messages. This test ensures UIController correctly shares its state, which is used by both UIThread and MainThreadUIQueue. There are multiple ways of checking that. One such way is to create a ProducerThreadMessage on the MainThreadUIQueue, simulate a upload with messages coming from the UIThread, and check if the output has the percentage done and number of files (both happen only when a ProducerThreadMessage or SeekAheadMessage is called). """ ui_thread_status_queue = Queue.Queue() stream = six.StringIO() # No time constraints for displaying messages. start_time = self.start_time ui_controller = UIController(0, 0, 0, 0, custom_time=start_time) main_thread_ui_queue = MainThreadUIQueue(stream, ui_controller) ui_thread = UIThread(ui_thread_status_queue, stream, ui_controller) PutToQueueWithTimeout( main_thread_ui_queue, ProducerThreadMessage(1, UPLOAD_SIZE, start_time, finished=True)) fpath = self.CreateTempFile(file_name='sample-file.txt', contents=b'foo') PutToQueueWithTimeout( ui_thread_status_queue, FileMessage(StorageUrlFromString(suri(fpath)), None, start_time + 10, size=UPLOAD_SIZE, message_type=FileMessage.FILE_UPLOAD, finished=False)) PutToQueueWithTimeout( ui_thread_status_queue, FileMessage(StorageUrlFromString(suri(fpath)), None, start_time + 20, size=UPLOAD_SIZE, message_type=FileMessage.FILE_UPLOAD, finished=True)) PutToQueueWithTimeout(ui_thread_status_queue, FinalMessage(start_time + 50)) PutToQueueWithTimeout(ui_thread_status_queue, ZERO_TASKS_TO_DO_ARGUMENT) JoinThreadAndRaiseOnTimeout(ui_thread) content = stream.getvalue() CheckUiOutputWithMFlag(self, content, 1, UPLOAD_SIZE) def test_ui_throughput_calculation_with_components(self): """Tests throughput calculation in the UI. This test takes two different values, both with a different size and different number of components, and see if throughput behaves as expected. """ status_queue = Queue.Queue() stream = six.StringIO() # Creates a UIController that has no time constraints for updating info, # except for having to wait at least 2 seconds (considering the time # informed by the messages) to update the throughput. We use a value # slightly smaller than 2 to ensure messages that are 2 seconds apart from # one another will be enough to calculate throughput. start_time = self.start_time ui_controller = UIController(sliding_throughput_period=2, update_message_period=1, first_throughput_latency=0, custom_time=start_time) # We use start_time to have a reasonable set of values for the time messages # processed by the UIController. However, the start_time does not influence # this test, as the throughput is calculated based on the time # difference between two messages, which is fixed in this test. ui_thread = UIThread(status_queue, stream, ui_controller) fpath1 = self.CreateTempFile(file_name='sample-file.txt', contents=b'foo') fpath2 = self.CreateTempFile(file_name='sample-file2.txt', contents=b'FOO') def _CreateFileVariables(alpha, component_number, src_url): """Creates size and component_size for a given file.""" size = 1024**2 * 60 * alpha # this is 60*alpha MiB component_size = size / component_number return (size, component_number, component_size, src_url) # Note: size1 and size2 do not actually correspond to the actual sizes of # fpath1 and fpath2. However, the UIController only uses the size sent on # the message, so we should be able to pretend they are much larger on size. (size1, component_num_file1, component_size_file1, src_url1) = (_CreateFileVariables(1, 3, StorageUrlFromString(suri(fpath1)))) (size2, component_num_file2, component_size_file2, src_url2) = (_CreateFileVariables(10, 4, StorageUrlFromString(suri(fpath2)))) for file_message_type, component_message_type, operation_name in ( (FileMessage.FILE_UPLOAD, FileMessage.COMPONENT_TO_UPLOAD, 'Uploading'), (FileMessage.FILE_DOWNLOAD, FileMessage.COMPONENT_TO_DOWNLOAD, 'Downloading')): # Testing for uploads and downloads PutToQueueWithTimeout( status_queue, FileMessage(src_url1, None, start_time + 100, size=size1, message_type=file_message_type)) PutToQueueWithTimeout( status_queue, FileMessage(src_url2, None, start_time + 150, size=size2, message_type=file_message_type)) for i in range(component_num_file1): PutToQueueWithTimeout( status_queue, FileMessage(src_url1, None, start_time + 200 + i, size=component_size_file1, component_num=i, message_type=component_message_type)) for i in range(component_num_file2): PutToQueueWithTimeout( status_queue, FileMessage(src_url2, None, start_time + 250 + i, size=component_size_file2, component_num=i, message_type=component_message_type)) progress_calls_number = 4 for j in range(1, progress_calls_number + 1): # We will send progress_calls_number ProgressMessages for each # component. base_start_time = (start_time + 300 + (j - 1) * (component_num_file1 + component_num_file2)) for i in range(component_num_file1): # Each component has size equal to # component_size_file1/progress_calls_number PutToQueueWithTimeout( status_queue, ProgressMessage(size1, j * component_size_file1 / progress_calls_number, src_url1, base_start_time + i, component_num=i, operation_name=operation_name)) for i in range(component_num_file2): # Each component has size equal to # component_size_file2/progress_calls_number PutToQueueWithTimeout( status_queue, ProgressMessage(size2, j * component_size_file2 / progress_calls_number, src_url2, base_start_time + component_num_file1 + i, component_num=i, operation_name=operation_name)) # Time to finish the components and files. for i in range(component_num_file1): PutToQueueWithTimeout( status_queue, FileMessage(src_url1, None, start_time + 500 + i, finished=True, size=component_size_file1, component_num=i, message_type=component_message_type)) for i in range(component_num_file2): PutToQueueWithTimeout( status_queue, FileMessage(src_url2, None, start_time + 600 + i, finished=True, size=component_size_file2, component_num=i, message_type=component_message_type)) PutToQueueWithTimeout( status_queue, FileMessage(src_url1, None, start_time + 700, size=size1, finished=True, message_type=file_message_type)) PutToQueueWithTimeout( status_queue, FileMessage(src_url2, None, start_time + 800, size=size2, finished=True, message_type=file_message_type)) PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT) JoinThreadAndRaiseOnTimeout(ui_thread) content = stream.getvalue() # There were 2-second periods when no progress was reported. The # throughput here will be 0. We will use BytesToFixedWidthString(0) # to ensure that any changes to the function are applied here as well. zero = BytesToFixedWidthString(0) self.assertIn(zero + '/s', content) file1_progress = (size1 / (component_num_file1 * progress_calls_number)) file2_progress = (size2 / (component_num_file2 * progress_calls_number)) # There were 2-second periods when only two progresses from file1 # were reported. The throughput here will be file1_progress. self.assertIn(BytesToFixedWidthString(file1_progress) + '/s', content) # There were 2-second periods when only two progresses from file2 # were reported. The throughput here will be file2_progress. self.assertIn(BytesToFixedWidthString(file2_progress) + '/s', content) # For each loop iteration, there are two 2-second periods when # one progress from each file is reported: in the middle of the # iteration, and in the end of the iteration along with the beginning # of the following iteration, on a total of # 2 * progress_calls_number - 1 occurrences (-1 due to only 1 occurrence # on the last iteration). # The throughput here will be (file1_progress + file2_progress) / 2. average_progress = BytesToFixedWidthString( (file1_progress + file2_progress) / 2) self.assertEqual(content.count(average_progress + '/s'), 2 * progress_calls_number - 1) def test_ui_throughput_calculation_with_no_components(self): """Tests throughput calculation in the UI. This test takes two different values, both with a different size and different number of components, and see if throughput behaves as expected. """ status_queue = Queue.Queue() stream = six.StringIO() # Creates a UIController that has no time constraints for updating info, # except for having to wait at least 2 seconds(considering the time informed # by the messages) to update the throughput. We use a value slightly smaller # than 2 to ensure messages that are 2 seconds apart from one another will # be enough to calculate throughput. start_time = self.start_time ui_controller = UIController(sliding_throughput_period=2, update_message_period=1, first_throughput_latency=0, custom_time=start_time) # We use start_time to have a reasonable set of values for the time messages # processed by the UIController. However, the start_time does not influence # much this test, as the throughput is calculated based on the time # difference between two messages, which is fixed in this text. ui_thread = UIThread(status_queue, stream, ui_controller) fpath1 = self.CreateTempFile(file_name='sample-file.txt', contents=b'foo') fpath2 = self.CreateTempFile(file_name='sample-file2.txt', contents=b'FOO') # Note: size1 and size2 do not actually correspond to the actual sizes of # fpath1 and fpath2. However, the UIController only uses the size sent on # the message, so we should be able to pretend they are much larger on size. size1 = 1024**2 * 60 src_url1 = StorageUrlFromString(suri(fpath1)) size2 = 1024**2 * 600 src_url2 = StorageUrlFromString(suri(fpath2)) for file_message_type, operation_name in ((FileMessage.FILE_UPLOAD, 'Uploading'), (FileMessage.FILE_DOWNLOAD, 'Downloading')): # Testing for uploads and downloads PutToQueueWithTimeout( status_queue, FileMessage(src_url1, None, start_time + 200, size=size1, message_type=file_message_type)) PutToQueueWithTimeout( status_queue, FileMessage(src_url2, None, start_time + 301, size=size2, message_type=file_message_type)) progress_calls_number = 4 for j in range(1, progress_calls_number + 1): # We will send progress_calls_number ProgressMessages for each file. PutToQueueWithTimeout( status_queue, ProgressMessage(size1, j * size1 / 4, src_url1, start_time + 300 + j * 2, operation_name=operation_name)) PutToQueueWithTimeout( status_queue, ProgressMessage(size2, j * size2 / 4, src_url2, start_time + 300 + j * 2 + 1, operation_name=operation_name)) # Time to finish the files. PutToQueueWithTimeout( status_queue, FileMessage(src_url1, None, start_time + 700, size=size1, finished=True, message_type=file_message_type)) PutToQueueWithTimeout( status_queue, FileMessage(src_url2, None, start_time + 800, size=size2, finished=True, message_type=file_message_type)) PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT) JoinThreadAndRaiseOnTimeout(ui_thread) content = stream.getvalue() # There were 2-second periods when no progress was reported. The # throughput here will be 0. We will use BytesToFixedWidthString(0) # to ensure that any changes to the function are applied here as well. zero = BytesToFixedWidthString(0) self.assertIn(zero + '/s', content) file1_progress = (size1 / progress_calls_number) file2_progress = (size2 / progress_calls_number) # For each loop iteration, there are two 2-second periods when # one progress from each file is reported: in the middle of the # iteration, and in the end of the iteration along with the beginning # of the following iteration, on a total of # 2 * progress_calls_number - 1 occurrences (-1 due to only 1 occurrence # on the last iteration). # The throughput here will be (file1_progress + file2_progress) / 2. average_progress = BytesToFixedWidthString( (file1_progress + file2_progress) / 2) self.assertEqual(content.count(average_progress + '/s'), 2 * progress_calls_number - 1) def test_ui_metadata_message_passing(self): """Tests that MetadataMessages are being correctly received and processed. This also tests the relation and hierarchy between different estimation sources, as represented by the EstimationSource class. """ status_queue = Queue.Queue() stream = six.StringIO() # Creates a UIController that has no time constraints for updating info, # except for having to wait at least 2 seconds(considering the time informed # by the messages) to update the throughput. We use a value slightly smaller # than 2 to ensure messages that are 2 seconds apart from one another will # be enough to calculate throughput. start_time = self.start_time ui_controller = UIController(sliding_throughput_period=2, update_message_period=1, first_throughput_latency=0, custom_time=start_time) num_objects = 200 ui_thread = UIThread(status_queue, stream, ui_controller) for i in range(num_objects): if i < 100: PutToQueueWithTimeout(status_queue, MetadataMessage(start_time + 0.1 * i)) elif i < 130: if i == 100: # Sends an estimation message PutToQueueWithTimeout( status_queue, ProducerThreadMessage(130, 0, start_time + 0.1 + 0.1 * i)) PutToQueueWithTimeout( status_queue, MetadataMessage(start_time + 10 + 0.2 * (i - 100))) elif i < 150: if i == 130: # Sends a SeekAheadMessage PutToQueueWithTimeout( status_queue, SeekAheadMessage(190, 0, start_time + 10.1 + 0.2 * (i - 100))) PutToQueueWithTimeout( status_queue, MetadataMessage(start_time + 16 + 0.5 * (i - 130))) elif i < num_objects: if i == 150: # Sends a final ProducerThreadMessage PutToQueueWithTimeout( status_queue, ProducerThreadMessage(200, 0, start_time + 16.1 + 0.5 * (i - 130), finished=True)) PutToQueueWithTimeout(status_queue, MetadataMessage(start_time + 26 + (i - 150))) PutToQueueWithTimeout(status_queue, FinalMessage(start_time + 100)) PutToQueueWithTimeout(status_queue, ZERO_TASKS_TO_DO_ARGUMENT) JoinThreadAndRaiseOnTimeout(ui_thread) content = stream.getvalue() # We should not have estimated the number of objects as 130 in the UI. self.assertNotIn('/130 objects', content) # We should have estimated the number of objects as 190 in the UI. self.assertIn('/190 objects', content) # We should have estimated the number of objects as 200 in the UI. self.assertIn('/200 objects', content) # We should have calculated the throughput at all moments. # First 100 elements. self.assertIn('10.00 objects/s', content) # At one exact point between first and second round of elements. self.assertEqual(content.count('7.50 objects/s'), 1) # Next 30 elements. self.assertIn('5.00 objects/s', content) # At one exact point between second and third round of elements. self.assertEqual(content.count('3.50 objects/s'), 1) # Next 20 elements. self.assertIn('2.00 objects/s', content) # At one exact point between third and fourth round of elements. self.assertEqual(content.count('1.50 objects/s'), 1) # Final 50 elements. self.assertIn('1.00 objects/s', content) CheckUiOutputWithMFlag(self, content, 200, metadata=True) def test_ui_manager(self): """Tests the correctness of the UI manager. This test ensures a DataManager is created whenever a data message appears, regardless of previous MetadataMessages. """ stream = six.StringIO() start_time = self.start_time ui_controller = UIController(custom_time=start_time) status_queue = MainThreadUIQueue(stream, ui_controller) # No manager has been created. self.assertEqual(ui_controller.manager, None) PutToQueueWithTimeout(status_queue, ProducerThreadMessage(2, 0, start_time)) # Still no manager has been created. self.assertEqual(ui_controller.manager, None) PutToQueueWithTimeout(status_queue, MetadataMessage(start_time + 1)) # Now we have a MetadataManager. self.assertIsInstance(ui_controller.manager, MetadataManager) PutToQueueWithTimeout( status_queue, FileMessage(StorageUrlFromString('foo'), None, start_time + 2)) # Now we have a DataManager since the DataManager overwrites the # MetadataManager. self.assertIsInstance(ui_controller.manager, DataManager) def test_ui_BytesToFixedWidthString(self): """Tests the correctness of BytesToFixedWidthString.""" self.assertEqual(' 0.0 B', BytesToFixedWidthString(0, decimal_places=1)) self.assertEqual(' 0.00 B', BytesToFixedWidthString(0, decimal_places=2)) self.assertEqual(' 2.3 KiB', BytesToFixedWidthString(2.27 * 1024, decimal_places=1)) self.assertEqual(' 1023 KiB', BytesToFixedWidthString(1023.2 * 1024, decimal_places=1)) self.assertEqual(' 1.0 MiB', BytesToFixedWidthString(1024**2, decimal_places=1)) self.assertEqual('999.1 MiB', BytesToFixedWidthString(999.1 * 1024**2, decimal_places=1)) def test_ui_spinner(self): stream = six.StringIO() start_time = self.start_time ui_controller = UIController(update_spinner_period=1, custom_time=start_time) status_queue = MainThreadUIQueue(stream, ui_controller) PutToQueueWithTimeout(status_queue, ProducerThreadMessage(1, len('foo'), start_time)) PutToQueueWithTimeout( status_queue, FileMessage(StorageUrlFromString('foo'), None, start_time, message_type=FileMessage.FILE_UPLOAD)) current_spinner = ui_controller.manager.GetSpinner() PutToQueueWithTimeout( status_queue, ProgressMessage(1, len('foo'), StorageUrlFromString('foo'), start_time + 1.2)) old_spinner1 = current_spinner current_spinner = ui_controller.manager.GetSpinner() # Spinner must have changed since more than 1 second has passed. self.assertNotEqual(old_spinner1, current_spinner) PutToQueueWithTimeout( status_queue, ProgressMessage(2, len('foo'), StorageUrlFromString('foo'), start_time + 2)) old_spinner2 = current_spinner current_spinner = ui_controller.manager.GetSpinner() # Spinner must not have changed since less than 1 second has passed. self.assertEqual(old_spinner2, current_spinner) PutToQueueWithTimeout( status_queue, ProgressMessage(3, len('foo'), StorageUrlFromString('foo'), start_time + 2.5)) old_spinner3 = current_spinner current_spinner = ui_controller.manager.GetSpinner() # Spinner must have changed since more than 1 second has passed. self.assertNotEqual(old_spinner3, current_spinner) PutToQueueWithTimeout( status_queue, FileMessage(StorageUrlFromString('foo'), None, start_time + 5, finished=True, message_type=FileMessage.FILE_UPLOAD)) old_spinner4 = current_spinner current_spinner = ui_controller.manager.GetSpinner() # Spinner must have changed since more than 1 second has passed. self.assertNotEqual(old_spinner4, current_spinner) # Moreover, since we have 4 spinner characters and were only supposed to # change it twice, current_spinner must be different from old_spinner1 and # old_spinner3. self.assertNotEqual(old_spinner3, current_spinner) self.assertNotEqual(old_spinner1, current_spinner) tests/test_logging.py 0000644 00000007005 15025013616 0010746 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for logging command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri @SkipForS3('Logging command requires S3 ACL configuration on target bucket.') class TestLogging(testcase.GsUtilIntegrationTestCase): """Integration tests for logging command.""" _enable_log_cmd = ['logging', 'set', 'on'] _disable_log_cmd = ['logging', 'set', 'off'] _get_log_cmd = ['logging', 'get'] def testLogging(self): """Tests enabling and disabling logging.""" bucket_uri = self.CreateBucket() bucket_suri = suri(bucket_uri) stderr = self.RunGsUtil(self._enable_log_cmd + ['-b', bucket_suri, bucket_suri], return_stderr=True) if self._use_gcloud_storage: self.assertIn('Updating', stderr) else: self.assertIn('Enabling logging', stderr) stdout = self.RunGsUtil(self._get_log_cmd + [bucket_suri], return_stdout=True) if self._use_gcloud_storage: _, _, prefixless_bucket = bucket_suri.partition('://') self.assertIn('"logBucket": "{}"'.format(prefixless_bucket), stdout) self.assertIn('"logObjectPrefix": "{}"'.format(prefixless_bucket), stdout) else: self.assertIn('LogObjectPrefix'.lower(), stdout.lower()) stderr = self.RunGsUtil(self._disable_log_cmd + [bucket_suri], return_stderr=True) if self._use_gcloud_storage: self.assertIn('Updating', stderr) else: self.assertIn('Disabling logging', stderr) def testTooFewArgumentsFails(self): """Ensures logging commands fail with too few arguments.""" # No arguments for enable, but valid subcommand. stderr = self.RunGsUtil(self._enable_log_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for disable, but valid subcommand. stderr = self.RunGsUtil(self._disable_log_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for get, but valid subcommand. stderr = self.RunGsUtil(self._get_log_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Neither arguments nor subcommand. stderr = self.RunGsUtil(['logging'], return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) class TestLoggingOldAlias(TestLogging): _enable_log_cmd = ['enablelogging'] _disable_log_cmd = ['disablelogging'] _get_log_cmd = ['getlogging'] tests/test_versioning.py 0000644 00000007161 15025013616 0011506 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for versioning command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import gslib.tests.testcase as testcase from gslib.tests.util import ObjectToURI as suri from gslib.utils.retry_util import Retry class TestVersioning(testcase.GsUtilIntegrationTestCase): """Integration tests for versioning command.""" _set_ver_cmd = ['versioning', 'set'] _get_ver_cmd = ['versioning', 'get'] def test_off_default(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil(self._get_ver_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout.strip(), '%s: Suspended' % suri(bucket_uri)) def test_turning_on(self): bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_ver_cmd + ['on', suri(bucket_uri)]) # Work around eventual consistency for S3 versioning. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(self._get_ver_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout.strip(), '%s: Enabled' % suri(bucket_uri)) _Check1() def test_turning_off(self): bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_ver_cmd + ['on', suri(bucket_uri)]) # Work around eventual consistency for S3 versioning. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(self._get_ver_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout.strip(), '%s: Enabled' % suri(bucket_uri)) _Check1() self.RunGsUtil(self._set_ver_cmd + ['off', suri(bucket_uri)]) # Work around eventual consistency for S3 versioning. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(self._get_ver_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout.strip(), '%s: Suspended' % suri(bucket_uri)) _Check2() def testTooFewArgumentsFails(self): """Ensures versioning commands fail with too few arguments.""" # No arguments for set, but valid subcommand. stderr = self.RunGsUtil(self._set_ver_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for get, but valid subcommand. stderr = self.RunGsUtil(self._get_ver_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Neither arguments nor subcommand. stderr = self.RunGsUtil(['versioning'], return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) class TestVersioningOldAlias(TestVersioning): _set_ver_cmd = ['setversioning'] _get_ver_cmd = ['getversioning'] tests/test_ls.py 0000644 00000152246 15025013616 0007746 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for ls command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from datetime import datetime import os import posixpath import re import stat import subprocess import sys import time import gslib from gslib.commands import ls from gslib.cs_api_map import ApiSelector from gslib.project_id import PopulateProjectId import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForGS from gslib.tests.testcase.integration_testcase import SkipForJSON from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import CaptureStdout from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import RUN_S3_TESTS from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import SkipForP12Creds from gslib.tests.util import TEST_ENCRYPTION_CONTENT1 from gslib.tests.util import TEST_ENCRYPTION_CONTENT1_CRC32C from gslib.tests.util import TEST_ENCRYPTION_CONTENT1_MD5 from gslib.tests.util import TEST_ENCRYPTION_CONTENT2 from gslib.tests.util import TEST_ENCRYPTION_CONTENT2_CRC32C from gslib.tests.util import TEST_ENCRYPTION_CONTENT2_MD5 from gslib.tests.util import TEST_ENCRYPTION_CONTENT3 from gslib.tests.util import TEST_ENCRYPTION_CONTENT3_CRC32C from gslib.tests.util import TEST_ENCRYPTION_CONTENT3_MD5 from gslib.tests.util import TEST_ENCRYPTION_CONTENT4 from gslib.tests.util import TEST_ENCRYPTION_CONTENT4_CRC32C from gslib.tests.util import TEST_ENCRYPTION_CONTENT4_MD5 from gslib.tests.util import TEST_ENCRYPTION_CONTENT5 from gslib.tests.util import TEST_ENCRYPTION_CONTENT5_CRC32C from gslib.tests.util import TEST_ENCRYPTION_CONTENT5_MD5 from gslib.tests.util import TEST_ENCRYPTION_KEY1 from gslib.tests.util import TEST_ENCRYPTION_KEY1_SHA256_B64 from gslib.tests.util import TEST_ENCRYPTION_KEY2 from gslib.tests.util import TEST_ENCRYPTION_KEY2_SHA256_B64 from gslib.tests.util import TEST_ENCRYPTION_KEY3 from gslib.tests.util import TEST_ENCRYPTION_KEY3_SHA256_B64 from gslib.tests.util import TEST_ENCRYPTION_KEY4 from gslib.tests.util import TEST_ENCRYPTION_KEY4_SHA256_B64 from gslib.tests.util import unittest from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import UTF8 from gslib.utils.ls_helper import PrintFullInfoAboutObject from gslib.utils.retry_util import Retry from gslib.utils.system_util import IS_WINDOWS from gslib.utils import shim_util from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock KMS_XML_SKIP_MSG = ('gsutil does not support KMS operations for S3 buckets, ' 'or listing KMS keys with the XML API.') BUCKET_LOCK_SKIP_MSG = ('gsutil does not support bucket lock operations for ' 'S3 buckets or listing retention policy with XML API.') class TestLsUnit(testcase.GsUtilUnitTestCase): """Unit tests for ls command.""" def test_one_object_with_L_storage_class_update(self): """Tests the JSON storage class update time field.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'XML API has no concept of storage class update time') # Case 1: Create an object message where Storage class update time is the # same as Creation time. current_time = datetime(2017, 1, 2, 3, 4, 5, 6, tzinfo=None) obj_metadata = apitools_messages.Object( name='foo', bucket='bar', timeCreated=current_time, updated=current_time, timeStorageClassUpdated=current_time, etag='12345') # Create mock object to point to obj_metadata. obj_ref = mock.Mock() obj_ref.root_object = obj_metadata obj_ref.url_string = 'foo' # Print out attributes of object message. with CaptureStdout() as output: PrintFullInfoAboutObject(obj_ref) output = '\n'.join(output) # Verify that no Storage class update time field displays since it's the # same as Creation time. find_stor_update_re = re.compile( r'^\s*Storage class update time:+(?P<stor_update_time_val>.+)$', re.MULTILINE) stor_update_time_match = re.search(find_stor_update_re, output) self.assertIsNone(stor_update_time_match) # Case 2: Create an object message where Storage class update time differs # from Creation time. new_update_time = datetime(2017, 2, 3, 4, 5, 6, 7, tzinfo=None) obj_metadata2 = apitools_messages.Object( name='foo2', bucket='bar2', timeCreated=current_time, updated=current_time, timeStorageClassUpdated=new_update_time, etag='12345') # Create mock object to point to obj_metadata2. obj_ref2 = mock.Mock() obj_ref2.root_object = obj_metadata2 obj_ref2.url_string = 'foo2' # Print out attributes of object message. with CaptureStdout() as output2: PrintFullInfoAboutObject(obj_ref2) output2 = '\n'.join(output2) # Verify that Creation time and Storage class update time fields display and # are the same as the times set in the object message. find_time_created_re = re.compile( r'^\s*Creation time:\s+(?P<time_created_val>.+)$', re.MULTILINE) time_created_match = re.search(find_time_created_re, output2) self.assertIsNotNone(time_created_match) time_created = time_created_match.group('time_created_val') self.assertEqual( time_created, datetime.strftime(current_time, '%a, %d %b %Y %H:%M:%S GMT')) find_stor_update_re_2 = re.compile( r'^\s*Storage class update time:+(?P<stor_update_time_val_2>.+)$', re.MULTILINE) stor_update_time_match_2 = re.search(find_stor_update_re_2, output2) self.assertIsNotNone(stor_update_time_match_2) stor_update_time = stor_update_time_match_2.group('stor_update_time_val_2') self.assertEqual( stor_update_time, datetime.strftime(new_update_time, '%a, %d %b %Y %H:%M:%S GMT')) @mock.patch.object(ls.LsCommand, 'WildcardIterator') def test_satisfies_pzs_is_displayed_if_present(self, mock_wildcard): bucket_uri = self.CreateBucket(bucket_name='foo') bucket_metadata = apitools_messages.Bucket(name='foo', satisfiesPZS=True) bucket_uri.root_object = bucket_metadata bucket_uri.url_string = 'foo' bucket_uri.storage_url = mock.Mock() mock_wildcard.return_value.IterBuckets.return_value = [bucket_uri] # MockKey doesn't support hash_algs, so the MD5 will not match. with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): stdout = self.RunCommand('ls', ['-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, 'Satisfies PZS:\t\t\tTrue') @mock.patch.object(ls.LsCommand, 'WildcardIterator') def test_placement_locations_not_displayed_for_normal_bucket( self, mock_wildcard): """Non custom dual region buckets should not display placement info.""" bucket_uri = self.CreateBucket(bucket_name='foo-non-cdr') bucket_metadata = apitools_messages.Bucket(name='foo-non-cdr') bucket_uri.root_object = bucket_metadata bucket_uri.url_string = 'foo-non-cdr' bucket_uri.storage_url = mock.Mock() mock_wildcard.return_value.IterBuckets.return_value = [bucket_uri] # MockKey doesn't support hash_algs, so the MD5 will not match. with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): stdout = self.RunCommand('ls', ['-Lb', suri(bucket_uri)], return_stdout=True) self.assertNotRegex(stdout, 'Placement locations:') class TestLsUnitWithShim(testcase.ShimUnitTestBase): """Unit tests for ls command with shim.""" def test_shim_translates_flags(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('ls', ['-rRlLbeah', '-p foo'], return_log_handler=True) self.assertIn( 'Gcloud Storage Command: {} storage ls' ' --fetch-encrypted-object-hashes' ' -r -r -l -L -b -e -a --readable-sizes --project foo'.format( shim_util._get_gcloud_binary_path('fake_dir')), mock_log_handler.messages['info']) class TestLs(testcase.GsUtilIntegrationTestCase): """Integration tests for ls command.""" def test_blank_ls(self): if not RUN_S3_TESTS: # Blank `ls` command lists GS buckets. self.RunGsUtil(['ls']) def test_empty_bucket(self): bucket_uri = self.CreateBucket() self.AssertNObjectsInBucket(bucket_uri, 0) def test_empty_bucket_with_b(self): bucket_uri = self.CreateBucket() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-b', suri(bucket_uri)], return_stdout=True) self.assertEqual('%s/\n' % suri(bucket_uri), stdout) _Check1() def test_bucket_with_Lb(self): """Tests ls -Lb.""" bucket_uri = self.CreateBucket() stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) # Check that the bucket URI is displayed. self.assertIn(suri(bucket_uri), stdout) # Check that we don't see output corresponding to listing objects rather # than buckets. self.assertNotIn('TOTAL:', stdout) # Toggle versioning on the bucket so that the modification time will be # greater than the creation time. self.RunGsUtil(['versioning', 'set', 'on', suri(bucket_uri)]) self.RunGsUtil(['versioning', 'set', 'off', suri(bucket_uri)]) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) find_metageneration_re = re.compile( r'^\s*Metageneration:\s+(?P<metageneration_val>.+)$', re.MULTILINE) find_time_created_re = re.compile( r'^\s*Time created:\s+(?P<time_created_val>.+)$', re.MULTILINE) find_time_updated_re = re.compile( r'^\s*Time updated:\s+(?P<time_updated_val>.+)$', re.MULTILINE) metageneration_match = re.search(find_metageneration_re, stdout) time_created_match = re.search(find_time_created_re, stdout) time_updated_match = re.search(find_time_updated_re, stdout) if self.test_api == ApiSelector.XML: # Check that lines for JSON-specific fields are not displayed. self.assertIsNone(metageneration_match) self.assertIsNone(time_created_match) self.assertIsNone(time_updated_match) elif self.test_api == ApiSelector.JSON: # Check that time created/updated lines are displayed. self.assertIsNotNone(metageneration_match) self.assertIsNotNone(time_created_match) self.assertIsNotNone(time_updated_match) # Check that updated time > created time. time_created = time_created_match.group('time_created_val') time_created = time.strptime(time_created, '%a, %d %b %Y %H:%M:%S %Z') time_updated = time_updated_match.group('time_updated_val') time_updated = time.strptime(time_updated, '%a, %d %b %Y %H:%M:%S %Z') self.assertGreater(time_updated, time_created) # Check that for bucket policy only fields. self._AssertBucketPolicyOnly(False, stdout) def test_bucket_with_Lb_bucket_policy_only(self): if self.test_api == ApiSelector.JSON: bucket_uri = self.CreateBucket(bucket_policy_only=True, prefer_json_api=True) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self._AssertBucketPolicyOnly(True, stdout) def _AssertBucketPolicyOnly(self, value, stdout): bucket_policy_only_re = re.compile( r'^\s*Bucket Policy Only enabled:\s+(?P<bpo_val>.+)$', re.MULTILINE) bucket_policy_only_match = re.search(bucket_policy_only_re, stdout) bucket_policy_only_val = bucket_policy_only_match.group('bpo_val') self.assertEqual(str(value), bucket_policy_only_val) def test_bucket_with_lb(self): """Tests ls -lb.""" bucket_uri = self.CreateBucket() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-lb', suri(bucket_uri)], return_stdout=True) self.assertIn(suri(bucket_uri), stdout) self.assertNotIn('TOTAL:', stdout) _Check1() def test_bucket_list_wildcard(self): """Tests listing multiple buckets with a wildcard.""" random_prefix = self.MakeRandomTestString() bucket1_name = self.MakeTempName('bucket', prefix=random_prefix) bucket2_name = self.MakeTempName('bucket', prefix=random_prefix) bucket1_uri = self.CreateBucket(bucket_name=bucket1_name) bucket2_uri = self.CreateBucket(bucket_name=bucket2_name) # This just double checks that the common prefix of the two buckets is what # we think it should be (based on implementation detail of CreateBucket). # We want to be careful when setting a wildcard on buckets to make sure we # don't step outside the test buckets to affect other buckets. common_prefix = posixpath.commonprefix( [suri(bucket1_uri), suri(bucket2_uri)]) self.assertTrue( common_prefix.startswith( '%s://%sgsutil-test-test-bucket-list-wildcard' % (self.default_provider, random_prefix))) wildcard = '%s*' % common_prefix # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-b', wildcard], return_stdout=True) expected = set([suri(bucket1_uri) + '/', suri(bucket2_uri) + '/']) actual = set(stdout.split()) self.assertEqual(expected, actual) _Check1() def test_nonexistent_bucket_with_ls(self): """Tests a bucket that is known not to exist.""" stderr = self.RunGsUtil( ['ls', '-lb', 'gs://%s' % self.nonexistent_bucket_name], return_stderr=True, expected_status=1) self.assertIn('404', stderr) stderr = self.RunGsUtil( ['ls', '-Lb', 'gs://%s' % self.nonexistent_bucket_name], return_stderr=True, expected_status=1) self.assertIn('404', stderr) stderr = self.RunGsUtil( ['ls', '-b', 'gs://%s' % self.nonexistent_bucket_name], return_stderr=True, expected_status=1) self.assertIn('404', stderr) def test_list_missing_object(self): """Tests listing a non-existent object.""" bucket_uri = self.CreateBucket() stderr = self.RunGsUtil(['ls', suri(bucket_uri, 'missing')], return_stderr=True, expected_status=1) self.assertIn('matched no objects', stderr) def test_with_one_object(self): bucket_uri = self.CreateBucket() obj_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', suri(bucket_uri)], return_stdout=True) self.assertEqual('%s\n' % obj_uri, stdout) _Check1() def location_redirect_test_helper(self, bucket_region, client_region): bucket_host = 's3.%s.amazonaws.com' % bucket_region client_host = 's3.%s.amazonaws.com' % client_region with SetBotoConfigForTest([('s3', 'host', bucket_host)]): bucket_uri = self.CreateBucket(location=bucket_region) obj_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(uri): stdout = self.RunGsUtil(['ls', uri], return_stdout=True) self.assertEqual('%s\n' % obj_uri, stdout) with SetBotoConfigForTest([('s3', 'host', client_host)]): # sends a GET request _Check1(suri(bucket_uri)) # sends a HEAD request, meaning error body is not included. _Check1(suri(obj_uri)) @SkipForGS('Only s3 V4 signatures error on location mismatches.') def test_400_location_redirect(self): # ap-east-1 used here since regions launched before March 20, 2019 do # some temporary redirecting for new buckets which suppresses 400 errors. self.location_redirect_test_helper('ap-east-1', 'us-east-2') @SkipForGS('Only s3 V4 signatures error on location mismatches.') def test_301_location_redirect(self): self.location_redirect_test_helper('eu-west-1', 'us-east-2') @SkipForS3('Not relevant for S3') @SkipForJSON('Only the XML API supports changing the calling format.') def test_default_gcs_calling_format_is_path_style(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') stderr = self.RunGsUtil(['-D', 'ls', suri(object_uri)], return_stdout=True) self.assertIn('Host: storage.googleapis.com', stderr) @SkipForS3('Not relevant for S3') @SkipForJSON('Only the XML API supports changing the calling format.') def test_gcs_calling_format_is_configurable(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') custom_calling_format = 'boto.s3.connection.SubdomainCallingFormat' with SetBotoConfigForTest([('s3', 'calling_format', custom_calling_format) ]): stderr = self.RunGsUtil(['-D', 'ls', suri(object_uri)], return_stdout=True) self.assertIn('Host: %s.storage.googleapis.com' % bucket_uri.bucket_name, stderr) @SkipForXML('Credstore file gets created only for json API') @SkipForP12Creds('P12 credentials are not cached, as they are supported via google-auth') def test_credfile_lock_permissions(self): tmpdir = self.CreateTempDir() filepath = os.path.join(tmpdir, 'credstore2') option = 'GSUtil:state_dir={}'.format(tmpdir) bucket_uri = self.CreateBucket() obj_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil( ['-o', option, 'ls', suri(bucket_uri)], return_stdout=True) self.assertEqual('%s\n' % obj_uri, stdout) if os.name == 'posix': self.assertTrue(os.path.exists(filepath)) mode = oct(stat.S_IMODE(os.stat(filepath).st_mode)) # Assert that only user has read/write permission self.assertEqual(oct(0o600), mode) _Check1() def test_one_object_with_l(self): """Tests listing one object with -l.""" obj_uri = self.CreateObject(contents=b'foo') stdout = self.RunGsUtil(['ls', '-l', suri(obj_uri)], return_stdout=True) output_items = stdout.split() self.assertTrue(output_items[0].isdigit()) # Throws exception if time string is not formatted correctly. time.strptime(stdout.split()[1], '%Y-%m-%dT%H:%M:%SZ') self.assertEqual(output_items[2], suri(obj_uri)) def test_one_object_with_L(self): """Tests listing one object with -L.""" obj_uri = self.CreateObject(contents=b'foo') # Ensure that creation and update don't take place in the same second. time.sleep(2) # Check that the creation time, rather than the updated time, is displayed. self.RunGsUtil(['setmeta', '-h', 'x-goog-meta-foo:bar', suri(obj_uri)]) find_time_created_re = re.compile( r'^\s*Creation time:\s+(?P<time_created_val>.+)$', re.MULTILINE) find_time_updated_re = re.compile( r'^\s*Update time:\s+(?P<time_updated_val>.+)$', re.MULTILINE) stdout = self.RunGsUtil(['ls', '-L', suri(obj_uri)], return_stdout=True) time_created_match = re.search(find_time_created_re, stdout) time_updated_match = re.search(find_time_updated_re, stdout) time_created = time_created_match.group('time_created_val') self.assertIsNotNone(time_created) time_created = time.strptime(time_created, '%a, %d %b %Y %H:%M:%S %Z') if self.test_api == ApiSelector.XML: # XML API has no concept of updated time. self.assertIsNone(time_updated_match) elif self.test_api == ApiSelector.JSON: time_updated = time_updated_match.group('time_updated_val') self.assertIsNotNone(time_updated) time_updated = time.strptime(time_updated, '%a, %d %b %Y %H:%M:%S %Z') self.assertGreater(time_updated, time_created) @SkipForS3('Integration test utils only support GCS JSON for versioning.') @SkipForXML('Integration test utils only support GCS JSON for versioning.') def test_one_object_with_generation(self): """Tests listing one object by generation when multiple versions exist.""" bucket = self.CreateBucketJson(versioning_enabled=True) object1 = self.CreateObjectJson(bucket_name=bucket.name, contents=b'1') object2 = self.CreateObjectJson(bucket_name=bucket.name, object_name=object1.name, contents=b'2') object_url_string1 = 'gs://{}/{}#{}'.format(object1.bucket, object1.name, object1.generation) object_url_string2 = 'gs://{}/{}#{}'.format(object2.bucket, object2.name, object2.generation) stdout = self.RunGsUtil(['ls', object_url_string2], return_stdout=True) self.assertNotIn(object_url_string1, stdout) self.assertIn(object_url_string2, stdout) def test_subdir(self): """Tests listing a bucket subdirectory.""" bucket_uri = self.CreateBucket(test_objects=1) k1_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo') self.StorageUriSetContentsFromString(k1_uri, 'baz') k2_uri = self.StorageUriCloneReplaceName(bucket_uri, 'dir/foo') self.StorageUriSetContentsFromString(k2_uri, 'bar') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '%s/dir' % suri(bucket_uri)], return_stdout=True) self.assertEqual('%s\n' % suri(k2_uri), stdout) stdout = self.RunGsUtil(['ls', suri(k1_uri)], return_stdout=True) self.assertEqual('%s\n' % suri(k1_uri), stdout) _Check1() def test_subdir_nocontents(self): """Tests listing a bucket subdirectory using -d. Result will display subdirectory names instead of contents. Uses a wildcard to show multiple matching subdirectories. """ bucket_uri = self.CreateBucket(test_objects=1) k1_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo') self.StorageUriSetContentsFromString(k1_uri, 'baz') k2_uri = self.StorageUriCloneReplaceName(bucket_uri, 'dir/foo') self.StorageUriSetContentsFromString(k2_uri, 'bar') k3_uri = self.StorageUriCloneReplaceName(bucket_uri, 'dir/foo2') self.StorageUriSetContentsFromString(k3_uri, 'foo') k4_uri = self.StorageUriCloneReplaceName(bucket_uri, 'dir2/foo3') self.StorageUriSetContentsFromString(k4_uri, 'foo2') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil( ['ls', '-d', '%s/dir*' % suri(bucket_uri)], return_stdout=True) self.assertEqual( '%s/dir/\n%s/dir2/\n' % (suri(bucket_uri), suri(bucket_uri)), stdout) stdout = self.RunGsUtil(['ls', suri(k1_uri)], return_stdout=True) self.assertEqual('%s\n' % suri(k1_uri), stdout) _Check1() def test_versioning(self): """Tests listing a versioned bucket.""" bucket1_uri = self.CreateBucket(test_objects=1) bucket2_uri = self.CreateVersionedBucket(test_objects=1) self.AssertNObjectsInBucket(bucket1_uri, 1, versioned=True) bucket_list = list(bucket1_uri.list_bucket()) objuri = [ self.StorageUriCloneReplaceKey(bucket1_uri, key).versionless_uri for key in bucket_list ][0] self.RunGsUtil(['cp', objuri, suri(bucket2_uri)]) self.RunGsUtil(['cp', objuri, suri(bucket2_uri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(['ls', '-a', suri(bucket2_uri)], return_stdout=True) self.assertNumLines(stdout, 3) stdout = self.RunGsUtil(['ls', '-la', suri(bucket2_uri)], return_stdout=True) self.assertIn( '%s#' % self.StorageUriCloneReplaceName(bucket2_uri, bucket_list[0].name), stdout) self.assertIn('metageneration=', stdout) _Check2() def test_etag(self): """Tests that listing an object with an etag.""" bucket_uri = self.CreateBucket() obj_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') # TODO: When testcase setup can use JSON, match against the exact JSON # etag. etag = obj_uri.get_key().etag.strip('"\'') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-l', suri(bucket_uri)], return_stdout=True) if self.test_api == ApiSelector.XML: self.assertNotIn(etag, stdout) else: self.assertNotIn('etag=', stdout) _Check1() def _Check2(): stdout = self.RunGsUtil(['ls', '-le', suri(bucket_uri)], return_stdout=True) if self.test_api == ApiSelector.XML: self.assertIn(etag, stdout) else: self.assertIn('etag=', stdout) _Check2() def _Check3(): stdout = self.RunGsUtil(['ls', '-ale', suri(bucket_uri)], return_stdout=True) if self.test_api == ApiSelector.XML: self.assertIn(etag, stdout) else: self.assertIn('etag=', stdout) _Check3() def test_labels(self): """Tests listing on a bucket with a label/tagging configuration.""" bucket_uri = self.CreateBucket() bucket_suri = suri(bucket_uri) stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) # No labels are present by default. self.assertRegex(stdout, r'Labels:\s+None') # Add a label and check that it shows up. self.RunGsUtil(['label', 'ch', '-l', 'labelkey:labelvalue', bucket_suri], force_gsutil=True) stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) label_regex = re.compile(r'Labels:\s+\{\s+"labelkey":\s+"labelvalue"\s+\}', re.MULTILINE) self.assertRegex(stdout, label_regex) @SkipForS3('S3 bucket configuration values are not supported via ls.') def test_location_constraint(self): """Tests listing a bucket with location constraint.""" bucket_uri = self.CreateBucket() bucket_suri = suri(bucket_uri) # No location constraint should be shown for `-lb` stdout = self.RunGsUtil(['ls', '-lb', bucket_suri], return_stdout=True) self.assertNotIn('Location constraint:', stdout) # Default location constraint is US stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) # Default location may vary between test environments; test that some # non-whitespace character is present after the whitespace: self.assertRegex(stdout, r'Location constraint:\s+\S') # TODO(b/135700569): Stop skipping this once this field is available to all # projects. @unittest.skip('b/135700569') @SkipForXML('Location type not available when using the GCS XML API.') @SkipForS3('Location type not printed for S3 buckets.') def test_location_type(self): """Tests listing a bucket with location constraint.""" bucket_uri = self.CreateBucket() bucket_suri = suri(bucket_uri) # No location type should be shown for `-lb` stdout = self.RunGsUtil(['ls', '-lb', bucket_suri], return_stdout=True) self.assertNotIn('Location type:', stdout) # Default location type may vary between test environments; test that some # non-whitespace character is present after the whitespace: stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) self.assertRegex(stdout, r'Location type:\s+\S') @SkipForS3('S3 bucket configuration values are not supported via ls.') def test_logging(self): """Tests listing a bucket with logging config.""" bucket_uri = self.CreateBucket() bucket_suri = suri(bucket_uri) # No logging info stdout = self.RunGsUtil(['ls', '-lb', bucket_suri], return_stdout=True) self.assertNotIn('Logging configuration', stdout) spacing = ' ' if self._use_gcloud_storage else '\t\t' # Logging configuration is absent by default stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) self.assertIn('Logging configuration:{}None'.format(spacing), stdout) # Enable and check self.RunGsUtil(['logging', 'set', 'on', '-b', bucket_suri, bucket_suri]) stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) self.assertIn('Logging configuration:{}Present'.format(spacing), stdout) # Disable and check self.RunGsUtil(['logging', 'set', 'off', bucket_suri]) stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) self.assertIn('Logging configuration:{}None'.format(spacing), stdout) @SkipForS3('S3 bucket configuration values are not supported via ls.') def test_web(self): """Tests listing a bucket with website config.""" bucket_uri = self.CreateBucket() bucket_suri = suri(bucket_uri) # No website configuration stdout = self.RunGsUtil(['ls', '-lb', bucket_suri], return_stdout=True) self.assertNotIn('Website configuration', stdout) spacing = ' ' if self._use_gcloud_storage else '\t\t' # Website configuration is absent by default stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) self.assertIn('Website configuration:{}None'.format(spacing), stdout) # Initialize and check self.RunGsUtil(['web', 'set', '-m', 'google.com', bucket_suri]) stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) self.assertIn('Website configuration:{}Present'.format(spacing), stdout) # Clear and check self.RunGsUtil(['web', 'set', bucket_suri]) stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) self.assertIn('Website configuration:{}None'.format(spacing), stdout) @SkipForS3('S3 bucket configuration values are not supported via ls.') @SkipForXML('Requester Pays is not supported for the XML API.') def test_requesterpays(self): """Tests listing a bucket with requester pays (billing) config.""" bucket_uri = self.CreateBucket() bucket_suri = suri(bucket_uri) spacing = ' ' if self._use_gcloud_storage else '\t\t' # No requester pays configuration stdout = self.RunGsUtil(['ls', '-lb', bucket_suri], return_stdout=True) self.assertNotIn('Requester Pays enabled', stdout) # Requester Pays configuration is absent by default stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) self.assertIn('Requester Pays enabled:{}None'.format(spacing), stdout) # Initialize and check self.RunGsUtil(['requesterpays', 'set', 'on', bucket_suri]) stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) self.assertIn('Requester Pays enabled:{}True'.format(spacing), stdout) # Clear and check self.RunGsUtil(['requesterpays', 'set', 'off', bucket_suri]) stdout = self.RunGsUtil(['ls', '-Lb', bucket_suri], return_stdout=True) self.assertIn('Requester Pays enabled:{}False'.format(spacing), stdout) def test_list_sizes(self): """Tests various size listing options.""" bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, contents=b'x' * 2048) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-l', suri(bucket_uri)], return_stdout=True) self.assertIn('2048', stdout) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(['ls', '-L', suri(bucket_uri)], return_stdout=True) self.assertIn('2048', stdout) _Check2() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check3(): stdout = self.RunGsUtil(['ls', '-al', suri(bucket_uri)], return_stdout=True) self.assertIn('2048', stdout) _Check3() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check4(): stdout = self.RunGsUtil(['ls', '-lh', suri(bucket_uri)], return_stdout=True) self.assertIn('2 KiB', stdout) _Check4() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check5(): stdout = self.RunGsUtil(['ls', '-alh', suri(bucket_uri)], return_stdout=True) self.assertIn('2 KiB', stdout) _Check5() @unittest.skipIf(IS_WINDOWS, 'Unicode handling on Windows requires mods to site-packages') def test_list_unicode_filename(self): """Tests listing an object with a unicode filename.""" # Note: This test fails on Windows (command.exe). I was able to get ls to # output Unicode filenames correctly by hacking the UniStream class code # shown at # http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/3259271 # into the start of gslib/commands/ls.py, along with no-op flush and # isastream functions (as an experiment). However, even with that change, # the current test still fails, since it also needs to run that # stdout/stderr-replacement code. That UniStream class replacement really # needs to be added to the site-packages on Windows python. object_name = u'Аудиоархив' bucket_uri = self.CreateVersionedBucket() key_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo', object_name=object_name) self.AssertNObjectsInBucket(bucket_uri, 1, versioned=True) stdout = self.RunGsUtil(['ls', '-ael', suri(key_uri)], return_stdout=True) self.assertIn(object_name, stdout) if self.default_provider == 'gs': self.assertIn(str(key_uri.generation), stdout) self.assertIn('metageneration=%s' % key_uri.get_key().metageneration, stdout) if self.test_api == ApiSelector.XML: self.assertIn(key_uri.get_key().etag.strip('"\''), stdout) else: # TODO: When testcase setup can use JSON, match against the exact JSON # etag. self.assertIn('etag=', stdout) elif self.default_provider == 's3': self.assertIn(key_uri.version_id, stdout) self.assertIn(key_uri.get_key().etag.strip('"\''), stdout) def test_list_acl(self): """Tests that long listing includes an ACL.""" key_uri = self.CreateObject(contents=b'foo') stdout = self.RunGsUtil(['ls', '-L', suri(key_uri)], return_stdout=True) self.assertIn('ACL:', stdout) self.assertNotIn('ACCESS DENIED', stdout) def test_list_gzip_content_length(self): """Tests listing a gzipped object.""" file_size = 10000 file_contents = b'x' * file_size fpath = self.CreateTempFile(contents=file_contents, file_name='foo.txt') key_uri = self.CreateObject() self.RunGsUtil(['cp', '-z', 'txt', suri(fpath), suri(key_uri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', suri(key_uri)], return_stdout=True) self.assertRegex(stdout, r'Content-Encoding:\s+gzip') find_content_length_re = r'Content-Length:\s+(?P<num>\d)' self.assertRegex(stdout, find_content_length_re) m = re.search(find_content_length_re, stdout) content_length = int(m.group('num')) self.assertGreater(content_length, 0) self.assertLess(content_length, file_size) _Check1() def test_output_chopped(self): """Tests that gsutil still succeeds with a truncated stdout.""" bucket_uri = self.CreateBucket(test_objects=2) # Run Python with the -u flag so output is not buffered. gsutil_cmd = [ sys.executable, '-u', gslib.GSUTIL_PATH, 'ls', suri(bucket_uri) ] # Set bufsize to 0 to make sure output is not buffered. p = subprocess.Popen(gsutil_cmd, stdout=subprocess.PIPE, bufsize=0) # Immediately close the stdout pipe so that gsutil gets a broken pipe error. p.stdout.close() p.wait() # Make sure it still exited cleanly. self.assertEqual(p.returncode, 0) @SkipForS3('Boto lib required for S3 does not handle paths ' 'starting with slash.') def test_recursive_list_slash_only(self): """Tests listing an object with a trailing slash.""" bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='/', contents=b'foo') self.AssertNObjectsInBucket(bucket_uri, 1) stdout = self.RunGsUtil(['ls', '-R', suri(bucket_uri)], return_stdout=True) # Note: The suri function normalizes the URI, so the double slash gets # removed. self.assertIn(suri(bucket_uri) + '/', stdout) def test_recursive_list_trailing_slash(self): """Tests listing an object with a trailing slash.""" bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='foo/', contents=b'foo') self.AssertNObjectsInBucket(bucket_uri, 1) stdout = self.RunGsUtil(['ls', '-R', suri(bucket_uri)], return_stdout=True) # Note: The suri function normalizes the URI, so the double slash gets # removed. self.assertIn(suri(bucket_uri) + '/foo/', stdout) @SkipForS3('Boto lib required for S3 does not handle paths ' 'starting with slash.') def test_recursive_list_trailing_two_slash(self): """Tests listing an object with two trailing slashes.""" bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='//', contents=b'foo') self.AssertNObjectsInBucket(bucket_uri, 1) stdout = self.RunGsUtil(['ls', '-R', suri(bucket_uri)], return_stdout=True) # Note: The suri function normalizes the URI, so the double slash gets # removed. self.assertIn(suri(bucket_uri) + '//', stdout) def test_wildcard_prefix(self): """Tests that an object name with a wildcard does not infinite loop.""" bucket_uri = self.CreateBucket() wildcard_folder_object = 'wildcard*/' object_matching_folder = 'wildcard10/foo' self.CreateObject(bucket_uri=bucket_uri, object_name=wildcard_folder_object, contents=b'foo') self.CreateObject(bucket_uri=bucket_uri, object_name=object_matching_folder, contents=b'foo') self.AssertNObjectsInBucket(bucket_uri, 2) stderr = self.RunGsUtil(['ls', suri(bucket_uri, 'wildcard*')], return_stderr=True, expected_status=1) if self._use_gcloud_storage: warning_message = ('Cloud folders named with wildcards are not supported.' ' API returned {}/wildcard*/') else: warning_message = 'Cloud folder {}/wildcard*/ contains a wildcard' self.assertIn(warning_message.format(suri(bucket_uri)), stderr) # Listing with a flat wildcard should still succeed. # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil(['ls', '-l', suri(bucket_uri, '**')], return_stdout=True) self.assertNumLines(stdout, 3) # 2 object lines, one summary line. _Check() @SkipForS3('S3 anonymous access is not supported.') def test_get_object_without_list_bucket_permission(self): # Bucket is not publicly readable by default. bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='permitted', contents=b'foo') # Set this object to be publicly readable. self.RunGsUtil(['acl', 'set', 'public-read', suri(object_uri)], force_gsutil=True) # Drop credentials. with self.SetAnonymousBotoCreds(): stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) self.assertIn(suri(object_uri), stdout) @SkipForS3('S3 customer-supplied encryption keys are not supported.') def test_list_encrypted_object(self): if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') object_uri = self.CreateObject(object_name='foo', contents=TEST_ENCRYPTION_CONTENT1, encryption_key=TEST_ENCRYPTION_KEY1) # Listing object with key should return unencrypted hashes. with SetBotoConfigForTest([('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]): # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _ListExpectDecrypted(): stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64.decode('ascii'), stdout) _ListExpectDecrypted() # Listing object without a key should return encrypted hashes. # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _ListExpectEncrypted(): stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) self.assertNotIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn('encrypted', stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64.decode('ascii'), stdout) _ListExpectEncrypted() # Listing object with a non-matching key should return encrypted hashes. with SetBotoConfigForTest([('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2)]): _ListExpectEncrypted() @SkipForS3('S3 customer-supplied encryption keys are not supported.') def test_list_mixed_encryption(self): """Tests listing objects with various encryption interactions.""" bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=TEST_ENCRYPTION_CONTENT1, encryption_key=TEST_ENCRYPTION_KEY1) self.CreateObject(bucket_uri=bucket_uri, object_name='foo2', contents=TEST_ENCRYPTION_CONTENT2, encryption_key=TEST_ENCRYPTION_KEY2) self.CreateObject(bucket_uri=bucket_uri, object_name='foo3', contents=TEST_ENCRYPTION_CONTENT3, encryption_key=TEST_ENCRYPTION_KEY3) self.CreateObject(bucket_uri=bucket_uri, object_name='foo4', contents=TEST_ENCRYPTION_CONTENT4, encryption_key=TEST_ENCRYPTION_KEY4) self.CreateObject(bucket_uri=bucket_uri, object_name='foo5', contents=TEST_ENCRYPTION_CONTENT5) # List 5 objects, one encrypted with each of four keys, and one # unencrypted. Supplying keys [1,3,4] should result in four unencrypted # listings and one encrypted listing (for key 2). with SetBotoConfigForTest([ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY3), ('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY4) ]): # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _ListExpectMixed(): """Validates object listing.""" stdout = self.RunGsUtil(['ls', '-L', suri(bucket_uri)], return_stdout=True) self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64.decode('ascii'), stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT2_MD5, stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT2_CRC32C, stdout) self.assertIn('encrypted', stdout) self.assertIn(TEST_ENCRYPTION_KEY2_SHA256_B64.decode('ascii'), stdout) self.assertIn(TEST_ENCRYPTION_CONTENT3_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT3_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY3_SHA256_B64.decode('ascii'), stdout) self.assertIn(TEST_ENCRYPTION_CONTENT4_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT4_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY4_SHA256_B64.decode('ascii'), stdout) self.assertIn(TEST_ENCRYPTION_CONTENT5_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT5_CRC32C, stdout) _ListExpectMixed() def test_non_ascii_project_fails(self): stderr = self.RunGsUtil(['ls', '-p', 'ã', 'gs://fobarbaz'], expected_status=1, return_stderr=True) self.assertIn('Invalid non-ASCII', stderr) def set_default_kms_key_on_bucket(self, bucket_uri): # Make sure our keyRing and cryptoKey exist. keyring_fqn = self.kms_api.CreateKeyRing( PopulateProjectId(None), testcase.KmsTestingResources.KEYRING_NAME, location=testcase.KmsTestingResources.KEYRING_LOCATION) key_fqn = self.kms_api.CreateCryptoKey( keyring_fqn, testcase.KmsTestingResources.CONSTANT_KEY_NAME) # Make sure that the service account for the desired bucket's parent project # is authorized to encrypt with the key above. self.RunGsUtil(['kms', 'encryption', '-k', key_fqn, suri(bucket_uri)]) return key_fqn @SkipForXML(KMS_XML_SKIP_MSG) @SkipForS3(KMS_XML_SKIP_MSG) def test_default_kms_key_listed_for_bucket(self): bucket_uri = self.CreateBucket() # Default KMS key is not set by default. stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'Default KMS key:\s+None') # Default KMS key's name should be listed after being set on the bucket. key_fqn = self.set_default_kms_key_on_bucket(bucket_uri) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'Default KMS key:\s+%s' % key_fqn) @SkipForXML(KMS_XML_SKIP_MSG) @SkipForS3(KMS_XML_SKIP_MSG) def test_kms_key_listed_for_kms_encrypted_object(self): bucket_uri = self.CreateBucket() key_fqn = self.set_default_kms_key_on_bucket(bucket_uri) # Copy an object into our bucket and encrypt using the key from above. obj_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'foo', kms_key_name=key_fqn) stdout = self.RunGsUtil(['ls', '-L', suri(obj_uri)], return_stdout=True) self.assertRegex(stdout, r'KMS key:\s+%s' % key_fqn) @SkipForXML(BUCKET_LOCK_SKIP_MSG) @SkipForS3(BUCKET_LOCK_SKIP_MSG) def test_list_retention_policy(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=1) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'Retention Policy\:\s*Present') # Clearing Retention Policy on the bucket. self.RunGsUtil(['retention', 'clear', suri(bucket_uri)]) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertNotRegex(stdout, r'Retention Policy:') @SkipForXML(BUCKET_LOCK_SKIP_MSG) @SkipForS3(BUCKET_LOCK_SKIP_MSG) def test_list_default_event_based_hold(self): bucket_uri = self.CreateBucket() self.RunGsUtil(['retention', 'event-default', 'set', suri(bucket_uri)]) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'Default Event-Based Hold:\t* *True') # Clearing the default Event-Based Hold on the bucket. self.RunGsUtil(['retention', 'event-default', 'release', suri(bucket_uri)]) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertNotRegex(stdout, r'Default Event-Based Hold') @SkipForXML(BUCKET_LOCK_SKIP_MSG) @SkipForS3(BUCKET_LOCK_SKIP_MSG) def test_list_temporary_hold(self): object_uri = self.CreateObject(contents=b'content') self.RunGsUtil(['retention', 'temp', 'set', suri(object_uri)]) stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) self.assertRegex(stdout, r'Temporary Hold') # Clearing the Temporary Hold on the object. self.RunGsUtil(['retention', 'temp', 'release', suri(object_uri)]) stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) self.assertNotRegex(stdout, r'Temporary Hold') @SkipForXML(BUCKET_LOCK_SKIP_MSG) @SkipForS3(BUCKET_LOCK_SKIP_MSG) def test_list_event_based_hold(self): object_uri = self.CreateObject(contents=b'content') self.RunGsUtil(['retention', 'event', 'set', suri(object_uri)]) stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) self.assertRegex(stdout, r'Event-Based Hold') # Clearing the Event-Based Hold on the object. self.RunGsUtil(['retention', 'event', 'release', suri(object_uri)]) stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) self.assertNotRegex(stdout, r'Event-Based Hold') @SkipForXML('public access prevention is not supported for the XML API.') @SkipForS3('public access prevention is not supported for S3 buckets.') def test_list_public_access_prevention(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'Public access prevention:\s*(unspecified|inherited)') # Enforce public access prevention. self.RunGsUtil(['pap', 'set', 'enforced', suri(bucket_uri)]) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'Public access prevention:\s*enforced') @SkipForXML('RPO is not supported for the XML API.') @SkipForS3('RPO is not supported for S3 buckets.') def test_list_Lb_displays_rpo(self): bucket_uri = self.CreateBucket(location='nam4') stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) # TODO: Uncomment this check once we have have consistent behavior from # the backend. Currently, both None and DEFAULT are valid values for # default replication and ls will not display the field if value is None. # self.assertRegex(stdout, r'RPO:\t\t\t\tDEFAULT') # Set RPO to ASYNC_TURBO self.RunGsUtil(['rpo', 'set', 'ASYNC_TURBO', suri(bucket_uri)]) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'RPO:\t\t\t\tASYNC_TURBO') @SkipForXML('Custom Dual Region is not supported for the XML API.') @SkipForS3('Custom Dual Region is not supported for S3 buckets.') def test_list_Lb_displays_custom_dual_region_placement_info(self): bucket_name = 'gs://' + self.MakeTempName('bucket') self.RunGsUtil(['mb', '--placement', 'us-central1,us-west1', bucket_name], expected_status=0) stdout = self.RunGsUtil(['ls', '-Lb', bucket_name], return_stdout=True) self.assertRegex(stdout, r"Placement locations:\t\t\['US-CENTRAL1', 'US-WEST1'\]") @SkipForXML('Autoclass is not supported for the XML API.') @SkipForS3('Autoclass is not supported for S3 buckets.') def test_list_autoclass(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertNotIn('Autoclass', stdout) # Enforce Autoclass. self.RunGsUtil(['autoclass', 'set', 'on', suri(bucket_uri)]) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'Autoclass:\t*Enabled on .+') tests/test_setmeta.py 0000644 00000031047 15025013616 0010765 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for setmeta command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import re from unittest import mock import six from gslib.commands import setmeta from gslib.cs_api_map import ApiSelector import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import unittest from gslib.utils.retry_util import Retry from gslib.utils import shim_util if six.PY3: long = int class TestSetMeta(testcase.GsUtilIntegrationTestCase): """Integration tests for setmeta command.""" def test_initial_metadata(self): """Tests copying file to an object with metadata.""" objuri = suri(self.CreateObject(contents=b'foo')) inpath = self.CreateTempFile() ct = 'image/gif' self.RunGsUtil([ '-h', 'x-%s-meta-xyz:abc' % self.provider_custom_meta, '-h', 'Content-Type:%s' % ct, 'cp', inpath, objuri ]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', objuri], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+%s' % ct) self.assertRegex(stdout, r'xyz:\s+abc') _Check1() def test_overwrite_existing(self): """Tests overwriting an object's metadata.""" objuri = suri(self.CreateObject(contents=b'foo')) inpath = self.CreateTempFile() self.RunGsUtil([ '-h', 'x-%s-meta-xyz:abc' % self.provider_custom_meta, '-h', 'Content-Type:image/gif', 'cp', inpath, objuri ]) self.RunGsUtil([ 'setmeta', '-h', 'Content-Type:text/html', '-h', 'x-%s-meta-xyz' % self.provider_custom_meta, objuri ]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', objuri], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+text/html') self.assertNotIn('xyz', stdout) _Check1() @SkipForS3('Preconditions not supported for s3 objects') def test_generation_precondition(self): """Tests setting metadata with a generation precondition.""" object_uri = self.CreateObject(contents=b'foo') generation = object_uri.generation ct = 'image/gif' stderr = self.RunGsUtil([ '-h', 'x-goog-if-generation-match:%d' % (long(generation) + 1), 'setmeta', '-h', 'x-%s-meta-xyz:abc' % self.provider_custom_meta, '-h', 'Content-Type:%s' % ct, suri(object_uri) ], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('pre-condition', stderr) else: self.assertIn('Precondition', stderr) self.RunGsUtil([ '-h', 'x-goog-generation-match:%s' % generation, 'setmeta', '-h', 'x-%s-meta-xyz:abc' % self.provider_custom_meta, '-h', 'Content-Type:%s' % ct, suri(object_uri) ]) stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+%s' % ct) self.assertRegex(stdout, r'xyz:\s+abc') @SkipForS3('Preconditions not supported for s3 objects') def test_metageneration_precondition(self): """Tests setting metadata with a metageneration precondition.""" object_uri = self.CreateObject(contents=b'foo') ct = 'image/gif' stderr = self.RunGsUtil([ '-h', 'x-goog-if-metageneration-match:5', 'setmeta', '-h', 'x-%s-meta-xyz:abc' % self.provider_custom_meta, '-h', 'Content-Type:%s' % ct, suri(object_uri) ], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('pre-condition', stderr) else: self.assertIn('Precondition', stderr) self.RunGsUtil([ '-h', 'x-goog-metageneration-match:1', 'setmeta', '-h', 'x-%s-meta-xyz:abc' % self.provider_custom_meta, '-h', 'Content-Type:%s' % ct, suri(object_uri) ]) stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+%s' % ct) self.assertRegex(stdout, r'xyz:\s+abc') def test_duplicate_header_removal(self): stderr = self.RunGsUtil([ 'setmeta', '-h', 'Content-Type:text/html', '-h', 'Content-Type', 'gs://foo/bar' ], expected_status=1, return_stderr=True) self.assertIn('Each header must appear at most once', stderr) def test_duplicate_header(self): stderr = self.RunGsUtil([ 'setmeta', '-h', 'Content-Type:text/html', '-h', 'Content-Type:foobar', 'gs://foo/bar' ], expected_status=1, return_stderr=True) self.assertIn('Each header must appear at most once', stderr) def test_setmeta_seek_ahead(self): object_uri = self.CreateObject(contents=b'foo') with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil( ['-m', 'setmeta', '-h', 'content-type:footype', suri(object_uri)], return_stderr=True) self.assertIn('Estimated work for this command: objects: 1\n', stderr) def test_recursion_works(self): bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') object2_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') self.RunGsUtil( ['setmeta', '-R', '-h', 'content-type:footype', suri(bucket_uri)]) for obj_uri in [object1_uri, object2_uri]: stdout = self.RunGsUtil(['stat', suri(obj_uri)], return_stdout=True) self.assertIn('footype', stdout) def test_metadata_parallelism(self): """Ensure that custom metadata works in the multi-thread/process case.""" # If this test hangs, it can indicate a pickling error. bucket_uri = self.CreateBucket(test_objects=2) self.AssertNObjectsInBucket(bucket_uri, 2) self.RunGsUtil([ 'setmeta', '-h', 'x-%s-meta-abc:123' % self.provider_custom_meta, suri(bucket_uri, '**') ]) def test_invalid_non_ascii_custom_header(self): unicode_header = 'x-%s-meta-soufflé:5' % self.provider_custom_meta stderr = self.RunGsUtil([ 'setmeta', '-h', unicode_header, '%s://foo/bar' % self.default_provider ], expected_status=1, return_stderr=True) self.assertIn('Invalid non-ASCII header', stderr) @SkipForS3('Only ASCII characters are supported for x-amz-meta headers') def test_valid_non_ascii_custom_header(self): """Tests setting custom metadata with a non-ASCII content.""" objuri = self.CreateObject(contents=b'foo') unicode_header = 'x-%s-meta-dessert:soufflé' % self.provider_custom_meta self.RunGsUtil(['setmeta', '-h', unicode_header, suri(objuri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', suri(objuri)], return_stdout=True) self.assertTrue(re.search('dessert:\\s+soufflé', stdout)) _Check1() @SkipForS3('XML header keys are case-insensitive') def test_uppercase_header(self): """Tests setting custom metadata with an uppercase value.""" if self.test_api == ApiSelector.XML: return unittest.skip('XML header keys are case-insensitive.') objuri = self.CreateObject(contents=b'foo') self.RunGsUtil([ 'setmeta', '-h', 'x-%s-meta-CaSe:SeNsItIvE' % self.provider_custom_meta, suri(objuri) ]) stdout = self.RunGsUtil(['stat', suri(objuri)], return_stdout=True) self.assertRegex(stdout, r'CaSe:\s+SeNsItIvE') def test_remove_header(self): """Tests removing a header""" objuri = self.CreateObject(contents=b'foo') def _Check1(): self.RunGsUtil(['setmeta', '-h', 'content-disposition:br', suri(objuri)]) stdout = self.RunGsUtil(['stat', suri(objuri)], return_stdout=True) self.assertRegex(stdout, r'Content-Disposition') def _Check2(): self.RunGsUtil(['setmeta', '-h', 'content-disposition', suri(objuri)]) stdout = self.RunGsUtil(['stat', suri(objuri)], return_stdout=True) self.assertRegex(stdout, r'(?!Content-Disposition)') _Check1() _Check2() def test_disallowed_header(self): stderr = self.RunGsUtil( ['setmeta', '-h', 'Content-Length:5', 'gs://foo/bar'], expected_status=1, return_stderr=True) self.assertIn('Invalid or disallowed header', stderr) def test_setmeta_bucket(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil([ 'setmeta', '-h', 'x-%s-meta-foo:5' % self.provider_custom_meta, suri(bucket_uri) ], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('ERROR', stderr) else: self.assertIn('must name an object', stderr) def test_setmeta_valid_with_multiple_colons_in_value(self): obj_uri = self.CreateObject(contents=b'foo') self.RunGsUtil([ 'setmeta', '-h', 'x-%s-meta-foo:bar:baz' % self.provider_custom_meta, suri(obj_uri) ]) stdout = self.RunGsUtil(['stat', suri(obj_uri)], return_stdout=True) self.assertRegex(stdout, r'foo:\s+bar:baz') def test_setmeta_with_canned_acl(self): stderr = self.RunGsUtil([ 'setmeta', '-h', 'x-%s-acl:public-read' % self.provider_custom_meta, 'gs://foo/bar' ], expected_status=1, return_stderr=True) self.assertIn('gsutil setmeta no longer allows canned ACLs', stderr) def test_invalid_non_ascii_header_value(self): unicode_header = 'Content-Type:dessert/soufflé' stderr = self.RunGsUtil(['setmeta', '-h', unicode_header, 'gs://foo/bar'], expected_status=1, return_stderr=True) self.assertIn('Invalid non-ASCII value', stderr) def test_setmeta_raises_error_if_not_provided_headers(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil(['setmeta', suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn( 'gsutil setmeta requires one or more headers to be provided with the' ' -h flag. See "gsutil help setmeta" for more information.', stderr) class TestSetMetaShim(testcase.ShimUnitTestBase): @mock.patch.object(setmeta.SetMetaCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_setmeta_set_and_clear_flags(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('setmeta', [ '-r', '-h', 'Cache-Control:', '-h', 'Content-Type:fake-content-type', 'gs://bucket/object', ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage objects update' ' --recursive --clear-cache-control' ' --content-type=fake-content-type gs://bucket/object').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) tests/test_util.py 0000644 00000047545 15025013616 0010312 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Tests for gsutil utility functions.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.utils import boto_util from gslib.utils import constants from gslib.utils import system_util from gslib.utils import ls_helper from gslib.utils import retry_util from gslib.utils import text_util from gslib.utils import unit_util import gslib.tests.testcase as testcase from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import TestParams from gslib.utils.text_util import CompareVersions from gslib.utils.unit_util import DecimalShort from gslib.utils.unit_util import HumanReadableWithDecimalPlaces from gslib.utils.unit_util import PrettyTime import httplib2 import os import six from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock # TODO: Split tests for <foo>_util methods out into their own test classes. class TestUtil(testcase.GsUtilUnitTestCase): """Tests for utility functions.""" def testMakeHumanReadable(self): """Tests converting byte counts to human-readable strings.""" self.assertEqual(unit_util.MakeHumanReadable(0), '0 B') self.assertEqual(unit_util.MakeHumanReadable(1023), '1023 B') self.assertEqual(unit_util.MakeHumanReadable(1024), '1 KiB') self.assertEqual(unit_util.MakeHumanReadable(1024**2), '1 MiB') self.assertEqual(unit_util.MakeHumanReadable(1024**3), '1 GiB') self.assertEqual(unit_util.MakeHumanReadable(1024**3 * 5.3), '5.3 GiB') self.assertEqual(unit_util.MakeHumanReadable(1024**4 * 2.7), '2.7 TiB') self.assertEqual(unit_util.MakeHumanReadable(1024**5), '1 PiB') self.assertEqual(unit_util.MakeHumanReadable(1024**6), '1 EiB') def testMakeBitsHumanReadable(self): """Tests converting bit counts to human-readable strings.""" self.assertEqual(unit_util.MakeBitsHumanReadable(0), '0 bit') self.assertEqual(unit_util.MakeBitsHumanReadable(1023), '1023 bit') self.assertEqual(unit_util.MakeBitsHumanReadable(1024), '1 Kibit') self.assertEqual(unit_util.MakeBitsHumanReadable(1024**2), '1 Mibit') self.assertEqual(unit_util.MakeBitsHumanReadable(1024**3), '1 Gibit') self.assertEqual(unit_util.MakeBitsHumanReadable(1024**3 * 5.3), '5.3 Gibit') self.assertEqual(unit_util.MakeBitsHumanReadable(1024**4 * 2.7), '2.7 Tibit') self.assertEqual(unit_util.MakeBitsHumanReadable(1024**5), '1 Pibit') self.assertEqual(unit_util.MakeBitsHumanReadable(1024**6), '1 Eibit') def testHumanReadableToBytes(self): """Tests converting human-readable strings to byte counts.""" self.assertEqual(unit_util.HumanReadableToBytes('1'), 1) self.assertEqual(unit_util.HumanReadableToBytes('15'), 15) self.assertEqual(unit_util.HumanReadableToBytes('15.3'), 15) self.assertEqual(unit_util.HumanReadableToBytes('15.7'), 16) self.assertEqual(unit_util.HumanReadableToBytes('1023'), 1023) self.assertEqual(unit_util.HumanReadableToBytes('1k'), 1024) self.assertEqual(unit_util.HumanReadableToBytes('2048'), 2048) self.assertEqual(unit_util.HumanReadableToBytes('1 k'), 1024) self.assertEqual(unit_util.HumanReadableToBytes('1 K'), 1024) self.assertEqual(unit_util.HumanReadableToBytes('1 KB'), 1024) self.assertEqual(unit_util.HumanReadableToBytes('1 KiB'), 1024) self.assertEqual(unit_util.HumanReadableToBytes('1 m'), 1024**2) self.assertEqual(unit_util.HumanReadableToBytes('1 M'), 1024**2) self.assertEqual(unit_util.HumanReadableToBytes('1 MB'), 1024**2) self.assertEqual(unit_util.HumanReadableToBytes('1 MiB'), 1024**2) self.assertEqual(unit_util.HumanReadableToBytes('1 g'), 1024**3) self.assertEqual(unit_util.HumanReadableToBytes('1 G'), 1024**3) self.assertEqual(unit_util.HumanReadableToBytes('1 GB'), 1024**3) self.assertEqual(unit_util.HumanReadableToBytes('1 GiB'), 1024**3) self.assertEqual(unit_util.HumanReadableToBytes('1t'), 1024**4) self.assertEqual(unit_util.HumanReadableToBytes('1T'), 1024**4) self.assertEqual(unit_util.HumanReadableToBytes('1TB'), 1024**4) self.assertEqual(unit_util.HumanReadableToBytes('1TiB'), 1024**4) self.assertEqual(unit_util.HumanReadableToBytes('1\t p'), 1024**5) self.assertEqual(unit_util.HumanReadableToBytes('1\t P'), 1024**5) self.assertEqual(unit_util.HumanReadableToBytes('1\t PB'), 1024**5) self.assertEqual(unit_util.HumanReadableToBytes('1\t PiB'), 1024**5) self.assertEqual(unit_util.HumanReadableToBytes('1e'), 1024**6) self.assertEqual(unit_util.HumanReadableToBytes('1E'), 1024**6) self.assertEqual(unit_util.HumanReadableToBytes('1EB'), 1024**6) self.assertEqual(unit_util.HumanReadableToBytes('1EiB'), 1024**6) def testCompareVersions(self): """Tests CompareVersions for various use cases.""" # CompareVersions(first, second) returns (g, m), where # g is True if first known to be greater than second, else False. # m is True if first known to be greater by at least 1 major version, (g, m) = CompareVersions('3.37', '3.2') self.assertTrue(g) self.assertFalse(m) (g, m) = CompareVersions('7', '2') self.assertTrue(g) self.assertTrue(m) (g, m) = CompareVersions('3.32', '3.32pre') self.assertTrue(g) self.assertFalse(m) (g, m) = CompareVersions('3.32pre', '3.31') self.assertTrue(g) self.assertFalse(m) (g, m) = CompareVersions('3.4pre', '3.3pree') self.assertTrue(g) self.assertFalse(m) (g, m) = CompareVersions('3.2', '3.37') self.assertFalse(g) self.assertFalse(m) (g, m) = CompareVersions('2', '7') self.assertFalse(g) self.assertFalse(m) (g, m) = CompareVersions('3.32pre', '3.32') self.assertFalse(g) self.assertFalse(m) (g, m) = CompareVersions('3.31', '3.32pre') self.assertFalse(g) self.assertFalse(m) (g, m) = CompareVersions('3.3pre', '3.3pre') self.assertFalse(g) self.assertFalse(m) (g, m) = CompareVersions('foobar', 'baz') self.assertFalse(g) self.assertFalse(m) (g, m) = CompareVersions('3.32', 'baz') self.assertFalse(g) self.assertFalse(m) (g, m) = CompareVersions('3.4', '3.3') self.assertTrue(g) self.assertFalse(m) (g, m) = CompareVersions('3.3', '3.4') self.assertFalse(g) self.assertFalse(m) (g, m) = CompareVersions('4.1', '3.33') self.assertTrue(g) self.assertTrue(m) (g, m) = CompareVersions('3.10', '3.1') self.assertTrue(g) self.assertFalse(m) def _AssertProxyInfosEqual(self, pi1, pi2): self.assertEqual(pi1.proxy_type, pi2.proxy_type) self.assertEqual(pi1.proxy_host, pi2.proxy_host) self.assertEqual(pi1.proxy_port, pi2.proxy_port) self.assertEqual(pi1.proxy_rdns, pi2.proxy_rdns) self.assertEqual(pi1.proxy_user, pi2.proxy_user) self.assertEqual(pi1.proxy_pass, pi2.proxy_pass) def testMakeMetadataLine(self): test_params = (TestParams(args=('AFairlyShortKey', 'Value'), expected=' AFairlyShortKey: Value'), TestParams(args=('', 'Value'), expected=' : Value'), TestParams(args=('AnotherKey', 'Value'), kwargs={'indent': 2}, expected=' AnotherKey: Value'), TestParams(args=('AKeyMuchLongerThanTheLast', 'Value'), expected=(' AKeyMuchLongerThanTheLast:Value'))) for params in test_params: line = ls_helper.MakeMetadataLine(*(params.args), **(params.kwargs)) self.assertEqual(line, params.expected) def testSetProxyInfo(self): """Tests SetProxyInfo for various proxy use cases in boto file.""" valid_proxy_types = ['socks4', 'socks5', 'http'] valid_proxy_host = ['hostname', '1.2.3.4', None] valid_proxy_port = [8888, 0] valid_proxy_user = ['foo', None] valid_proxy_pass = ['Bar', None] valid_proxy_rdns = [True, False, None] proxy_type_spec = { 'socks4': httplib2.socks.PROXY_TYPE_SOCKS4, 'socks5': httplib2.socks.PROXY_TYPE_SOCKS5, 'http': httplib2.socks.PROXY_TYPE_HTTP, 'https': httplib2.socks.PROXY_TYPE_HTTP } #Generate all input combination values boto_proxy_config_test_values = [{ 'proxy_host': p_h, 'proxy_type': p_t, 'proxy_port': p_p, 'proxy_user': p_u, 'proxy_pass': p_s, 'proxy_rdns': p_d } for p_h in valid_proxy_host for p_s in valid_proxy_pass for p_p in valid_proxy_port for p_u in valid_proxy_user for p_t in valid_proxy_types for p_d in valid_proxy_rdns] #Test all input combination values with SetEnvironmentForTest({'http_proxy': 'http://host:50'}): for test_values in boto_proxy_config_test_values: proxy_type = proxy_type_spec.get(test_values.get('proxy_type')) proxy_host = test_values.get('proxy_host') proxy_port = test_values.get('proxy_port') proxy_user = test_values.get('proxy_user') proxy_pass = test_values.get('proxy_pass') proxy_rdns = bool(test_values.get('proxy_rdns')) # Added to force socks proxies not to use rdns as in SetProxyInfo() if not (proxy_type == proxy_type_spec['http']): proxy_rdns = False expected = httplib2.ProxyInfo(proxy_host=proxy_host, proxy_type=proxy_type, proxy_port=proxy_port, proxy_user=proxy_user, proxy_pass=proxy_pass, proxy_rdns=proxy_rdns) # Checks to make sure environment variable fallbacks are working if not (expected.proxy_host and expected.proxy_port): expected = httplib2.ProxyInfo(proxy_type_spec['http'], 'host', 50) # Assume proxy_rnds is True if a proxy environment variable exists. if test_values.get('proxy_rdns') == None: expected.proxy_rdns = True self._AssertProxyInfosEqual(boto_util.SetProxyInfo(test_values), expected) def testProxyInfoFromEnvironmentVar(self): """Tests ProxyInfoFromEnvironmentVar for various cases.""" valid_variables = ['http_proxy', 'https_proxy'] if not system_util.IS_WINDOWS: # Dynamically set Windows environment variables are case-insensitive. valid_variables.append('HTTPS_PROXY') # Clear any existing environment variables for the duration of the test. clear_dict = {} for key in valid_variables: clear_dict[key] = None with SetEnvironmentForTest(clear_dict): for env_var in valid_variables: for url_string in ['hostname', 'http://hostname', 'https://hostname']: with SetEnvironmentForTest({env_var: url_string}): self._AssertProxyInfosEqual( boto_util.ProxyInfoFromEnvironmentVar(env_var), httplib2.ProxyInfo( httplib2.socks.PROXY_TYPE_HTTP, 'hostname', 443 if env_var.lower().startswith('https') else 80)) # Shouldn't populate info for other variables for other_env_var in valid_variables: if other_env_var == env_var: continue self._AssertProxyInfosEqual( boto_util.ProxyInfoFromEnvironmentVar(other_env_var), httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, None, 0)) for url_string in [ '1.2.3.4:50', 'http://1.2.3.4:50', 'https://1.2.3.4:50' ]: with SetEnvironmentForTest({env_var: url_string}): self._AssertProxyInfosEqual( boto_util.ProxyInfoFromEnvironmentVar(env_var), httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, '1.2.3.4', 50)) for url_string in [ 'foo:bar@1.2.3.4:50', 'http://foo:bar@1.2.3.4:50', 'https://foo:bar@1.2.3.4:50' ]: with SetEnvironmentForTest({env_var: url_string}): self._AssertProxyInfosEqual( boto_util.ProxyInfoFromEnvironmentVar(env_var), httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, '1.2.3.4', 50, proxy_user='foo', proxy_pass='bar')) for url_string in [ 'bar@1.2.3.4:50', 'http://bar@1.2.3.4:50', 'https://bar@1.2.3.4:50' ]: with SetEnvironmentForTest({env_var: url_string}): self._AssertProxyInfosEqual( boto_util.ProxyInfoFromEnvironmentVar(env_var), httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, '1.2.3.4', 50, proxy_user='bar')) for env_var in ['proxy', 'noproxy', 'garbage']: for url_string in ['1.2.3.4:50', 'http://1.2.3.4:50']: with SetEnvironmentForTest({env_var: url_string}): self._AssertProxyInfosEqual( boto_util.ProxyInfoFromEnvironmentVar(env_var), httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, None, 0)) # We want to make sure the wrapped function is called without executing it. @mock.patch.object(retry_util.http_wrapper, 'HandleExceptionsAndRebuildHttpConnections') @mock.patch.object(retry_util.logging, 'info') def testWarnAfterManyRetriesHandler(self, mock_log_info_fn, mock_wrapped_fn): # The only ExceptionRetryArgs attributes that the function cares about are # num_retries and total_wait_sec; we can pass None for the other values. retry_args_over_threshold = retry_util.http_wrapper.ExceptionRetryArgs( None, None, None, 3, None, constants.LONG_RETRY_WARN_SEC + 1) retry_args_under_threshold = retry_util.http_wrapper.ExceptionRetryArgs( None, None, None, 2, None, constants.LONG_RETRY_WARN_SEC - 1) retry_util.LogAndHandleRetries()(retry_args_under_threshold) self.assertTrue(mock_wrapped_fn.called) # Check that we didn't emit a message. self.assertFalse(mock_log_info_fn.called) retry_util.LogAndHandleRetries()(retry_args_over_threshold) self.assertEqual(mock_wrapped_fn.call_count, 2) # Check that we did emit a message. self.assertTrue(mock_log_info_fn.called) def testUIDecimalShort(self): """Tests DecimalShort for UI.""" self.assertEqual('12.3b', DecimalShort(12345678910)) self.assertEqual('123.5m', DecimalShort(123456789)) self.assertEqual('1.2k', DecimalShort(1234)) self.assertEqual('1.0k', DecimalShort(1000)) self.assertEqual('432', DecimalShort(432)) self.assertEqual('43.2t', DecimalShort(43.25 * 10**12)) self.assertEqual('43.2q', DecimalShort(43.25 * 10**15)) self.assertEqual('43250.0q', DecimalShort(43.25 * 10**18)) def testUIPrettyTime(self): """Tests PrettyTime for UI.""" self.assertEqual('25:02:10', PrettyTime(90130)) self.assertEqual('01:00:00', PrettyTime(3600)) self.assertEqual('00:59:59', PrettyTime(3599)) self.assertEqual('100+ hrs', PrettyTime(3600 * 100)) self.assertEqual('999+ hrs', PrettyTime(3600 * 10000)) def testUIHumanReadableWithDecimalPlaces(self): """Tests HumanReadableWithDecimalPlaces for UI.""" self.assertEqual('1.0 GiB', HumanReadableWithDecimalPlaces(1024**3 + 1024**2 * 10, 1)) self.assertEqual('1.0 GiB', HumanReadableWithDecimalPlaces(1024**3), 1) self.assertEqual('1.01 GiB', HumanReadableWithDecimalPlaces(1024**3 + 1024**2 * 10, 2)) self.assertEqual('1.000 GiB', HumanReadableWithDecimalPlaces(1024**3 + 1024**2 * 5, 3)) self.assertEqual('1.10 GiB', HumanReadableWithDecimalPlaces(1024**3 + 1024**2 * 100, 2)) self.assertEqual('1.100 GiB', HumanReadableWithDecimalPlaces(1024**3 + 1024**2 * 100, 3)) self.assertEqual('10.00 MiB', HumanReadableWithDecimalPlaces(1024**2 * 10, 2)) # The test below is good for rounding. self.assertEqual('2.01 GiB', HumanReadableWithDecimalPlaces(2157969408, 2)) self.assertEqual('2.0 GiB', HumanReadableWithDecimalPlaces(2157969408, 1)) self.assertEqual('0 B', HumanReadableWithDecimalPlaces(0, 0)) self.assertEqual('0.00 B', HumanReadableWithDecimalPlaces(0, 2)) self.assertEqual('0.00000 B', HumanReadableWithDecimalPlaces(0, 5)) def testAmzGenerationTypeConversions(self): amz_gen_as_str = six.ensure_str('9PpsRjBGjBh90IvIS96dgRc_UL6NyGqD') amz_gen_as_long = 25923956239092482442895228561437790190304192615858167521375267910356975448388 self.assertEqual(text_util.DecodeLongAsString(amz_gen_as_long), amz_gen_as_str) self.assertEqual(text_util.EncodeStringAsLong(amz_gen_as_str), amz_gen_as_long) def DoTestAddQueryParamToUrl(self, url, param_name, param_val, expected_url): new_url = text_util.AddQueryParamToUrl(url, param_name, param_val) self.assertEqual(new_url, expected_url) def testAddQueryParamToUrlWorksForASCIIValues(self): # Note that the params here contain empty values and duplicates. old_url = 'http://foo.bar/path/endpoint?a=1&a=2&b=3&c=' param_name = 'newparam' param_val = 'nevalue' expected_url = '{}&{}={}'.format(old_url, param_name, param_val) self.DoTestAddQueryParamToUrl(old_url, param_name, param_val, expected_url) def testAddQueryParamToUrlWorksForUTF8Values(self): old_url = 'http://foo.bar/path/êndpoint?Â=1&a=2&ß=3&c=' param_name = 'nêwparam' param_val = 'nêwvalue' # Expected return value is a UTF-8 encoded `str`. expected_url = '{}&{}={}'.format(old_url, param_name, param_val) self.DoTestAddQueryParamToUrl(old_url, param_name, param_val, expected_url) def testAddQueryParamToUrlWorksForRawUnicodeValues(self): old_url = 'http://foo.bar/path/êndpoint?Â=1&a=2&ß=3&c=' param_name = 'nêwparam' param_val = 'nêwvalue' # Since the original URL was a `unicode`, the returned URL should also be. expected_url = '{}&{}={}'.format(old_url, param_name, param_val) self.DoTestAddQueryParamToUrl(old_url, param_name, param_val, expected_url) @mock.patch.object(boto_util, 'GetMaxUploadCompressionBufferSize') def testGetMaxConcurrentCompressedUploadsMinimum(self, mock_config): """Test GetMaxConcurrentCompressedUploads returns at least 1.""" mock_config.return_value = 0 self.assertEqual(boto_util.GetMaxConcurrentCompressedUploads(), 1) mock_config.return_value = -1 self.assertEqual(boto_util.GetMaxConcurrentCompressedUploads(), 1) tests/test_psc.py 0000644 00000013601 15025013616 0010104 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for private service connect custom endpoints.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from boto import config from gslib.gcs_json_api import DEFAULT_HOST from gslib.tests import testcase from gslib.tests.testcase import integration_testcase from gslib.tests.util import ObjectToURI from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import unittest # Get full output from stdout, which might not be piped in Python 3. PYTHON_UNBUFFERED_ENV_VAR = {'PYTHONUNBUFFERED': '1'} class TestPsc(testcase.GsUtilIntegrationTestCase): """Integration tests for PSC custom endpoints.""" @integration_testcase.SkipForXML('JSON test.') @integration_testcase.SkipForS3('Custom endpoints not available for S3.') def test_persists_custom_endpoint_through_json_sliced_download(self): gs_host = config.get('Credentials', 'gs_json_host', DEFAULT_HOST) if gs_host == DEFAULT_HOST: # Skips test when run without a custom endpoint configured. return temporary_directory = self.CreateTempDir() with SetBotoConfigForTest([ ('GSUtil', 'sliced_object_download_threshold', '1B'), ('GSUtil', 'sliced_object_download_component_size', '1B') ]): bucket_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') stdout = self.RunGsUtil( ['-DD', 'cp', ObjectToURI(key_uri), temporary_directory], env_vars=PYTHON_UNBUFFERED_ENV_VAR, return_stdout=True) self.assertIn(gs_host, stdout) self.assertNotIn(DEFAULT_HOST, stdout) @integration_testcase.SkipForJSON('XML test.') @integration_testcase.SkipForS3('Custom endpoints not available for S3.') def test_persists_custom_endpoint_through_xml_sliced_download(self): gs_host = config.get('Credentials', 'gs_host', DEFAULT_HOST) if gs_host == DEFAULT_HOST: # Skips test when run without a custom endpoint configured. return temporary_directory = self.CreateTempDir() with SetBotoConfigForTest([ ('GSUtil', 'sliced_object_download_threshold', '1B'), ('GSUtil', 'sliced_object_download_component_size', '1B') ]): bucket_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') stdout, stderr = self.RunGsUtil( ['-D', 'cp', ObjectToURI(key_uri), temporary_directory], return_stdout=True, return_stderr=True) output = stdout + stderr self.assertIn(gs_host, output) self.assertNotIn('hostname=' + DEFAULT_HOST, output) @integration_testcase.SkipForXML('JSON test.') @integration_testcase.SkipForS3('Custom endpoints not available for S3.') def test_persists_custom_endpoint_through_json_parallel_composite_upload( self): gs_host = config.get('Credentials', 'gs_json_host', DEFAULT_HOST) if gs_host == DEFAULT_HOST: # Skips test when run without a custom endpoint configured. return temporary_file = self.CreateTempFile(contents=b'foo') with SetBotoConfigForTest([ ('GSUtil', 'parallel_composite_upload_threshold', '1B'), ('GSUtil', 'parallel_composite_upload_component_size', '1B') ]): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil( ['-DD', 'cp', temporary_file, ObjectToURI(bucket_uri)], env_vars=PYTHON_UNBUFFERED_ENV_VAR, return_stdout=True) self.assertIn(gs_host, stdout) self.assertNotIn(DEFAULT_HOST, stdout) @integration_testcase.SkipForJSON('XML test.') @integration_testcase.SkipForS3('Custom endpoints not available for S3.') def test_persists_custom_endpoint_through_xml_parallel_composite_upload(self): gs_host = config.get('Credentials', 'gs_host', DEFAULT_HOST) if gs_host == DEFAULT_HOST: # Skips test when run without a custom endpoint configured. return temporary_file = self.CreateTempFile(contents=b'foo') with SetBotoConfigForTest([ ('GSUtil', 'parallel_composite_upload_threshold', '1B'), ('GSUtil', 'parallel_composite_upload_component_size', '1B') ]): bucket_uri = self.CreateBucket() stdout, stderr = self.RunGsUtil( ['-D', 'cp', temporary_file, ObjectToURI(bucket_uri)], return_stdout=True, return_stderr=True) output = stdout + stderr self.assertIn(gs_host, output) self.assertNotIn('hostname=' + DEFAULT_HOST, output) @integration_testcase.SkipForJSON('XML test.') @integration_testcase.SkipForS3('Custom endpoints not available for S3.') def test_persists_custom_endpoint_through_resumable_upload(self): gs_host = config.get('Credentials', 'gs_host', DEFAULT_HOST) if gs_host == DEFAULT_HOST: # Skips test when run without a custom endpoint configured. return temporary_file = self.CreateTempFile(contents=b'foo') with SetBotoConfigForTest([('GSUtil', 'resumable_threshold', '1')]): bucket_uri = self.CreateBucket() stdout, stderr = self.RunGsUtil( ['-D', 'cp', temporary_file, ObjectToURI(bucket_uri)], return_stdout=True, return_stderr=True) output = stdout + stderr self.assertIn(gs_host, output) self.assertNotIn('hostname=' + DEFAULT_HOST, output) tests/mock_cloud_api.py 0000644 00000017743 15025013616 0011243 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implements a simple mock gsutil Cloud API for unit testing. gsutil 4 was primarily unit-tested using boto/gsutil 3's mock storage_uri class, since it was possible that changing out the underlying mocks would have had subtly different behavior and increased the risk of breaking back-compat. Most unit and integration tests in gsutil 4 still set up the test objects with storage_uris and boto, and the unit tests interact with test objects via storage uris and boto. This testing approach ties our tests heavily to boto; extending the boto mocks is difficult because it requires checking into boto. This also makes the unit test coverage boto-specific in several cases. MockCloudApi was initially written to cover some parallel composite upload cases that the boto mocks couldn't handle. It is not yet a full implementation. Eventually, we can move to full a mock Cloud API implementation. However, we need to ensure we don't lose boto coverage from mock storage_uri. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import six from gslib.cloud_api import ServiceException from gslib.discard_messages_queue import DiscardMessagesQueue from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.translation_helper import CreateBucketNotFoundException from gslib.utils.translation_helper import CreateObjectNotFoundException if six.PY3: long = int class MockObject(object): """Defines a mock cloud storage provider object.""" def __init__(self, root_object, contents=''): self.root_object = root_object self.contents = contents def __str__(self): return '%s/%s#%s' % (self.root_object.bucket, self.root_object.name, self.root_object.generation) def __repr__(self): return str(self) class MockBucket(object): """Defines a mock cloud storage provider bucket.""" def __init__(self, bucket_name, versioned=False): self.root_object = apitools_messages.Bucket( name=bucket_name, versioning=apitools_messages.Bucket.VersioningValue(enabled=versioned)) # Dict of object_name: (dict of 'live': MockObject # 'versioned': ordered list of MockObject). self.objects = {} def CreateObject(self, object_name, contents=''): return self.CreateObjectWithMetadata( MockObject(apitools_messages.Object(name=object_name, contents=contents))) def CreateObjectWithMetadata(self, apitools_object, contents=''): """Creates an object in the bucket according to the input metadata. This will create a new object version (ignoring the generation specified in the input object). Args: apitools_object: apitools Object. contents: optional object contents. Returns: apitools Object representing created object. """ # This modifies the apitools_object with a generation number. object_name = apitools_object.name if (self.root_object.versioning and self.root_object.versioning.enabled and apitools_object.name in self.objects): if 'live' in self.objects[object_name]: # Versioning enabled and object exists, create an object with a # generation 1 higher. apitools_object.generation = ( self.objects[object_name]['live'].root_object.generation + 1) # Move the live object to versioned. if 'versioned' not in self.objects[object_name]: self.objects[object_name]['versioned'] = [] self.objects[object_name]['versioned'].append( self.objects[object_name]['live']) elif ('versioned' in self.objects[object_name] and self.objects[object_name]['versioned']): # Versioning enabled but only archived objects exist, pick a generation # higher than the highest versioned object (which will be at the end). apitools_object.generation = ( self.objects[object_name]['versioned'][-1].root_object.generation + 1) else: # Versioning disabled or no objects exist yet with this name. apitools_object.generation = 1 self.objects[object_name] = {} new_object = MockObject(apitools_object, contents=contents) self.objects[object_name]['live'] = new_object return new_object class MockCloudApi(object): """Simple mock service for buckets/objects that implements Cloud API. Also includes some setup functions for tests. """ def __init__(self, provider='gs'): self.buckets = {} self.provider = provider self.status_queue = DiscardMessagesQueue() def MockCreateBucket(self, bucket_name): """Creates a simple bucket without exercising the API directly.""" if bucket_name in self.buckets: raise ServiceException('Bucket %s already exists.' % bucket_name, status=409) self.buckets[bucket_name] = MockBucket(bucket_name) def MockCreateVersionedBucket(self, bucket_name): """Creates a simple bucket without exercising the API directly.""" if bucket_name in self.buckets: raise ServiceException('Bucket %s already exists.' % bucket_name, status=409) self.buckets[bucket_name] = MockBucket(bucket_name, versioned=True) def MockCreateObject(self, bucket_name, object_name, contents=''): """Creates an object without exercising the API directly.""" if bucket_name not in self.buckets: self.MockCreateBucket(bucket_name) self.buckets[bucket_name].CreateObject(object_name, contents=contents) def MockCreateObjectWithMetadata(self, apitools_object, contents=''): """Creates an object without exercising the API directly.""" assert apitools_object.bucket, 'No bucket specified for mock object' assert apitools_object.name, 'No object name specified for mock object' if apitools_object.bucket not in self.buckets: self.MockCreateBucket(apitools_object.bucket) return self.buckets[apitools_object.bucket].CreateObjectWithMetadata( apitools_object, contents=contents).root_object # pylint: disable=unused-argument def GetObjectMetadata(self, bucket_name, object_name, generation=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" if generation: generation = long(generation) if bucket_name in self.buckets: bucket = self.buckets[bucket_name] if object_name in bucket.objects and bucket.objects[object_name]: if generation: if 'versioned' in bucket.objects[object_name]: for obj in bucket.objects[object_name]['versioned']: if obj.root_object.generation == generation: return obj.root_object if 'live' in bucket.objects[object_name]: if (bucket.objects[object_name]['live'].root_object.generation == generation): return bucket.objects[object_name]['live'].root_object else: # Return live object. if 'live' in bucket.objects[object_name]: return bucket.objects[object_name]['live'].root_object raise CreateObjectNotFoundException(404, self.provider, bucket_name, object_name) raise CreateBucketNotFoundException(404, self.provider, bucket_name) tests/test_user_agent_helper.py 0000644 00000012540 15025013616 0013013 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for hashing helper functions and classes.""" from gslib.utils import system_util from gslib.utils.user_agent_helper import GetUserAgent import gslib.tests.testcase as testcase import six from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock class TestUserAgentHelper(testcase.GsUtilUnitTestCase): """Unit tests for the GetUserAgent helper function.""" @mock.patch('gslib.VERSION', '4_test') def testNoArgs(self): self.assertRegex(GetUserAgent([]), r"^ gsutil/4_test \([^\)]+\)") def testAnalyticsFlag(self): self.assertRegex(GetUserAgent([], False), r"analytics/enabled") self.assertRegex(GetUserAgent([], True), r"analytics/disabled") @mock.patch.object(system_util, 'IsRunningInteractively') def testInteractiveFlag(self, mock_interactive): mock_interactive.return_value = True self.assertRegex(GetUserAgent([]), r"interactive/True") mock_interactive.return_value = False self.assertRegex(GetUserAgent([]), r"interactive/False") def testHelp(self): self.assertRegex(GetUserAgent(['help']), r"command/help") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testCp(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['cp', '-r', '-Z', '1.txt', 'gs://dst']), r"command/cp$") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testCpNotEnoughArgs(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['cp']), r"command/cp$") self.assertRegex(GetUserAgent(['cp', '1.txt']), r"command/cp$") self.assertRegex(GetUserAgent(['cp', '-r', '1.ts']), r"command/cp$") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testCpEncoding(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['cp', 'öne', 'twö']), r"command/cp$") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testRsync(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['rsync', '1.txt', 'gs://dst']), r"command/rsync$") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testMv(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['mv', 'gs://src/1.txt', 'gs://dst/1.txt']), r"command/mv$") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testCpCloudToCloud(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['cp', '-r', 'gs://src', 'gs://dst']), r"command/cp$") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testCpForcedDaisyChain(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['cp', '-D', 'gs://src', 'gs://dst']), r"command/cp$") def testCpDaisyChain(self): self.assertRegex(GetUserAgent(['cp', '-r', '-Z', 'gs://src', 's3://dst']), r"command/cp-DaisyChain") self.assertRegex(GetUserAgent(['mv', 'gs://src/1.txt', 's3://dst/1.txt']), r"command/mv-DaisyChain") self.assertRegex(GetUserAgent(['rsync', '-r', 'gs://src', 's3://dst']), r"command/rsync-DaisyChain") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testPassOnInvalidUrlError(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['cp', '-r', '-Z', 'bad://src', 's3://dst']), r"command/cp$") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testRewriteEncryptionKey(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['rewrite', '-k', 'gs://dst']), r"command/rewrite-k$") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testRewriteStorageClass(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['rewrite', '-s', 'gs://dst']), r"command/rewrite-s$") @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testRewriteEncryptionKeyAndStorageClass(self, mock_invoked): mock_invoked.return_value = False self.assertRegex(GetUserAgent(['rewrite', '-k', '-s', 'gs://dst']), r"command/rewrite-k-s$") @mock.patch.object(system_util, 'CloudSdkVersion') @mock.patch.object(system_util, 'InvokedViaCloudSdk') def testCloudSdk(self, mock_invoked, mock_version): mock_invoked.return_value = True mock_version.return_value = '500.1' self.assertRegex(GetUserAgent(['help']), r"google-cloud-sdk/500.1$") mock_invoked.return_value = False mock_version.return_value = '500.1' self.assertRegex(GetUserAgent(['help']), r"command/help$") tests/signurl_signatures.py 0000644 00000013311 15025013616 0012205 0 ustar 00 TEST_SIGN_PUT_SIG = ( 'https://storage.googleapis.com/test/test.txt?x-goog-signature=' '9701aa8c722487256002e5a7532ad991703075e6f71a26af8107147635cc2a' '61680dc33548b3e8b43575a5d30b06b36365207bc96bda0ec3dc5605928cce' 'd393178307de10e96b6f38bd2686178e4a20dff3022e5fb938e6f85e7a433c' '7a7ba9d7a64d889a484464e6f9bf4a8582e93c82e5c7b1ad9cfa793be2db72' '14bbcd8ab0aba70ec7043eb496202185a265738c000555880b1e3f8a73cd19' '8c0e805b73fe4ac21a102f262ea5e10103fa4ecdc69e6bc280212f9f1773cc' 'd603b854f4c7df8c47325023641e06cb5afac537f3fe68a5786715745005cb' 'fb38750d7a9f0ae9de7a77466428baa668e507b10bdcf3d0eb596134318292' '15bb532c2e174160&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-cred' 'ential=test.apps.googleusercontent.com%2F19000101%2Fus-east%2F' 'storage%2Fgoog4_request&x-goog-date=19000101T000555Z&x-goog-ex' 'pires=3600&x-goog-signedheaders=host%3Bx-goog-resumable') TEST_SIGN_RESUMABLE = ( 'https://storage.googleapis.com/test/test.txt?x-goog-signature=' '9701aa8c722487256002e5a7532ad991703075e6f71a26af8107147635cc2a' '61680dc33548b3e8b43575a5d30b06b36365207bc96bda0ec3dc5605928cce' 'd393178307de10e96b6f38bd2686178e4a20dff3022e5fb938e6f85e7a433c' '7a7ba9d7a64d889a484464e6f9bf4a8582e93c82e5c7b1ad9cfa793be2db72' '14bbcd8ab0aba70ec7043eb496202185a265738c000555880b1e3f8a73cd19' '8c0e805b73fe4ac21a102f262ea5e10103fa4ecdc69e6bc280212f9f1773cc' 'd603b854f4c7df8c47325023641e06cb5afac537f3fe68a5786715745005cb' 'fb38750d7a9f0ae9de7a77466428baa668e507b10bdcf3d0eb596134318292' '15bb532c2e174160&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-cred' 'ential=test.apps.googleusercontent.com%2F19000101%2Fus-east%2F' 'storage%2Fgoog4_request&x-goog-date=19000101T000555Z&x-goog-ex' 'pires=3600&x-goog-signedheaders=host%3Bx-goog-resumable') TEST_SIGN_URL_PUT_CONTENT = ( 'https://storage.googleapis.com/test/test.txt?x-goog-signature=' '9743fbea283c92e23753e740b9d1541876ed606be21d12c70d3b4bccd63552' '71bade73a86c846af0bfe7c53eef18132851569378cb21795be6a46e60a086' '4c57e80986cc1952e75e2237bde0d619741fe191646a40a9be593a3a3bf0e6' '855efefe6c39ebdbb82e49bfc0cae3492d400d4d50909861398a02032ad771' 'f446c63ead6c5e35d6bf8c3f47e7d1f6b076c4cf2e5f33e30e2a560830a947' 'c4ff7ba40c2dfab7e9e1a3f8a7fcf9d7715f94b8b3b19e4c1cd3a954142521' '7ec95bbdf484c56779623066f77756af67375f27607c25ec92de51d6ed1fa0' '0d4f0b901412ebd963e08ed328812be2bd6742871f7272ca681a7e3170684b' '53f8142aeb39b704&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-cred' 'ential=test.apps.googleusercontent.com%2F19000101%2Feu%2Fstora' 'ge%2Fgoog4_request&x-goog-date=19000101T000555Z&x-goog-expires' '=3600&x-goog-signedheaders=content-type%3Bhost') TEST_SIGN_URL_GET = ( 'https://storage.googleapis.com/test/test.txt?x-goog-signature=' '3ee945ed5fdc7da8a88f391aa0b51e69b3abef015f8271cdc767856f0bc1ae' 'b6a1e3dd9afc9cfdd769e2e6af1256f7d6a84ad52e02e422fb69e05a88fd6a' 'cdb92d971ab2db7f9ad368b414ea912d31118abd94932842cce1aa98ecb21d' 'c2f2507c9f6231db617163a012185b08c3eca1be9aef8adde61c5a44a4d2a3' '52e1b5d8180594ea826ea099645f90dd3629ee7a89357bfc36c9097fb6b774' '92c692d64aaab63611162b3c43c88ff9f49921fa4ee4638ce7a9eaac455106' '948d6ab7bb08ae9fc7312541c92720edcd64620d37e2f49e0ea112cea783db' 'aaa8b08d81a5208c405d7547e715198325dd7ef1e5db16dd315addd8cf28f6' '8b65007b0ef82026&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-cred' 'ential=test.apps.googleusercontent.com%2F19000101%2Fasia%2Fsto' 'rage%2Fgoog4_request&x-goog-date=19000101T000555Z&x-goog-expir' 'es=0&x-goog-signedheaders=host') TEST_SIGN_URL_GET_WITH_JSON_KEY = ( 'https://storage.googleapis.com/test/test.txt?x-goog-signature=' '2ed227f18d31cdf2b01da7cd4fcea45330fbfcc0dda1d327a8c27124a276ee' 'e0de835e9cd4b0bee609d6b4b21a88a8092a9c089574a300243dde38351f0d' '183df007211ded41f2f0854290b995be6c9d0367d9c00976745ba27740238b' '0dd49fee7c41e7ed1569bbab8ffbb00a2078e904ebeeec2f8e55e93d4baba1' '3db5dc670b1b16183a15d5067f1584db88b3dc55e3edd3c97c0f31fec99ea4' 'ce96ddb8235b0352c9ce5110dad1a580072d955fe9203b6701364ddd85226b' '55bec84ac46e48cd324fd5d8d8ad264d1aa0b7dbad3ac04b87b2a6c2c8ef95' '3285cbe3b431e5def84552e112899459fcb64d2d84320c06faa1e8efa26eca' 'cce2eff41f2d2364&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-cred' 'ential=test%40developer.gserviceaccount.com%2F19000101%2Fasia%' '2Fstorage%2Fgoog4_request&x-goog-date=19000101T000555Z&x-goog-' 'expires=0&x-goog-signedheaders=host') TEST_SIGN_URL_PUT_WITH_SERVICE_ACCOUNT = ( 'https://storage.googleapis.com/test/test.txt?x-goog-signature=' '66616b655f7369676e6174757265&x-goog-algorithm=GOOG4-RSA-SHA256' '&x-goog-credential=fake_service_account_email%2F19000101%2Fus-' 'east1%2Fstorage%2Fgoog4_request&x-goog-date=19000101T000555Z&x' '-goog-expires=3600&x-goog-signedheaders=host') TEST_SIGN_URL_GET_USERPROJECT = ( 'https://storage.googleapis.com/test/test.txt?x-goog-signature=' '2a461e23321fbc6d6346848836a0b82fd53b796a58421e9978ac6ca011e2b4' '557aa7db63328c9d3f2dacd98b3c3b13d849a25df813b1efefd448ce7a2c27' '9851383b772f84b461dfafb4f2af4bec98d224880ea3b1be61ab447614ec1f' '735b45aaedca9b257969d7bac69d64f7e2c4ea4b5c18cb2b04afff03fe0b8f' 'ba9d85bfce6ba81f5354070938bfd6bce614204a63e58098d47b66a711a9ac' '44e2dfd06d3183e8007d8fb9c221ac79fee306c8b6a123d5918a586bb0fde9' 'e945a14ae1264496fe4146fcf92f9f1ed4a13880ec13543ffbcf4c985db2f2' '01b287184f1f89dd14d2c801c26275c8f7f348e3eb3456c4ff42dcc0ae7fee' '4e8c03b1e8a57570&userProject=myproject&x-goog-algorithm=GOOG4-' 'RSA-SHA256&x-goog-credential=test.apps.googleusercontent.com%2' 'F19000101%2Fasia%2Fstorage%2Fgoog4_request&x-goog-date=1900010' '1T000555Z&x-goog-expires=0&x-goog-signedheaders=host') tests/test_mb.py 0000644 00000047217 15025013616 0007727 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for mb command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from random import randint import boto import gslib.tests.testcase as testcase from gslib.project_id import PopulateProjectId from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.testcase.integration_testcase import SkipForJSON from gslib.tests.util import ObjectToURI as suri from gslib.utils.retention_util import SECONDS_IN_DAY from gslib.utils.retention_util import SECONDS_IN_MONTH from gslib.utils.retention_util import SECONDS_IN_YEAR from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.utils.retry_util import Retry from gslib.utils import shim_util BUCKET_LOCK_SKIP_MSG = ('gsutil does not support bucket lock operations for ' 'S3 buckets.') KMS_SKIP_MSG = 'gsutil KMS operations only run on GCS JSON API.' class TestMb(testcase.GsUtilIntegrationTestCase): """Integration tests for mb command.""" def GetKey(self, mutable=False): # Make sure our keyRing exists (only needs to be done once, but subsequent # attempts will receive a 409 and be treated as a success). keyring_fqn = self.kms_api.CreateKeyRing( PopulateProjectId(None), testcase.KmsTestingResources.KEYRING_NAME, location=testcase.KmsTestingResources.KEYRING_LOCATION) key_name = testcase.KmsTestingResources.CONSTANT_KEY_NAME_DO_NOT_AUTHORIZE if mutable: # Randomly pick 1 of 1000 key names. key_name = testcase.KmsTestingResources.MUTABLE_KEY_NAME_TEMPLATE % ( randint(0, 9), randint(0, 9), randint(0, 9)) # Make sure the key with that name has been created. key_fqn = self.kms_api.CreateCryptoKey(keyring_fqn, key_name) # The key may have already been created and used in a previous test # invocation; make sure it doesn't contain the IAM policy binding that # allows our project to encrypt/decrypt with it. key_policy = self.kms_api.GetKeyIamPolicy(key_fqn) if key_policy.bindings: key_policy.bindings = [] self.kms_api.SetKeyIamPolicy(key_fqn, key_policy) return key_fqn @SkipForS3('S3 returns success when bucket already exists.') def test_mb_bucket_exists(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil(['mb', suri(bucket_uri)], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn( 'HTTPError 409: The requested bucket name is not available.', stderr) else: self.assertIn('already exists', stderr) def test_non_ascii_project_fails(self): stderr = self.RunGsUtil(['mb', '-p', 'ã', 'gs://fobarbaz'], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('The project property must be set to a valid project ID', stderr) else: self.assertIn('Invalid non-ASCII', stderr) @SkipForS3(BUCKET_LOCK_SKIP_MSG) def test_create_with_retention_seconds(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) self.RunGsUtil(['mb', '--retention', '60s', suri(bucket_uri)]) self.VerifyRetentionPolicy(bucket_uri, expected_retention_period_in_seconds=60) @SkipForS3(BUCKET_LOCK_SKIP_MSG) def test_create_with_retention_days(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) self.RunGsUtil(['mb', '--retention', '1d', suri(bucket_uri)]) self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=SECONDS_IN_DAY) @SkipForS3(BUCKET_LOCK_SKIP_MSG) def test_create_with_retention_months(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) self.RunGsUtil(['mb', '--retention', '1m', suri(bucket_uri)]) self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=SECONDS_IN_MONTH) @SkipForS3(BUCKET_LOCK_SKIP_MSG) def test_create_with_retention_years(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) self.RunGsUtil(['mb', '--retention', '1y', suri(bucket_uri)]) self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=SECONDS_IN_YEAR) @SkipForS3(BUCKET_LOCK_SKIP_MSG) def test_create_with_retention_invalid_arg(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) stderr = self.RunGsUtil(['mb', '--retention', '1second', suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertRegex(stderr, r'Incorrect retention period specified') def test_create_with_retention_on_s3_urls_fails(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('s3://%s' % (bucket_name.lower()), suppress_consec_slashes=False) stderr = self.RunGsUtil( ['mb', '--retention', '1y', suri(bucket_uri)], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('Features disallowed for S3: Setting Retention Period', stderr) else: self.assertRegex( stderr, r'Retention policy can only be specified for GCS buckets.') @SkipForXML('Public access prevention only runs on GCS JSON API.') def test_create_with_pap_enforced(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) self.RunGsUtil(['mb', '--pap', 'enforced', suri(bucket_uri)]) self.VerifyPublicAccessPreventionValue(bucket_uri, 'enforced') @SkipForXML('Public access prevention only runs on GCS JSON API.') def test_create_with_pap_inherited(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) self.RunGsUtil(['mb', '--pap', 'inherited', suri(bucket_uri)]) stdout = self.RunGsUtil(['publicaccessprevention', 'get', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'%s:\s+inherited' % suri(bucket_uri)) @SkipForXML('Public access prevention only runs on GCS JSON API.') def test_create_with_pap_invalid_arg(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) stderr = self.RunGsUtil(['mb', '--pap', 'invalid_arg', suri(bucket_uri)], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn( 'Flag value not in translation map for "--pap": invalid_arg', stderr) else: self.assertRegex(stderr, r'invalid_arg is not a valid value') @SkipForXML('RPO flag only works for GCS JSON API.') def test_create_with_rpo_async_turbo(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) self.RunGsUtil( ['mb', '-l', 'nam4', '--rpo', 'ASYNC_TURBO', suri(bucket_uri)]) self.VerifyCommandGet(bucket_uri, 'rpo', 'ASYNC_TURBO') @SkipForXML('RPO flag only works for GCS JSON API.') def test_create_sets_rpo_to_default(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) self.RunGsUtil(['mb', '-l', 'nam4', suri(bucket_uri)]) try: self.VerifyCommandGet(bucket_uri, 'rpo', 'DEFAULT') except AssertionError: # TODO: Remove the try/except block once we have consistent results # returned from the backend for rpo get. self.VerifyCommandGet(bucket_uri, 'rpo', 'None') @SkipForXML('RPO flag only works for GCS JSON API.') def test_create_with_rpo_async_turbo_fails_for_regional_bucket(self): """Turbo replication is only meant for dual-region.""" bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) stderr = self.RunGsUtil( ['mb', '-l', 'us-central1', '--rpo', 'ASYNC_TURBO', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('ASYNC_TURBO cannot be enabled on REGION bucket', stderr) @SkipForXML('RPO flag only works for GCS JSON API.') def test_create_with_rpo_incorrect_value_raises_error(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) expected_status = 2 if self._use_gcloud_storage else 1 # Location nam4 is used for dual-region. stderr = self.RunGsUtil( ['mb', '-l', 'nam4', '--rpo', 'incorrect_value', suri(bucket_uri)], return_stderr=True, expected_status=expected_status) if self._use_gcloud_storage: self.assertIn( '--recovery-point-objective: Invalid choice: \'incorrect_value\'', stderr) else: self.assertIn( 'Invalid value for --rpo. Must be one of: (ASYNC_TURBO|DEFAULT),' ' provided: incorrect_value', stderr) @SkipForXML(KMS_SKIP_MSG) @SkipForS3(KMS_SKIP_MSG) def test_create_with_k_flag_not_authorized(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) key = self.GetKey() stderr = self.RunGsUtil([ 'mb', '-l', testcase.KmsTestingResources.KEYRING_LOCATION, '-k', key, suri(bucket_uri) ], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('HTTPError 403: Permission denied on Cloud KMS key.', stderr) else: self.assertIn('To authorize, run:', stderr) self.assertIn('-k %s' % key, stderr) @SkipForXML(KMS_SKIP_MSG) @SkipForS3(KMS_SKIP_MSG) def test_create_with_k_flag_p_flag_not_authorized(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) key = self.GetKey() stderr = self.RunGsUtil([ 'mb', '-l', testcase.KmsTestingResources.KEYRING_LOCATION, '-k', key, '-p', PopulateProjectId(), suri(bucket_uri) ], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('HTTPError 403: Permission denied on Cloud KMS key.', stderr) else: self.assertIn('To authorize, run:', stderr) self.assertIn('-p %s' % PopulateProjectId(), stderr) @SkipForXML(KMS_SKIP_MSG) @SkipForS3(KMS_SKIP_MSG) @Retry(AssertionError, tries=3, timeout_secs=1) def test_create_with_k_flag_authorized(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) key = self.GetKey(mutable=True) self.RunGsUtil(['kms', 'authorize', '-k', key]) self.RunGsUtil([ 'mb', '-l', testcase.KmsTestingResources.KEYRING_LOCATION, '-k', key, suri(bucket_uri) ], expected_status=0) @SkipForXML('Custom Dual Region is not supported for the XML API.') @SkipForS3('Custom Dual Region is not supported for S3 buckets.') def test_create_with_custom_dual_regions_via_l_flag(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) self.RunGsUtil(['mb', '-l', 'us-east1+us-east4', suri(bucket_uri)]) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r"ocations:\s*\[\s*.US-EAST1.,\s*.US-EAST4") @SkipForXML('Custom Dual Region is not supported for the XML API.') @SkipForS3('Custom Dual Region is not supported for S3 buckets.') def test_create_with_invalid_dual_regions_via_l_flag_raises_error(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) stderr = self.RunGsUtil( ['mb', '-l', 'invalid_reg1+invalid_reg2', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('The specified location constraint is not valid', stderr) @SkipForXML('The --placement flag only works for GCS JSON API.') def test_create_with_placement_flag(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) self.RunGsUtil( ['mb', '--placement', 'us-central1,us-west1', suri(bucket_uri)]) stdout = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r"ocations:\s*\[\s*.US-CENTRAL1.,\s*.US-WEST1") @SkipForXML('The --placement flag only works for GCS JSON API.') def test_create_with_invalid_placement_flag_raises_error(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) stderr = self.RunGsUtil( ['mb', '--placement', 'invalid_reg1,invalid_reg2', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertRegex( stderr, r'.*400.*(Invalid custom placement config|' r'One or more unrecognized regions in dual-region, received:' r' INVALID_REG1, INVALID_REG2).*') @SkipForXML('The --placement flag only works for GCS JSON API.') def test_create_with_incorrect_number_of_placement_values_raises_error(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) # Location nam4 is used for dual-region. expected_status = 2 if self._use_gcloud_storage else 1 stderr = self.RunGsUtil( ['mb', '--placement', 'val1,val2,val3', suri(bucket_uri)], return_stderr=True, expected_status=expected_status) if self._use_gcloud_storage: self.assertIn('--placement: too many args', stderr) else: self.assertIn( 'CommandException: Please specify two regions separated by comma' ' without space. Specified: val1,val2,val3', stderr) @SkipForJSON('Testing XML only behavior.') def test_single_json_only_flag_raises_error_with_xml_api(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) stderr = self.RunGsUtil(['mb', '--rpo', 'ASYNC_TURBO', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn( 'CommandException: The --rpo option(s) can only be used' ' for GCS Buckets with the JSON API', stderr) @SkipForJSON('Testing XML only behavior.') def test_multiple_json_only_flags_raise_error_with_xml_api(self): bucket_name = self.MakeTempName('bucket') bucket_uri = boto.storage_uri('gs://%s' % (bucket_name.lower()), suppress_consec_slashes=False) stderr = self.RunGsUtil([ 'mb', '--autoclass', '--pap', 'enabled', '--placement', 'uscentral-1,us-asia1', '--rpo', 'ASYNC_TURBO', '-b', 'on', suri(bucket_uri) ], return_stderr=True, expected_status=1) self.assertIn( 'CommandException: The --autoclass, --pap, --placement, --rpo,' ' -b option(s) can only be used for GCS Buckets with the JSON API', stderr) class TestMbUnitTestsWithShim(testcase.ShimUnitTestBase): """Unit tests for gsutil mb with shim.""" def test_shim_translates_retention_seconds_flags(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('mb', args=[ '--retention', '1y', 'gs://fake-bucket', ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage buckets create' ' --retention-period 31557600s gs://fake-bucket').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @SkipForXML('The --rpo flag only works for GCS JSON API.') def test_shim_translates_recovery_point_objective_flag(self): fake_cloudsdk_dir = 'fake_dir' with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': fake_cloudsdk_dir, }): mock_log_handler = self.RunCommand( 'mb', args=['--rpo', 'DEFAULT', 'gs://fake-bucket-1'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage' ' buckets create --recovery-point-objective DEFAULT').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) tests/test_seek_ahead_thread.py 0000644 00000021452 15025013616 0012722 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for gsutil seek_ahead_thread.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import threading import six from six.moves import queue as Queue from six.moves import range from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.seek_ahead_thread import SeekAheadResult from gslib.seek_ahead_thread import SeekAheadThread import gslib.tests.testcase as testcase from gslib.ui_controller import UIController from gslib.ui_controller import UIThread from gslib.utils import constants from gslib.utils import parallelism_framework_util from gslib.utils import unit_util _ZERO_TASKS_TO_DO_ARGUMENT = ( parallelism_framework_util.ZERO_TASKS_TO_DO_ARGUMENT) class TestSeekAheadThread(testcase.GsUtilUnitTestCase): """Unit tests for SeekAheadThread class and surrounding functionality.""" # After waiting this long, assume the SeekAheadThread is hung. thread_wait_time = 5 def testCancellation(self): """Tests cancellation of SeekAheadThread.""" class TrackingCancellationIterator(object): """Yields dummy results and sends cancellation after some # of yields.""" def __init__(self, num_iterations, num_iterations_before_cancel, cancel_event): """Initializes the iterator. Args: num_iterations: Total number of results to yield. num_iterations_before_cancel: Set cancel event before yielding on the given iteration. cancel_event: threading.Event() to signal SeekAheadThread to stop. """ self.num_iterations_before_cancel = num_iterations_before_cancel self.iterated_results = 0 self.num_iterations = num_iterations self.cancel_issued = False self.cancel_event = cancel_event def __iter__(self): while self.iterated_results < self.num_iterations: if (not self.cancel_issued and self.iterated_results >= self.num_iterations_before_cancel): self.cancel_event.set() self.cancel_issued = True yield SeekAheadResult() self.iterated_results += 1 # We expect to get up to the nearest NUM_OBJECTS_PER_LIST_PAGE results. noplp = constants.NUM_OBJECTS_PER_LIST_PAGE for num_iterations, num_iterations_before_cancel, expected_iterations in ( (noplp, 0, 0), (noplp + 1, 1, noplp), (noplp + 1, noplp, noplp), (noplp * 2 + 1, noplp + 1, noplp * 2), (2, 1, 2), (noplp, 1, noplp), (noplp * 2, noplp + 1, noplp * 2)): cancel_event = threading.Event() status_queue = Queue.Queue() stream = six.StringIO() ui_controller = UIController() ui_thread = UIThread(status_queue, stream, ui_controller) seek_ahead_iterator = TrackingCancellationIterator( num_iterations, num_iterations_before_cancel, cancel_event) seek_ahead_thread = SeekAheadThread(seek_ahead_iterator, cancel_event, status_queue) seek_ahead_thread.join(self.thread_wait_time) status_queue.put(_ZERO_TASKS_TO_DO_ARGUMENT) ui_thread.join(self.thread_wait_time) if seek_ahead_thread.is_alive(): seek_ahead_thread.terminate = True self.fail( 'Cancellation issued after %s iterations, but SeekAheadThread ' 'is still alive.' % num_iterations_before_cancel) self.assertEqual( expected_iterations, seek_ahead_iterator.iterated_results, 'Cancellation issued after %s iterations, SeekAheadThread iterated ' '%s results, expected: %s results.' % (num_iterations_before_cancel, seek_ahead_iterator.iterated_results, expected_iterations)) message = stream.getvalue() if message: self.fail('Status queue should be empty but contains message: %s' % message) def testEstimateWithoutSize(self): """Tests SeekAheadThread providing an object count.""" class SeekAheadResultIterator(object): def __init__(self, num_results): self.num_results = num_results self.yielded = 0 def __iter__(self): while self.yielded < self.num_results: yield SeekAheadResult() self.yielded += 1 cancel_event = threading.Event() status_queue = Queue.Queue() stream = six.StringIO() ui_controller = UIController() ui_thread = UIThread(status_queue, stream, ui_controller) num_objects = 5 seek_ahead_iterator = SeekAheadResultIterator(num_objects) seek_ahead_thread = SeekAheadThread(seek_ahead_iterator, cancel_event, status_queue) seek_ahead_thread.join(self.thread_wait_time) status_queue.put(_ZERO_TASKS_TO_DO_ARGUMENT) ui_thread.join(self.thread_wait_time) if seek_ahead_thread.is_alive(): seek_ahead_thread.terminate = True self.fail('SeekAheadThread is still alive.') message = stream.getvalue() if not message: self.fail('Status queue empty but SeekAheadThread should have posted ' 'summary message') self.assertEqual( message, 'Estimated work for this command: objects: %s\n' % num_objects) def testEstimateWithSize(self): """Tests SeekAheadThread providing an object count and total size.""" class SeekAheadResultIteratorWithSize(object): """Yields dummy result of the given size.""" def __init__(self, num_objects, size): self.num_objects = num_objects self.size = size self.yielded = 0 def __iter__(self): while self.yielded < self.num_objects: yield SeekAheadResult(data_bytes=self.size) self.yielded += 1 cancel_event = threading.Event() status_queue = Queue.Queue() stream = six.StringIO() ui_controller = UIController() ui_thread = UIThread(status_queue, stream, ui_controller) num_objects = 5 object_size = 10 seek_ahead_iterator = SeekAheadResultIteratorWithSize( num_objects, object_size) seek_ahead_thread = SeekAheadThread(seek_ahead_iterator, cancel_event, status_queue) seek_ahead_thread.join(self.thread_wait_time) status_queue.put(_ZERO_TASKS_TO_DO_ARGUMENT) ui_thread.join(self.thread_wait_time) if seek_ahead_thread.is_alive(): seek_ahead_thread.terminate = True self.fail('SeekAheadThread is still alive.') message = stream.getvalue() if not message: self.fail('Status queue empty but SeekAheadThread should have posted ' 'summary message') total_size = num_objects * object_size self.assertEqual( message, 'Estimated work for this command: objects: %s, total size: %s\n' % (num_objects, unit_util.MakeHumanReadable(total_size))) def testWithLocalFiles(self): """Tests SeekAheadThread with an actual directory.""" tmpdir = self.CreateTempDir() num_files = 5 total_size = 0 # Create 5 files with sizes 0, 1, 2, 3, 4. for i in range(num_files): self.CreateTempFile(tmpdir=tmpdir, file_name='obj%s' % str(i), contents=b'a' * i) total_size += i # Recursively "copy" tmpdir. seek_ahead_iterator = SeekAheadNameExpansionIterator( 'cp', 0, None, [tmpdir], True) cancel_event = threading.Event() status_queue = Queue.Queue() stream = six.StringIO() ui_controller = UIController() ui_thread = UIThread(status_queue, stream, ui_controller) seek_ahead_thread = SeekAheadThread(seek_ahead_iterator, cancel_event, status_queue) seek_ahead_thread.join(self.thread_wait_time) status_queue.put(_ZERO_TASKS_TO_DO_ARGUMENT) ui_thread.join(self.thread_wait_time) if seek_ahead_thread.is_alive(): seek_ahead_thread.terminate = True self.fail('SeekAheadThread is still alive.') message = stream.getvalue() if not message: self.fail('Status queue empty but SeekAheadThread should have posted ' 'summary message') self.assertEqual( message, 'Estimated work for this command: objects: %s, total size: %s\n' % (num_files, unit_util.MakeHumanReadable(total_size))) tests/test_cp.py 0000644 00000660003 15025013616 0007725 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for cp command.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals import ast import base64 import binascii import datetime import gzip import logging import os import pickle import pkgutil import random import re import stat import string import sys import threading from unittest import mock from apitools.base.py import exceptions as apitools_exceptions import boto from boto import storage_uri from boto.exception import ResumableTransferDisposition from boto.exception import StorageResponseError from boto.storage_uri import BucketStorageUri from gslib import command from gslib import exception from gslib import name_expansion from gslib.cloud_api import ResumableUploadStartOverException from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD from gslib.commands.cp import ShimTranslatePredefinedAclSubOptForCopy from gslib.cs_api_map import ApiSelector from gslib.daisy_chain_wrapper import _DEFAULT_DOWNLOAD_CHUNK_SIZE from gslib.discard_messages_queue import DiscardMessagesQueue from gslib.exception import InvalidUrlError from gslib.gcs_json_api import GcsJsonApi from gslib.parallel_tracker_file import ObjectFromTracker from gslib.parallel_tracker_file import WriteParallelUploadTrackerFile from gslib.project_id import PopulateProjectId from gslib.storage_url import StorageUrlFromString from gslib.tests.rewrite_helper import EnsureRewriteResumeCallbackHandler from gslib.tests.rewrite_helper import HaltingRewriteCallbackHandler from gslib.tests.rewrite_helper import RewriteHaltException import gslib.tests.testcase as testcase from gslib.tests.testcase.base import NotParallelizable from gslib.tests.testcase.integration_testcase import SkipForGS from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.testcase.integration_testcase import SkipForJSON from gslib.tests.util import AuthorizeProjectToUseTestingKmsKey from gslib.tests.util import BuildErrorRegex from gslib.tests.util import GenerationFromURI as urigen from gslib.tests.util import HaltingCopyCallbackHandler from gslib.tests.util import HaltOneComponentCopyCallbackHandler from gslib.tests.util import HAS_GS_PORT from gslib.tests.util import HAS_S3_CREDS from gslib.tests.util import KmsTestingResources from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import ORPHANED_FILE from gslib.tests.util import POSIX_GID_ERROR from gslib.tests.util import POSIX_INSUFFICIENT_ACCESS_ERROR from gslib.tests.util import POSIX_MODE_ERROR from gslib.tests.util import POSIX_UID_ERROR from gslib.tests.util import SequentialAndParallelTransfer from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import TailSet from gslib.tests.util import TEST_ENCRYPTION_KEY1 from gslib.tests.util import TEST_ENCRYPTION_KEY1_SHA256_B64 from gslib.tests.util import TEST_ENCRYPTION_KEY2 from gslib.tests.util import TEST_ENCRYPTION_KEY3 from gslib.tests.util import unittest from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.tracker_file import DeleteTrackerFile from gslib.tracker_file import GetRewriteTrackerFilePath from gslib.tracker_file import GetSlicedDownloadTrackerFilePaths from gslib.ui_controller import BytesToFixedWidthString from gslib.utils import hashing_helper from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.constants import START_CALLBACK_PER_BYTES from gslib.utils.constants import UTF8 from gslib.utils.copy_helper import GetTrackerFilePath from gslib.utils.copy_helper import PARALLEL_UPLOAD_STATIC_SALT from gslib.utils.copy_helper import PARALLEL_UPLOAD_TEMP_NAMESPACE from gslib.utils.copy_helper import TrackerFileType from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents from gslib.utils.hashing_helper import CalculateMd5FromContents from gslib.utils.hashing_helper import GetMd5 from gslib.utils.metadata_util import CreateCustomMetadata from gslib.utils.posix_util import GID_ATTR from gslib.utils.posix_util import MODE_ATTR from gslib.utils.posix_util import NA_ID from gslib.utils.posix_util import NA_MODE from gslib.utils.posix_util import UID_ATTR from gslib.utils.posix_util import ParseAndSetPOSIXAttributes from gslib.utils.posix_util import ValidateFilePermissionAccess from gslib.utils.posix_util import ValidatePOSIXMode from gslib.utils.retry_util import Retry from gslib.utils.system_util import IS_WINDOWS from gslib.utils.text_util import get_random_ascii_chars from gslib.utils.unit_util import EIGHT_MIB from gslib.utils.unit_util import HumanReadableToBytes from gslib.utils.unit_util import MakeHumanReadable from gslib.utils.unit_util import ONE_KIB from gslib.utils.unit_util import ONE_MIB from gslib.utils import shim_util import six from six.moves import http_client from six.moves import range from six.moves import range if six.PY3: long = int # pylint: disable=redefined-builtin,invalid-name # These POSIX-specific variables aren't defined for Windows. # pylint: disable=g-import-not-at-top if not IS_WINDOWS: from gslib.tests import util from gslib.tests.util import DEFAULT_MODE from gslib.tests.util import GetInvalidGid from gslib.tests.util import GetNonPrimaryGid from gslib.tests.util import GetPrimaryGid from gslib.tests.util import INVALID_UID from gslib.tests.util import USER_ID # pylint: enable=g-import-not-at-top # (status_code, error_prefix, error_substring) _GCLOUD_STORAGE_GZIP_FLAG_CONFLICT_OUTPUT = ( 2, 'ERROR', 'At most one of --gzip-in-flight | --gzip-in-flight-all | --gzip-local |' ' --gzip-local-all can be specified') def TestCpMvPOSIXBucketToLocalErrors(cls, bucket_uri, obj, tmpdir, is_cp=True): """Helper function for preserve_posix_errors tests in test_cp and test_mv. Args: cls: An instance of either TestCp or TestMv. bucket_uri: The uri of the bucket that the object is in. obj: The object to run the tests on. tmpdir: The local file path to cp to. is_cp: Whether or not the calling test suite is cp or mv. """ error_key = 'error_regex' if cls._use_gcloud_storage: insufficient_access_error = no_read_access_error = re.compile( r"User \d+ owns file, but owner does not have read permission") missing_gid_error = re.compile( r"GID in .* metadata doesn't exist on current system") missing_uid_error = re.compile( r"UID in .* metadata doesn't exist on current system") else: insufficient_access_error = BuildErrorRegex( obj, POSIX_INSUFFICIENT_ACCESS_ERROR) missing_gid_error = BuildErrorRegex(obj, POSIX_GID_ERROR) missing_uid_error = BuildErrorRegex(obj, POSIX_UID_ERROR) no_read_access_error = BuildErrorRegex(obj, POSIX_MODE_ERROR) # A dict of test_name: attrs_dict. # attrs_dict holds the different attributes that we want for the object in a # specific test. # To minimize potential test flakes from the system's GID mapping changing # mid-test, we use the GID-related methods that fetch GID info each time, # rather than reusing the LazyWrapper-wrapped constants across operations. test_params = { 'test1': { MODE_ATTR: '333', error_key: no_read_access_error, }, 'test2': { GID_ATTR: GetInvalidGid, error_key: missing_gid_error, }, 'test3': { GID_ATTR: GetInvalidGid, MODE_ATTR: '420', error_key: missing_gid_error, }, 'test4': { UID_ATTR: INVALID_UID, error_key: missing_uid_error, }, 'test5': { UID_ATTR: INVALID_UID, MODE_ATTR: '530', error_key: missing_uid_error, }, 'test6': { UID_ATTR: INVALID_UID, GID_ATTR: GetInvalidGid, error_key: missing_uid_error, }, 'test7': { UID_ATTR: INVALID_UID, GID_ATTR: GetInvalidGid, MODE_ATTR: '640', error_key: missing_uid_error, }, 'test8': { UID_ATTR: INVALID_UID, GID_ATTR: GetPrimaryGid, error_key: missing_uid_error, }, 'test9': { UID_ATTR: INVALID_UID, GID_ATTR: GetNonPrimaryGid, error_key: missing_uid_error, }, 'test10': { UID_ATTR: INVALID_UID, GID_ATTR: GetPrimaryGid, MODE_ATTR: '640', error_key: missing_uid_error, }, 'test11': { UID_ATTR: INVALID_UID, GID_ATTR: GetNonPrimaryGid, MODE_ATTR: '640', error_key: missing_uid_error, }, 'test12': { UID_ATTR: USER_ID, GID_ATTR: GetInvalidGid, error_key: missing_gid_error, }, 'test13': { UID_ATTR: USER_ID, GID_ATTR: GetInvalidGid, MODE_ATTR: '640', error_key: missing_gid_error, }, 'test14': { GID_ATTR: GetPrimaryGid, MODE_ATTR: '240', error_key: insufficient_access_error, } } # The first variable below can be used to help debug the test if there is a # problem. for test_name, attrs_dict in six.iteritems(test_params): cls.ClearPOSIXMetadata(obj) # Attributes default to None if they are not in attrs_dict; some attrs are # functions or LazyWrapper objects that should be called. uid = attrs_dict.get(UID_ATTR) if uid is not None and callable(uid): uid = uid() gid = attrs_dict.get(GID_ATTR) if gid is not None and callable(gid): gid = gid() mode = attrs_dict.get(MODE_ATTR) cls.SetPOSIXMetadata(cls.default_provider, bucket_uri.bucket_name, obj.object_name, uid=uid, gid=gid, mode=mode) stderr = cls.RunGsUtil([ 'cp' if is_cp else 'mv', '-P', suri(bucket_uri, obj.object_name), tmpdir ], expected_status=1, return_stderr=True) if cls._use_gcloud_storage: general_posix_error = 'ERROR' else: general_posix_error = ORPHANED_FILE cls.assertIn( general_posix_error, stderr, 'Error during test "%s": %s not found in stderr:\n%s' % (test_name, general_posix_error, stderr)) error_regex = attrs_dict[error_key] cls.assertTrue( error_regex.search(stderr), 'Test %s did not match expected error; could not find a match for ' '%s\n\nin stderr:\n%s' % (test_name, error_regex.pattern, stderr)) listing1 = TailSet(suri(bucket_uri), cls.FlatListBucket(bucket_uri)) listing2 = TailSet(tmpdir, cls.FlatListDir(tmpdir)) # Bucket should have un-altered content. cls.assertEqual(listing1, set(['/%s' % obj.object_name])) # Dir should have un-altered content. cls.assertEqual(listing2, set([''])) def TestCpMvPOSIXBucketToLocalNoErrors(cls, bucket_uri, tmpdir, is_cp=True): """Helper function for preserve_posix_no_errors tests in test_cp and test_mv. Args: cls: An instance of either TestCp or TestMv. bucket_uri: The uri of the bucket that the object is in. tmpdir: The local file path to cp to. is_cp: Whether or not the calling test suite is cp or mv. """ primary_gid = os.stat(tmpdir).st_gid non_primary_gid = util.GetNonPrimaryGid() test_params = { 'obj1': { GID_ATTR: primary_gid }, 'obj2': { GID_ATTR: non_primary_gid }, 'obj3': { GID_ATTR: primary_gid, MODE_ATTR: '440' }, 'obj4': { GID_ATTR: non_primary_gid, MODE_ATTR: '444' }, 'obj5': { UID_ATTR: USER_ID }, 'obj6': { UID_ATTR: USER_ID, MODE_ATTR: '420' }, 'obj7': { UID_ATTR: USER_ID, GID_ATTR: primary_gid }, 'obj8': { UID_ATTR: USER_ID, GID_ATTR: non_primary_gid }, 'obj9': { UID_ATTR: USER_ID, GID_ATTR: primary_gid, MODE_ATTR: '433' }, 'obj10': { UID_ATTR: USER_ID, GID_ATTR: non_primary_gid, MODE_ATTR: '442' } } for obj_name, attrs_dict in six.iteritems(test_params): uid = attrs_dict.get(UID_ATTR) gid = attrs_dict.get(GID_ATTR) mode = attrs_dict.get(MODE_ATTR) cls.CreateObject(bucket_uri=bucket_uri, object_name=obj_name, contents=obj_name.encode(UTF8), uid=uid, gid=gid, mode=mode) for obj_name in six.iterkeys(test_params): # Move objects one at a time to avoid listing consistency. cls.RunGsUtil( ['cp' if is_cp else 'mv', '-P', suri(bucket_uri, obj_name), tmpdir]) listing = TailSet(tmpdir, cls.FlatListDir(tmpdir)) cls.assertEqual( listing, set([ '/obj1', '/obj2', '/obj3', '/obj4', '/obj5', '/obj6', '/obj7', '/obj8', '/obj9', '/obj10' ])) cls.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj1'), gid=primary_gid, mode=DEFAULT_MODE) cls.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj2'), gid=non_primary_gid, mode=DEFAULT_MODE) cls.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj3'), gid=primary_gid, mode=0o440) cls.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj4'), gid=non_primary_gid, mode=0o444) cls.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj5'), uid=USER_ID, gid=primary_gid, mode=DEFAULT_MODE) cls.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj6'), uid=USER_ID, gid=primary_gid, mode=0o420) cls.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj7'), uid=USER_ID, gid=primary_gid, mode=DEFAULT_MODE) cls.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj8'), uid=USER_ID, gid=non_primary_gid, mode=DEFAULT_MODE) cls.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj9'), uid=USER_ID, gid=primary_gid, mode=0o433) cls.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj10'), uid=USER_ID, gid=non_primary_gid, mode=0o442) def TestCpMvPOSIXLocalToBucketNoErrors(cls, bucket_uri, is_cp=True): """Helper function for testing local to bucket POSIX preservation. Args: cls: An instance of either TestCp or TestMv. bucket_uri: The uri of the bucket to cp/mv to. is_cp: Whether or not the calling test suite is cp or mv. """ primary_gid = os.getgid() non_primary_gid = util.GetNonPrimaryGid() test_params = { 'obj1': { GID_ATTR: primary_gid }, 'obj2': { GID_ATTR: non_primary_gid }, 'obj3': { GID_ATTR: primary_gid, MODE_ATTR: '440' }, 'obj4': { GID_ATTR: non_primary_gid, MODE_ATTR: '444' }, 'obj5': { UID_ATTR: USER_ID }, 'obj6': { UID_ATTR: USER_ID, MODE_ATTR: '420' }, 'obj7': { UID_ATTR: USER_ID, GID_ATTR: primary_gid }, 'obj8': { UID_ATTR: USER_ID, GID_ATTR: non_primary_gid }, 'obj9': { UID_ATTR: USER_ID, GID_ATTR: primary_gid, MODE_ATTR: '433' }, 'obj10': { UID_ATTR: USER_ID, GID_ATTR: non_primary_gid, MODE_ATTR: '442' } } for obj_name, attrs_dict in six.iteritems(test_params): uid = attrs_dict.get(UID_ATTR, NA_ID) gid = attrs_dict.get(GID_ATTR, NA_ID) mode = attrs_dict.get(MODE_ATTR, NA_MODE) if mode != NA_MODE: ValidatePOSIXMode(int(mode, 8)) ValidateFilePermissionAccess(obj_name, uid=uid, gid=int(gid), mode=int(mode)) fpath = cls.CreateTempFile(contents=b'foo', uid=uid, gid=gid, mode=mode) cls.RunGsUtil( ['cp' if is_cp else 'mv', '-P', fpath, suri(bucket_uri, obj_name)]) if uid != NA_ID: cls.VerifyObjectCustomAttribute(bucket_uri.bucket_name, obj_name, UID_ATTR, str(uid)) if gid != NA_ID: cls.VerifyObjectCustomAttribute(bucket_uri.bucket_name, obj_name, GID_ATTR, str(gid)) if mode != NA_MODE: cls.VerifyObjectCustomAttribute(bucket_uri.bucket_name, obj_name, MODE_ATTR, str(mode)) def _ReadContentsFromFifo(fifo_path, list_for_output): with open(fifo_path, 'rb') as f: list_for_output.append(f.read()) def _WriteContentsToFifo(contents, fifo_path): with open(fifo_path, 'wb') as f: f.write(contents) class _JSONForceHTTPErrorCopyCallbackHandler(object): """Test callback handler that raises an arbitrary HTTP error exception.""" def __init__(self, startover_at_byte, http_error_num): self._startover_at_byte = startover_at_byte self._http_error_num = http_error_num self.started_over_once = False # pylint: disable=invalid-name def call(self, total_bytes_transferred, total_size): """Forcibly exits if the transfer has passed the halting point.""" if (total_bytes_transferred >= self._startover_at_byte and not self.started_over_once): sys.stderr.write('Forcing HTTP error %s after byte %s. ' '%s/%s transferred.\r\n' % (self._http_error_num, self._startover_at_byte, MakeHumanReadable(total_bytes_transferred), MakeHumanReadable(total_size))) self.started_over_once = True raise apitools_exceptions.HttpError({'status': self._http_error_num}, None, None) class _XMLResumableUploadStartOverCopyCallbackHandler(object): """Test callback handler that raises start-over exception during upload.""" def __init__(self, startover_at_byte): self._startover_at_byte = startover_at_byte self.started_over_once = False # pylint: disable=invalid-name def call(self, total_bytes_transferred, total_size): """Forcibly exits if the transfer has passed the halting point.""" if (total_bytes_transferred >= self._startover_at_byte and not self.started_over_once): sys.stderr.write( 'Forcing ResumableUpload start over error after byte %s. ' '%s/%s transferred.\r\n' % (self._startover_at_byte, MakeHumanReadable(total_bytes_transferred), MakeHumanReadable(total_size))) self.started_over_once = True raise boto.exception.ResumableUploadException( 'Forcing upload start over', ResumableTransferDisposition.START_OVER) class _DeleteBucketThenStartOverCopyCallbackHandler(object): """Test callback handler that deletes bucket then raises start-over.""" def __init__(self, startover_at_byte, bucket_uri): self._startover_at_byte = startover_at_byte self._bucket_uri = bucket_uri self.started_over_once = False # pylint: disable=invalid-name def call(self, total_bytes_transferred, total_size): """Forcibly exits if the transfer has passed the halting point.""" if (total_bytes_transferred >= self._startover_at_byte and not self.started_over_once): sys.stderr.write('Deleting bucket (%s)' % (self._bucket_uri.bucket_name)) @Retry(StorageResponseError, tries=5, timeout_secs=1) def DeleteBucket(): bucket_list = list(self._bucket_uri.list_bucket(all_versions=True)) for k in bucket_list: self._bucket_uri.get_bucket().delete_key(k.name, version_id=k.version_id) self._bucket_uri.delete_bucket() DeleteBucket() sys.stderr.write( 'Forcing ResumableUpload start over error after byte %s. ' '%s/%s transferred.\r\n' % (self._startover_at_byte, MakeHumanReadable(total_bytes_transferred), MakeHumanReadable(total_size))) self.started_over_once = True raise ResumableUploadStartOverException('Artificially forcing start-over') class _ResumableUploadRetryHandler(object): """Test callback handler for causing retries during a resumable transfer.""" def __init__(self, retry_at_byte, exception_to_raise, exc_args, num_retries=1): self._retry_at_byte = retry_at_byte self._exception_to_raise = exception_to_raise self._exception_args = exc_args self._num_retries = num_retries self._retries_made = 0 # pylint: disable=invalid-name def call(self, total_bytes_transferred, unused_total_size): """Cause a single retry at the retry point.""" if (total_bytes_transferred >= self._retry_at_byte and self._retries_made < self._num_retries): self._retries_made += 1 raise self._exception_to_raise(*self._exception_args) class TestCp(testcase.GsUtilIntegrationTestCase): """Integration tests for cp command.""" # For tests that artificially halt, we need to ensure at least one callback # occurs. halt_size = START_CALLBACK_PER_BYTES * 2 def _get_test_file(self, name): contents = pkgutil.get_data('gslib', 'tests/test_data/%s' % name) return self.CreateTempFile(file_name=name, contents=contents) def _CpWithFifoViaGsUtilAndAppendOutputToList(self, src_path_tuple, dst_path, list_for_return_value, **kwargs): arg_list = ['cp'] arg_list.extend(src_path_tuple) arg_list.append(dst_path) # Append stderr, stdout, or return status (if specified in kwargs) to the # given list. list_for_return_value.append(self.RunGsUtil(arg_list, **kwargs)) @SequentialAndParallelTransfer def test_noclobber(self): key_uri = self.CreateObject(contents=b'foo') fpath = self.CreateTempFile(contents=b'bar') stderr = self.RunGsUtil( ['cp', '-n', fpath, suri(key_uri)], return_stderr=True) self.assertRegex(stderr, r'Skipping.*: {}'.format(re.escape(suri(key_uri)))) self.assertEqual(key_uri.get_contents_as_string(), b'foo') stderr = self.RunGsUtil(['cp', '-n', suri(key_uri), fpath], return_stderr=True) with open(fpath, 'rb') as f: self.assertRegex(stderr, r'Skipping.*: {}'.format(re.escape(suri(f)))) self.assertEqual(f.read(), b'bar') @SequentialAndParallelTransfer def test_noclobber_different_size(self): key_uri = self.CreateObject(contents=b'foo') fpath = self.CreateTempFile(contents=b'quux') stderr = self.RunGsUtil( ['cp', '-n', fpath, suri(key_uri)], return_stderr=True) self.assertRegex(stderr, r'Skipping.*: {}'.format(re.escape(suri(key_uri)))) self.assertEqual(key_uri.get_contents_as_string(), b'foo') stderr = self.RunGsUtil(['cp', '-n', suri(key_uri), fpath], return_stderr=True) with open(fpath, 'rb') as f: self.assertRegex(stderr, r'Skipping.*: {}'.format(re.escape(suri(f)))) self.assertEqual(f.read(), b'quux') def test_dest_bucket_not_exist(self): fpath = self.CreateTempFile(contents=b'foo') invalid_bucket_uri = ('%s://%s' % (self.default_provider, self.nonexistent_bucket_name)) # TODO(b/135780661): Remove retry after bug resolved @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stderr = self.RunGsUtil(['cp', fpath, invalid_bucket_uri], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('not found: 404', stderr) else: self.assertIn('does not exist', stderr) _Check() def test_copy_in_cloud_noclobber(self): bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket1_uri, contents=b'foo') stderr = self.RunGsUtil( ['cp', suri(key_uri), suri(bucket2_uri)], return_stderr=True) # Rewrite API may output an additional 'Copying' progress notification. self.assertGreaterEqual(stderr.count('Copying'), 1) self.assertLessEqual(stderr.count('Copying'), 2) stderr = self.RunGsUtil( ['cp', '-n', suri(key_uri), suri(bucket2_uri)], return_stderr=True) self.assertRegex( stderr, r'Skipping.*: {}'.format(suri(bucket2_uri, key_uri.object_name))) @SequentialAndParallelTransfer @SkipForXML('Boto library does not handle objects with .. in them.') def test_skip_object_with_parent_directory_symbol_in_name(self): bucket_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='dir/../../../file', contents=b'data', prefer_json_api=True) self.CreateObject(bucket_uri=bucket_uri, object_name='file2', contents=b'data') directory = self.CreateTempDir() stderr = self.RunGsUtil( ['cp', '-r', suri(bucket_uri), directory], return_stderr=True) # By default, deletes in the tearDown method run with the XML API. Boto # does not handle names with '..', so we need to delete problematic # objects with the json API. Delete happens before assertions, in case they # raise errors and prevent cleanup. self.json_api.DeleteObject(bucket_uri.bucket_name, key_uri.object_name) self.assertIn( 'Skipping copy of source URL %s because it would be copied ' 'outside the expected destination directory: %s.' % (suri(key_uri), os.path.abspath(directory)), stderr) self.assertFalse(os.path.exists(os.path.join(directory, 'file'))) self.assertTrue( os.path.exists(os.path.join(directory, bucket_uri.bucket_name, 'file2'))) @SequentialAndParallelTransfer @SkipForXML('Boto library does not handle objects with .. in them.') def test_skip_parent_directory_symbol_in_name_is_reflected_in_manifest(self): bucket_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='dir/../../../file', contents=b'data', prefer_json_api=True) directory = self.CreateTempDir() log_path = os.path.join(directory, 'log.csv') stderr = self.RunGsUtil( ['cp', '-r', '-L', log_path, suri(bucket_uri), directory], return_stderr=True) # By default, deletes in the tearDown method run with the XML API. Boto # does not handle names with '..', so we need to delete problematic # objects with the json API. Delete happens before assertions, in case they # raise errors and prevent cleanup. self.json_api.DeleteObject(bucket_uri.bucket_name, key_uri.object_name) self.assertIn( 'Skipping copy of source URL %s because it would be copied ' 'outside the expected destination directory: %s.' % (suri(key_uri), os.path.abspath(directory)), stderr) self.assertFalse(os.path.exists(os.path.join(directory, 'file'))) with open(log_path, 'r') as f: lines = f.readlines() results = lines[1].strip().split(',') self.assertEqual(results[0], suri(key_uri)) # The 'Source' column. self.assertEqual(results[8], 'skip') # The 'Result' column. @SequentialAndParallelTransfer @SkipForXML('Boto library does not handle objects with .. in them.') @unittest.skipIf(IS_WINDOWS, 'os.symlink() is not available on Windows.') def test_skip_parent_directory_symbol_object_with_symlink_destination(self): bucket_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='dir/../../../file', contents=b'data', prefer_json_api=True) second_key_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='file2', contents=b'data') directory = self.CreateTempDir() linked_destination = os.path.join(directory, 'linked_destination') destination = os.path.join(directory, 'destination') os.mkdir(destination) os.symlink(destination, linked_destination) stderr = self.RunGsUtil([ '-D', 'cp', '-r', suri(bucket_uri), suri(second_key_uri), linked_destination ], return_stderr=True) # By default, deletes in the tearDown method run with the XML API. Boto # does not handle names with '..', so we need to delete problematic # objects with the json API. Delete happens before assertions, in case they # raise errors and prevent cleanup. self.json_api.DeleteObject(bucket_uri.bucket_name, key_uri.object_name) self.assertIn( 'Skipping copy of source URL %s because it would be copied ' 'outside the expected destination directory: %s.' % (suri(key_uri), linked_destination), stderr) self.assertFalse(os.path.exists(os.path.join(linked_destination, 'file'))) self.assertTrue(os.path.exists(os.path.join(linked_destination, 'file2'))) @unittest.skipIf(IS_WINDOWS, 'os.mkfifo not available on Windows.') @SequentialAndParallelTransfer def test_cp_from_local_file_to_fifo(self): contents = b'bar' fifo_path = self.CreateTempFifo() file_path = self.CreateTempFile(contents=contents) list_for_output = [] read_thread = threading.Thread(target=_ReadContentsFromFifo, args=(fifo_path, list_for_output)) read_thread.start() write_thread = threading.Thread( target=self._CpWithFifoViaGsUtilAndAppendOutputToList, args=((file_path,), fifo_path, [])) write_thread.start() write_thread.join(120) read_thread.join(120) if not list_for_output: self.fail('Reading/writing to the fifo timed out.') self.assertEqual(list_for_output[0].strip(), contents) @unittest.skipIf(IS_WINDOWS, 'os.mkfifo not available on Windows.') @SequentialAndParallelTransfer def test_cp_from_one_object_to_fifo(self): fifo_path = self.CreateTempFifo() bucket_uri = self.CreateBucket() contents = b'bar' obj_uri = self.CreateObject(bucket_uri=bucket_uri, contents=contents) list_for_output = [] read_thread = threading.Thread(target=_ReadContentsFromFifo, args=(fifo_path, list_for_output)) read_thread.start() write_thread = threading.Thread( target=self._CpWithFifoViaGsUtilAndAppendOutputToList, args=((suri(obj_uri),), fifo_path, [])) write_thread.start() write_thread.join(120) read_thread.join(120) if not list_for_output: self.fail('Reading/writing to the fifo timed out.') self.assertEqual(list_for_output[0].strip(), contents) @unittest.skipIf(IS_WINDOWS, 'os.mkfifo not available on Windows.') @SequentialAndParallelTransfer def test_cp_from_multiple_objects_to_fifo(self): fifo_path = self.CreateTempFifo() bucket_uri = self.CreateBucket() contents1 = b'foo and bar' contents2 = b'baz and qux' obj1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=contents1) obj2_uri = self.CreateObject(bucket_uri=bucket_uri, contents=contents2) list_for_output = [] read_thread = threading.Thread(target=_ReadContentsFromFifo, args=(fifo_path, list_for_output)) read_thread.start() write_thread = threading.Thread( target=self._CpWithFifoViaGsUtilAndAppendOutputToList, args=((suri(obj1_uri), suri(obj2_uri)), fifo_path, [])) write_thread.start() write_thread.join(120) read_thread.join(120) if not list_for_output: self.fail('Reading/writing to the fifo timed out.') self.assertIn(contents1, list_for_output[0]) self.assertIn(contents2, list_for_output[0]) @SequentialAndParallelTransfer def test_streaming(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil( ['cp', '-', '%s' % suri(bucket_uri, 'foo')], stdin='bar', return_stderr=True) if self._use_gcloud_storage: self.assertIn('Copying file://- to ' + suri(bucket_uri, 'foo'), stderr) else: self.assertIn('Copying from <STDIN>', stderr) key_uri = self.StorageUriCloneReplaceName(bucket_uri, 'foo') self.assertEqual(key_uri.get_contents_as_string(), b'bar') @unittest.skipIf(IS_WINDOWS, 'os.mkfifo not available on Windows.') @SequentialAndParallelTransfer def test_streaming_from_fifo_to_object(self): bucket_uri = self.CreateBucket() fifo_path = self.CreateTempFifo() object_name = 'foo' object_contents = b'bar' list_for_output = [] # Start writer in the background, which won't finish until a corresponding # read operation is performed on the fifo. write_thread = threading.Thread(target=_WriteContentsToFifo, args=(object_contents, fifo_path)) write_thread.start() # The fifo requires both a pending read and write before either operation # will complete. Regardless of which operation occurs first, the # corresponding subsequent operation will unblock the first one. # We run gsutil in a thread so that it can timeout rather than hang forever # if the write thread fails. read_thread = threading.Thread( target=self._CpWithFifoViaGsUtilAndAppendOutputToList, args=((fifo_path,), suri(bucket_uri, object_name), list_for_output), kwargs={'return_stderr': True}) read_thread.start() read_thread.join(120) write_thread.join(120) if not list_for_output: self.fail('Reading/writing to the fifo timed out.') if self._use_gcloud_storage: self.assertIn( 'Copying file://{} to {}'.format(fifo_path, suri(bucket_uri, object_name)), list_for_output[0]) else: self.assertIn('Copying from named pipe', list_for_output[0]) key_uri = self.StorageUriCloneReplaceName(bucket_uri, object_name) self.assertEqual(key_uri.get_contents_as_string(), object_contents) @unittest.skipIf(IS_WINDOWS, 'os.mkfifo not available on Windows.') @SequentialAndParallelTransfer def test_streaming_from_fifo_to_stdout(self): fifo_path = self.CreateTempFifo() contents = b'bar' list_for_output = [] write_thread = threading.Thread(target=_WriteContentsToFifo, args=(contents, fifo_path)) write_thread.start() read_thread = threading.Thread( target=self._CpWithFifoViaGsUtilAndAppendOutputToList, args=((fifo_path,), '-', list_for_output), kwargs={'return_stdout': True}) read_thread.start() read_thread.join(120) write_thread.join(120) if not list_for_output: self.fail('Reading/writing to the fifo timed out.') self.assertEqual(list_for_output[0].strip().encode('ascii'), contents) @unittest.skipIf(IS_WINDOWS, 'os.mkfifo not available on Windows.') @SequentialAndParallelTransfer def test_streaming_from_stdout_to_fifo(self): fifo_path = self.CreateTempFifo() contents = b'bar' list_for_output = [] list_for_gsutil_output = [] read_thread = threading.Thread(target=_ReadContentsFromFifo, args=(fifo_path, list_for_output)) read_thread.start() write_thread = threading.Thread( target=self._CpWithFifoViaGsUtilAndAppendOutputToList, args=(('-',), fifo_path, list_for_gsutil_output), kwargs={ 'return_stderr': True, 'stdin': contents }) write_thread.start() write_thread.join(120) read_thread.join(120) if not list_for_output: self.fail('Reading/writing to the fifo timed out.') self.assertEqual(list_for_output[0].strip(), contents) def test_streaming_multiple_arguments(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil(['cp', '-', '-', suri(bucket_uri)], stdin='bar', return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn( 'Multiple URL strings are not supported when transferring' ' from stdin.', stderr) else: self.assertIn('Multiple URL strings are not supported with streaming', stderr) # TODO: Implement a way to test both with and without using magic file. @SequentialAndParallelTransfer def test_detect_content_type(self): """Tests local detection of content type.""" bucket_uri = self.CreateBucket() dsturi = suri(bucket_uri, 'foo') self.RunGsUtil(['cp', self._get_test_file('test.mp3'), dsturi]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) if IS_WINDOWS: self.assertTrue( re.search(r'Content-Type:\s+audio/x-mpg', stdout) or re.search(r'Content-Type:\s+audio/mpeg', stdout)) else: self.assertRegex(stdout, r'Content-Type:\s+audio/mpeg') _Check1() self.RunGsUtil(['cp', self._get_test_file('test.gif'), dsturi]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+image/gif') _Check2() def test_content_type_override_default(self): """Tests overriding content type with the default value.""" bucket_uri = self.CreateBucket() dsturi = suri(bucket_uri, 'foo') self.RunGsUtil( ['-h', 'Content-Type:', 'cp', self._get_test_file('test.mp3'), dsturi]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+application/octet-stream') _Check1() self.RunGsUtil( ['-h', 'Content-Type:', 'cp', self._get_test_file('test.gif'), dsturi]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+application/octet-stream') _Check2() def test_content_type_override(self): """Tests overriding content type with a value.""" bucket_uri = self.CreateBucket() dsturi = suri(bucket_uri, 'foo') self.RunGsUtil([ '-h', 'Content-Type:text/plain', 'cp', self._get_test_file('test.mp3'), dsturi ]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+text/plain') _Check1() self.RunGsUtil([ '-h', 'Content-Type:text/plain', 'cp', self._get_test_file('test.gif'), dsturi ]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+text/plain') _Check2() @unittest.skipIf(IS_WINDOWS, 'magicfile is not available on Windows.') @SequentialAndParallelTransfer def test_magicfile_override(self): """Tests content type override with magicfile value.""" bucket_uri = self.CreateBucket() dsturi = suri(bucket_uri, 'foo') fpath = self.CreateTempFile(contents=b'foo/bar\n') self.RunGsUtil(['cp', fpath, dsturi]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) use_magicfile = boto.config.getbool('GSUtil', 'use_magicfile', False) content_type = ('text/plain' if use_magicfile else 'application/octet-stream') self.assertRegex(stdout, r'Content-Type:\s+%s' % content_type) _Check1() @SequentialAndParallelTransfer def test_content_type_mismatches(self): """Tests overriding content type when it does not match the file type.""" bucket_uri = self.CreateBucket() dsturi = suri(bucket_uri, 'foo') fpath = self.CreateTempFile(contents=b'foo/bar\n') self.RunGsUtil([ '-h', 'Content-Type:image/gif', 'cp', self._get_test_file('test.mp3'), dsturi ]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+image/gif') _Check1() self.RunGsUtil([ '-h', 'Content-Type:image/gif', 'cp', self._get_test_file('test.gif'), dsturi ]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+image/gif') _Check2() self.RunGsUtil(['-h', 'Content-Type:image/gif', 'cp', fpath, dsturi]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check3(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+image/gif') _Check3() @SequentialAndParallelTransfer def test_content_type_header_case_insensitive(self): """Tests that content type header is treated with case insensitivity.""" bucket_uri = self.CreateBucket() dsturi = suri(bucket_uri, 'foo') fpath = self._get_test_file('test.gif') self.RunGsUtil(['-h', 'content-Type:text/plain', 'cp', fpath, dsturi]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+text/plain') self.assertNotRegex(stdout, r'image/gif') _Check1() self.RunGsUtil([ '-h', 'CONTENT-TYPE:image/gif', '-h', 'content-type:image/gif', 'cp', fpath, dsturi ]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(['ls', '-L', dsturi], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+image/gif') self.assertNotRegex(stdout, r'image/gif,\s*image/gif') _Check2() @SequentialAndParallelTransfer def test_other_headers(self): """Tests that non-content-type headers are applied successfully on copy.""" bucket_uri = self.CreateBucket() dst_uri = suri(bucket_uri, 'foo') fpath = self._get_test_file('test.gif') self.RunGsUtil([ '-h', 'Cache-Control:public,max-age=12', '-h', 'x-%s-meta-1:abcd' % self.provider_custom_meta, 'cp', fpath, dst_uri ]) stdout = self.RunGsUtil(['ls', '-L', dst_uri], return_stdout=True) self.assertRegex(stdout, r'Cache-Control\s*:\s*public,max-age=12') self.assertRegex(stdout, r'Metadata:\s*1:\s*abcd') dst_uri2 = suri(bucket_uri, 'bar') self.RunGsUtil(['cp', dst_uri, dst_uri2]) # Ensure metadata was preserved across copy. stdout = self.RunGsUtil(['ls', '-L', dst_uri2], return_stdout=True) self.assertRegex(stdout, r'Cache-Control\s*:\s*public,max-age=12') self.assertRegex(stdout, r'Metadata:\s*1:\s*abcd') @SequentialAndParallelTransfer def test_request_reason_header(self): """Test that x-goog-request-header can be set using the environment variable.""" os.environ['CLOUDSDK_CORE_REQUEST_REASON'] = 'b/this_is_env_reason' bucket_uri = self.CreateBucket() dst_uri = suri(bucket_uri, 'foo') fpath = self._get_test_file('test.gif') # Ensure x-goog-request-header is set in cp command stderr = self.RunGsUtil(['-DD', 'cp', fpath, dst_uri], return_stderr=True) if self._use_gcloud_storage: reason_regex = r"b'X-Goog-Request-Reason': b'b/this_is_env_reason'" else: reason_regex = r"'x-goog-request-reason': 'b/this_is_env_reason'" self.assertRegex(stderr, reason_regex) # Ensure x-goog-request-header is set in ls command stderr = self.RunGsUtil(['-DD', 'ls', '-L', dst_uri], return_stderr=True) self.assertRegex(stderr, reason_regex) @SequentialAndParallelTransfer @SkipForXML('XML APIs use a different debug log format.') def test_request_reason_header_persists_multiple_requests_json(self): """Test that x-goog-request-header works when cp sends multiple requests.""" os.environ['CLOUDSDK_CORE_REQUEST_REASON'] = 'b/this_is_env_reason' bucket_uri = self.CreateBucket() dst_uri = suri(bucket_uri, 'foo') fpath = self._get_test_file('test.gif') boto_config_for_test = ('GSUtil', 'resumable_threshold', '0') with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil(['-DD', 'cp', fpath, dst_uri], return_stderr=True) if self._use_gcloud_storage: reason_regex = r'X-Goog-Request-Reason\': b\'b/this_is_env_reason' else: reason_regex = r'x-goog-request-reason\': \'b/this_is_env_reason' self.assertRegex( stderr, # POST follows GET request. Both need the request-reason header. r'GET[\s\S]*' + reason_regex + r'[\s\S]*POST[\s\S]*' + reason_regex) @SequentialAndParallelTransfer @SkipForJSON('JSON API uses a different debug log format.') def test_request_reason_header_persists_multiple_requests_xml(self): """Test that x-goog-request-header works when cp sends multiple requests.""" os.environ['CLOUDSDK_CORE_REQUEST_REASON'] = 'b/this_is_env_reason' bucket_uri = self.CreateBucket() dst_uri = suri(bucket_uri, 'foo') fpath = self._get_test_file('test.gif') boto_config_for_test = ('GSUtil', 'resumable_threshold', '0') with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil(['-D', 'cp', fpath, dst_uri], return_stderr=True) reason_regex = ( r'Final headers: \{[\s\S]*\'' r'x-goog-request-reason\': \'b/this_is_env_reason\'[\s\S]*}') # Pattern should match twice since two requests should have a reason header. self.assertRegex(stderr, reason_regex + r'[\s\S]*' + reason_regex) @SequentialAndParallelTransfer def test_versioning(self): """Tests copy with versioning.""" bucket_uri = self.CreateVersionedBucket() k1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'data2') k2_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'data1') g1 = urigen(k2_uri) self.RunGsUtil(['cp', suri(k1_uri), suri(k2_uri)]) k2_uri = self.StorageUriCloneReplaceName(bucket_uri, k2_uri.object_name) k2_uri = self.StorageUriCloneReplaceKey(bucket_uri, k2_uri.get_key()) g2 = urigen(k2_uri) self.StorageUriSetContentsFromString(k2_uri, 'data3') g3 = urigen(k2_uri) fpath = self.CreateTempFile() # Check to make sure current version is data3. self.RunGsUtil(['cp', k2_uri.versionless_uri, fpath]) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'data3') # Check contents of all three versions self.RunGsUtil(['cp', '%s#%s' % (k2_uri.versionless_uri, g1), fpath]) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'data1') self.RunGsUtil(['cp', '%s#%s' % (k2_uri.versionless_uri, g2), fpath]) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'data2') self.RunGsUtil(['cp', '%s#%s' % (k2_uri.versionless_uri, g3), fpath]) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'data3') # Copy first version to current and verify. self.RunGsUtil( ['cp', '%s#%s' % (k2_uri.versionless_uri, g1), k2_uri.versionless_uri]) self.RunGsUtil(['cp', k2_uri.versionless_uri, fpath]) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'data1') # Attempt to specify a version-specific URI for destination. stderr = self.RunGsUtil(['cp', fpath, k2_uri.uri], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn( 'destination argument of the cp command cannot' ' be a version-specific URL', stderr) else: self.assertIn('cannot be the destination for gsutil cp', stderr) def test_versioning_no_parallelism(self): """Tests that copy all-versions errors when parallelism is enabled.""" # TODO(b/135780661): Remove retry after bug resolved @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stderr = self.RunGsUtil([ '-m', 'cp', '-A', suri(self.nonexistent_bucket_name, 'foo'), suri(self.nonexistent_bucket_name, 'bar') ], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('sequential instead of parallel task execution', stderr) else: self.assertIn('-m option is not supported with the cp -A flag', stderr) _Check() @SkipForS3('S3 lists versioned objects in reverse timestamp order.') def test_recursive_copying_versioned_bucket(self): """Tests cp -R with versioned buckets.""" bucket1_uri = self.CreateVersionedBucket() bucket2_uri = self.CreateVersionedBucket() bucket3_uri = self.CreateVersionedBucket() # Write two versions of an object to the bucket1. v1_uri = self.CreateObject(bucket_uri=bucket1_uri, object_name='k', contents=b'data0') self.CreateObject(bucket_uri=bucket1_uri, object_name='k', contents=b'longer_data1', gs_idempotent_generation=urigen(v1_uri)) self.AssertNObjectsInBucket(bucket1_uri, 2, versioned=True) self.AssertNObjectsInBucket(bucket2_uri, 0, versioned=True) self.AssertNObjectsInBucket(bucket3_uri, 0, versioned=True) # Recursively copy to second versioned bucket. # -A flag should copy all versions in order. self.RunGsUtil( ['cp', '-R', '-A', suri(bucket1_uri, '*'), suri(bucket2_uri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): """Validates the results of the cp -R.""" listing1 = self.RunGsUtil(['ls', '-la', suri(bucket1_uri)], return_stdout=True).split('\n') listing2 = self.RunGsUtil(['ls', '-la', suri(bucket2_uri)], return_stdout=True).split('\n') # 2 lines of listing output, 1 summary line, 1 empty line from \n split. self.assertEqual(len(listing1), 4) self.assertEqual(len(listing2), 4) # First object in each bucket should match in size and version-less name. size1, _, uri_str1, _ = listing1[0].split() self.assertEqual(size1, str(len('data0'))) self.assertEqual(storage_uri(uri_str1).object_name, 'k') size2, _, uri_str2, _ = listing2[0].split() self.assertEqual(size2, str(len('data0'))) self.assertEqual(storage_uri(uri_str2).object_name, 'k') # Similarly for second object in each bucket. size1, _, uri_str1, _ = listing1[1].split() self.assertEqual(size1, str(len('longer_data1'))) self.assertEqual(storage_uri(uri_str1).object_name, 'k') size2, _, uri_str2, _ = listing2[1].split() self.assertEqual(size2, str(len('longer_data1'))) self.assertEqual(storage_uri(uri_str2).object_name, 'k') _Check2() # Recursively copy to second versioned bucket with no -A flag. # This should copy only the live object. self.RunGsUtil(['cp', '-R', suri(bucket1_uri, '*'), suri(bucket3_uri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check3(): """Validates the results of the cp -R.""" listing1 = self.RunGsUtil(['ls', '-la', suri(bucket1_uri)], return_stdout=True).split('\n') listing2 = self.RunGsUtil(['ls', '-la', suri(bucket3_uri)], return_stdout=True).split('\n') # 2 lines of listing output, 1 summary line, 1 empty line from \n split. self.assertEqual(len(listing1), 4) # 1 lines of listing output, 1 summary line, 1 empty line from \n split. self.assertEqual(len(listing2), 3) # Live (second) object in bucket 1 should match the single live object. size1, _, uri_str1, _ = listing2[0].split() self.assertEqual(size1, str(len('longer_data1'))) self.assertEqual(storage_uri(uri_str1).object_name, 'k') _Check3() @SequentialAndParallelTransfer @SkipForS3('Preconditions not supported for S3.') def test_cp_generation_zero_match(self): """Tests that cp handles an object-not-exists precondition header.""" bucket_uri = self.CreateBucket() fpath1 = self.CreateTempFile(contents=b'data1') # Match 0 means only write the object if it doesn't already exist. gen_match_header = 'x-goog-if-generation-match:0' # First copy should succeed. # TODO: This can fail (rarely) if the server returns a 5xx but actually # commits the bytes. If we add restarts on small uploads, handle this # case. self.RunGsUtil(['-h', gen_match_header, 'cp', fpath1, suri(bucket_uri)]) # Second copy should fail with a precondition error. stderr = self.RunGsUtil( ['-h', gen_match_header, 'cp', fpath1, suri(bucket_uri)], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn( 'HTTPError 412: At least one of the pre-conditions you specified' ' did not hold.', stderr) else: self.assertIn('PreconditionException', stderr) @SequentialAndParallelTransfer @SkipForS3('Preconditions not supported for S3.') def test_cp_v_generation_match(self): """Tests that cp -v option handles the if-generation-match header.""" bucket_uri = self.CreateVersionedBucket() k1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'data1') g1 = k1_uri.generation tmpdir = self.CreateTempDir() fpath1 = self.CreateTempFile(tmpdir=tmpdir, contents=b'data2') gen_match_header = 'x-goog-if-generation-match:%s' % g1 # First copy should succeed. self.RunGsUtil(['-h', gen_match_header, 'cp', fpath1, suri(k1_uri)]) # Second copy should fail the precondition. stderr = self.RunGsUtil( ['-h', gen_match_header, 'cp', fpath1, suri(k1_uri)], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('pre-condition', stderr) else: self.assertIn('PreconditionException', stderr) # Specifiying a generation with -n should fail before the request hits the # server. stderr = self.RunGsUtil( ['-h', gen_match_header, 'cp', '-n', fpath1, suri(k1_uri)], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn( 'Cannot specify both generation precondition and no-clobber', stderr) else: self.assertIn('ArgumentException', stderr) self.assertIn( 'Specifying x-goog-if-generation-match is not supported ' 'with cp -n', stderr) @SequentialAndParallelTransfer def test_cp_nv(self): """Tests that cp -nv works when skipping existing file.""" bucket_uri = self.CreateVersionedBucket() k1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'data1') tmpdir = self.CreateTempDir() fpath1 = self.CreateTempFile(tmpdir=tmpdir, contents=b'data2') # First copy should succeed. self.RunGsUtil(['cp', '-nv', fpath1, suri(k1_uri)]) # Second copy should skip copying. stderr = self.RunGsUtil( ['cp', '-nv', fpath1, suri(k1_uri)], return_stderr=True) self.assertIn('Skipping existing', stderr) @SequentialAndParallelTransfer @SkipForS3('S3 lists versioned objects in reverse timestamp order.') def test_cp_v_option(self): """"Tests that cp -v returns the created object's version-specific URI.""" bucket_uri = self.CreateVersionedBucket() k1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'data1') k2_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'data2') # Case 1: Upload file to object using one-shot PUT. tmpdir = self.CreateTempDir() fpath1 = self.CreateTempFile(tmpdir=tmpdir, contents=b'data1') self._run_cp_minus_v_test('-v', fpath1, k2_uri.uri) # Case 2: Upload file to object using resumable upload. size_threshold = ONE_KIB boto_config_for_test = ('GSUtil', 'resumable_threshold', str(size_threshold)) with SetBotoConfigForTest([boto_config_for_test]): file_as_string = os.urandom(size_threshold) tmpdir = self.CreateTempDir() fpath1 = self.CreateTempFile(tmpdir=tmpdir, contents=file_as_string) self._run_cp_minus_v_test('-v', fpath1, k2_uri.uri) # Case 3: Upload stream to object. self._run_cp_minus_v_test('-v', '-', k2_uri.uri) # Case 4: Download object to file. For this case we just expect output of # gsutil cp -v to be the URI of the file. tmpdir = self.CreateTempDir() fpath1 = self.CreateTempFile(tmpdir=tmpdir) dst_uri = storage_uri(fpath1) stderr = self.RunGsUtil( ['cp', '-v', suri(k1_uri), suri(dst_uri)], return_stderr=True) # TODO: Add ordering assertion (should be in stderr.split('\n)[-2]) back # once both the creation and status messages are handled by the UI thread. self.assertIn('Created: %s\n' % dst_uri.uri, stderr) # Case 5: Daisy-chain from object to object. self._run_cp_minus_v_test('-Dv', k1_uri.uri, k2_uri.uri) # Case 6: Copy object to object in-the-cloud. self._run_cp_minus_v_test('-v', k1_uri.uri, k2_uri.uri) def _run_cp_minus_v_test(self, opt, src_str, dst_str): """Runs cp -v with the options and validates the results.""" stderr = self.RunGsUtil(['cp', opt, src_str, dst_str], return_stderr=True) match = re.search(r'Created: (.*)\n', stderr) self.assertIsNotNone(match) created_uri = match.group(1) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-a', dst_str], return_stdout=True) lines = stdout.split('\n') # Final (most recent) object should match the "Created:" URI. This is # in second-to-last line (last line is '\n'). self.assertGreater(len(lines), 2) self.assertEqual(created_uri, lines[-2]) _Check1() @SequentialAndParallelTransfer def test_stdin_args(self): """Tests cp with the -I option.""" tmpdir = self.CreateTempDir() fpath1 = self.CreateTempFile(tmpdir=tmpdir, contents=b'data1') fpath2 = self.CreateTempFile(tmpdir=tmpdir, contents=b'data2') bucket_uri = self.CreateBucket() self.RunGsUtil(['cp', '-I', suri(bucket_uri)], stdin='\n'.join((fpath1, fpath2))) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', suri(bucket_uri)], return_stdout=True) self.assertIn(os.path.basename(fpath1), stdout) self.assertIn(os.path.basename(fpath2), stdout) self.assertNumLines(stdout, 2) _Check1() def test_cross_storage_class_cloud_cp(self): bucket1_uri = self.CreateBucket(storage_class='standard') bucket2_uri = self.CreateBucket( storage_class='durable_reduced_availability') key_uri = self.CreateObject(bucket_uri=bucket1_uri, contents=b'foo') # Server now allows copy-in-the-cloud across storage classes. self.RunGsUtil(['cp', suri(key_uri), suri(bucket2_uri)]) @unittest.skipUnless(HAS_S3_CREDS, 'Test requires both S3 and GS credentials') def test_cross_provider_cp(self): s3_bucket = self.CreateBucket(provider='s3') gs_bucket = self.CreateBucket(provider='gs') s3_key = self.CreateObject(bucket_uri=s3_bucket, contents=b'foo') gs_key = self.CreateObject(bucket_uri=gs_bucket, contents=b'bar') self.RunGsUtil(['cp', suri(s3_key), suri(gs_bucket)]) self.RunGsUtil(['cp', suri(gs_key), suri(s3_bucket)]) @unittest.skipUnless(HAS_S3_CREDS, 'Test requires both S3 and GS credentials') @unittest.skip('This test performs a large copy but remains here for ' 'debugging purposes.') def test_cross_provider_large_cp(self): s3_bucket = self.CreateBucket(provider='s3') gs_bucket = self.CreateBucket(provider='gs') s3_key = self.CreateObject(bucket_uri=s3_bucket, contents=b'f' * 1024 * 1024) gs_key = self.CreateObject(bucket_uri=gs_bucket, contents=b'b' * 1024 * 1024) self.RunGsUtil(['cp', suri(s3_key), suri(gs_bucket)]) self.RunGsUtil(['cp', suri(gs_key), suri(s3_bucket)]) with SetBotoConfigForTest([('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'json_resumable_chunk_size', str(ONE_KIB * 256))]): # Ensure copy also works across json upload chunk boundaries. self.RunGsUtil(['cp', suri(s3_key), suri(gs_bucket)]) @unittest.skipUnless(HAS_S3_CREDS, 'Test requires both S3 and GS credentials') def test_gs_to_s3_multipart_cp(self): """Ensure daisy_chain works for an object that is downloaded in 2 parts.""" s3_bucket = self.CreateBucket(provider='s3') gs_bucket = self.CreateBucket(provider='gs', prefer_json_api=True) num_bytes = int(_DEFAULT_DOWNLOAD_CHUNK_SIZE * 1.1) gs_key = self.CreateObject(bucket_uri=gs_bucket, contents=b'b' * num_bytes, prefer_json_api=True) self.RunGsUtil([ '-o', 's3:use-sigv4=True', '-o', 's3:host=s3.amazonaws.com', 'cp', suri(gs_key), suri(s3_bucket) ]) @unittest.skip('This test is slow due to creating many objects, ' 'but remains here for debugging purposes.') def test_daisy_chain_cp_file_sizes(self): """Ensure daisy chain cp works with a wide of file sizes.""" bucket_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() exponent_cap = 28 # Up to 256 MiB in size. for i in range(exponent_cap): one_byte_smaller = 2**i - 1 normal = 2**i one_byte_larger = 2**i + 1 self.CreateObject(bucket_uri=bucket_uri, contents=b'a' * one_byte_smaller) self.CreateObject(bucket_uri=bucket_uri, contents=b'b' * normal) self.CreateObject(bucket_uri=bucket_uri, contents=b'c' * one_byte_larger) self.AssertNObjectsInBucket(bucket_uri, exponent_cap * 3) self.RunGsUtil( ['-m', 'cp', '-D', suri(bucket_uri, '**'), suri(bucket2_uri)]) self.AssertNObjectsInBucket(bucket2_uri, exponent_cap * 3) def test_daisy_chain_cp(self): """Tests cp with the -D option.""" bucket1_uri = self.CreateBucket(storage_class='standard') bucket2_uri = self.CreateBucket( storage_class='durable_reduced_availability') key_uri = self.CreateObject(bucket_uri=bucket1_uri, contents=b'foo') # Set some headers on source object so we can verify that headers are # presereved by daisy-chain copy. self.RunGsUtil([ 'setmeta', '-h', 'Cache-Control:public,max-age=12', '-h', 'Content-Type:image/gif', '-h', 'x-%s-meta-1:abcd' % self.provider_custom_meta, suri(key_uri) ]) # Set public-read (non-default) ACL so we can verify that cp -D -p works. self.RunGsUtil(['acl', 'set', 'public-read', suri(key_uri)]) acl_json = self.RunGsUtil(['acl', 'get', suri(key_uri)], return_stdout=True) # Perform daisy-chain copy and verify that source object headers and ACL # were preserved. Also specify -n option to test that gsutil correctly # removes the x-goog-if-generation-match:0 header that was set at uploading # time when updating the ACL. stderr = self.RunGsUtil( ['cp', '-Dpn', suri(key_uri), suri(bucket2_uri)], return_stderr=True) self.assertNotIn('Copy-in-the-cloud disallowed', stderr) @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): uri = suri(bucket2_uri, key_uri.object_name) stdout = self.RunGsUtil(['ls', '-L', uri], return_stdout=True) self.assertRegex(stdout, r'Cache-Control:\s+public,max-age=12') self.assertRegex(stdout, r'Content-Type:\s+image/gif') self.assertRegex(stdout, r'Metadata:\s+1:\s+abcd') new_acl_json = self.RunGsUtil(['acl', 'get', uri], return_stdout=True) self.assertEqual(acl_json, new_acl_json) _Check() @unittest.skipUnless( not HAS_GS_PORT, 'gs_port is defined in config which can cause ' 'problems when uploading and downloading to the same local host port') def test_daisy_chain_cp_download_failure(self): """Tests cp with the -D option when the download thread dies.""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket1_uri, contents=b'a' * self.halt_size) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, '-D', suri(key_uri), suri(bucket2_uri) ], expected_status=1, return_stderr=True) # Should have three exception traces; one from the download thread and # two from the upload thread (expection message is repeated in main's # _OutputAndExit). self.assertEqual( stderr.count( 'ResumableDownloadException: Artifically halting download'), 3) def test_streaming_gzip_upload(self): """Tests error when compression flag is requested on a streaming source.""" bucket_uri = self.CreateBucket() stderr = self.RunGsUtil( ['cp', '-Z', '-', suri(bucket_uri, 'foo')], return_stderr=True, expected_status=1, stdin='streaming data') if self._use_gcloud_storage: self.assertIn( 'Gzip content encoding is not currently supported for streaming ' 'uploads.', stderr) else: self.assertIn( 'gzip compression is not currently supported on streaming uploads', stderr) def test_seek_ahead_upload_cp(self): """Tests that the seek-ahead iterator estimates total upload work.""" tmpdir = self.CreateTempDir(test_files=3) bucket_uri = self.CreateBucket() with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil( ['-m', 'cp', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) self.assertIn( 'Estimated work for this command: objects: 3, total size: 18', stderr) with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '0'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil( ['-m', 'cp', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) self.assertNotIn('Estimated work', stderr) def test_seek_ahead_download_cp(self): tmpdir = self.CreateTempDir() bucket_uri = self.CreateBucket(test_objects=3) self.AssertNObjectsInBucket(bucket_uri, 3) with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil( ['-m', 'cp', '-r', suri(bucket_uri), tmpdir], return_stderr=True) self.assertIn( 'Estimated work for this command: objects: 3, total size: 18', stderr) with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '0'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil( ['-m', 'cp', '-r', suri(bucket_uri), tmpdir], return_stderr=True) self.assertNotIn('Estimated work', stderr) def test_canned_acl_cp(self): """Tests copying with a canned ACL.""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket1_uri, contents=b'foo') self.RunGsUtil( ['cp', '-a', 'public-read', suri(key_uri), suri(bucket2_uri)]) # Set public-read on the original key after the copy so we can compare # the ACLs. self.RunGsUtil(['acl', 'set', 'public-read', suri(key_uri)]) public_read_acl = self.RunGsUtil(['acl', 'get', suri(key_uri)], return_stdout=True) @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): uri = suri(bucket2_uri, key_uri.object_name) new_acl_json = self.RunGsUtil(['acl', 'get', uri], return_stdout=True) self.assertEqual(public_read_acl, new_acl_json) _Check() @SequentialAndParallelTransfer def test_canned_acl_upload(self): """Tests uploading a file with a canned ACL.""" bucket1_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket1_uri, contents=b'foo') # Set public-read on the object so we can compare the ACLs. self.RunGsUtil(['acl', 'set', 'public-read', suri(key_uri)]) public_read_acl = self.RunGsUtil(['acl', 'get', suri(key_uri)], return_stdout=True) file_name = 'bar' fpath = self.CreateTempFile(file_name=file_name, contents=b'foo') self.RunGsUtil(['cp', '-a', 'public-read', fpath, suri(bucket1_uri)]) new_acl_json = self.RunGsUtil( ['acl', 'get', suri(bucket1_uri, file_name)], return_stdout=True) self.assertEqual(public_read_acl, new_acl_json) resumable_size = ONE_KIB boto_config_for_test = ('GSUtil', 'resumable_threshold', str(resumable_size)) with SetBotoConfigForTest([boto_config_for_test]): resumable_file_name = 'resumable_bar' resumable_contents = os.urandom(resumable_size) resumable_fpath = self.CreateTempFile(file_name=resumable_file_name, contents=resumable_contents) self.RunGsUtil( ['cp', '-a', 'public-read', resumable_fpath, suri(bucket1_uri)]) new_resumable_acl_json = self.RunGsUtil( ['acl', 'get', suri(bucket1_uri, resumable_file_name)], return_stdout=True) self.assertEqual(public_read_acl, new_resumable_acl_json) def test_cp_key_to_local_stream(self): bucket_uri = self.CreateBucket() contents = b'foo' key_uri = self.CreateObject(bucket_uri=bucket_uri, contents=contents) stdout = self.RunGsUtil(['cp', suri(key_uri), '-'], return_stdout=True) self.assertIn(contents, stdout.encode('ascii')) def test_cp_local_file_to_local_stream(self): contents = b'content' fpath = self.CreateTempFile(contents=contents) stdout = self.RunGsUtil(['cp', fpath, '-'], return_stdout=True) self.assertIn(contents, stdout.encode(UTF8)) @SequentialAndParallelTransfer def test_cp_zero_byte_file(self): dst_bucket_uri = self.CreateBucket() src_dir = self.CreateTempDir() fpath = os.path.join(src_dir, 'zero_byte') with open(fpath, 'w') as unused_out_file: pass # Write a zero byte file self.RunGsUtil(['cp', fpath, suri(dst_bucket_uri)]) @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', suri(dst_bucket_uri)], return_stdout=True) self.assertIn(os.path.basename(fpath), stdout) _Check1() download_path = os.path.join(src_dir, 'zero_byte_download') self.RunGsUtil(['cp', suri(dst_bucket_uri, 'zero_byte'), download_path]) self.assertTrue(os.stat(download_path)) def test_copy_bucket_to_bucket(self): """Tests recursively copying from bucket to bucket. This should produce identically named objects (and not, in particular, destination objects named by the version-specific URI from source objects). """ src_bucket_uri = self.CreateVersionedBucket() dst_bucket_uri = self.CreateVersionedBucket() self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj0', contents=b'abc') self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj1', contents=b'def') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _CopyAndCheck(): self.RunGsUtil(['cp', '-R', suri(src_bucket_uri), suri(dst_bucket_uri)]) stdout = self.RunGsUtil(['ls', '-R', dst_bucket_uri.uri], return_stdout=True) self.assertIn( '%s%s/obj0\n' % (dst_bucket_uri, src_bucket_uri.bucket_name), stdout) self.assertIn( '%s%s/obj1\n' % (dst_bucket_uri, src_bucket_uri.bucket_name), stdout) _CopyAndCheck() def test_copy_duplicate_nested_object_names_to_new_cloud_dir(self): """Tests copying from bucket to same bucket preserves file structure.""" bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='dir1/file.txt', contents=b'data') self.CreateObject(bucket_uri=bucket_uri, object_name='dir2/file.txt', contents=b'data') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _CopyAndCheck(): self.RunGsUtil( ['cp', '-R', suri(bucket_uri) + '/*', suri(bucket_uri) + '/dst']) stdout = self.RunGsUtil(['ls', '-R', bucket_uri.uri], return_stdout=True) self.assertIn(suri(bucket_uri) + '/dst/dir1/file.txt', stdout) self.assertIn(suri(bucket_uri) + '/dst/dir2/file.txt', stdout) _CopyAndCheck() def test_copy_duplicate_nested_object_names_to_existing_cloud_dir(self): """Tests copying from bucket to same bucket preserves file structure.""" bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='dir1/file.txt', contents=b'data') self.CreateObject(bucket_uri=bucket_uri, object_name='dir2/file.txt', contents=b'data') self.CreateObject(bucket_uri=bucket_uri, object_name='dst/existing_file.txt', contents=b'data') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _CopyAndCheck(): self.RunGsUtil( ['cp', '-R', suri(bucket_uri) + '/*', suri(bucket_uri) + '/dst']) stdout = self.RunGsUtil(['ls', '-R', bucket_uri.uri], return_stdout=True) self.assertIn(suri(bucket_uri) + '/dst/dir1/file.txt', stdout) self.assertIn(suri(bucket_uri) + '/dst/dir2/file.txt', stdout) self.assertIn(suri(bucket_uri) + '/dst/existing_file.txt', stdout) _CopyAndCheck() @SkipForGS('Only s3 V4 signatures error on location mismatches.') def test_copy_bucket_to_bucket_with_location_redirect(self): # cp uses a sender function that raises an exception on location mismatches, # instead of returning a response. This integration test ensures retries # from exceptions work correctly. src_bucket_region = 'ap-east-1' dest_bucket_region = 'us-east-2' src_bucket_host = 's3.%s.amazonaws.com' % src_bucket_region dest_bucket_host = 's3.%s.amazonaws.com' % dest_bucket_region client_host = 's3.eu-west-1.amazonaws.com' with SetBotoConfigForTest([('s3', 'host', src_bucket_host)]): src_bucket_uri = self.CreateBucket(location=src_bucket_region) self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj0', contents=b'abc') self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj1', contents=b'def') with SetBotoConfigForTest([('s3', 'host', dest_bucket_host)]): dst_bucket_uri = self.CreateBucket(location=dest_bucket_region) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _CopyAndCheck(): self.RunGsUtil(['cp', '-R', suri(src_bucket_uri), suri(dst_bucket_uri)]) stdout = self.RunGsUtil(['ls', '-R', dst_bucket_uri.uri], return_stdout=True) self.assertIn( '%s%s/obj0\n' % (dst_bucket_uri, src_bucket_uri.bucket_name), stdout) self.assertIn( '%s%s/obj1\n' % (dst_bucket_uri, src_bucket_uri.bucket_name), stdout) with SetBotoConfigForTest([('s3', 'host', client_host)]): _CopyAndCheck() def test_copy_bucket_to_dir(self): """Tests recursively copying from bucket to a directory. This should produce identically named objects (and not, in particular, destination objects named by the version- specific URI from source objects). """ src_bucket_uri = self.CreateBucket() dst_dir = self.CreateTempDir() self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj0', contents=b'abc') self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj1', contents=b'def') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _CopyAndCheck(): """Copies the bucket recursively and validates the results.""" self.RunGsUtil(['cp', '-R', suri(src_bucket_uri), dst_dir]) dir_list = [] for dirname, _, filenames in os.walk(dst_dir): for filename in filenames: dir_list.append(os.path.join(dirname, filename)) dir_list = sorted(dir_list) self.assertEqual(len(dir_list), 2) self.assertEqual( os.path.join(dst_dir, src_bucket_uri.bucket_name, 'obj0'), dir_list[0]) self.assertEqual( os.path.join(dst_dir, src_bucket_uri.bucket_name, 'obj1'), dir_list[1]) _CopyAndCheck() @unittest.skipUnless(HAS_S3_CREDS, 'Test requires both S3 and GS credentials') def test_copy_object_to_dir_s3_v4(self): """Tests copying object from s3 to local dir with v4 signature. Regions like us-east2 accept only V4 signature, hence we will create the bucket in us-east2 region to enforce testing with V4 signature. """ src_bucket_uri = self.CreateBucket(provider='s3', location='us-east-2') dst_dir = self.CreateTempDir() self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj0', contents=b'abc') self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj1', contents=b'def') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _CopyAndCheck(): """Copies the bucket recursively and validates the results.""" self.RunGsUtil(['cp', '-R', suri(src_bucket_uri), dst_dir]) dir_list = [] for dirname, _, filenames in os.walk(dst_dir): for filename in filenames: dir_list.append(os.path.join(dirname, filename)) dir_list = sorted(dir_list) self.assertEqual(len(dir_list), 2) self.assertEqual( os.path.join(dst_dir, src_bucket_uri.bucket_name, 'obj0'), dir_list[0]) self.assertEqual( os.path.join(dst_dir, src_bucket_uri.bucket_name, 'obj1'), dir_list[1]) _CopyAndCheck() @SkipForS3('The boto lib used for S3 does not handle objects ' 'starting with slashes if we use V4 signature') def test_recursive_download_with_leftover_slash_only_dir_placeholder(self): """Tests that we correctly handle leftover dir placeholders.""" src_bucket_uri = self.CreateBucket() dst_dir = self.CreateTempDir() self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj0', contents=b'abc') self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj1', contents=b'def') # Create a placeholder like what can be left over by web GUI tools. key_uri = self.StorageUriCloneReplaceName(src_bucket_uri, '/') self.StorageUriSetContentsFromString(key_uri, '') self.AssertNObjectsInBucket(src_bucket_uri, 3) self.RunGsUtil(['cp', '-R', suri(src_bucket_uri), dst_dir]) dir_list = [] for dirname, _, filenames in os.walk(dst_dir): for filename in filenames: dir_list.append(os.path.join(dirname, filename)) dir_list = sorted(dir_list) self.assertEqual(len(dir_list), 2) self.assertEqual(os.path.join(dst_dir, src_bucket_uri.bucket_name, 'obj0'), dir_list[0]) self.assertEqual(os.path.join(dst_dir, src_bucket_uri.bucket_name, 'obj1'), dir_list[1]) def test_recursive_download_with_leftover_dir_placeholder(self): """Tests that we correctly handle leftover dir placeholders.""" src_bucket_uri = self.CreateBucket() dst_dir = self.CreateTempDir() self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj0', contents=b'abc') self.CreateObject(bucket_uri=src_bucket_uri, object_name='obj1', contents=b'def') # Create a placeholder like what can be left over by web GUI tools. key_uri = self.StorageUriCloneReplaceName(src_bucket_uri, 'foo/') self.StorageUriSetContentsFromString(key_uri, '') self.AssertNObjectsInBucket(src_bucket_uri, 3) self.RunGsUtil(['cp', '-R', suri(src_bucket_uri), dst_dir]) dir_list = [] for dirname, _, filenames in os.walk(dst_dir): for filename in filenames: dir_list.append(os.path.join(dirname, filename)) dir_list = sorted(dir_list) self.assertEqual(len(dir_list), 2) self.assertEqual(os.path.join(dst_dir, src_bucket_uri.bucket_name, 'obj0'), dir_list[0]) self.assertEqual(os.path.join(dst_dir, src_bucket_uri.bucket_name, 'obj1'), dir_list[1]) def test_copy_quiet(self): bucket_uri = self.CreateBucket() key_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') stderr = self.RunGsUtil([ '-q', 'cp', suri(key_uri), suri(self.StorageUriCloneReplaceName(bucket_uri, 'o2')) ], return_stderr=True) self.assertEqual(stderr.count('Copying '), 0) def test_cp_md5_match(self): """Tests that the uploaded object has the expected MD5. Note that while this does perform a file to object upload, MD5's are not supported for composite objects so we don't use the decorator in this case. """ bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'bar') with open(fpath, 'rb') as f_in: md5 = binascii.unhexlify(CalculateMd5FromContents(f_in)) try: encoded_bytes = base64.encodebytes(md5) except AttributeError: # For Python 2 compatability. encoded_bytes = base64.encodestring(md5) file_md5 = encoded_bytes.rstrip(b'\n') self.RunGsUtil(['cp', fpath, suri(bucket_uri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', '-L', suri(bucket_uri)], return_stdout=True) self.assertRegex( stdout, r'Hash\s+\(md5\):\s+%s' % re.escape(file_md5.decode('ascii'))) _Check1() @unittest.skipIf(IS_WINDOWS, 'Unicode handling on Windows requires mods to site-packages') @SequentialAndParallelTransfer def test_cp_manifest_upload_unicode(self): return self._ManifestUpload('foo-unicöde'.encode(UTF8), 'bar-unicöde'.encode(UTF8), 'manifest-unicöde'.encode(UTF8)) @SequentialAndParallelTransfer def test_cp_manifest_upload(self): """Tests uploading with a mnifest file.""" return self._ManifestUpload('foo', 'bar', 'manifest') def _ManifestUpload(self, file_name, object_name, manifest_name): """Tests uploading with a manifest file.""" bucket_uri = self.CreateBucket() dsturi = suri(bucket_uri, object_name) fpath = self.CreateTempFile(file_name=file_name, contents=b'bar') logpath = self.CreateTempFile(file_name=manifest_name, contents=b'') # Ensure the file is empty. open(logpath, 'w').close() self.RunGsUtil(['cp', '-L', logpath, fpath, dsturi]) with open(logpath, 'r') as f: lines = f.readlines() if six.PY2: lines = [six.text_type(line, UTF8) for line in lines] self.assertEqual(len(lines), 2) expected_headers = [ 'Source', 'Destination', 'Start', 'End', 'Md5', 'UploadId', 'Source Size', 'Bytes Transferred', 'Result', 'Description' ] self.assertEqual(expected_headers, lines[0].strip().split(',')) results = lines[1].strip().split(',') results = dict(zip(expected_headers, results)) self.assertEqual( results['Source'], 'file://' + fpath, ) self.assertEqual( results['Destination'], dsturi, ) date_format = '%Y-%m-%dT%H:%M:%S.%fZ' start_date = datetime.datetime.strptime(results['Start'], date_format) end_date = datetime.datetime.strptime(results['End'], date_format) self.assertEqual(end_date > start_date, True) if self.RunGsUtil == testcase.GsUtilIntegrationTestCase.RunGsUtil: # Check that we didn't do automatic parallel uploads - compose doesn't # calculate the MD5 hash. Since RunGsUtil is overriden in # TestCpParallelUploads to force parallel uploads, we can check which # method was used. self.assertEqual(results['Md5'], 'rL0Y20zC+Fzt72VPzMSk2A==') self.assertEqual(int(results['Source Size']), 3) self.assertEqual(int(results['Bytes Transferred']), 3) self.assertEqual(results['Result'], 'OK') @SequentialAndParallelTransfer def test_cp_manifest_download(self): """Tests downloading with a manifest file.""" key_uri = self.CreateObject(contents=b'foo') fpath = self.CreateTempFile(contents=b'') logpath = self.CreateTempFile(contents=b'') # Ensure the file is empty. open(logpath, 'w').close() self.RunGsUtil( ['cp', '-L', logpath, suri(key_uri), fpath], return_stdout=True) with open(logpath, 'r') as f: lines = f.readlines() if six.PY3: decode_lines = [] for line in lines: if line.startswith("b'"): some_strs = line.split(',') line_parts = [] for some_str in some_strs: if some_str.startswith("b'"): line_parts.append(ast.literal_eval(some_str).decode(UTF8)) else: line_parts.append(some_str) decode_lines.append(','.join(line_parts)) else: decode_lines.append(line) lines = decode_lines self.assertEqual(len(lines), 2) expected_headers = [ 'Source', 'Destination', 'Start', 'End', 'Md5', 'UploadId', 'Source Size', 'Bytes Transferred', 'Result', 'Description' ] self.assertEqual(expected_headers, lines[0].strip().split(',')) results = lines[1].strip().split(',') self.assertEqual(results[0][:5], '%s://' % self.default_provider) # source self.assertEqual(results[1][:7], 'file://') # destination date_format = '%Y-%m-%dT%H:%M:%S.%fZ' start_date = datetime.datetime.strptime(results[2], date_format) end_date = datetime.datetime.strptime(results[3], date_format) self.assertEqual(end_date > start_date, True) self.assertEqual(int(results[6]), 3) # Source Size # Bytes transferred might be more than 3 if the file was gzipped, since # the minimum gzip header is 10 bytes. self.assertGreaterEqual(int(results[7]), 3) # Bytes Transferred self.assertEqual(results[8], 'OK') # Result @SequentialAndParallelTransfer def test_copy_unicode_non_ascii_filename(self): key_uri = self.CreateObject() # Try with and without resumable upload threshold, to ensure that each # scenario works. In particular, resumable uploads have tracker filename # logic. file_contents = b'x' * START_CALLBACK_PER_BYTES * 2 fpath = self.CreateTempFile(file_name='Аудиоархив', contents=file_contents) with SetBotoConfigForTest([('GSUtil', 'resumable_threshold', '1')]): # fpath_bytes = fpath.encode(UTF8) self.RunGsUtil(['cp', fpath, suri(key_uri)], return_stderr=True) stdout = self.RunGsUtil(['cat', suri(key_uri)], return_stdout=True, force_gsutil=True) self.assertEqual(stdout.encode('ascii'), file_contents) with SetBotoConfigForTest([('GSUtil', 'resumable_threshold', str(START_CALLBACK_PER_BYTES * 3))]): self.RunGsUtil(['cp', fpath, suri(key_uri)], return_stderr=True) stdout = self.RunGsUtil(['cat', suri(key_uri)], return_stdout=True, force_gsutil=True) self.assertEqual(stdout.encode('ascii'), file_contents) # Note: We originally one time implemented a test # (test_copy_invalid_unicode_filename) that invalid unicode filenames were # skipped, but it turns out os.walk() on macOS doesn't have problems with # such files (so, failed that test). Given that, we decided to remove the # test. @SequentialAndParallelTransfer def test_gzip_upload_and_download(self): bucket_uri = self.CreateBucket() contents = b'x' * 10000 tmpdir = self.CreateTempDir() self.CreateTempFile(file_name='test.html', tmpdir=tmpdir, contents=contents) self.CreateTempFile(file_name='test.js', tmpdir=tmpdir, contents=contents) self.CreateTempFile(file_name='test.txt', tmpdir=tmpdir, contents=contents) # Test that copying specifying only 2 of the 3 prefixes gzips the correct # files, and test that including whitespace in the extension list works. self.RunGsUtil([ 'cp', '-z', 'js, html', os.path.join(tmpdir, 'test.*'), suri(bucket_uri) ]) self.AssertNObjectsInBucket(bucket_uri, 3) uri1 = suri(bucket_uri, 'test.html') uri2 = suri(bucket_uri, 'test.js') uri3 = suri(bucket_uri, 'test.txt') stdout = self.RunGsUtil(['stat', uri1], return_stdout=True) self.assertRegex(stdout, r'Content-Encoding:\s+gzip') stdout = self.RunGsUtil(['stat', uri2], return_stdout=True) self.assertRegex(stdout, r'Content-Encoding:\s+gzip') stdout = self.RunGsUtil(['stat', uri3], return_stdout=True) self.assertNotRegex(stdout, r'Content-Encoding:\s+gzip') fpath4 = self.CreateTempFile() for uri in (uri1, uri2, uri3): self.RunGsUtil(['cp', uri, suri(fpath4)]) with open(fpath4, 'rb') as f: self.assertEqual(f.read(), contents) @SkipForS3('No compressed transport encoding support for S3.') @SkipForXML('No compressed transport encoding support for the XML API.') @SequentialAndParallelTransfer def test_gzip_transport_encoded_upload_and_download(self): """Test gzip encoded files upload correctly. This checks that files are not tagged with a gzip content encoding and that the contents of the files are uncompressed in GCS. This test uses the -j flag to target specific extensions. """ def _create_test_data(): # pylint: disable=invalid-name """Setup the bucket and local data to test with. Returns: Triplet containing the following values: bucket_uri: String URI of cloud storage bucket to upload mock data to. tmpdir: String, path of a temporary directory to write mock data to. local_uris: Tuple of three strings; each is the file path to a file containing mock data. """ bucket_uri = self.CreateBucket() contents = b'x' * 10000 tmpdir = self.CreateTempDir() local_uris = [] for filename in ('test.html', 'test.js', 'test.txt'): local_uris.append( self.CreateTempFile(file_name=filename, tmpdir=tmpdir, contents=contents)) return (bucket_uri, tmpdir, local_uris) def _upload_test_data(tmpdir, bucket_uri): # pylint: disable=invalid-name """Upload local test data. Args: tmpdir: String, path of a temporary directory to write mock data to. bucket_uri: String URI of cloud storage bucket to upload mock data to. Returns: stderr: String output from running the gsutil command to upload mock data. """ if self._use_gcloud_storage: extension_list_string = 'js,html' else: extension_list_string = 'js, html' stderr = self.RunGsUtil([ '-D', 'cp', '-j', extension_list_string, os.path.join(tmpdir, 'test*'), suri(bucket_uri) ], return_stderr=True) self.AssertNObjectsInBucket(bucket_uri, 3) return stderr def _assert_sent_compressed(local_uris, stderr): # pylint: disable=invalid-name """Ensure the correct files were marked for compression. Args: local_uris: Tuple of three strings; each is the file path to a file containing mock data. stderr: String output from running the gsutil command to upload mock data. """ local_uri_html, local_uri_js, local_uri_txt = local_uris assert_base_string = 'Using compressed transport encoding for file://{}.' self.assertIn(assert_base_string.format(local_uri_html), stderr) self.assertIn(assert_base_string.format(local_uri_js), stderr) self.assertNotIn(assert_base_string.format(local_uri_txt), stderr) def _assert_stored_uncompressed(bucket_uri, contents=b'x' * 10000): # pylint: disable=invalid-name """Ensure the files are not compressed when they are stored in the bucket. Args: bucket_uri: String with URI for bucket containing uploaded test data. contents: Byte string that are stored in each file in the bucket. """ local_uri_html = suri(bucket_uri, 'test.html') local_uri_js = suri(bucket_uri, 'test.js') local_uri_txt = suri(bucket_uri, 'test.txt') fpath4 = self.CreateTempFile() for uri in (local_uri_html, local_uri_js, local_uri_txt): stdout = self.RunGsUtil(['stat', uri], return_stdout=True) self.assertNotRegex(stdout, r'Content-Encoding:\s+gzip') self.RunGsUtil(['cp', uri, suri(fpath4)]) with open(fpath4, 'rb') as f: self.assertEqual(f.read(), contents) # Get mock data, run tests bucket_uri, tmpdir, local_uris = _create_test_data() stderr = _upload_test_data(tmpdir, bucket_uri) _assert_sent_compressed(local_uris, stderr) _assert_stored_uncompressed(bucket_uri) @SkipForS3('No compressed transport encoding support for S3.') @SkipForXML('No compressed transport encoding support for the XML API.') @SequentialAndParallelTransfer def test_gzip_transport_encoded_parallel_upload_non_resumable(self): """Test non resumable, gzip encoded files upload correctly in parallel. This test generates a small amount of data (e.g. 100 chars) to upload. Due to the small size, it will be below the resumable threshold, and test the behavior of non-resumable uploads. """ # Setup the bucket and local data. bucket_uri = self.CreateBucket() contents = b'x' * 100 tmpdir = self.CreateTempDir(test_files=10, contents=contents) # Upload the data. with SetBotoConfigForTest([('GSUtil', 'resumable_threshold', str(ONE_KIB)) ]): stderr = self.RunGsUtil( ['-D', '-m', 'cp', '-J', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) # Ensure all objects are uploaded. self.AssertNObjectsInBucket(bucket_uri, 10) if not self._use_gcloud_storage: # Ensure the progress logger sees a gzip encoding. self.assertIn('send: Using gzip transport encoding for the request.', stderr) @SkipForS3('No compressed transport encoding support for S3.') @SkipForXML('No compressed transport encoding support for the XML API.') @SequentialAndParallelTransfer def test_gzip_transport_encoded_parallel_upload_resumable(self): """Test resumable, gzip encoded files upload correctly in parallel. This test generates a large amount of data (e.g. halt_size amount of chars) to upload. Due to the large size, it will be above the resumable threshold, and test the behavior of resumable uploads. """ # Setup the bucket and local data. bucket_uri = self.CreateBucket() contents = get_random_ascii_chars(size=self.halt_size) tmpdir = self.CreateTempDir(test_files=10, contents=contents) # Upload the data. with SetBotoConfigForTest([('GSUtil', 'resumable_threshold', str(ONE_KIB)) ]): stderr = self.RunGsUtil( ['-D', '-m', 'cp', '-J', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) # Ensure all objects are uploaded. self.AssertNObjectsInBucket(bucket_uri, 10) if not self._use_gcloud_storage: # Ensure the progress logger sees a gzip encoding. self.assertIn('send: Using gzip transport encoding for the request.', stderr) @SequentialAndParallelTransfer def test_gzip_all_upload_and_download(self): bucket_uri = self.CreateBucket() contents = b'x' * 10000 tmpdir = self.CreateTempDir() self.CreateTempFile(file_name='test.html', tmpdir=tmpdir, contents=contents) self.CreateTempFile(file_name='test.js', tmpdir=tmpdir, contents=contents) self.CreateTempFile(file_name='test.txt', tmpdir=tmpdir, contents=contents) self.CreateTempFile(file_name='test', tmpdir=tmpdir, contents=contents) # Test that all files are compressed. self.RunGsUtil( ['cp', '-Z', os.path.join(tmpdir, 'test*'), suri(bucket_uri)]) self.AssertNObjectsInBucket(bucket_uri, 4) uri1 = suri(bucket_uri, 'test.html') uri2 = suri(bucket_uri, 'test.js') uri3 = suri(bucket_uri, 'test.txt') uri4 = suri(bucket_uri, 'test') stdout = self.RunGsUtil(['stat', uri1], return_stdout=True) self.assertRegex(stdout, r'Content-Encoding:\s+gzip') stdout = self.RunGsUtil(['stat', uri2], return_stdout=True) self.assertRegex(stdout, r'Content-Encoding:\s+gzip') stdout = self.RunGsUtil(['stat', uri3], return_stdout=True) self.assertRegex(stdout, r'Content-Encoding:\s+gzip') stdout = self.RunGsUtil(['stat', uri4], return_stdout=True) self.assertRegex(stdout, r'Content-Encoding:\s+gzip') fpath4 = self.CreateTempFile() for uri in (uri1, uri2, uri3, uri4): self.RunGsUtil(['cp', uri, suri(fpath4)]) with open(fpath4, 'rb') as f: self.assertEqual(f.read(), contents) @SkipForS3('No compressed transport encoding support for S3.') @SkipForXML('No compressed transport encoding support for the XML API.') @SequentialAndParallelTransfer def test_gzip_transport_encoded_all_upload_and_download(self): """Test gzip encoded files upload correctly. This checks that files are not tagged with a gzip content encoding and that the contents of the files are uncompressed in GCS. This test uses the -J flag to target all files. """ # Setup the bucket and local data. bucket_uri = self.CreateBucket() contents = b'x' * 10000 tmpdir = self.CreateTempDir() local_uri1 = self.CreateTempFile(file_name='test.txt', tmpdir=tmpdir, contents=contents) local_uri2 = self.CreateTempFile(file_name='test', tmpdir=tmpdir, contents=contents) # Upload the data. stderr = self.RunGsUtil( ['-D', 'cp', '-J', os.path.join(tmpdir, 'test*'), suri(bucket_uri)], return_stderr=True) self.AssertNObjectsInBucket(bucket_uri, 2) # Ensure the correct files were marked for compression. self.assertIn( 'Using compressed transport encoding for file://%s.' % (local_uri1), stderr) self.assertIn( 'Using compressed transport encoding for file://%s.' % (local_uri2), stderr) if not self._use_gcloud_storage: # Ensure the progress logger sees a gzip encoding. self.assertIn('send: Using gzip transport encoding for the request.', stderr) # Ensure the files do not have a stored encoding of gzip and are stored # uncompressed. remote_uri1 = suri(bucket_uri, 'test.txt') remote_uri2 = suri(bucket_uri, 'test') fpath4 = self.CreateTempFile() for uri in (remote_uri1, remote_uri2): stdout = self.RunGsUtil(['stat', uri], return_stdout=True) self.assertNotRegex(stdout, r'Content-Encoding:\s+gzip') self.RunGsUtil(['cp', uri, suri(fpath4)]) with open(fpath4, 'rb') as f: self.assertEqual(f.read(), contents) def test_both_gzip_options_error(self): """Test that mixing compression flags error.""" cases = ( # Test with -Z and -z ['cp', '-Z', '-z', 'html, js', 'a.js', 'b.js'], # Same test, but with arguments in the opposite order. ['cp', '-z', 'html, js', '-Z', 'a.js', 'b.js']) if self._use_gcloud_storage: expected_status, expected_error_prefix, expected_error_substring = ( _GCLOUD_STORAGE_GZIP_FLAG_CONFLICT_OUTPUT) else: expected_status = 1 expected_error_prefix = 'CommandException' expected_error_substring = ( 'Specifying both the -z and -Z options together is invalid.') for case in cases: stderr = self.RunGsUtil(case, return_stderr=True, expected_status=expected_status) self.assertIn(expected_error_prefix, stderr) self.assertIn(expected_error_substring, stderr) def test_both_gzip_transport_encoding_options_error(self): """Test that mixing transport encoding flags error.""" cases = ( # Test with -J and -j ['cp', '-J', '-j', 'html, js', 'a.js', 'b.js'], # Same test, but with arguments in the opposite order. ['cp', '-j', 'html, js', '-J', 'a.js', 'b.js']) if self._use_gcloud_storage: expected_status, expected_error_prefix, expected_error_substring = ( _GCLOUD_STORAGE_GZIP_FLAG_CONFLICT_OUTPUT) else: expected_status = 1 expected_error_prefix = 'CommandException' expected_error_substring = ( 'Specifying both the -j and -J options together is invalid.') for case in cases: stderr = self.RunGsUtil(case, return_stderr=True, expected_status=expected_status) self.assertIn(expected_error_prefix, stderr) self.assertIn(expected_error_substring, stderr) def test_combined_gzip_options_error(self): """Test that mixing transport encoding and compression flags error.""" cases = (['cp', '-Z', '-j', 'html, js', 'a.js', 'b.js'], ['cp', '-J', '-z', 'html, js', 'a.js', 'b.js'], ['cp', '-j', 'html, js', '-Z', 'a.js', 'b.js'], ['cp', '-z', 'html, js', '-J', 'a.js', 'b.js']) if self._use_gcloud_storage: expected_status, expected_error_prefix, expected_error_substring = ( _GCLOUD_STORAGE_GZIP_FLAG_CONFLICT_OUTPUT) else: expected_status = 1 expected_error_prefix = 'CommandException' expected_error_substring = ( 'Specifying both the -j/-J and -z/-Z options together is invalid.') for case in cases: stderr = self.RunGsUtil(case, return_stderr=True, expected_status=expected_status) self.assertIn(expected_error_prefix, stderr) self.assertIn(expected_error_substring, stderr) def test_upload_with_subdir_and_unexpanded_wildcard(self): fpath1 = self.CreateTempFile(file_name=('tmp', 'x', 'y', 'z')) bucket_uri = self.CreateBucket() wildcard_uri = '%s*' % fpath1[:-5] stderr = self.RunGsUtil( ['cp', '-R', wildcard_uri, suri(bucket_uri)], return_stderr=True) self.assertIn('Copying file:', stderr) self.AssertNObjectsInBucket(bucket_uri, 1) def test_upload_does_not_raise_with_content_md5_and_check_hashes_never(self): fpath1 = self.CreateTempFile(file_name=('foo')) bucket_uri = self.CreateBucket() with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): stderr = self.RunGsUtil( ['-h', 'Content-MD5: invalid-md5', 'cp', fpath1, suri(bucket_uri)], return_stderr=True) self.assertIn('Copying file:', stderr) self.AssertNObjectsInBucket(bucket_uri, 1) @SequentialAndParallelTransfer def test_cp_object_ending_with_slash(self): """Tests that cp works with object names ending with slash.""" tmpdir = self.CreateTempDir() bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='abc/', contents=b'dir') self.CreateObject(bucket_uri=bucket_uri, object_name='abc/def', contents=b'def') self.AssertNObjectsInBucket(bucket_uri, 2) self.RunGsUtil(['cp', '-R', suri(bucket_uri), tmpdir]) # Check that files in the subdir got copied even though subdir object # download was skipped. with open(os.path.join(tmpdir, bucket_uri.bucket_name, 'abc', 'def')) as f: self.assertEqual('def', '\n'.join(f.readlines())) def test_cp_without_read_access(self): """Tests that cp fails without read access to the object.""" # TODO: With 401's triggering retries in apitools, this test will take # a long time. Ideally, make apitools accept a num_retries config for this # until we stop retrying the 401's. bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') # Use @Retry as hedge against bucket listing eventual consistency. self.AssertNObjectsInBucket(bucket_uri, 1) if self.default_provider == 's3': expected_error_regex = r'AccessDenied' else: expected_error_regex = r'Anonymous \S+ do(es)? not have' with self.SetAnonymousBotoCreds(): stderr = self.RunGsUtil(['cp', suri(object_uri), 'foo'], return_stderr=True, expected_status=1) self.assertRegex(stderr, expected_error_regex) @unittest.skipIf(IS_WINDOWS, 'os.symlink() is not available on Windows.') def test_cp_minus_r_minus_e(self): """Tests that cp -e -r ignores symlinks when recursing.""" bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() # Create a valid file, since cp expects to copy at least one source URL # successfully. self.CreateTempFile(tmpdir=tmpdir, contents=b'foo') subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) os.mkdir(os.path.join(tmpdir, 'missing')) # Create a blank directory that is a broken symlink to ensure that we # don't fail recursive enumeration with a bad symlink. os.symlink(os.path.join(tmpdir, 'missing'), os.path.join(subdir, 'missing')) os.rmdir(os.path.join(tmpdir, 'missing')) self.RunGsUtil(['cp', '-r', '-e', tmpdir, suri(bucket_uri)]) @unittest.skipIf(IS_WINDOWS, 'os.symlink() is not available on Windows.') def test_cp_minus_e(self): fpath_dir = self.CreateTempDir() fpath1 = self.CreateTempFile(tmpdir=fpath_dir) fpath2 = os.path.join(fpath_dir, 'cp_minus_e') bucket_uri = self.CreateBucket() os.symlink(fpath1, fpath2) # We also use -c to continue on errors. One of the expanded glob entries # should be the symlinked file, which should throw a CommandException since # no valid (non-symlinked) files could be found at that path; we don't want # the command to terminate if that's the first file we attempt to copy. stderr = self.RunGsUtil([ '-m', 'cp', '-e', '%s%s*' % (fpath_dir, os.path.sep), suri(bucket_uri, 'files') ], return_stderr=True) self.assertIn('Copying file', stderr) if self._use_gcloud_storage: self.assertIn('Skipping symlink', stderr) else: self.assertIn('Skipping symbolic link', stderr) # Ensure that top-level arguments are ignored if they are symlinks. The file # at fpath1 should be successfully copied, then copying the symlink at # fpath2 should fail. stderr = self.RunGsUtil( ['cp', '-e', '-r', fpath1, fpath2, suri(bucket_uri, 'files')], return_stderr=True, expected_status=1) self.assertIn('Copying file', stderr) if self._use_gcloud_storage: self.assertIn('Skipping symlink', stderr) self.assertIn('URL matched no objects or files: %s' % fpath2, stderr) else: self.assertIn('Skipping symbolic link', stderr) self.assertIn('CommandException: No URLs matched: %s' % fpath2, stderr) def test_cp_multithreaded_wildcard(self): """Tests that cp -m works with a wildcard.""" num_test_files = 5 tmp_dir = self.CreateTempDir(test_files=num_test_files) bucket_uri = self.CreateBucket() wildcard_uri = '%s%s*' % (tmp_dir, os.sep) self.RunGsUtil(['-m', 'cp', wildcard_uri, suri(bucket_uri)]) self.AssertNObjectsInBucket(bucket_uri, num_test_files) @SequentialAndParallelTransfer def test_cp_duplicate_source_args(self): """Tests that cp -m works when a source argument is provided twice.""" object_contents = b'edge' object_uri = self.CreateObject(object_name='foo', contents=object_contents) tmp_dir = self.CreateTempDir() self.RunGsUtil(['-m', 'cp', suri(object_uri), suri(object_uri), tmp_dir]) with open(os.path.join(tmp_dir, 'foo'), 'rb') as in_fp: contents = in_fp.read() # Contents should be not duplicated. self.assertEqual(contents, object_contents) @SkipForS3('gsutil doesn\'t support S3 customer-supplied encryption keys.') @SequentialAndParallelTransfer def test_cp_download_encrypted_object(self): """Tests downloading an encrypted object.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') object_contents = b'bar' object_uri = self.CreateObject(object_name='foo', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY1) fpath = self.CreateTempFile() boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): self.RunGsUtil(['cp', suri(object_uri), suri(fpath)]) with open(fpath, 'rb') as f: self.assertEqual(f.read(), object_contents) # If multiple keys are supplied and one is correct, download should succeed. fpath2 = self.CreateTempFile() boto_config_for_test2 = [ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY3), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key2', TEST_ENCRYPTION_KEY1) ] with SetBotoConfigForTest(boto_config_for_test2): self.RunGsUtil(['cp', suri(object_uri), suri(fpath2)]) with open(fpath2, 'rb') as f: self.assertEqual(f.read(), object_contents) @SkipForS3('gsutil doesn\'t support S3 customer-supplied encryption keys.') @SequentialAndParallelTransfer def test_cp_download_encrypted_object_without_key(self): """Tests downloading an encrypted object without the necessary key.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') object_contents = b'bar' object_uri = self.CreateObject(object_name='foo', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY1) fpath = self.CreateTempFile() stderr = self.RunGsUtil( ['cp', suri(object_uri), suri(fpath)], expected_status=1, return_stderr=True) self.assertIn( 'Missing decryption key with SHA256 hash %s' % TEST_ENCRYPTION_KEY1_SHA256_B64, stderr) @SkipForS3('gsutil doesn\'t support S3 customer-supplied encryption keys.') @SequentialAndParallelTransfer def test_cp_upload_encrypted_object(self): """Tests uploading an encrypted object.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object_uri = suri(bucket_uri, 'foo') file_contents = b'bar' fpath = self.CreateTempFile(contents=file_contents, file_name='foo') boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] # Uploading the object should succeed. with SetBotoConfigForTest(boto_config_for_test): self.RunGsUtil(['cp', suri(fpath), suri(bucket_uri)]) self.AssertObjectUsesCSEK(object_uri, TEST_ENCRYPTION_KEY1) with SetBotoConfigForTest(boto_config_for_test): # Reading the object back should succeed. fpath2 = self.CreateTempFile() self.RunGsUtil(['cp', suri(bucket_uri, 'foo'), suri(fpath2)]) with open(fpath2, 'rb') as f: self.assertEqual(f.read(), file_contents) @SkipForS3('No resumable upload or encryption support for S3.') def test_cp_resumable_upload_encrypted_object_break(self): """Tests that an encrypted upload resumes after a connection break.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object_uri_str = suri(bucket_uri, 'foo') fpath = self.CreateTempFile(contents=b'a' * self.halt_size) boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5))) with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, object_uri_str ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting upload', stderr) stderr = self.RunGsUtil(['cp', fpath, object_uri_str], return_stderr=True) self.assertIn('Resuming upload', stderr) stdout = self.RunGsUtil(['stat', object_uri_str], return_stdout=True) with open(fpath, 'rb') as fp: self.assertIn(CalculateB64EncodedMd5FromContents(fp), stdout) self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY1) @SkipForS3('No resumable upload or encryption support for S3.') def test_cp_resumable_upload_encrypted_object_different_key(self): """Tests that an encrypted upload resume uses original encryption key.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object_uri_str = suri(bucket_uri, 'foo') file_contents = b'a' * self.halt_size fpath = self.CreateTempFile(contents=file_contents) boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5))) with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, object_uri_str ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting upload', stderr) # Resume the upload with multiple keys, including the original. boto_config_for_test2 = [('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test2): stderr = self.RunGsUtil(['cp', fpath, object_uri_str], return_stderr=True) self.assertIn('Resuming upload', stderr) # Object should have the original key. self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY1) @SkipForS3('No resumable upload or encryption support for S3.') def test_cp_resumable_upload_encrypted_object_missing_key(self): """Tests that an encrypted upload does not resume without original key.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object_uri_str = suri(bucket_uri, 'foo') file_contents = b'a' * self.halt_size fpath = self.CreateTempFile(contents=file_contents) boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5))) with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, object_uri_str ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting upload', stderr) # Resume the upload without the original key. boto_config_for_test2 = [('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2)] with SetBotoConfigForTest(boto_config_for_test2): stderr = self.RunGsUtil(['cp', fpath, object_uri_str], return_stderr=True) self.assertNotIn('Resuming upload', stderr) self.assertIn('does not match current encryption key', stderr) self.assertIn('Restarting upload from scratch', stderr) # Object should have the new key. self.AssertObjectUsesCSEK(object_uri_str, TEST_ENCRYPTION_KEY2) def _ensure_object_unencrypted(self, object_uri_str): """Strongly consistent check that the object is unencrypted.""" stdout = self.RunGsUtil(['stat', object_uri_str], return_stdout=True) self.assertNotIn('Encryption Key', stdout) @SkipForS3('No resumable upload support for S3.') def test_cp_resumable_upload_break(self): """Tests that an upload can be resumed after a connection break.""" bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'a' * self.halt_size) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5))) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting upload', stderr) stderr = self.RunGsUtil(['cp', fpath, suri(bucket_uri)], return_stderr=True) self.assertIn('Resuming upload', stderr) @SkipForS3('No compressed transport encoding support for S3.') @SkipForXML('No compressed transport encoding support for the XML API.') @SequentialAndParallelTransfer def test_cp_resumable_upload_gzip_encoded_break(self): """Tests that a gzip encoded upload can be resumed.""" # Setup the bucket and local data. File contents are randomized to prevent # them from compressing below the resumable-threshold and failing the test. bucket_uri = self.CreateBucket() contents = get_random_ascii_chars(size=self.halt_size) local_uri = self.CreateTempFile(file_name='test.txt', contents=contents) # Configure boto boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5))) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil([ '-D', 'cp', '-J', '--testcallbackfile', test_callback_file, local_uri, suri(bucket_uri) ], expected_status=1, return_stderr=True) # Ensure the progress logger sees a gzip encoding. self.assertIn('send: Using gzip transport encoding for the request.', stderr) self.assertIn('Artifically halting upload', stderr) stderr = self.RunGsUtil(['-D', 'cp', '-J', local_uri, suri(bucket_uri)], return_stderr=True) self.assertIn('Resuming upload', stderr) # Ensure the progress logger is still seeing a gzip encoding. self.assertIn('send: Using gzip transport encoding for the request.', stderr) # Ensure the files do not have a stored encoding of gzip and are stored # uncompressed. temp_uri = self.CreateTempFile() remote_uri = suri(bucket_uri, 'test.txt') stdout = self.RunGsUtil(['stat', remote_uri], return_stdout=True) self.assertNotRegex(stdout, r'Content-Encoding:\s+gzip') self.RunGsUtil(['cp', remote_uri, suri(temp_uri)]) with open(temp_uri, 'rb') as f: self.assertEqual(f.read(), contents) @SkipForS3('No resumable upload support for S3.') def test_cp_resumable_upload_retry(self): """Tests that a resumable upload completes with one retry.""" bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'a' * self.halt_size) # TODO: Raising an httplib or socket error blocks bucket teardown # in JSON for 60-120s on a multiprocessing lock acquire. Figure out why; # until then, raise an apitools retryable exception. if self.test_api == ApiSelector.XML: test_callback_file = self.CreateTempFile(contents=pickle.dumps( _ResumableUploadRetryHandler(5, http_client.BadStatusLine, ( 'unused',)))) else: test_callback_file = self.CreateTempFile(contents=pickle.dumps( _ResumableUploadRetryHandler( 5, apitools_exceptions.BadStatusCodeError, ('unused', 'unused', 'unused')))) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil([ '-D', 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ], return_stderr=1) if self.test_api == ApiSelector.XML: self.assertIn('Got retryable failure', stderr) else: self.assertIn('Retrying', stderr) @SkipForS3('No resumable upload support for S3.') def test_cp_resumable_streaming_upload_retry(self): """Tests that a streaming resumable upload completes with one retry.""" if self.test_api == ApiSelector.XML: return unittest.skip('XML does not support resumable streaming uploads.') bucket_uri = self.CreateBucket() test_callback_file = self.CreateTempFile(contents=pickle.dumps( _ResumableUploadRetryHandler(5, apitools_exceptions.BadStatusCodeError, ('unused', 'unused', 'unused')))) # Need to reduce the JSON chunk size since streaming uploads buffer a # full chunk. boto_configs_for_test = [('GSUtil', 'json_resumable_chunk_size', str(256 * ONE_KIB)), ('Boto', 'num_retries', '2')] with SetBotoConfigForTest(boto_configs_for_test): stderr = self.RunGsUtil([ '-D', 'cp', '--testcallbackfile', test_callback_file, '-', suri(bucket_uri, 'foo') ], stdin='a' * 512 * ONE_KIB, return_stderr=1) self.assertIn('Retrying', stderr) @SkipForS3('preserve_acl flag not supported for S3.') def test_cp_preserve_no_owner(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') # Anonymous user can read the object and write to the bucket, but does # not own the object. self.RunGsUtil(['acl', 'ch', '-u', 'AllUsers:R', suri(object_uri)]) self.RunGsUtil(['acl', 'ch', '-u', 'AllUsers:W', suri(bucket_uri)]) with self.SetAnonymousBotoCreds(): stderr = self.RunGsUtil( ['cp', '-p', suri(object_uri), suri(bucket_uri, 'foo')], return_stderr=True, expected_status=1) self.assertIn('OWNER permission is required for preserving ACLs', stderr) @SkipForS3('No resumable upload support for S3.') def test_cp_progress_callbacks(self): bucket_uri = self.CreateBucket() final_size_string = BytesToFixedWidthString(1024**2) final_progress_callback = final_size_string + '/' + final_size_string fpath = self.CreateTempFile(contents=b'a' * ONE_MIB, file_name='foo') boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil(['cp', fpath, suri(bucket_uri)], return_stderr=True) self.assertEqual(1, stderr.count(final_progress_callback)) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(2 * ONE_MIB)) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil(['cp', fpath, suri(bucket_uri)], return_stderr=True) self.assertEqual(1, stderr.count(final_progress_callback)) stderr = self.RunGsUtil(['cp', suri(bucket_uri, 'foo'), fpath], return_stderr=True) self.assertEqual(1, stderr.count(final_progress_callback)) @SkipForS3('No resumable upload support for S3.') def test_cp_resumable_upload(self): """Tests that a basic resumable upload completes successfully.""" bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'a' * self.halt_size) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): self.RunGsUtil(['cp', fpath, suri(bucket_uri)]) @SkipForS3('No resumable upload support for S3.') def test_resumable_upload_break_leaves_tracker(self): """Tests that a tracker file is created with a resumable upload.""" bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(file_name='foo', contents=b'a' * self.halt_size) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): tracker_filename = GetTrackerFilePath( StorageUrlFromString(suri(bucket_uri, 'foo')), TrackerFileType.UPLOAD, self.test_api) test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5))) try: stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri, 'foo') ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting upload', stderr) self.assertTrue(os.path.exists(tracker_filename), 'Tracker file %s not present.' % tracker_filename) # Test the permissions if os.name == 'posix': mode = oct(stat.S_IMODE(os.stat(tracker_filename).st_mode)) # Assert that only user has read/write permission self.assertEqual(oct(0o600), mode) finally: DeleteTrackerFile(tracker_filename) @SkipForS3('No resumable upload support for S3.') def test_cp_resumable_upload_break_file_size_change(self): """Tests a resumable upload where the uploaded file changes size. This should fail when we read the tracker data. """ bucket_uri = self.CreateBucket() tmp_dir = self.CreateTempDir() fpath = self.CreateTempFile(file_name='foo', tmpdir=tmp_dir, contents=b'a' * self.halt_size) test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(True, 5))) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting upload', stderr) fpath = self.CreateTempFile(file_name='foo', tmpdir=tmp_dir, contents=b'a' * self.halt_size * 2) stderr = self.RunGsUtil(['cp', fpath, suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn('ResumableUploadAbortException', stderr) @SkipForS3('No resumable upload support for S3.') def test_cp_resumable_upload_break_file_content_change(self): """Tests a resumable upload where the uploaded file changes content.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'XML doesn\'t make separate HTTP calls at fixed-size boundaries for ' 'resumable uploads, so we can\'t guarantee that the server saves a ' 'specific part of the upload.') bucket_uri = self.CreateBucket() tmp_dir = self.CreateTempDir() fpath = self.CreateTempFile(file_name='foo', tmpdir=tmp_dir, contents=b'a' * ONE_KIB * ONE_KIB) test_callback_file = self.CreateTempFile(contents=pickle.dumps( HaltingCopyCallbackHandler(True, int(ONE_KIB) * 512))) resumable_threshold_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) resumable_chunk_size_for_test = ('GSUtil', 'json_resumable_chunk_size', str(ONE_KIB * 256)) with SetBotoConfigForTest( [resumable_threshold_for_test, resumable_chunk_size_for_test]): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting upload', stderr) fpath = self.CreateTempFile(file_name='foo', tmpdir=tmp_dir, contents=b'b' * ONE_KIB * ONE_KIB) stderr = self.RunGsUtil(['cp', fpath, suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn('doesn\'t match cloud-supplied digest', stderr) @SkipForS3('No resumable upload support for S3.') def test_cp_resumable_upload_break_file_smaller_size(self): """Tests a resumable upload where the uploaded file changes content. This should fail hash validation. """ bucket_uri = self.CreateBucket() tmp_dir = self.CreateTempDir() fpath = self.CreateTempFile(file_name='foo', tmpdir=tmp_dir, contents=b'a' * ONE_KIB * ONE_KIB) test_callback_file = self.CreateTempFile(contents=pickle.dumps( HaltingCopyCallbackHandler(True, int(ONE_KIB) * 512))) resumable_threshold_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) resumable_chunk_size_for_test = ('GSUtil', 'json_resumable_chunk_size', str(ONE_KIB * 256)) with SetBotoConfigForTest( [resumable_threshold_for_test, resumable_chunk_size_for_test]): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting upload', stderr) fpath = self.CreateTempFile(file_name='foo', tmpdir=tmp_dir, contents=b'a' * ONE_KIB) stderr = self.RunGsUtil(['cp', fpath, suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn('ResumableUploadAbortException', stderr) @SkipForS3('No resumable upload support for S3.') def test_cp_composite_encrypted_upload_resume(self): """Tests that an encrypted composite upload resumes successfully.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() dst_url = StorageUrlFromString(suri(bucket_uri, 'foo')) file_contents = b'foobar' file_name = 'foobar' source_file = self.CreateTempFile(contents=file_contents, file_name=file_name) src_url = StorageUrlFromString(source_file) # Simulate an upload that had occurred by writing a tracker file # that points to a previously uploaded component. tracker_file_name = GetTrackerFilePath(dst_url, TrackerFileType.PARALLEL_UPLOAD, self.test_api, src_url) tracker_prefix = '123' # Create component 0 to be used in the resume; it must match the name # that will be generated in copy_helper, so we use the same scheme. encoded_name = (PARALLEL_UPLOAD_STATIC_SALT + source_file).encode(UTF8) content_md5 = GetMd5() content_md5.update(encoded_name) digest = content_md5.hexdigest() component_object_name = (tracker_prefix + PARALLEL_UPLOAD_TEMP_NAMESPACE + digest + '_0') component_size = 3 object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name=component_object_name, contents=file_contents[:component_size], encryption_key=TEST_ENCRYPTION_KEY1) existing_component = ObjectFromTracker(component_object_name, str(object_uri.generation)) existing_components = [existing_component] enc_key_sha256 = TEST_ENCRYPTION_KEY1_SHA256_B64 WriteParallelUploadTrackerFile(tracker_file_name, tracker_prefix, existing_components, encryption_key_sha256=enc_key_sha256) try: # Now "resume" the upload using the original encryption key. with SetBotoConfigForTest([ ('GSUtil', 'parallel_composite_upload_threshold', '1'), ('GSUtil', 'parallel_composite_upload_component_size', str(component_size)), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1) ]): stderr = self.RunGsUtil( ['cp', source_file, suri(bucket_uri, 'foo')], return_stderr=True) self.assertIn('Found 1 existing temporary components to reuse.', stderr) self.assertFalse( os.path.exists(tracker_file_name), 'Tracker file %s should have been deleted.' % tracker_file_name) read_contents = self.RunGsUtil(['cat', suri(bucket_uri, 'foo')], return_stdout=True) self.assertEqual(read_contents.encode('ascii'), file_contents) finally: # Clean up if something went wrong. DeleteTrackerFile(tracker_file_name) @SkipForS3('No resumable upload support for S3.') def test_cp_composite_encrypted_upload_restart(self): """Tests that encrypted composite upload restarts given a different key.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() dst_url = StorageUrlFromString(suri(bucket_uri, 'foo')) file_contents = b'foobar' source_file = self.CreateTempFile(contents=file_contents, file_name='foo') src_url = StorageUrlFromString(source_file) # Simulate an upload that had occurred by writing a tracker file. tracker_file_name = GetTrackerFilePath(dst_url, TrackerFileType.PARALLEL_UPLOAD, self.test_api, src_url) tracker_prefix = '123' existing_component_name = 'foo_1' object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo_1', contents=b'foo', encryption_key=TEST_ENCRYPTION_KEY1) existing_component = ObjectFromTracker(existing_component_name, str(object_uri.generation)) existing_components = [existing_component] enc_key_sha256 = TEST_ENCRYPTION_KEY1_SHA256_B64 WriteParallelUploadTrackerFile(tracker_file_name, tracker_prefix, existing_components, enc_key_sha256.decode('ascii')) try: # Now "resume" the upload using the original encryption key. with SetBotoConfigForTest([ ('GSUtil', 'parallel_composite_upload_threshold', '1'), ('GSUtil', 'parallel_composite_upload_component_size', '3'), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2) ]): stderr = self.RunGsUtil( ['cp', source_file, suri(bucket_uri, 'foo')], return_stderr=True) self.assertIn( 'does not match current encryption key. ' 'Deleting old components and restarting upload', stderr) self.assertNotIn('existing temporary components to reuse.', stderr) self.assertFalse( os.path.exists(tracker_file_name), 'Tracker file %s should have been deleted.' % tracker_file_name) read_contents = self.RunGsUtil(['cat', suri(bucket_uri, 'foo')], return_stdout=True) self.assertEqual(read_contents.encode('ascii'), file_contents) finally: # Clean up if something went wrong. DeleteTrackerFile(tracker_file_name) @SkipForS3('Test uses gs-specific KMS encryption') def test_kms_key_correctly_applied_to_composite_upload(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'abcd') obj_suri = suri(bucket_uri, 'composed') key_fqn = AuthorizeProjectToUseTestingKmsKey() with SetBotoConfigForTest([ ('GSUtil', 'encryption_key', key_fqn), ('GSUtil', 'parallel_composite_upload_threshold', '1'), ('GSUtil', 'parallel_composite_upload_component_size', '1') ]): self.RunGsUtil(['cp', fpath, obj_suri]) with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): self.AssertObjectUsesCMEK(obj_suri, key_fqn) @SkipForS3('No composite upload support for S3.') def test_nearline_applied_to_parallel_composite_upload(self): bucket_uri = self.CreateBucket(storage_class='standard') fpath = self.CreateTempFile(contents=b'abcd') obj_suri = suri(bucket_uri, 'composed') with SetBotoConfigForTest([ ('GSUtil', 'parallel_composite_upload_threshold', '1'), ('GSUtil', 'parallel_composite_upload_component_size', '1') ]): self.RunGsUtil(['cp', '-s', 'nearline', fpath, obj_suri]) stdout = self.RunGsUtil(['ls', '-L', obj_suri], return_stdout=True) if self._use_gcloud_storage: self.assertRegexpMatchesWithFlags( stdout, r'Storage class: NEARLINE', flags=re.IGNORECASE) else: self.assertRegexpMatchesWithFlags(stdout, r'Storage class: NEARLINE', flags=re.IGNORECASE) # This temporarily changes the tracker directory to unwritable which # interferes with any parallel running tests that use the tracker directory. @NotParallelizable @SkipForS3('No resumable upload support for S3.') @unittest.skipIf(IS_WINDOWS, 'chmod on dir unsupported on Windows.') @SequentialAndParallelTransfer def test_cp_unwritable_tracker_file(self): """Tests a resumable upload with an unwritable tracker file.""" bucket_uri = self.CreateBucket() tracker_filename = GetTrackerFilePath( StorageUrlFromString(suri(bucket_uri, 'foo')), TrackerFileType.UPLOAD, self.test_api) tracker_dir = os.path.dirname(tracker_filename) fpath = self.CreateTempFile(file_name='foo', contents=b'a' * ONE_KIB) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) save_mod = os.stat(tracker_dir).st_mode try: os.chmod(tracker_dir, 0) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil(['cp', fpath, suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn('Couldn\'t write tracker file', stderr) finally: os.chmod(tracker_dir, save_mod) if os.path.exists(tracker_filename): os.unlink(tracker_filename) # This temporarily changes the tracker directory to unwritable which # interferes with any parallel running tests that use the tracker directory. @NotParallelizable @unittest.skipIf(IS_WINDOWS, 'chmod on dir unsupported on Windows.') @SequentialAndParallelTransfer def test_cp_unwritable_tracker_file_download(self): """Tests downloads with an unwritable tracker file.""" object_uri = self.CreateObject(contents=b'foo' * ONE_KIB) tracker_filename = GetTrackerFilePath( StorageUrlFromString(suri(object_uri)), TrackerFileType.DOWNLOAD, self.test_api) tracker_dir = os.path.dirname(tracker_filename) fpath = self.CreateTempFile() save_mod = os.stat(tracker_dir).st_mode try: os.chmod(tracker_dir, 0) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(EIGHT_MIB)) with SetBotoConfigForTest([boto_config_for_test]): # Should succeed because we are below the threshold. self.RunGsUtil(['cp', suri(object_uri), fpath]) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], expected_status=1, return_stderr=True) self.assertIn('Couldn\'t write tracker file', stderr) finally: os.chmod(tracker_dir, save_mod) if os.path.exists(tracker_filename): os.unlink(tracker_filename) def _test_cp_resumable_download_break_helper(self, boto_config, encryption_key=None): """Helper function for different modes of resumable download break. Args: boto_config: List of boto configuration tuples for use with SetBotoConfigForTest. encryption_key: Base64 encryption key for object encryption (if any). """ bucket_uri = self.CreateBucket() file_contents = b'a' * self.halt_size object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=file_contents, encryption_key=encryption_key) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) with SetBotoConfigForTest(boto_config): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), fpath ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting download.', stderr) tracker_filename = GetTrackerFilePath(StorageUrlFromString(fpath), TrackerFileType.DOWNLOAD, self.test_api) self.assertTrue(os.path.isfile(tracker_filename)) stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) self.assertIn('Resuming download', stderr) with open(fpath, 'rb') as f: self.assertEqual(f.read(), file_contents, 'File contents differ') def test_cp_resumable_download_break(self): """Tests that a download can be resumed after a connection break.""" self._test_cp_resumable_download_break_helper([ ('GSUtil', 'resumable_threshold', str(ONE_KIB)) ]) @SkipForS3('gsutil doesn\'t support S3 customer-supplied encryption keys.') def test_cp_resumable_encrypted_download_break(self): """Tests that an encrypted download resumes after a connection break.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') self._test_cp_resumable_download_break_helper( [('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)], encryption_key=TEST_ENCRYPTION_KEY1) @SkipForS3('gsutil doesn\'t support S3 customer-supplied encryption keys.') def test_cp_resumable_encrypted_download_key_rotation(self): """Tests that a download restarts with a rotated encryption key.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() file_contents = b'a' * self.halt_size object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=file_contents, encryption_key=TEST_ENCRYPTION_KEY1) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), fpath ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting download.', stderr) tracker_filename = GetTrackerFilePath(StorageUrlFromString(fpath), TrackerFileType.DOWNLOAD, self.test_api) self.assertTrue(os.path.isfile(tracker_filename)) # After simulated connection break, rotate the key on the object. boto_config_for_test2 = [('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2)] with SetBotoConfigForTest(boto_config_for_test2): self.RunGsUtil(['rewrite', '-k', suri(object_uri)]) # Now resume the download using only the new encryption key. Since its # generation changed, we must restart it. boto_config_for_test3 = [('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2)] with SetBotoConfigForTest(boto_config_for_test3): stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) self.assertIn('Restarting download', stderr) with open(fpath, 'rb') as f: self.assertEqual(f.read(), file_contents, 'File contents differ') @SequentialAndParallelTransfer def test_cp_resumable_download_etag_differs(self): """Tests that download restarts the file when the source object changes. This causes the etag not to match. """ bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'abc' * self.halt_size) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): # This will create a tracker file with an ETag. stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), fpath ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting download.', stderr) # Create a new object with different contents - it should have a # different ETag since the content has changed. object_uri = self.CreateObject( bucket_uri=bucket_uri, object_name='foo', contents=b'b' * self.halt_size, gs_idempotent_generation=object_uri.generation) stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) self.assertNotIn('Resuming download', stderr) # TODO: Enable this test for sequential downloads when their tracker files are # modified to contain the source object generation. @unittest.skipUnless(UsingCrcmodExtension(), 'Sliced download requires fast crcmod.') @SkipForS3('No sliced download support for S3.') def test_cp_resumable_download_generation_differs(self): """Tests that a resumable download restarts if the generation differs.""" bucket_uri = self.CreateBucket() file_contents = b'abcd' * self.halt_size object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=file_contents) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_max_components', '3') ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), suri(fpath) ], return_stderr=True, expected_status=1) self.assertIn('Artifically halting download.', stderr) # Overwrite the object with an identical object, increasing # the generation but leaving other metadata the same. identical_file = self.CreateTempFile(contents=file_contents) self.RunGsUtil(['cp', suri(identical_file), suri(object_uri)]) stderr = self.RunGsUtil( ['cp', suri(object_uri), suri(fpath)], return_stderr=True) self.assertIn('Restarting download from scratch', stderr) with open(fpath, 'rb') as f: self.assertEqual(f.read(), file_contents, 'File contents differ') def test_cp_resumable_download_file_larger(self): """Tests download deletes the tracker file when existing file is larger.""" bucket_uri = self.CreateBucket() fpath = self.CreateTempFile() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'a' * self.halt_size) test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), fpath ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting download.', stderr) with open(fpath + '_.gstmp', 'w') as larger_file: for _ in range(self.halt_size * 2): larger_file.write('a') stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], expected_status=1, return_stderr=True) self.assertNotIn('Resuming download', stderr) self.assertIn('Deleting tracker file', stderr) def test_cp_resumable_download_content_differs(self): """Tests that we do not re-download when tracker file matches existing file. We only compare size, not contents, so re-download should not occur even though the contents are technically different. However, hash validation on the file should still occur and we will delete the file then because the hashes differ. """ bucket_uri = self.CreateBucket() tmp_dir = self.CreateTempDir() fpath = self.CreateTempFile(tmpdir=tmp_dir) temp_download_file = fpath + '_.gstmp' with open(temp_download_file, 'w') as fp: fp.write('abcd' * ONE_KIB) object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'efgh' * ONE_KIB) stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) etag_match = re.search(r'\s*ETag:\s*(.*)', stdout) self.assertIsNotNone(etag_match, 'Could not get object ETag') self.assertEqual(len(etag_match.groups()), 1, 'Did not match expected single ETag') etag = etag_match.group(1) tracker_filename = GetTrackerFilePath(StorageUrlFromString(fpath), TrackerFileType.DOWNLOAD, self.test_api) try: with open(tracker_filename, 'w') as tracker_fp: tracker_fp.write(etag) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True, expected_status=1) self.assertIn('Download already complete', stderr) self.assertIn('doesn\'t match cloud-supplied digest', stderr) # File and tracker file should be deleted. self.assertFalse(os.path.isfile(temp_download_file)) self.assertFalse(os.path.isfile(tracker_filename)) # Permanent file should not have been created. self.assertFalse(os.path.isfile(fpath)) finally: if os.path.exists(tracker_filename): os.unlink(tracker_filename) def test_cp_resumable_download_content_matches(self): """Tests download no-ops when tracker file matches existing file.""" bucket_uri = self.CreateBucket() tmp_dir = self.CreateTempDir() fpath = self.CreateTempFile(tmpdir=tmp_dir) matching_contents = b'abcd' * ONE_KIB temp_download_file = fpath + '_.gstmp' with open(temp_download_file, 'wb') as fp: fp.write(matching_contents) object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=matching_contents) stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) etag_match = re.search(r'\s*ETag:\s*(.*)', stdout) self.assertIsNotNone(etag_match, 'Could not get object ETag') self.assertEqual(len(etag_match.groups()), 1, 'Did not match expected single ETag') etag = etag_match.group(1) tracker_filename = GetTrackerFilePath(StorageUrlFromString(fpath), TrackerFileType.DOWNLOAD, self.test_api) with open(tracker_filename, 'w') as tracker_fp: tracker_fp.write(etag) try: boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) self.assertIn('Download already complete', stderr) # Tracker file should be removed after successful hash validation. self.assertFalse(os.path.isfile(tracker_filename)) finally: if os.path.exists(tracker_filename): os.unlink(tracker_filename) def test_cp_resumable_download_tracker_file_not_matches(self): """Tests that download overwrites when tracker file etag does not match.""" bucket_uri = self.CreateBucket() tmp_dir = self.CreateTempDir() fpath = self.CreateTempFile(tmpdir=tmp_dir, contents=b'abcd' * ONE_KIB) object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'efgh' * ONE_KIB) stdout = self.RunGsUtil(['ls', '-L', suri(object_uri)], return_stdout=True) etag_match = re.search(r'\s*ETag:\s*(.*)', stdout) self.assertIsNotNone(etag_match, 'Could not get object ETag') self.assertEqual(len(etag_match.groups()), 1, 'Did not match regex for exactly one object ETag') etag = etag_match.group(1) etag += 'nonmatching' tracker_filename = GetTrackerFilePath(StorageUrlFromString(fpath), TrackerFileType.DOWNLOAD, self.test_api) with open(tracker_filename, 'w') as tracker_fp: tracker_fp.write(etag) try: boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) self.assertNotIn('Resuming download', stderr) # Ensure the file was overwritten. with open(fpath, 'r') as in_fp: contents = in_fp.read() self.assertEqual( contents, 'efgh' * ONE_KIB, 'File not overwritten when it should have been ' 'due to a non-matching tracker file.') self.assertFalse(os.path.isfile(tracker_filename)) finally: if os.path.exists(tracker_filename): os.unlink(tracker_filename) def test_cp_double_gzip(self): """Tests that upload and download of a doubly-gzipped file succeeds.""" bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(file_name='looks-zipped.gz', contents=b'foo') self.RunGsUtil([ '-h', 'content-type:application/gzip', 'cp', '-Z', suri(fpath), suri(bucket_uri, 'foo') ]) self.RunGsUtil(['cp', suri(bucket_uri, 'foo'), fpath]) @SkipForS3('No compressed transport encoding support for S3.') @SkipForXML('No compressed transport encoding support for the XML API.') @SequentialAndParallelTransfer def test_cp_double_gzip_transport_encoded(self): """Tests that upload and download of a doubly-gzipped file succeeds.""" bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(file_name='looks-zipped.gz', contents=b'foo') stderr = self.RunGsUtil([ '-DD', '-h', 'content-type:application/gzip', 'cp', '-J', suri(fpath), suri(bucket_uri, 'foo') ], return_stderr=True) if self._use_gcloud_storage: self.assertIn("b\'Content-Encoding\': b\'gzip\'", stderr) self.assertIn('"contentType": "application/gzip"', stderr) else: self.assertIn("\'Content-Encoding\': \'gzip\'", stderr) self.assertIn('contentType: \'application/gzip\'', stderr) self.RunGsUtil(['cp', suri(bucket_uri, 'foo'), fpath]) @unittest.skipIf(IS_WINDOWS, 'TODO(b/293885158) Timeout on Windows.') @SequentialAndParallelTransfer def test_cp_resumable_download_gzip(self): """Tests that download can be resumed successfully with a gzipped file.""" # Generate some reasonably incompressible data. This compresses to a bit # around 128K in practice, but we assert specifically below that it is # larger than self.halt_size to guarantee that we can halt the download # partway through. object_uri = self.CreateObject() random.seed(0) contents = str([ random.choice(string.ascii_letters) for _ in range(self.halt_size) ]).encode('ascii') random.seed() # Reset the seed for any other tests. fpath1 = self.CreateTempFile(file_name='unzipped.txt', contents=contents) self.RunGsUtil(['cp', '-z', 'txt', suri(fpath1), suri(object_uri)]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _GetObjectSize(): stdout = self.RunGsUtil(['du', suri(object_uri)], return_stdout=True) size_match = re.search(r'(\d+)\s+.*', stdout) self.assertIsNotNone(size_match, 'Could not get object size') self.assertEqual(len(size_match.groups()), 1, 'Did not match regex for exactly one object size.') return long(size_match.group(1)) object_size = _GetObjectSize() self.assertGreaterEqual( object_size, self.halt_size, 'Compresed object size was not large enough to ' 'allow for a halted download, so the test results ' 'would be invalid. Please increase the compressed ' 'object size in the test.') fpath2 = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), suri(fpath2) ], return_stderr=True, expected_status=1) self.assertIn('Artifically halting download.', stderr) self.assertIn('Downloading to temp gzip filename', stderr) # Tracker files will have different names depending on if we are # downloading sequentially or in parallel. sliced_download_threshold = HumanReadableToBytes( boto.config.get('GSUtil', 'sliced_object_download_threshold', DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD)) sliced_download = (len(contents) > sliced_download_threshold and sliced_download_threshold > 0 and UsingCrcmodExtension()) if sliced_download: trackerfile_type = TrackerFileType.SLICED_DOWNLOAD else: trackerfile_type = TrackerFileType.DOWNLOAD tracker_filename = GetTrackerFilePath(StorageUrlFromString(fpath2), trackerfile_type, self.test_api) # We should have a temporary gzipped file, a tracker file, and no # final file yet. self.assertTrue(os.path.isfile(tracker_filename)) self.assertTrue(os.path.isfile('%s_.gztmp' % fpath2)) stderr = self.RunGsUtil( ['cp', suri(object_uri), suri(fpath2)], return_stderr=True) self.assertIn('Resuming download', stderr) with open(fpath2, 'rb') as f: self.assertEqual(f.read(), contents, 'File contents did not match.') self.assertFalse(os.path.isfile(tracker_filename)) self.assertFalse(os.path.isfile('%s_.gztmp' % fpath2)) def _GetFaviconFile(self): # Make a temp file from favicon.ico.gz. Finding the location of our test # data varies depending on how/where gsutil was installed, so we get the # data via pkgutil and use this workaround. if not hasattr(self, 'test_data_favicon_file'): contents = pkgutil.get_data('gslib', 'tests/test_data/favicon.ico.gz') self.test_data_favicon_file = self.CreateTempFile(contents=contents) return self.test_data_favicon_file def test_cp_download_transfer_encoded(self): """Tests chunked transfer encoded download handling. Tests that download works correctly with a gzipped chunked transfer-encoded object (which therefore lacks Content-Length) of a size that gets fetched in a single chunk (exercising downloading of objects lacking a length response header). """ # Upload a file / content-encoding / content-type that triggers this flow. # Note: We need to use the file with pre-zipped format and manually set the # content-encoding and content-type because the Python gzip module (used by # gsutil cp -Z) won't reproduce the bytes that trigger this problem. bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo') input_filename = self._GetFaviconFile() self.RunGsUtil([ '-h', 'Content-Encoding:gzip', '-h', 'Content-Type:image/x-icon', 'cp', suri(input_filename), suri(object_uri) ]) # Compute the MD5 of the uncompressed bytes. with gzip.open(input_filename) as fp: hash_dict = {'md5': GetMd5()} hashing_helper.CalculateHashesFromContents(fp, hash_dict) in_file_md5 = hash_dict['md5'].digest() # Downloading this file triggers the flow. fpath2 = self.CreateTempFile() self.RunGsUtil(['cp', suri(object_uri), suri(fpath2)]) # Compute MD5 of the downloaded (uncompressed) file, and validate it. with open(fpath2, 'rb') as fp: hash_dict = {'md5': GetMd5()} hashing_helper.CalculateHashesFromContents(fp, hash_dict) out_file_md5 = hash_dict['md5'].digest() self.assertEqual(in_file_md5, out_file_md5) @SequentialAndParallelTransfer def test_cp_resumable_download_check_hashes_never(self): """Tests that resumble downloads work with check_hashes = never.""" bucket_uri = self.CreateBucket() contents = b'abcd' * self.halt_size object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=contents) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = [('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'check_hashes', 'never')] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), fpath ], expected_status=1, return_stderr=True) self.assertIn('Artifically halting download.', stderr) stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) self.assertIn('Resuming download', stderr) self.assertIn('Found no hashes to validate object downloaded', stderr) with open(fpath, 'rb') as f: self.assertEqual(f.read(), contents, 'File contents did not match.') @SkipForS3('No resumable upload support for S3.') def test_cp_resumable_upload_bucket_deleted(self): """Tests that a not found exception is raised if bucket no longer exists.""" bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'a' * 2 * ONE_KIB) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) test_callback_file = self.CreateTempFile(contents=pickle.dumps( _DeleteBucketThenStartOverCopyCallbackHandler(5, bucket_uri))) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ], return_stderr=True, expected_status=1) self.assertIn('Deleting bucket', stderr) self.assertIn('bucket does not exist', stderr) @SkipForS3('No sliced download support for S3.') def test_cp_sliced_download(self): """Tests that sliced object download works in the general case.""" bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'abc' * ONE_KIB) fpath = self.CreateTempFile() # Force fast crcmod to return True to test the basic sliced download # scenario, ensuring that if the user installs crcmod, it will work. boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(ONE_KIB)), ('GSUtil', 'test_assume_fast_crcmod', 'True'), ('GSUtil', 'sliced_object_download_threshold', str(ONE_KIB)), ('GSUtil', 'sliced_object_download_max_components', '3') ] with SetBotoConfigForTest(boto_config_for_test): self.RunGsUtil(['cp', suri(object_uri), fpath]) # Each tracker file should have been deleted. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertFalse(os.path.isfile(tracker_filename)) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'abc' * ONE_KIB, 'File contents differ') @unittest.skipUnless(UsingCrcmodExtension(), 'Sliced download requires fast crcmod.') @SkipForS3('No sliced download support for S3.') def test_cp_unresumable_sliced_download(self): """Tests sliced download works when resumability is disabled.""" bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'abcd' * self.halt_size) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(self.halt_size * 5)), ('GSUtil', 'sliced_object_download_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_max_components', '4') ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), suri(fpath) ], return_stderr=True, expected_status=1) self.assertIn('not downloaded successfully', stderr) # Temporary download file should exist. self.assertTrue(os.path.isfile(fpath + '_.gstmp')) # No tracker files should exist. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertFalse(os.path.isfile(tracker_filename)) # Perform the entire download, without resuming. with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil( ['cp', suri(object_uri), suri(fpath)], return_stderr=True) self.assertNotIn('Resuming download', stderr) # Temporary download file should have been deleted. self.assertFalse(os.path.isfile(fpath + '_.gstmp')) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'abcd' * self.halt_size, 'File contents differ') @unittest.skipUnless(UsingCrcmodExtension(), 'Sliced download requires fast crcmod.') @SkipForS3('No sliced download support for S3.') def test_cp_sliced_download_resume(self): """Tests that sliced object download is resumable.""" bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'abc' * self.halt_size) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_max_components', '3') ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), suri(fpath) ], return_stderr=True, expected_status=1) self.assertIn('not downloaded successfully', stderr) # Each tracker file should exist. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertTrue(os.path.isfile(tracker_filename)) stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) self.assertIn('Resuming download', stderr) # Each tracker file should have been deleted. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertFalse(os.path.isfile(tracker_filename)) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'abc' * self.halt_size, 'File contents differ') @unittest.skipUnless(UsingCrcmodExtension(), 'Sliced download requires fast crcmod.') @SkipForS3('No sliced download support for S3.') def test_cp_sliced_download_partial_resume(self): """Test sliced download resumability when some components are finished.""" bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'abc' * self.halt_size) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltOneComponentCopyCallbackHandler(5))) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_max_components', '3') ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), suri(fpath) ], return_stderr=True, expected_status=1) self.assertIn('not downloaded successfully', stderr) # Each tracker file should exist. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertTrue(os.path.isfile(tracker_filename)) stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) self.assertIn('Resuming download', stderr) self.assertIn('Download already complete', stderr) # Each tracker file should have been deleted. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertFalse(os.path.isfile(tracker_filename)) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'abc' * self.halt_size, 'File contents differ') @unittest.skipUnless(UsingCrcmodExtension(), 'Sliced download requires fast crcmod.') @SkipForS3('No sliced download support for S3.') def test_cp_sliced_download_resume_content_differs(self): """Tests differing file contents are detected by sliced downloads.""" bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'abc' * self.halt_size) fpath = self.CreateTempFile(contents=b'') test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_max_components', '3') ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), suri(fpath) ], return_stderr=True, expected_status=1) self.assertIn('not downloaded successfully', stderr) # Temporary download file should exist. self.assertTrue(os.path.isfile(fpath + '_.gstmp')) # Each tracker file should exist. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertTrue(os.path.isfile(tracker_filename)) with open(fpath + '_.gstmp', 'r+b') as f: f.write(b'altered file contents') stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True, expected_status=1) self.assertIn('Resuming download', stderr) self.assertIn('doesn\'t match cloud-supplied digest', stderr) self.assertIn('HashMismatchException: crc32c', stderr) # Each tracker file should have been deleted. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertFalse(os.path.isfile(tracker_filename)) # Temporary file should have been deleted due to hash mismatch. self.assertFalse(os.path.isfile(fpath + '_.gstmp')) # Final file should not exist. self.assertFalse(os.path.isfile(fpath)) @unittest.skipUnless(UsingCrcmodExtension(), 'Sliced download requires fast crcmod.') @SkipForS3('No sliced download support for S3.') def test_cp_sliced_download_component_size_changed(self): """Tests sliced download doesn't break when the boto config changes. If the number of components used changes cross-process, the download should be restarted. """ bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'abcd' * self.halt_size) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_component_size', str(self.halt_size // 4)), ('GSUtil', 'sliced_object_download_max_components', '4') ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), suri(fpath) ], return_stderr=True, expected_status=1) self.assertIn('not downloaded successfully', stderr) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_component_size', str(self.halt_size // 2)), ('GSUtil', 'sliced_object_download_max_components', '2') ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) self.assertIn('Sliced download tracker file doesn\'t match ', stderr) self.assertIn('Restarting download from scratch', stderr) self.assertNotIn('Resuming download', stderr) @unittest.skipUnless(UsingCrcmodExtension(), 'Sliced download requires fast crcmod.') @SkipForS3('No sliced download support for S3.') def test_cp_sliced_download_disabled_cross_process(self): """Tests temporary files are not orphaned if sliced download is disabled. Specifically, temporary files should be deleted when the corresponding non-sliced download is completed. """ bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'abcd' * self.halt_size) fpath = self.CreateTempFile() test_callback_file = self.CreateTempFile( contents=pickle.dumps(HaltingCopyCallbackHandler(False, 5))) boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_max_components', '4') ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, suri(object_uri), suri(fpath) ], return_stderr=True, expected_status=1) self.assertIn('not downloaded successfully', stderr) # Temporary download file should exist. self.assertTrue(os.path.isfile(fpath + '_.gstmp')) # Each tracker file should exist. tracker_filenames = GetSlicedDownloadTrackerFilePaths( StorageUrlFromString(fpath), self.test_api) for tracker_filename in tracker_filenames: self.assertTrue(os.path.isfile(tracker_filename)) # Disable sliced downloads by increasing the threshold boto_config_for_test = [ ('GSUtil', 'resumable_threshold', str(self.halt_size)), ('GSUtil', 'sliced_object_download_threshold', str(self.halt_size * 5)), ('GSUtil', 'sliced_object_download_max_components', '4') ] with SetBotoConfigForTest(boto_config_for_test): stderr = self.RunGsUtil(['cp', suri(object_uri), fpath], return_stderr=True) self.assertNotIn('Resuming download', stderr) # Temporary download file should have been deleted. self.assertFalse(os.path.isfile(fpath + '_.gstmp')) # Each tracker file should have been deleted. for tracker_filename in tracker_filenames: self.assertFalse(os.path.isfile(tracker_filename)) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'abcd' * self.halt_size) @SkipForS3('No resumable upload support for S3.') def test_cp_resumable_upload_start_over_http_error(self): for start_over_error in ( 403, # If user doesn't have storage.buckets.get access to dest bucket. 404, # If the dest bucket exists, but the dest object does not. 410): # If the service tells us to restart the upload from scratch. self.start_over_error_test_helper(start_over_error) def start_over_error_test_helper(self, http_error_num): bucket_uri = self.CreateBucket() # The object contents need to be fairly large to avoid the race condition # where the contents finish uploading before we artifically halt the copy. rand_chars = get_random_ascii_chars(size=(ONE_MIB * 4)) fpath = self.CreateTempFile(contents=rand_chars) boto_config_for_test = ('GSUtil', 'resumable_threshold', str(ONE_KIB)) if self.test_api == ApiSelector.JSON: test_callback_file = self.CreateTempFile( contents=pickle.dumps(_JSONForceHTTPErrorCopyCallbackHandler(5, 404))) elif self.test_api == ApiSelector.XML: test_callback_file = self.CreateTempFile(contents=pickle.dumps( _XMLResumableUploadStartOverCopyCallbackHandler(5))) with SetBotoConfigForTest([boto_config_for_test]): stderr = self.RunGsUtil([ 'cp', '--testcallbackfile', test_callback_file, fpath, suri(bucket_uri) ], return_stderr=True) self.assertIn('Restarting upload of', stderr) def test_cp_minus_c(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'foo') cp_command = [ 'cp', '-c', suri(bucket_uri) + '/foo2', suri(object_uri), suri(bucket_uri) + '/dir/', ] self.RunGsUtil(cp_command, expected_status=1) self.RunGsUtil(['stat', '%s/dir/foo' % suri(bucket_uri)]) def test_rewrite_cp(self): """Tests the JSON Rewrite API.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'bar') gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(), DiscardMessagesQueue(), self.default_provider) key = object_uri.get_key() src_obj_metadata = apitools_messages.Object(name=key.name, bucket=key.bucket.name, contentType=key.content_type) dst_obj_metadata = apitools_messages.Object( bucket=src_obj_metadata.bucket, name=self.MakeTempName('object'), contentType=src_obj_metadata.contentType) gsutil_api.CopyObject(src_obj_metadata, dst_obj_metadata) self.assertEqual( gsutil_api.GetObjectMetadata(src_obj_metadata.bucket, src_obj_metadata.name, fields=['customerEncryption', 'md5Hash']).md5Hash, gsutil_api.GetObjectMetadata(dst_obj_metadata.bucket, dst_obj_metadata.name, fields=['customerEncryption', 'md5Hash']).md5Hash, 'Error: Rewritten object\'s hash doesn\'t match source object.') def test_rewrite_cp_resume(self): """Tests the JSON Rewrite API, breaking and resuming via a tracker file.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() # Second bucket needs to be a different storage class so the service # actually rewrites the bytes. bucket_uri2 = self.CreateBucket( storage_class='durable_reduced_availability') # maxBytesPerCall must be >= 1 MiB, so create an object > 2 MiB because we # need 2 response from the service: 1 success, 1 failure prior to # completion. object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=(b'12' * ONE_MIB) + b'bar', prefer_json_api=True) gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(), DiscardMessagesQueue(), self.default_provider) key = object_uri.get_key() src_obj_metadata = apitools_messages.Object(name=key.name, bucket=key.bucket.name, contentType=key.content_type, etag=key.etag.strip('"\'')) dst_obj_name = self.MakeTempName('object') dst_obj_metadata = apitools_messages.Object( bucket=bucket_uri2.bucket_name, name=dst_obj_name, contentType=src_obj_metadata.contentType) tracker_file_name = GetRewriteTrackerFilePath(src_obj_metadata.bucket, src_obj_metadata.name, dst_obj_metadata.bucket, dst_obj_metadata.name, self.test_api) try: try: gsutil_api.CopyObject(src_obj_metadata, dst_obj_metadata, progress_callback=HaltingRewriteCallbackHandler( ONE_MIB * 2).call, max_bytes_per_call=ONE_MIB) self.fail('Expected RewriteHaltException.') except RewriteHaltException: pass # Tracker file should be left over. self.assertTrue(os.path.exists(tracker_file_name)) # Now resume. Callback ensures we didn't start over. gsutil_api.CopyObject( src_obj_metadata, dst_obj_metadata, progress_callback=EnsureRewriteResumeCallbackHandler(ONE_MIB * 2).call, max_bytes_per_call=ONE_MIB) # Copy completed; tracker file should be deleted. self.assertFalse(os.path.exists(tracker_file_name)) self.assertEqual( gsutil_api.GetObjectMetadata(src_obj_metadata.bucket, src_obj_metadata.name, fields=['customerEncryption', 'md5Hash']).md5Hash, gsutil_api.GetObjectMetadata(dst_obj_metadata.bucket, dst_obj_metadata.name, fields=['customerEncryption', 'md5Hash']).md5Hash, 'Error: Rewritten object\'s hash doesn\'t match source object.') finally: # Clean up if something went wrong. DeleteTrackerFile(tracker_file_name) def test_rewrite_cp_resume_source_changed(self): """Tests that Rewrite starts over when the source object has changed.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() # Second bucket needs to be a different storage class so the service # actually rewrites the bytes. bucket_uri2 = self.CreateBucket( storage_class='durable_reduced_availability') # maxBytesPerCall must be >= 1 MiB, so create an object > 2 MiB because we # need 2 response from the service: 1 success, 1 failure prior to # completion. object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=(b'12' * ONE_MIB) + b'bar', prefer_json_api=True) gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(), DiscardMessagesQueue(), self.default_provider) key = object_uri.get_key() src_obj_metadata = apitools_messages.Object(name=key.name, bucket=key.bucket.name, contentType=key.content_type, etag=key.etag.strip('"\'')) dst_obj_name = self.MakeTempName('object') dst_obj_metadata = apitools_messages.Object( bucket=bucket_uri2.bucket_name, name=dst_obj_name, contentType=src_obj_metadata.contentType) tracker_file_name = GetRewriteTrackerFilePath(src_obj_metadata.bucket, src_obj_metadata.name, dst_obj_metadata.bucket, dst_obj_metadata.name, self.test_api) try: try: gsutil_api.CopyObject(src_obj_metadata, dst_obj_metadata, progress_callback=HaltingRewriteCallbackHandler( ONE_MIB * 2).call, max_bytes_per_call=ONE_MIB) self.fail('Expected RewriteHaltException.') except RewriteHaltException: pass # Overwrite the original object. object_uri2 = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'bar', prefer_json_api=True) key2 = object_uri2.get_key() src_obj_metadata2 = apitools_messages.Object( name=key2.name, bucket=key2.bucket.name, contentType=key2.content_type, etag=key2.etag.strip('"\'')) # Tracker file for original object should still exist. self.assertTrue(os.path.exists(tracker_file_name)) # Copy the new object. gsutil_api.CopyObject(src_obj_metadata2, dst_obj_metadata, max_bytes_per_call=ONE_MIB) # Copy completed; original tracker file should be deleted. self.assertFalse(os.path.exists(tracker_file_name)) self.assertEqual( gsutil_api.GetObjectMetadata(src_obj_metadata2.bucket, src_obj_metadata2.name, fields=['customerEncryption', 'md5Hash']).md5Hash, gsutil_api.GetObjectMetadata(dst_obj_metadata.bucket, dst_obj_metadata.name, fields=['customerEncryption', 'md5Hash']).md5Hash, 'Error: Rewritten object\'s hash doesn\'t match source object.') finally: # Clean up if something went wrong. DeleteTrackerFile(tracker_file_name) def test_rewrite_cp_resume_command_changed(self): """Tests that Rewrite starts over when the arguments changed.""" if self.test_api == ApiSelector.XML: return unittest.skip('Rewrite API is only supported in JSON.') bucket_uri = self.CreateBucket() # Second bucket needs to be a different storage class so the service # actually rewrites the bytes. bucket_uri2 = self.CreateBucket( storage_class='durable_reduced_availability') # maxBytesPerCall must be >= 1 MiB, so create an object > 2 MiB because we # need 2 response from the service: 1 success, 1 failure prior to # completion. object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=(b'12' * ONE_MIB) + b'bar', prefer_json_api=True) gsutil_api = GcsJsonApi(BucketStorageUri, logging.getLogger(), DiscardMessagesQueue(), self.default_provider) key = object_uri.get_key() src_obj_metadata = apitools_messages.Object(name=key.name, bucket=key.bucket.name, contentType=key.content_type, etag=key.etag.strip('"\'')) dst_obj_name = self.MakeTempName('object') dst_obj_metadata = apitools_messages.Object( bucket=bucket_uri2.bucket_name, name=dst_obj_name, contentType=src_obj_metadata.contentType) tracker_file_name = GetRewriteTrackerFilePath(src_obj_metadata.bucket, src_obj_metadata.name, dst_obj_metadata.bucket, dst_obj_metadata.name, self.test_api) try: try: gsutil_api.CopyObject(src_obj_metadata, dst_obj_metadata, canned_acl='private', progress_callback=HaltingRewriteCallbackHandler( ONE_MIB * 2).call, max_bytes_per_call=ONE_MIB) self.fail('Expected RewriteHaltException.') except RewriteHaltException: pass # Tracker file for original object should still exist. self.assertTrue(os.path.exists(tracker_file_name)) # Copy the same object but with different call parameters. gsutil_api.CopyObject(src_obj_metadata, dst_obj_metadata, canned_acl='public-read', max_bytes_per_call=ONE_MIB) # Copy completed; original tracker file should be deleted. self.assertFalse(os.path.exists(tracker_file_name)) new_obj_metadata = gsutil_api.GetObjectMetadata( dst_obj_metadata.bucket, dst_obj_metadata.name, fields=['acl', 'customerEncryption', 'md5Hash']) self.assertEqual( gsutil_api.GetObjectMetadata(src_obj_metadata.bucket, src_obj_metadata.name, fields=['customerEncryption', 'md5Hash']).md5Hash, new_obj_metadata.md5Hash, 'Error: Rewritten object\'s hash doesn\'t match source object.') # New object should have a public-read ACL from the second command. found_public_acl = False for acl_entry in new_obj_metadata.acl: if acl_entry.entity == 'allUsers': found_public_acl = True self.assertTrue(found_public_acl, 'New object was not written with a public ACL.') finally: # Clean up if something went wrong. DeleteTrackerFile(tracker_file_name) @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_cp_preserve_posix_bucket_to_dir_no_errors(self): """Tests use of the -P flag with cp from a bucket to a local dir. Specifically tests combinations of POSIX attributes in metadata that will pass validation. """ bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() TestCpMvPOSIXBucketToLocalNoErrors(self, bucket_uri, tmpdir, is_cp=True) @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') def test_cp_preserve_posix_bucket_to_dir_errors(self): """Tests use of the -P flag with cp from a bucket to a local dir. Specifically, combinations of POSIX attributes in metadata that will fail validation. """ bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() obj = self.CreateObject(bucket_uri=bucket_uri, object_name='obj', contents=b'obj') TestCpMvPOSIXBucketToLocalErrors(self, bucket_uri, obj, tmpdir, is_cp=True) @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') def test_cp_preseve_posix_dir_to_bucket_no_errors(self): """Tests use of the -P flag with cp from a local dir to a bucket.""" bucket_uri = self.CreateBucket() TestCpMvPOSIXLocalToBucketNoErrors(self, bucket_uri, is_cp=True) def test_cp_minus_s_to_non_cloud_dest_fails(self): """Test that cp -s operations to a non-cloud destination are prevented.""" local_file = self.CreateTempFile(contents=b'foo') dest_dir = self.CreateTempDir() stderr = self.RunGsUtil(['cp', '-s', 'standard', local_file, dest_dir], expected_status=1, return_stderr=True) self.assertIn('Cannot specify storage class for a non-cloud destination:', stderr) # TODO: Remove @skip annotation from this test once we upgrade to the Boto # version that parses the storage class header for HEAD Object responses. @SkipForXML('Need Boto version > 2.46.1') def test_cp_specify_nondefault_storage_class(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'foo') object2_suri = suri(object_uri) + 'bar' # Specify storage class name as mixed case here to ensure that it # gets normalized to uppercase (S3 would return an error otherwise), and # that using the normalized case is accepted by each API. nondefault_storage_class = { 's3': 'Standard_iA', 'gs': 'durable_REDUCED_availability' } storage_class = nondefault_storage_class[self.default_provider] self.RunGsUtil(['cp', '-s', storage_class, suri(object_uri), object2_suri]) stdout = self.RunGsUtil(['stat', object2_suri], return_stdout=True) self.assertRegexpMatchesWithFlags(stdout, r'Storage class:\s+%s' % storage_class, flags=re.IGNORECASE) @SkipForS3('Test uses gs-specific storage classes.') def test_cp_sets_correct_dest_storage_class(self): """Tests that object storage class is set correctly with and without -s.""" # Use a non-default storage class as the default for the bucket. bucket_uri = self.CreateBucket(storage_class='nearline') # Ensure storage class is set correctly for a local-to-cloud copy. local_fname = 'foo-orig' local_fpath = self.CreateTempFile(contents=b'foo', file_name=local_fname) foo_cloud_suri = suri(bucket_uri) + '/' + local_fname self.RunGsUtil(['cp', '-s', 'standard', local_fpath, foo_cloud_suri]) with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): stdout = self.RunGsUtil(['stat', foo_cloud_suri], return_stdout=True) self.assertRegexpMatchesWithFlags(stdout, r'Storage class:\s+STANDARD', flags=re.IGNORECASE) # Ensure storage class is set correctly for a cloud-to-cloud copy when no # destination storage class is specified. foo_nl_suri = suri(bucket_uri) + '/foo-nl' self.RunGsUtil(['cp', foo_cloud_suri, foo_nl_suri]) # TODO: Remove with-clause after adding storage class parsing in Boto. with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): stdout = self.RunGsUtil(['stat', foo_nl_suri], return_stdout=True) self.assertRegexpMatchesWithFlags(stdout, r'Storage class:\s+NEARLINE', flags=re.IGNORECASE) # Ensure storage class is set correctly for a cloud-to-cloud copy when a # non-bucket-default storage class is specified. foo_std_suri = suri(bucket_uri) + '/foo-std' self.RunGsUtil(['cp', '-s', 'standard', foo_nl_suri, foo_std_suri]) # TODO: Remove with-clause after adding storage class parsing in Boto. with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): stdout = self.RunGsUtil(['stat', foo_std_suri], return_stdout=True) self.assertRegexpMatchesWithFlags(stdout, r'Storage class:\s+STANDARD', flags=re.IGNORECASE) @SkipForS3('Test uses gs-specific KMS encryption') def test_kms_key_correctly_applied_to_dst_obj_from_src_with_no_key(self): bucket_uri = self.CreateBucket() obj1_name = 'foo' obj2_name = 'bar' key_fqn = AuthorizeProjectToUseTestingKmsKey() # Create the unencrypted object, then copy it, specifying a KMS key for the # new object. obj_uri = self.CreateObject(bucket_uri=bucket_uri, object_name=obj1_name, contents=b'foo') with SetBotoConfigForTest([('GSUtil', 'encryption_key', key_fqn)]): self.RunGsUtil( ['cp', suri(obj_uri), '%s/%s' % (suri(bucket_uri), obj2_name)]) # Make sure the new object is encrypted with the specified KMS key. with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): self.AssertObjectUsesCMEK('%s/%s' % (suri(bucket_uri), obj2_name), key_fqn) @SkipForS3('Test uses gs-specific KMS encryption') def test_kms_key_correctly_applied_to_dst_obj_from_local_file(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'abcd') obj_name = 'foo' obj_suri = suri(bucket_uri) + '/' + obj_name key_fqn = AuthorizeProjectToUseTestingKmsKey() with SetBotoConfigForTest([('GSUtil', 'encryption_key', key_fqn)]): self.RunGsUtil(['cp', fpath, obj_suri]) with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): self.AssertObjectUsesCMEK(obj_suri, key_fqn) @SkipForS3('Test uses gs-specific KMS encryption') def test_kms_key_works_with_resumable_upload(self): resumable_threshold = 1024 * 1024 # 1M bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'a' * resumable_threshold) obj_name = 'foo' obj_suri = suri(bucket_uri) + '/' + obj_name key_fqn = AuthorizeProjectToUseTestingKmsKey() with SetBotoConfigForTest([('GSUtil', 'encryption_key', key_fqn), ('GSUtil', 'resumable_threshold', str(resumable_threshold))]): self.RunGsUtil(['cp', fpath, obj_suri]) with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): self.AssertObjectUsesCMEK(obj_suri, key_fqn) @SkipForS3('Test uses gs-specific KMS encryption') def test_kms_key_correctly_applied_to_dst_obj_from_src_with_diff_key(self): bucket_uri = self.CreateBucket() obj1_name = 'foo' obj2_name = 'bar' key1_fqn = AuthorizeProjectToUseTestingKmsKey() key2_fqn = AuthorizeProjectToUseTestingKmsKey( key_name=KmsTestingResources.CONSTANT_KEY_NAME2) obj1_suri = suri( self.CreateObject(bucket_uri=bucket_uri, object_name=obj1_name, contents=b'foo', kms_key_name=key1_fqn)) # Copy the object to the same bucket, specifying a different key to be used. obj2_suri = '%s/%s' % (suri(bucket_uri), obj2_name) with SetBotoConfigForTest([('GSUtil', 'encryption_key', key2_fqn)]): self.RunGsUtil(['cp', obj1_suri, obj2_suri]) # Ensure the new object has the different key. with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): self.AssertObjectUsesCMEK(obj2_suri, key2_fqn) @SkipForS3('Test uses gs-specific KMS encryption') @SkipForXML('Copying KMS-encrypted objects prohibited with XML API') def test_kms_key_not_applied_to_nonkms_dst_obj_from_src_with_kms_key(self): bucket_uri = self.CreateBucket() obj1_name = 'foo' obj2_name = 'bar' key1_fqn = AuthorizeProjectToUseTestingKmsKey() obj1_suri = suri( self.CreateObject(bucket_uri=bucket_uri, object_name=obj1_name, contents=b'foo', kms_key_name=key1_fqn)) # Copy the object to the same bucket, not specifying any KMS key. obj2_suri = '%s/%s' % (suri(bucket_uri), obj2_name) self.RunGsUtil(['cp', obj1_suri, obj2_suri]) # Ensure the new object has no KMS key. with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): self.AssertObjectUnencrypted(obj2_suri) @unittest.skipUnless( IS_WINDOWS, 'Only Windows paths need to be normalized to use backslashes instead of ' 'forward slashes.') def test_windows_path_with_back_and_forward_slash_is_normalized(self): # Prior to this test and its corresponding fix, running # `gsutil cp dir/./file gs://bucket` would result in an object whose name # was "dir/./file", rather than just "file", as Windows tried to split on # the path component separator "\" intead of "/". tmp_dir = self.CreateTempDir() self.CreateTempFile(tmpdir=tmp_dir, file_name='obj1', contents=b'foo') bucket_uri = self.CreateBucket() self.RunGsUtil(['cp', '%s\\./obj1' % tmp_dir, suri(bucket_uri)]) # If the destination path was not created correctly, this stat call should # fail with a non-zero exit code because the specified object won't exist. self.RunGsUtil(['stat', '%s/obj1' % suri(bucket_uri)]) def test_cp_minus_m_streaming_upload(self): """Tests that cp -m - anything is disallowed.""" stderr = self.RunGsUtil(['-m', 'cp', '-', 'file'], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn( 'WARNING: Using sequential instead of parallel task execution to' ' transfer from stdin', stderr) else: self.assertIn( 'CommandException: Cannot upload from a stream when using gsutil -m', stderr) @SequentialAndParallelTransfer def test_cp_overwrites_existing_destination(self): key_uri = self.CreateObject(contents=b'foo') fpath = self.CreateTempFile(contents=b'bar') stderr = self.RunGsUtil(['cp', suri(key_uri), fpath], return_stderr=True) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'foo') @SequentialAndParallelTransfer def test_downloads_are_reliable_with_more_than_one_gsutil_instance(self): test_file_count = 10 temporary_directory = self.CreateTempDir() bucket_uri = self.CreateBucket(test_objects=test_file_count) cp_args = ['cp', suri(bucket_uri, '*'), temporary_directory] threads = [] for _ in range(2): thread = threading.Thread(target=self.RunGsUtil, args=[cp_args]) thread.start() threads.append(thread) [t.join() for t in threads] self.assertEqual(len(os.listdir(temporary_directory)), test_file_count) class TestCpUnitTests(testcase.GsUtilUnitTestCase): """Unit tests for gsutil cp.""" def testDownloadWithNoHashAvailable(self): """Tests a download with no valid server-supplied hash.""" # S3 should have a special message for non-MD5 etags. bucket_uri = self.CreateBucket(provider='s3') object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') object_uri.get_key().etag = '12345' # Not an MD5 dst_dir = self.CreateTempDir() log_handler = self.RunCommand('cp', [suri(object_uri), dst_dir], return_log_handler=True) warning_messages = log_handler.messages['warning'] self.assertEqual(2, len(warning_messages)) self.assertRegex( warning_messages[0], r'Non-MD5 etag \(12345\) present for key .*, ' r'data integrity checks are not possible') self.assertIn('Integrity cannot be assured', warning_messages[1]) def testDownloadWithDestinationEndingWithDelimiterRaisesError(self): """Tests a download with no valid server-supplied hash.""" # S3 should have a special message for non-MD5 etags. bucket_uri = self.CreateBucket(provider='s3') object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') destination_path = 'random_dir' + os.path.sep with self.assertRaises(InvalidUrlError) as error: self.RunCommand('cp', [suri(object_uri), destination_path]) self.assertEqual(str(error), 'Invalid destination path: random_dir/') def test_object_and_prefix_same_name(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'foo') self.CreateObject(bucket_uri=bucket_uri, object_name='foo/bar', contents=b'bar') fpath = self.CreateTempFile() # MockKey doesn't support hash_algs, so the MD5 will not match. with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): self.RunCommand('cp', [suri(object_uri), fpath]) with open(fpath, 'rb') as f: self.assertEqual(f.read(), b'foo') def test_cp_upload_respects_no_hashes(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'abcd') with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): log_handler = self.RunCommand('cp', [fpath, suri(bucket_uri)], return_log_handler=True) warning_messages = log_handler.messages['warning'] self.assertEqual(1, len(warning_messages)) self.assertIn('Found no hashes to validate object upload', warning_messages[0]) @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') @mock.patch('os.geteuid', new=mock.Mock(return_value=0)) @mock.patch.object(os, 'chown', autospec=True) def test_posix_runs_chown_as_super_user(self, mock_chown): fpath = self.CreateTempFile(contents=b'abcd') obj = apitools_messages.Object() obj.metadata = CreateCustomMetadata(entries={UID_ATTR: USER_ID}) ParseAndSetPOSIXAttributes(fpath, obj, False, True) mock_chown.assert_called_once_with(fpath, USER_ID, -1) @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') @mock.patch('os.geteuid', new=mock.Mock(return_value=1)) @mock.patch.object(os, 'chown', autospec=True) def test_posix_skips_chown_when_not_super_user(self, mock_chown): fpath = self.CreateTempFile(contents=b'abcd') obj = apitools_messages.Object() obj.metadata = CreateCustomMetadata(entries={UID_ATTR: USER_ID}) ParseAndSetPOSIXAttributes(fpath, obj, False, True) mock_chown.assert_not_called() @mock.patch( 'gslib.utils.copy_helper.TriggerReauthForDestinationProviderIfNecessary') @mock.patch('gslib.command.Command._GetProcessAndThreadCount') @mock.patch('gslib.command.Command.Apply', new=mock.Mock(spec=command.Command.Apply)) def test_cp_triggers_reauth(self, mock_get_process_and_thread_count, mock_trigger_reauth): path = self.CreateTempFile(file_name=('foo')) bucket_uri = self.CreateBucket() mock_get_process_and_thread_count.return_value = 2, 3 self.RunCommand('cp', [path, suri(bucket_uri)]) mock_trigger_reauth.assert_called_once_with( StorageUrlFromString(suri(bucket_uri)), mock.ANY, # Gsutil API. 6, # Worker count. ) mock_get_process_and_thread_count.assert_called_once_with( process_count=None, thread_count=None, parallel_operations_override=None, print_macos_warning=False, ) def test_translates_predefined_acl_sub_opts(self): sub_opts = [('--flag-key', 'flag-value'), ('-a', 'public-read'), ('-a', 'does-not-exist')] ShimTranslatePredefinedAclSubOptForCopy(sub_opts) self.assertEqual(sub_opts, [('--flag-key', 'flag-value'), ('-a', 'publicRead'), ('-a', 'does-not-exist')]) class TestCpShimUnitTests(testcase.ShimUnitTestBase): """Unit tests for shimming cp flags""" def test_shim_translates_flags(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'abcd') with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('cp', [ '-e', '-n', '-r', '-R', '-s', 'some-class', '-v', '-a', 'public-read', fpath, suri(bucket_uri) ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage cp' ' --ignore-symlinks --no-clobber -r -r --storage-class some-class' ' --print-created-message --predefined-acl publicRead {} {}'.format( shim_util._get_gcloud_binary_path('fake_dir'), fpath, suri(bucket_uri)), info_lines) warn_lines = '\n'.join(mock_log_handler.messages['warning']) self.assertIn('Use the -m flag to enable parallelism', warn_lines) tests/test_iam.py 0000644 00000265254 15025013616 0010102 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for the iam command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import defaultdict import json import os import subprocess from gslib.commands import iam from gslib.exception import CommandException from gslib.project_id import PopulateProjectId import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import GenerationFromURI as urigen from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import unittest from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import shim_util from gslib.utils.constants import UTF8 from gslib.utils.iam_helper import BindingsMessageToUpdateDict from gslib.utils.iam_helper import BindingsDictToUpdateDict from gslib.utils.iam_helper import BindingStringToTuple as bstt from gslib.utils.iam_helper import DiffBindings from gslib.utils.iam_helper import IsEqualBindings from gslib.utils.iam_helper import PatchBindings from gslib.utils.retry_util import Retry from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock bvle = apitools_messages.Policy.BindingsValueListEntry # Feature iam_bucket_roles must be turned on in bigstore dev config for setting # the new IAM policies on buckets. IAM_BUCKET_READ_ROLE_ABBREV = 'legacyBucketReader' IAM_BUCKET_READ_ROLE = 'roles/storage.%s' % IAM_BUCKET_READ_ROLE_ABBREV # GCS IAM does not currently support new object-level roles. IAM_OBJECT_READ_ROLE = 'roles/storage.legacyObjectReader' IAM_OBJECT_VIEWER_ROLE = 'roles/storage.objectViewer' TEST_CONDITION_DESCRIPTION = 'Description for our test condition.' TEST_CONDITION_EXPR_RESOURCE_IS_OBJECT = ( 'resource.type == "google.cloud.storage.Object"') TEST_CONDITION_TITLE = 'Test Condition Title' def gen_binding(role, members=None, condition=None): """Generate the "bindings" portion of an IAM Policy dictionary. Generates list of dicts which each represent a storage_v1_messages.Policy.BindingsValueListEntry object. The list will contain a single dict which has attributes corresponding to arguments passed to this method. Args: role: (str) An IAM policy role (e.g. "roles/storage.objectViewer"). Fully specified in BindingsValueListEntry. members: (List[str]) A list of members (e.g. ["user:foo@bar.com"]). If None, bind to ["allUsers"]. Fully specified in BindingsValueListEntry. condition: (Dict) A dictionary representing the JSON used to define a binding condition, containing the keys "description", "expression", and "title". Returns: (List[Dict[str, Any]]) A Python representation of the "bindings" portion of an IAM Policy. """ binding = { 'members': ['allUsers'] if members is None else members, 'role': role, } if condition: binding['condition'] = condition return [binding] def patch_binding(policy, role, new_policy): """Returns a patched Python object representation of a Policy. Given replaces the original role:members binding in policy with new_policy. Args: policy: Python dict representation of a Policy instance. role: An IAM policy role (e.g. "roles/storage.objectViewer"). Fully specified in BindingsValueListEntry. new_policy: A Python dict representation of a Policy instance, with a single BindingsValueListEntry entry. Returns: A Python dict representation of the patched IAM Policy object. """ bindings = [ b for b in policy.get('bindings', []) if b.get('role', '') != role ] bindings.extend(new_policy) policy = dict(policy) policy['bindings'] = bindings return policy class TestIamIntegration(testcase.GsUtilIntegrationTestCase): """Superclass for iam integration test cases.""" def assertEqualsPoliciesString(self, a, b): """Asserts two serialized policy bindings are equal.""" expected = [ bvle(members=binding_dict['members'], role=binding_dict['role']) for binding_dict in json.loads(a)['bindings'] ] result = [ bvle(members=binding_dict['members'], role=binding_dict['role']) for binding_dict in json.loads(b)['bindings'] ] self.assertTrue(IsEqualBindings(expected, result)) @SkipForS3('Tests use GS IAM model.') @SkipForXML('XML IAM control is not supported.') class TestIamHelpers(testcase.GsUtilUnitTestCase): """Unit tests for iam command helper.""" def test_convert_bindings_simple(self): """Tests that Policy.bindings lists are converted to dicts properly.""" self.assertEqual(BindingsMessageToUpdateDict([]), defaultdict(set)) expected = defaultdict(set, {'x': set(['y'])}) self.assertEqual( BindingsMessageToUpdateDict([bvle(role='x', members=['y'])]), expected) def test_convert_bindings_duplicates(self): """Test that role and member duplication are converted correctly.""" expected = defaultdict(set, {'x': set(['y', 'z'])}) duplicate_roles = [ bvle(role='x', members=['y']), bvle(role='x', members=['z']) ] duplicate_members = [ bvle(role='x', members=['z', 'y']), bvle(role='x', members=['z']) ] self.assertEqual(BindingsMessageToUpdateDict(duplicate_roles), expected) self.assertEqual(BindingsMessageToUpdateDict(duplicate_members), expected) def test_convert_bindings_dict_simple(self): """Tests that Policy.bindings lists are converted to dicts properly.""" self.assertEqual(BindingsDictToUpdateDict([]), defaultdict(set)) expected = defaultdict(set, {'x': set(['y'])}) self.assertEqual( BindingsDictToUpdateDict([{ 'role': 'x', 'members': ['y'] }]), expected) def test_convert_bindings_dict_duplicates(self): """Test that role and member duplication are converted correctly.""" expected = defaultdict(set, {'x': set(['y', 'z'])}) duplicate_roles = [{ 'role': 'x', 'members': ['y'] }, { 'role': 'x', 'members': ['z'] }] duplicate_members = [{ 'role': 'x', 'members': ['z', 'y'] }, { 'role': 'x', 'members': ['z'] }] self.assertEqual(BindingsDictToUpdateDict(duplicate_roles), expected) self.assertEqual(BindingsDictToUpdateDict(duplicate_members), expected) def test_equality_bindings_literal(self): """Tests an easy case of identical bindings.""" bindings = [bvle(role='x', members=['y'])] self.assertTrue(IsEqualBindings([], [])) self.assertTrue(IsEqualBindings(bindings, bindings)) def test_equality_bindings_extra_roles(self): """Tests bindings equality when duplicate roles are added.""" bindings = [bvle(role='x', members=['x', 'y'])] bindings2 = bindings * 2 bindings3 = [ bvle(role='x', members=['y']), bvle(role='x', members=['x']), ] self.assertTrue(IsEqualBindings(bindings, bindings2)) self.assertTrue(IsEqualBindings(bindings, bindings3)) def test_diff_bindings_add_role(self): """Tests simple grant behavior of Policy.bindings diff.""" expected = [bvle(role='x', members=['y'])] (granted, removed) = DiffBindings([], expected) self.assertEqual(granted.bindings, expected) self.assertEqual(removed.bindings, []) def test_diff_bindings_drop_role(self): """Tests simple remove behavior of Policy.bindings diff.""" expected = [bvle(role='x', members=['y'])] (granted, removed) = DiffBindings(expected, []) self.assertEqual(granted.bindings, []) self.assertEqual(removed.bindings, expected) def test_diff_bindings_swap_role(self): """Tests expected behavior of switching a role.""" old = [bvle(role='x', members=['y'])] new = [bvle(role='a', members=['b'])] (granted, removed) = DiffBindings(old, new) self.assertEqual(granted.bindings, new) self.assertEqual(removed.bindings, old) def test_diff_bindings_add_member(self): """Tests expected behavior of adding a member to a role.""" old = [bvle(role='x', members=['y'])] new = [bvle(role='x', members=['z', 'y'])] expected = [bvle(role='x', members=['z'])] (granted, removed) = DiffBindings(old, new) self.assertEqual(granted.bindings, expected) self.assertEqual(removed.bindings, []) def test_diff_bindings_drop_member(self): """Tests expected behavior of dropping a member from a role.""" old = [bvle(role='x', members=['z', 'y'])] new = [bvle(role='x', members=['y'])] expected = [bvle(role='x', members=['z'])] (granted, removed) = DiffBindings(old, new) self.assertEqual(granted.bindings, []) self.assertEqual(removed.bindings, expected) def test_diff_bindings_swap_member(self): """Tests expected behavior of switching a member in a role.""" old = [bvle(role='x', members=['z'])] new = [bvle(role='x', members=['y'])] (granted, removed) = DiffBindings(old, new) self.assertEqual(granted.bindings, new) self.assertEqual(removed.bindings, old) def test_patch_bindings_grant(self): """Tests patching a grant binding.""" base_list = [ bvle(role='a', members=['user:foo@bar.com']), bvle(role='b', members=['user:foo@bar.com']), bvle(role='c', members=['user:foo@bar.com']), ] base = BindingsMessageToUpdateDict(base_list) diff_list = [ bvle(role='d', members=['user:foo@bar.com']), ] diff = BindingsMessageToUpdateDict(diff_list) expected = BindingsMessageToUpdateDict(base_list + diff_list) res = PatchBindings(base, diff, True) self.assertEqual(res, expected) def test_patch_bindings_remove(self): """Tests patching a remove binding.""" base = BindingsMessageToUpdateDict([ bvle(members=['user:foo@bar.com'], role='a'), bvle(members=['user:foo@bar.com'], role='b'), bvle(members=['user:foo@bar.com'], role='c'), ]) diff = BindingsMessageToUpdateDict([ bvle(members=['user:foo@bar.com'], role='a'), ]) expected = BindingsMessageToUpdateDict([ bvle(members=['user:foo@bar.com'], role='b'), bvle(members=['user:foo@bar.com'], role='c'), ]) res = PatchBindings(base, diff, False) self.assertEqual(res, expected) def test_patch_bindings_remove_all(self): """Tests removing all roles from a member.""" base = BindingsMessageToUpdateDict([ bvle(members=['user:foo@bar.com'], role='a'), bvle(members=['user:foo@bar.com'], role='b'), bvle(members=['user:foo@bar.com'], role='c'), ]) diff = BindingsMessageToUpdateDict([ bvle(members=['user:foo@bar.com'], role=''), ]) res = PatchBindings(base, diff, False) self.assertEqual(res, {}) diff = BindingsMessageToUpdateDict([ bvle(members=['user:foo@bar.com'], role='a'), bvle(members=['user:foo@bar.com'], role='b'), bvle(members=['user:foo@bar.com'], role='c'), ]) res = PatchBindings(base, diff, False) self.assertEqual(res, {}) def test_patch_bindings_multiple_users(self): """Tests expected behavior when multiple users exist.""" expected = BindingsMessageToUpdateDict([ bvle(members=['user:fii@bar.com'], role='b'), ]) base = BindingsMessageToUpdateDict([ bvle(members=['user:foo@bar.com'], role='a'), bvle(members=['user:foo@bar.com', 'user:fii@bar.com'], role='b'), bvle(members=['user:foo@bar.com'], role='c'), ]) diff = BindingsMessageToUpdateDict([ bvle(members=['user:foo@bar.com'], role='a'), bvle(members=['user:foo@bar.com'], role='b'), bvle(members=['user:foo@bar.com'], role='c'), ]) res = PatchBindings(base, diff, False) self.assertEqual(res, expected) def test_patch_bindings_grant_all_users(self): """Tests a public member grant.""" base = BindingsMessageToUpdateDict([ bvle(role='a', members=['user:foo@bar.com']), bvle(role='b', members=['user:foo@bar.com']), bvle(role='c', members=['user:foo@bar.com']), ]) diff = BindingsMessageToUpdateDict([ bvle(role='a', members=['allUsers']), ]) expected = BindingsMessageToUpdateDict([ bvle(role='a', members=['allUsers', 'user:foo@bar.com']), bvle(role='b', members=['user:foo@bar.com']), bvle(role='c', members=['user:foo@bar.com']), ]) res = PatchBindings(base, diff, True) self.assertEqual(res, expected) def test_patch_bindings_public_member_overwrite(self): """Tests public member vs. public member interaction.""" base_list = [ bvle(role='a', members=['allUsers']), ] base = BindingsMessageToUpdateDict(base_list) diff_list = [ bvle(role='a', members=['allAuthenticatedUsers']), ] diff = BindingsMessageToUpdateDict(diff_list) res = PatchBindings(base, diff, True) self.assertEqual(res, BindingsMessageToUpdateDict(base_list + diff_list)) def test_valid_public_member_single_role(self): """Tests parsing single role (case insensitive).""" (_, bindings) = bstt(True, 'allusers:admin') self.assertEqual(len(bindings), 1) self.assertIn({ 'members': ['allUsers'], 'role': 'roles/storage.admin' }, bindings) def test_grant_no_role_error(self): """Tests that an error is raised when no role is specified for a grant.""" with self.assertRaises(CommandException): bstt(True, 'allUsers') with self.assertRaises(CommandException): bstt(True, 'user:foo@bar.com') with self.assertRaises(CommandException): bstt(True, 'user:foo@bar.com:') with self.assertRaises(CommandException): bstt(True, 'deleted:user:foo@bar.com?uid=1234:') def test_remove_all_roles(self): """Tests parsing a -d allUsers or -d user:foo@bar.com request.""" # Input specifies remove all roles from allUsers. (is_grant, bindings) = bstt(False, 'allUsers') self.assertEqual(len(bindings), 1) self.assertIn({'members': ['allUsers'], 'role': ''}, bindings) self.assertEqual((is_grant, bindings), bstt(False, 'allUsers:')) # Input specifies remove all roles from a user. (_, bindings) = bstt(False, 'user:foo@bar.com') self.assertEqual(len(bindings), 1) def test_valid_multiple_roles(self): """Tests parsing of multiple roles bound to one user.""" (_, bindings) = bstt(True, 'allUsers:a,b,c,roles/custom') self.assertEqual(len(bindings), 4) self.assertIn({ 'members': ['allUsers'], 'role': 'roles/storage.a' }, bindings) self.assertIn({ 'members': ['allUsers'], 'role': 'roles/storage.b' }, bindings) self.assertIn({ 'members': ['allUsers'], 'role': 'roles/storage.c' }, bindings) self.assertIn({'members': ['allUsers'], 'role': 'roles/custom'}, bindings) def test_valid_custom_roles(self): """Tests parsing of custom roles bound to one user.""" (_, bindings) = bstt(True, 'user:foo@bar.com:roles/custom1,roles/custom2') self.assertEqual(len(bindings), 2) self.assertIn({ 'members': ['user:foo@bar.com'], 'role': 'roles/custom1' }, bindings) self.assertIn({ 'members': ['user:foo@bar.com'], 'role': 'roles/custom2' }, bindings) def test_valid_member(self): """Tests member parsing (case insensitive).""" (_, bindings) = bstt(True, 'User:foo@bar.com:admin') self.assertEqual(len(bindings), 1) self.assertIn( { 'members': ['user:foo@bar.com'], 'role': 'roles/storage.admin' }, bindings) def test_valid_deleted_member(self): """Tests deleted member parsing (case insensitive).""" (_, bindings) = bstt(False, 'Deleted:User:foo@bar.com?uid=123') self.assertEqual(len(bindings), 1) self.assertIn({ 'members': ['deleted:user:foo@bar.com?uid=123'], 'role': '' }, bindings) (_, bindings) = bstt(True, 'deleted:User:foo@bar.com?uid=123:admin') self.assertEqual(len(bindings), 1) self.assertIn( { 'members': ['deleted:user:foo@bar.com?uid=123'], 'role': 'roles/storage.admin' }, bindings) # These emails can actually have multiple query params (_, bindings) = bstt( True, 'deleted:user:foo@bar.com?query=param,uid=123?uid=456:admin,admin2') self.assertEqual(len(bindings), 2) self.assertIn( { 'members': ['deleted:user:foo@bar.com?query=param,uid=123?uid=456'], 'role': 'roles/storage.admin' }, bindings) self.assertIn( { 'members': ['deleted:user:foo@bar.com?query=param,uid=123?uid=456'], 'role': 'roles/storage.admin2' }, bindings) def test_duplicate_roles(self): """Tests that duplicate roles are ignored.""" (_, bindings) = bstt(True, 'allUsers:a,a') self.assertEqual(len(bindings), 1) self.assertIn({ 'members': ['allUsers'], 'role': 'roles/storage.a' }, bindings) def test_removing_project_convenience_groups(self): """Tests that project convenience roles can be removed.""" (_, bindings) = bstt(False, 'projectViewer:123424:admin') self.assertEqual(len(bindings), 1) self.assertIn( { 'members': ['projectViewer:123424'], 'role': 'roles/storage.admin' }, bindings) (_, bindings) = bstt(False, 'projectViewer:123424') self.assertEqual(len(bindings), 1) self.assertIn({'members': ['projectViewer:123424'], 'role': ''}, bindings) def test_adding_project_convenience_groups(self): """Tests that project convenience roles cannot be added.""" with self.assertRaises(CommandException): bstt(True, 'projectViewer:123424:admin') def test_invalid_input(self): """Tests invalid input handling.""" with self.assertRaises(CommandException): bstt(True, 'non_valid_public_member:role') with self.assertRaises(CommandException): bstt(True, 'non_valid_type:id:role') with self.assertRaises(CommandException): bstt(True, 'user:r') with self.assertRaises(CommandException): bstt(True, 'deleted:user') with self.assertRaises(CommandException): bstt(True, 'deleted:not_a_type') with self.assertRaises(CommandException): bstt(True, 'deleted:user:foo@no_uid_suffix') def test_invalid_n_args(self): """Tests invalid input due to too many colons.""" with self.assertRaises(CommandException): bstt(True, 'allUsers:some_id:some_role') with self.assertRaises(CommandException): bstt(True, 'user:foo@bar.com:r:nonsense') with self.assertRaises(CommandException): bstt(True, 'deleted:user:foo@bar.com?uid=1234:r:nonsense') @SkipForS3('Tests use GS IAM model.') @SkipForXML('XML IAM control is not supported.') class TestIamCh(TestIamIntegration): """Integration tests for iam ch command.""" def setUp(self): super(TestIamCh, self).setUp() self.bucket = self.CreateBucket() self.bucket2 = self.CreateBucket() self.object = self.CreateObject(bucket_uri=self.bucket, contents=b'foo') self.object2 = self.CreateObject(bucket_uri=self.bucket, contents=b'bar') self.bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.object_iam_string = self.RunGsUtil(['iam', 'get', self.object.uri], return_stdout=True) self.object2_iam_string = self.RunGsUtil(['iam', 'get', self.object2.uri], return_stdout=True) self.user = 'user:foo@bar.com' self.user2 = 'user:bar@foo.com' def test_patch_no_role(self): """Tests expected failure if no bindings are listed.""" stderr = self.RunGsUtil(['iam', 'ch', self.bucket.uri], return_stderr=True, expected_status=1) self.assertIn('CommandException', stderr) def test_raises_error_message_for_d_flag_missing_argument(self): """Tests expected failure if no bindings are listed.""" stderr = self.RunGsUtil( ['iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), '-d'], return_stderr=True, expected_status=1) self.assertIn( 'A -d flag is missing an argument specifying bindings to remove.', stderr) def test_path_mix_of_buckets_and_objects(self): """Tests expected failure if both buckets and objects are provided.""" stderr = self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri, self.object.uri ], return_stderr=True, expected_status=1) self.assertIn('CommandException', stderr) def test_path_file_url(self): """Tests expected failure is caught when a file url is provided.""" stderr = self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), 'file://somefile' ], return_stderr=True, expected_status=1) self.assertIn('AttributeError', stderr) def test_patch_single_grant_single_bucket(self): """Tests granting single role.""" self.assertHasNo(self.bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri ]) bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertHas(bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) def test_patch_repeated_grant(self): """Granting multiple times for the same member will have no effect.""" self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri ]) self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri ]) bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertHas(bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) def test_patch_single_remove_single_bucket(self): """Tests removing a single role.""" self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri ]) self.RunGsUtil([ 'iam', 'ch', '-d', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri ]) bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertHasNo(bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) def test_patch_null_remove(self): """Removing a non-existent binding will have no effect.""" self.RunGsUtil([ 'iam', 'ch', '-d', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri ]) bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertHasNo(bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) self.assertEqualsPoliciesString(bucket_iam_string, self.bucket_iam_string) def test_patch_mixed_grant_remove_single_bucket(self): """Tests that mixing grant and remove requests will succeed.""" self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user2, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri ]) self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), '-d', '%s:%s' % (self.user2, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri ]) bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertHas(bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) self.assertHasNo(bucket_iam_string, self.user2, IAM_BUCKET_READ_ROLE) def test_patch_public_grant_single_bucket(self): """Test public grant request interacts properly with existing members.""" self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri ]) self.RunGsUtil([ 'iam', 'ch', 'allUsers:%s' % IAM_BUCKET_READ_ROLE_ABBREV, self.bucket.uri ]) bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertHas(bucket_iam_string, 'allUsers', IAM_BUCKET_READ_ROLE) self.assertHas(bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) def test_patch_remove_all_roles(self): """Remove with no roles specified will remove member from all bindings.""" self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri ]) self.RunGsUtil(['iam', 'ch', '-d', self.user, self.bucket.uri]) bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertHasNo(bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) def test_patch_single_object(self): """Tests object IAM patch behavior.""" self.assertHasNo(self.object_iam_string, self.user, IAM_OBJECT_READ_ROLE) self.RunGsUtil( ['iam', 'ch', '%s:legacyObjectReader' % self.user, self.object.uri]) object_iam_string = self.RunGsUtil(['iam', 'get', self.object.uri], return_stdout=True) self.assertHas(object_iam_string, self.user, IAM_OBJECT_READ_ROLE) def test_patch_multithreaded_single_object(self): """Tests the edge-case behavior of multithreaded execution.""" self.assertHasNo(self.object_iam_string, self.user, IAM_OBJECT_READ_ROLE) self.RunGsUtil([ '-m', 'iam', 'ch', '%s:legacyObjectReader' % self.user, self.object.uri ]) object_iam_string = self.RunGsUtil(['iam', 'get', self.object.uri], return_stdout=True) self.assertHas(object_iam_string, self.user, IAM_OBJECT_READ_ROLE) def test_patch_invalid_input(self): """Tests that listing bindings after a bucket will throw an error.""" stderr = self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri, '%s:%s' % (self.user2, IAM_BUCKET_READ_ROLE_ABBREV) ], return_stderr=True, expected_status=1) self.assertIn('CommandException', stderr) bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertHas(bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) self.assertHasNo(bucket_iam_string, self.user2, IAM_BUCKET_READ_ROLE) def test_patch_disallowed_binding_type(self): """Tests that we disallow certain binding types with appropriate err.""" stderr = self.RunGsUtil( ['iam', 'ch', 'projectOwner:my-project:admin', self.bucket.uri], return_stderr=True, expected_status=1) self.assertIn('not supported', stderr) def test_patch_remove_disallowed_binding_type(self): """Tests that we can remove project convenience values.""" # Set up the bucket to include a disallowed member which we can then remove. disallowed_member = 'projectViewer:%s' % PopulateProjectId() policy_file_path = self.CreateTempFile(contents=json.dumps( patch_binding( json.loads(self.bucket_iam_string), IAM_OBJECT_READ_ROLE, gen_binding(IAM_OBJECT_READ_ROLE, members=[disallowed_member ]))).encode(UTF8)) self.RunGsUtil(['iam', 'set', policy_file_path, self.bucket.uri]) # Confirm the disallowed member was actually added. iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertHas(iam_string, disallowed_member, IAM_OBJECT_READ_ROLE) # Use iam ch to remove the disallowed member. self.RunGsUtil(['iam', 'ch', '-d', disallowed_member, self.bucket.uri]) # Confirm the disallowed member was actually removed. iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertHasNo(iam_string, disallowed_member, IAM_OBJECT_READ_ROLE) def test_patch_multiple_objects(self): """Tests IAM patch against multiple objects.""" self.RunGsUtil([ 'iam', 'ch', '-r', '%s:legacyObjectReader' % self.user, self.bucket.uri ]) object_iam_string = self.RunGsUtil(['iam', 'get', self.object.uri], return_stdout=True) object2_iam_string = self.RunGsUtil(['iam', 'get', self.object2.uri], return_stdout=True) self.assertHas(object_iam_string, self.user, IAM_OBJECT_READ_ROLE) self.assertHas(object2_iam_string, self.user, IAM_OBJECT_READ_ROLE) def test_patch_multithreaded_multiple_objects(self): """Tests multithreaded behavior against multiple objects.""" self.RunGsUtil([ '-m', 'iam', 'ch', '-r', '%s:legacyObjectReader' % self.user, self.bucket.uri ]) object_iam_string = self.RunGsUtil(['iam', 'get', self.object.uri], return_stdout=True) object2_iam_string = self.RunGsUtil(['iam', 'get', self.object2.uri], return_stdout=True) self.assertHas(object_iam_string, self.user, IAM_OBJECT_READ_ROLE) self.assertHas(object2_iam_string, self.user, IAM_OBJECT_READ_ROLE) def test_patch_error(self): """See TestIamSet.test_set_error.""" stderr = self.RunGsUtil([ 'iam', 'ch', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri, 'gs://%s' % self.nonexistent_bucket_name, self.bucket2.uri ], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('not found: 404.', stderr) else: self.assertIn('BucketNotFoundException', stderr) bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) bucket2_iam_string = self.RunGsUtil(['iam', 'get', self.bucket2.uri], return_stdout=True) self.assertHas(bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) self.assertEqualsPoliciesString(bucket2_iam_string, self.bucket_iam_string) def test_patch_force_error(self): """See TestIamSet.test_set_force_error.""" stderr = self.RunGsUtil([ 'iam', 'ch', '-f', '%s:%s' % (self.user, IAM_BUCKET_READ_ROLE_ABBREV), self.bucket.uri, 'gs://%s' % self.nonexistent_bucket_name, self.bucket2.uri ], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('not found: 404.', stderr) else: self.assertIn('CommandException', stderr) bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) bucket2_iam_string = self.RunGsUtil(['iam', 'get', self.bucket2.uri], return_stdout=True) self.assertHas(bucket_iam_string, self.user, IAM_BUCKET_READ_ROLE) self.assertHas(bucket2_iam_string, self.user, IAM_BUCKET_READ_ROLE) def test_patch_multithreaded_error(self): """See TestIamSet.test_set_multithreaded_error.""" # TODO(b/135780661): Remove retry after bug resolved @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stderr = self.RunGsUtil([ '-m', 'iam', 'ch', '-r', '%s:legacyObjectReader' % self.user, 'gs://%s' % self.nonexistent_bucket_name, self.bucket.uri ], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('not found: 404.', stderr) else: self.assertIn('BucketNotFoundException', stderr) # TODO(b/135780661): Remove retry after bug resolved @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): object_iam_string = self.RunGsUtil(['iam', 'get', self.object.uri], return_stdout=True) object2_iam_string = self.RunGsUtil(['iam', 'get', self.object2.uri], return_stdout=True) self.assertEqualsPoliciesString(self.object_iam_string, object_iam_string) self.assertEqualsPoliciesString(self.object_iam_string, object2_iam_string) _Check1() _Check2() def test_assert_has(self): test_policy = { 'bindings': [{ 'members': ['allUsers'], 'role': 'roles/storage.admin' }, { 'members': ['user:foo@bar.com', 'serviceAccount:bar@foo.com'], 'role': IAM_BUCKET_READ_ROLE }] } self.assertHas(json.dumps(test_policy), 'allUsers', 'roles/storage.admin') self.assertHas(json.dumps(test_policy), 'user:foo@bar.com', IAM_BUCKET_READ_ROLE) self.assertHasNo(json.dumps(test_policy), 'allUsers', IAM_BUCKET_READ_ROLE) self.assertHasNo(json.dumps(test_policy), 'user:foo@bar.com', 'roles/storage.admin') def assertHas(self, policy, member, role): """Asserts a member has permission for role. Given an IAM policy, check if the specified member is bound to the specified role. Does not check group inheritence -- that is, if checking against the [{'member': ['allUsers'], 'role': X}] policy, this function will still raise an exception when testing for any member other than 'allUsers' against role X. This function does not invoke the TestIamPolicy endpoints to smartly check IAM policy resolution. This function is simply to assert the expected IAM policy is returned, not whether or not the IAM policy is being invoked as expected. Args: policy: Policy object as formatted by IamCommand._GetIam() member: A member string (e.g. 'user:foo@bar.com'). role: A fully specified role (e.g. 'roles/storage.admin') Raises: AssertionError if member is not bound to role. """ policy = json.loads(policy) bindings = dict((p['role'], p) for p in policy.get('bindings', [])) if role in bindings: if member in bindings[role]['members']: return raise AssertionError('Member \'%s\' does not have permission \'%s\' in ' 'policy %s' % (member, role, policy)) def assertHasNo(self, policy, member, role): """Functions as logical compliment of TestIamCh.assertHas().""" try: self.assertHas(policy, member, role) except AssertionError: pass else: raise AssertionError('Member \'%s\' has permission \'%s\' in ' 'policy %s' % (member, role, policy)) @SkipForS3('Tests use GS IAM model.') @SkipForXML('XML IAM control is not supported.') class TestIamSet(TestIamIntegration): """Integration tests for iam set command.""" # TODO(iam-beta): Replace gen_binding, _patch_binding with generators from # iam_helper. def setUp(self): super(TestIamSet, self).setUp() self.public_bucket_read_binding = gen_binding(IAM_BUCKET_READ_ROLE) self.public_object_read_binding = gen_binding(IAM_OBJECT_READ_ROLE) self.project_viewer_objectviewer_with_cond_binding = gen_binding( IAM_OBJECT_VIEWER_ROLE, # Note: We use projectViewer:some-project-id here because conditions # cannot be applied to a binding that only has allUsers in the members # list; the API gives back a 400 error if you try. members=['projectViewer:%s' % PopulateProjectId()], condition={ 'title': TEST_CONDITION_TITLE, 'description': TEST_CONDITION_DESCRIPTION, 'expression': TEST_CONDITION_EXPR_RESOURCE_IS_OBJECT, }) self.bucket = self.CreateBucket() self.versioned_bucket = self.CreateVersionedBucket() # Create a bucket to fetch its policy, used as a base for other policies. self.bucket_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.old_bucket_iam_path = self.CreateTempFile( contents=self.bucket_iam_string.encode(UTF8)) # Using the existing bucket's policy, make an altered policy that allows # allUsers to be "legacyBucketReader"s. Some tests will later apply this # policy. self.new_bucket_iam_policy = patch_binding( json.loads(self.bucket_iam_string), IAM_BUCKET_READ_ROLE, self.public_bucket_read_binding) self.new_bucket_iam_path = self.CreateTempFile( contents=json.dumps(self.new_bucket_iam_policy).encode(UTF8)) # Using the existing bucket's policy, make an altered policy that contains # a binding with a condition in it. Some tests will later apply this policy. self.new_bucket_policy_with_conditions_policy = json.loads( self.bucket_iam_string) self.new_bucket_policy_with_conditions_policy['bindings'].append( self.project_viewer_objectviewer_with_cond_binding[0]) self.new_bucket_policy_with_conditions_path = self.CreateTempFile( contents=json.dumps(self.new_bucket_policy_with_conditions_policy)) # Create an object to fetch its policy, used as a base for other policies. self.object = self.CreateObject(contents='foobar') self.object_iam_string = self.RunGsUtil(['iam', 'get', self.object.uri], return_stdout=True) self.old_object_iam_path = self.CreateTempFile( contents=self.object_iam_string.encode(UTF8)) # Using the existing object's policy, make an altered policy that allows # allUsers to be "legacyObjectReader"s. Some tests will later apply this # policy. self.new_object_iam_policy = patch_binding( json.loads(self.object_iam_string), IAM_OBJECT_READ_ROLE, self.public_object_read_binding) self.new_object_iam_path = self.CreateTempFile( contents=json.dumps(self.new_object_iam_policy).encode(UTF8)) def test_seek_ahead_iam(self): """Ensures that the seek-ahead iterator is being used with iam commands.""" gsutil_object = self.CreateObject(bucket_uri=self.bucket, contents=b'foobar') # This forces the seek-ahead iterator to be utilized. with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil( ['-m', 'iam', 'set', self.new_object_iam_path, gsutil_object.uri], return_stderr=True) self.assertIn('Estimated work for this command: objects: 1\n', stderr) def test_set_mix_of_buckets_and_objects(self): """Tests that failure is thrown when buckets and objects are provided.""" stderr = self.RunGsUtil([ 'iam', 'set', self.new_object_iam_path, self.bucket.uri, self.object.uri ], return_stderr=True, expected_status=1) self.assertIn('CommandException', stderr) def test_set_file_url(self): """Tests that failure is thrown when a file url is provided.""" stderr = self.RunGsUtil( ['iam', 'set', self.new_object_iam_path, 'file://somefile'], return_stderr=True, expected_status=1) self.assertIn('AttributeError', stderr) def test_set_invalid_iam_bucket(self): """Ensures invalid content returns error on input check.""" # TODO(b/135780661): Remove retry after bug resolved @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): inpath = self.CreateTempFile(contents=b'badIam') stderr = self.RunGsUtil(['iam', 'set', inpath, self.bucket.uri], return_stderr=True, expected_status=1) error_message = ('Found invalid JSON/YAML' if self._use_gcloud_storage else 'ArgumentException') self.assertIn(error_message, stderr) # TODO(b/135780661): Remove retry after bug resolved @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Tests that setting with a non-existent file will also return error. stderr = self.RunGsUtil( ['iam', 'set', 'nonexistent/path', self.bucket.uri], return_stderr=True, expected_status=1) error_message = ('No such file or directory' if self._use_gcloud_storage else 'ArgumentException') self.assertIn(error_message, stderr) _Check1() _Check2() def test_get_invalid_bucket(self): """Ensures that invalid bucket names returns an error.""" stderr = self.RunGsUtil(['iam', 'get', self.nonexistent_bucket_name], return_stderr=True, expected_status=1) error_message = ('AttributeError' if self._use_gcloud_storage else 'CommandException') self.assertIn(error_message, stderr) stderr = self.RunGsUtil( ['iam', 'get', 'gs://%s' % self.nonexistent_bucket_name], return_stderr=True, expected_status=1) error_message = ('not found' if self._use_gcloud_storage else 'BucketNotFoundException') self.assertIn(error_message, stderr) # N.B.: The call to wildcard_iterator.WildCardIterator here will invoke # ListBucket, which only promises eventual consistency. We use @Retry here # to mitigate errors due to this. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): # pylint: disable=invalid-name # There are at least two buckets in the project # due to TestIamSet.setUp(). stderr = self.RunGsUtil(['iam', 'get', 'gs://*'], return_stderr=True, expected_status=1) error_message = ('must match a single cloud resource' if self._use_gcloud_storage else 'CommandException') self.assertIn(error_message, stderr) _Check() def test_set_valid_iam_bucket(self): """Tests setting a valid IAM on a bucket.""" self.RunGsUtil( ['iam', 'set', '-e', '', self.new_bucket_iam_path, self.bucket.uri]) set_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.RunGsUtil( ['iam', 'set', '-e', '', self.old_bucket_iam_path, self.bucket.uri]) reset_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertEqualsPoliciesString(self.bucket_iam_string, reset_iam_string) self.assertIn(self.public_bucket_read_binding[0], json.loads(set_iam_string)['bindings']) @unittest.skip('Disabled until all projects whitelisted for conditions.') def test_set_and_get_valid_bucket_policy_with_conditions(self): """Tests setting and getting an IAM policy with conditions on a bucket.""" self.RunGsUtil([ 'iam', 'set', '-e', '', self.new_bucket_policy_with_conditions_path, self.bucket.uri ]) get_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertIn(TEST_CONDITION_DESCRIPTION, get_iam_string) self.assertIn(TEST_CONDITION_EXPR_RESOURCE_IS_OBJECT, get_iam_string.replace('\\', '')) self.assertIn(TEST_CONDITION_TITLE, get_iam_string) # Note: We only test this for buckets, since objects cannot currently have # conditions in their policy bindings. @unittest.skip('Disabled until all projects whitelisted for conditions.') def test_ch_fails_after_setting_conditions(self): """Tests that if we "set" a policy with conditions, "ch" won't patch it.""" print() self.RunGsUtil([ 'iam', 'set', '-e', '', self.new_bucket_policy_with_conditions_path, self.bucket.uri ]) # Assert that we get an error both with and without ch's `-f` option. # Without `-f`: stderr = self.RunGsUtil( ['iam', 'ch', 'allUsers:objectViewer', self.bucket.uri], return_stderr=True, expected_status=1) self.assertIn('CommandException: Could not patch IAM policy for', stderr) # Also make sure we print the workaround message. self.assertIn('The resource had conditions present', stderr) # With `-f`: stderr = self.RunGsUtil( ['iam', 'ch', '-f', 'allUsers:objectViewer', self.bucket.uri], return_stderr=True, expected_status=1) self.assertIn('CommandException: Some IAM policies could not be patched', stderr) # Also make sure we print the workaround message. self.assertIn('Some resources had conditions', stderr) def test_set_blank_etag(self): """Tests setting blank etag behaves appropriately.""" self.RunGsUtil( ['iam', 'set', '-e', '', self.new_bucket_iam_path, self.bucket.uri]) set_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.RunGsUtil([ 'iam', 'set', '-e', json.loads(set_iam_string)['etag'], self.old_bucket_iam_path, self.bucket.uri ]) reset_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertEqualsPoliciesString(self.bucket_iam_string, reset_iam_string) self.assertIn(self.public_bucket_read_binding[0], json.loads(set_iam_string)['bindings']) def test_set_valid_etag(self): """Tests setting valid etag behaves correctly.""" get_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.RunGsUtil([ 'iam', 'set', '-e', json.loads(get_iam_string)['etag'], self.new_bucket_iam_path, self.bucket.uri ]) set_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.RunGsUtil([ 'iam', 'set', '-e', json.loads(set_iam_string)['etag'], self.old_bucket_iam_path, self.bucket.uri ]) reset_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.assertEqualsPoliciesString(self.bucket_iam_string, reset_iam_string) self.assertIn(self.public_bucket_read_binding[0], json.loads(set_iam_string)['bindings']) def test_set_invalid_etag(self): """Tests setting an invalid etag format raises an error.""" self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) stderr = self.RunGsUtil([ 'iam', 'set', '-e', 'some invalid etag', self.new_bucket_iam_path, self.bucket.uri ], return_stderr=True, expected_status=1) error_message = ('DecodeError' if self._use_gcloud_storage else 'ArgumentException') self.assertIn(error_message, stderr) def test_set_mismatched_etag(self): """Tests setting mismatched etag raises an error.""" get_iam_string = self.RunGsUtil(['iam', 'get', self.bucket.uri], return_stdout=True) self.RunGsUtil([ 'iam', 'set', '-e', json.loads(get_iam_string)['etag'], self.new_bucket_iam_path, self.bucket.uri ]) stderr = self.RunGsUtil([ 'iam', 'set', '-e', json.loads(get_iam_string)['etag'], self.new_bucket_iam_path, self.bucket.uri ], return_stderr=True, expected_status=1) error_message = ('pre-conditions you specified did not hold' if self._use_gcloud_storage else 'PreconditionException') self.assertIn(error_message, stderr) def _create_multiple_objects(self): """Creates two versioned objects and return references to all versions. Returns: A four-tuple (a, b, a*, b*) of storage_uri.BucketStorageUri instances. """ old_gsutil_object = self.CreateObject(bucket_uri=self.versioned_bucket, contents=b'foo') old_gsutil_object2 = self.CreateObject(bucket_uri=self.versioned_bucket, contents=b'bar') gsutil_object = self.CreateObject( bucket_uri=self.versioned_bucket, object_name=old_gsutil_object.object_name, contents=b'new_foo', gs_idempotent_generation=urigen(old_gsutil_object)) gsutil_object2 = self.CreateObject( bucket_uri=self.versioned_bucket, object_name=old_gsutil_object2.object_name, contents=b'new_bar', gs_idempotent_generation=urigen(old_gsutil_object2)) return (old_gsutil_object, old_gsutil_object2, gsutil_object, gsutil_object2) def test_set_valid_iam_multiple_objects(self): """Tests setting a valid IAM on multiple objects.""" (old_gsutil_object, old_gsutil_object2, gsutil_object, gsutil_object2) = self._create_multiple_objects() # Set IAM policy on newest versions of all objects. self.RunGsUtil([ 'iam', 'set', '-r', self.new_object_iam_path, self.versioned_bucket.uri ]) set_iam_string = self.RunGsUtil(['iam', 'get', gsutil_object.uri], return_stdout=True) set_iam_string2 = self.RunGsUtil(['iam', 'get', gsutil_object2.uri], return_stdout=True) self.assertEqualsPoliciesString(set_iam_string, set_iam_string2) self.assertIn(self.public_object_read_binding[0], json.loads(set_iam_string)['bindings']) # Check that old versions are not affected by the set IAM call. iam_string_old = self.RunGsUtil( ['iam', 'get', old_gsutil_object.version_specific_uri], return_stdout=True) iam_string_old2 = self.RunGsUtil( ['iam', 'get', old_gsutil_object2.version_specific_uri], return_stdout=True) self.assertEqualsPoliciesString(iam_string_old, iam_string_old2) self.assertEqualsPoliciesString(self.object_iam_string, iam_string_old) def test_set_valid_iam_multithreaded_multiple_objects(self): """Tests setting a valid IAM on multiple objects.""" (old_gsutil_object, old_gsutil_object2, gsutil_object, gsutil_object2) = self._create_multiple_objects() # Set IAM policy on newest versions of all objects. self.RunGsUtil([ '-m', 'iam', 'set', '-r', self.new_object_iam_path, self.versioned_bucket.uri ]) set_iam_string = self.RunGsUtil(['iam', 'get', gsutil_object.uri], return_stdout=True) set_iam_string2 = self.RunGsUtil(['iam', 'get', gsutil_object2.uri], return_stdout=True) self.assertEqualsPoliciesString(set_iam_string, set_iam_string2) self.assertIn(self.public_object_read_binding[0], json.loads(set_iam_string)['bindings']) # Check that old versions are not affected by the set IAM call. iam_string_old = self.RunGsUtil( ['iam', 'get', old_gsutil_object.version_specific_uri], return_stdout=True) iam_string_old2 = self.RunGsUtil( ['iam', 'get', old_gsutil_object2.version_specific_uri], return_stdout=True) self.assertEqualsPoliciesString(iam_string_old, iam_string_old2) self.assertEqualsPoliciesString(self.object_iam_string, iam_string_old) def test_set_valid_iam_multiple_objects_all_versions(self): """Tests set IAM policy on all versions of all objects.""" (old_gsutil_object, old_gsutil_object2, gsutil_object, gsutil_object2) = self._create_multiple_objects() self.RunGsUtil([ 'iam', 'set', '-ra', self.new_object_iam_path, self.versioned_bucket.uri ]) set_iam_string = self.RunGsUtil( ['iam', 'get', gsutil_object.version_specific_uri], return_stdout=True) set_iam_string2 = self.RunGsUtil( ['iam', 'get', gsutil_object2.version_specific_uri], return_stdout=True) set_iam_string_old = self.RunGsUtil( ['iam', 'get', old_gsutil_object.version_specific_uri], return_stdout=True) set_iam_string_old2 = self.RunGsUtil( ['iam', 'get', old_gsutil_object2.version_specific_uri], return_stdout=True) self.assertEqualsPoliciesString(set_iam_string, set_iam_string2) self.assertEqualsPoliciesString(set_iam_string, set_iam_string_old) self.assertEqualsPoliciesString(set_iam_string, set_iam_string_old2) self.assertIn(self.public_object_read_binding[0], json.loads(set_iam_string)['bindings']) def test_set_error(self): """Tests fail-fast behavior of iam set. We initialize two buckets (bucket, bucket2) and attempt to set both along with a third, non-existent bucket in between, self.nonexistent_bucket_name. We want to ensure 1.) Bucket "bucket" IAM policy has been set appropriately, 2.) Bucket self.nonexistent_bucket_name has caused an error, and 3.) gsutil has exited and "bucket2"'s IAM policy is unaltered. """ bucket = self.CreateBucket() bucket2 = self.CreateBucket() stderr = self.RunGsUtil([ 'iam', 'set', '-e', '', self.new_bucket_iam_path, bucket.uri, 'gs://%s' % self.nonexistent_bucket_name, bucket2.uri ], return_stderr=True, expected_status=1) # The program has exited due to a bucket lookup 404. error_message = ('not found' if self._use_gcloud_storage else 'BucketNotFoundException') self.assertIn(error_message, stderr) set_iam_string = self.RunGsUtil(['iam', 'get', bucket.uri], return_stdout=True) set_iam_string2 = self.RunGsUtil(['iam', 'get', bucket2.uri], return_stdout=True) # The IAM policy has been set on Bucket "bucket". self.assertIn(self.public_bucket_read_binding[0], json.loads(set_iam_string)['bindings']) # The IAM policy for Bucket "bucket2" remains unchanged. self.assertEqualsPoliciesString(self.bucket_iam_string, set_iam_string2) def test_set_force_error(self): """Tests ignoring failure behavior of iam set. Similar to TestIamSet.test_set_error, except here we want to ensure 1.) Bucket "bucket" IAM policy has been set appropriately, 2.) Bucket self.nonexistent_bucket_name has caused an error, BUT 3.) gsutil has continued and "bucket2"'s IAM policy has been set as well. """ bucket = self.CreateBucket() bucket2 = self.CreateBucket() stderr = self.RunGsUtil([ 'iam', 'set', '-f', self.new_bucket_iam_path, bucket.uri, 'gs://%s' % self.nonexistent_bucket_name, bucket2.uri ], return_stderr=True, expected_status=1) # The program asserts that an error has occured (due to 404). error_message = ('not found' if self._use_gcloud_storage else 'CommandException') self.assertIn(error_message, stderr) set_iam_string = self.RunGsUtil(['iam', 'get', bucket.uri], return_stdout=True) set_iam_string2 = self.RunGsUtil(['iam', 'get', bucket2.uri], return_stdout=True) # The IAM policy has been set appropriately on Bucket "bucket". self.assertIn(self.public_bucket_read_binding[0], json.loads(set_iam_string)['bindings']) # The IAM policy has also been set on Bucket "bucket2". self.assertEqualsPoliciesString(set_iam_string, set_iam_string2) def test_set_multithreaded_error(self): """Tests fail-fast behavior of multithreaded iam set. This is testing gsutil iam set with the -m and -r flags present in invocation. N.B.: Currently, (-m, -r) behaves identically to (-m, -fr) and (-fr,). However, (-m, -fr) and (-fr,) behavior is not as expected due to name_expansion.NameExpansionIterator.next raising problematic e.g. 404 or 403 errors. More details on this issue can be found in comments in commands.iam.IamCommand._SetIam. Thus, the following command gsutil -m iam set -fr <object_policy> gs://bad_bucket gs://good_bucket will NOT set policies on objects in gs://good_bucket due to an error when iterating over gs://bad_bucket. """ # TODO(b/135780661): Remove retry after bug resolved @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stderr = self.RunGsUtil([ '-m', 'iam', 'set', '-r', self.new_object_iam_path, 'gs://%s' % self.nonexistent_bucket_name, self.bucket.uri ], return_stderr=True, expected_status=1) error_message = ('not found' if self._use_gcloud_storage else 'BucketNotFoundException') self.assertIn(error_message, stderr) # TODO(b/135780661): Remove retry after bug resolved @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): gsutil_object = self.CreateObject(bucket_uri=self.bucket, contents=b'foobar') gsutil_object2 = self.CreateObject(bucket_uri=self.bucket, contents=b'foobar') set_iam_string = self.RunGsUtil(['iam', 'get', gsutil_object.uri], return_stdout=True) set_iam_string2 = self.RunGsUtil(['iam', 'get', gsutil_object2.uri], return_stdout=True) self.assertEqualsPoliciesString(set_iam_string, set_iam_string2) self.assertEqualsPoliciesString(self.object_iam_string, set_iam_string) _Check1() _Check2() def test_set_valid_iam_single_unversioned_object(self): """Tests setting a valid IAM on an object.""" gsutil_object = self.CreateObject(bucket_uri=self.bucket, contents=b'foobar') lookup_uri = gsutil_object.uri self.RunGsUtil(['iam', 'set', self.new_object_iam_path, lookup_uri]) set_iam_string = self.RunGsUtil(['iam', 'get', lookup_uri], return_stdout=True) self.RunGsUtil([ 'iam', 'set', '-e', json.loads(set_iam_string)['etag'], self.old_object_iam_path, lookup_uri ]) reset_iam_string = self.RunGsUtil(['iam', 'get', lookup_uri], return_stdout=True) self.assertEqualsPoliciesString(self.object_iam_string, reset_iam_string) self.assertIn(self.public_object_read_binding[0], json.loads(set_iam_string)['bindings']) def test_set_valid_iam_single_versioned_object(self): """Tests setting a valid IAM on a versioned object.""" gsutil_object = self.CreateObject(bucket_uri=self.bucket, contents=b'foobar') lookup_uri = gsutil_object.version_specific_uri self.RunGsUtil(['iam', 'set', self.new_object_iam_path, lookup_uri]) set_iam_string = self.RunGsUtil(['iam', 'get', lookup_uri], return_stdout=True) self.RunGsUtil([ 'iam', 'set', '-e', json.loads(set_iam_string)['etag'], self.old_object_iam_path, lookup_uri ]) reset_iam_string = self.RunGsUtil(['iam', 'get', lookup_uri], return_stdout=True) self.assertEqualsPoliciesString(self.object_iam_string, reset_iam_string) self.assertIn(self.public_object_read_binding[0], json.loads(set_iam_string)['bindings']) def test_set_valid_iam_multithreaded_single_object(self): """Tests setting a valid IAM on a single object with multithreading.""" gsutil_object = self.CreateObject(bucket_uri=self.bucket, contents=b'foobar') lookup_uri = gsutil_object.version_specific_uri self.RunGsUtil( ['-m', 'iam', 'set', '-e', '', self.new_object_iam_path, lookup_uri]) set_iam_string = self.RunGsUtil(['iam', 'get', lookup_uri], return_stdout=True) self.RunGsUtil( ['-m', 'iam', 'set', '-e', '', self.old_object_iam_path, lookup_uri]) reset_iam_string = self.RunGsUtil(['iam', 'get', lookup_uri], return_stdout=True) self.assertEqualsPoliciesString(self.object_iam_string, reset_iam_string) self.assertIn(self.public_object_read_binding[0], json.loads(set_iam_string)['bindings']) # Test multithreading on single object, specified with wildcards. lookup_uri = '%s*' % self.bucket.uri self.RunGsUtil( ['-m', 'iam', 'set', '-e', '', self.new_object_iam_path, lookup_uri]) set_iam_string = self.RunGsUtil(['iam', 'get', lookup_uri], return_stdout=True) self.RunGsUtil( ['-m', 'iam', 'set', '-e', '', self.old_object_iam_path, lookup_uri]) reset_iam_string = self.RunGsUtil(['iam', 'get', lookup_uri], return_stdout=True) self.assertEqualsPoliciesString(self.object_iam_string, reset_iam_string) self.assertIn(self.public_object_read_binding[0], json.loads(set_iam_string)['bindings']) class TestIamShim(testcase.ShimUnitTestBase): _FAKE_CONFIG_GET_ACCOUNT_PROCESS = subprocess.CompletedProcess( args=[], returncode=0, stdout='fake_account@gmail.com') _MOCK_CONFIG_GET_ACCOUNT_CALL = mock.call( [ shim_util._get_gcloud_binary_path('fake_dir'), 'config', 'get', 'account' ], stderr=-1, stdout=-1, encoding='utf-8', ) @mock.patch.object(iam.IamCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_iam_get_object(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('iam', ['get', 'gs://bucket/object'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage objects get-iam-policy' ' --format=json gs://bucket/object').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(iam.IamCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_iam_get_bucket(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('iam', ['get', 'gs://bucket'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage buckets get-iam-policy' ' --format=json gs://bucket').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(iam.IamCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_iam_set_object(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'iam', ['set', 'policy-file', 'gs://b/o1', 'gs://b/o2'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage objects set-iam-policy' ' --format=json gs://b/o1 gs://b/o2 policy-file').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(iam.IamCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_iam_set_bucket(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'iam', ['set', 'policy-file', 'gs://b1', 'gs://b2'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage buckets set-iam-policy' ' --format=json gs://b1 gs://b2 policy-file').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(iam.IamCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_iam_set_mix_of_bucket_and_objects_if_recursive(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'iam', ['set', '-r', 'policy-file', 'gs://b1', 'gs://b2/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage objects set-iam-policy' ' --format=json --recursive gs://b1 gs://b2/o policy-file').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(iam.IamCommand, 'RunCommand', new=mock.Mock()) def test_shim_raises_for_iam_set_mix_of_bucket_and_objects(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with self.assertRaisesRegex( CommandException, 'Cannot operate on a mix of buckets and objects.'): self.RunCommand('iam', ['set', 'policy-file', 'gs://b', 'gs://b/o']) @mock.patch.object(iam.IamCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_iam_set_handles_valid_etag(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'iam', ['set', '-e', 'abc=', 'policy-file', 'gs://b'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage buckets set-iam-policy' ' --format=json --etag abc= gs://b policy-file').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(iam.IamCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_iam_set_handles_empty_etag(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'iam', ['set', '-e', '', 'policy-file', 'gs://b'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage buckets set-iam-policy' ' --format=json --etag= gs://b policy-file').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(iam.IamCommand, 'RunCommand', new=mock.Mock()) def test_shim_warns_with_dry_run_mode_for_iam_ch(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('iam', ['ch', '-d', 'allUsers', 'gs://b'], return_log_handler=True) warning_lines = '\n'.join(mock_log_handler.messages['warning']) self.assertIn( 'The shim maps iam ch commands to several gcloud storage commands,' ' which cannot be determined without running gcloud storage.', warning_lines) def _get_run_call(self, command, env=mock.ANY, stdin=None, stderr=subprocess.PIPE, stdout=subprocess.PIPE, text=True): return mock.call(command, env=env, input=stdin, stderr=stderr, stdout=stdout, text=text) def test_iam_ch_adds_updates_and_deletes_bucket_policies(self): original_policy = { 'bindings': [{ 'role': 'preserved-role', 'members': ['allUsers'], }, { 'role': 'roles/storage.modified-role', 'members': ['allUsers', 'user:deleted-user@example.com'], }, { 'role': 'roles/storage.deleted-role', 'members': ['allUsers'], }] } new_policy = { 'bindings': [{ 'role': 'preserved-role', 'members': ['allUsers'], }, { 'role': 'roles/storage.modified-role', 'members': ['allAuthenticatedUsers', 'allUsers'], }] } with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): get_process = subprocess.CompletedProcess( args=[], returncode=0, stdout=json.dumps(original_policy)) set_process = subprocess.CompletedProcess(args=[], returncode=0) self._mock_subprocess_run.side_effect = [ self._FAKE_CONFIG_GET_ACCOUNT_PROCESS, get_process, set_process ] with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self.RunCommand('iam', [ 'ch', 'allAuthenticatedUsers:modified-role', '-d', 'user:deleted-user@example.com', '-d', 'allUsers:deleted-role', 'gs://b' ]) self.assertEqual(self._mock_subprocess_run.call_args_list, [ self._MOCK_CONFIG_GET_ACCOUNT_CALL, self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'buckets', 'get-iam-policy', 'gs://b/', '--format=json' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'buckets', 'set-iam-policy', 'gs://b/', '-', ], stdin=json.dumps(new_policy, sort_keys=True)) ]) def test_iam_ch_updates_bucket_policies_for_multiple_urls(self): original_policy1 = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['user:test-user1@example.com'], }] } original_policy2 = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['user:test-user2@example.com'], }] } new_policy1 = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['allAuthenticatedUsers', 'user:test-user1@example.com'], }] } new_policy2 = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['allAuthenticatedUsers', 'user:test-user2@example.com'], }] } with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): get_process1 = subprocess.CompletedProcess( args=[], returncode=0, stdout=json.dumps(original_policy1)) get_process2 = subprocess.CompletedProcess( args=[], returncode=0, stdout=json.dumps(original_policy2)) set_process = subprocess.CompletedProcess(args=[], returncode=0) self._mock_subprocess_run.side_effect = [ self._FAKE_CONFIG_GET_ACCOUNT_PROCESS, get_process1, set_process, get_process2, set_process ] with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self.RunCommand( 'iam', ['ch', 'allAuthenticatedUsers:modified-role', 'gs://b1', 'gs://b2']) self.assertEqual(self._mock_subprocess_run.call_args_list, [ self._MOCK_CONFIG_GET_ACCOUNT_CALL, self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'buckets', 'get-iam-policy', 'gs://b1/', '--format=json' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'buckets', 'set-iam-policy', 'gs://b1/', '-', ], stdin=json.dumps(new_policy1, sort_keys=True)), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'buckets', 'get-iam-policy', 'gs://b2/', '--format=json' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'buckets', 'set-iam-policy', 'gs://b2/', '-', ], stdin=json.dumps(new_policy2, sort_keys=True)) ]) def test_iam_ch_updates_object_policies(self): original_policy = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['allUsers'], }] } new_policy = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['allAuthenticatedUsers', 'allUsers'], }] } with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): ls_process = subprocess.CompletedProcess(args=[], returncode=0, stdout=json.dumps([{ 'url': 'gs://b/o', 'type': 'cloud_object' }])) get_process = subprocess.CompletedProcess( args=[], returncode=0, stdout=json.dumps(original_policy)) set_process = subprocess.CompletedProcess(args=[], returncode=0) self._mock_subprocess_run.side_effect = [ self._FAKE_CONFIG_GET_ACCOUNT_PROCESS, ls_process, get_process, set_process ] with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self.RunCommand( 'iam', ['ch', 'allAuthenticatedUsers:modified-role', 'gs://b/o']) self.assertEqual(self._mock_subprocess_run.call_args_list, [ self._MOCK_CONFIG_GET_ACCOUNT_CALL, self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'ls', '--json', 'gs://b/o' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'objects', 'get-iam-policy', 'gs://b/o', '--format=json' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'objects', 'set-iam-policy', 'gs://b/o', '-', ], stdin=json.dumps(new_policy, sort_keys=True)) ]) def test_iam_ch_expands_urls_with_recursion_and_ignores_container_headers( self): original_policy = { 'bindings': [{ 'role': 'modified-role', 'members': ['allUsers'], }] } with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): ls_process = subprocess.CompletedProcess(args=[], returncode=0, stdout=json.dumps([{ 'url': 'gs://b/dir/', 'type': 'prefix' }, { 'url': 'gs://b/dir/:', 'type': 'cloud_object' }, { 'url': 'gs://b/dir2/', 'type': 'prefix' }, { 'url': 'gs://b/dir2/o', 'type': 'cloud_object' }])) get_process = subprocess.CompletedProcess( args=[], returncode=0, stdout=json.dumps(original_policy)) set_process = subprocess.CompletedProcess(args=[], returncode=0) self._mock_subprocess_run.side_effect = ( [self._FAKE_CONFIG_GET_ACCOUNT_PROCESS, ls_process] + [get_process, set_process] * 3) with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): self.RunCommand( 'iam', ['ch', '-r', 'allAuthenticatedUsers:modified-role', 'gs://b']) self.assertEqual(self._mock_subprocess_run.call_args_list, [ self._MOCK_CONFIG_GET_ACCOUNT_CALL, self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'ls', '--json', '-r', 'gs://b/' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'objects', 'get-iam-policy', 'gs://b/dir/:', '--format=json' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'objects', 'set-iam-policy', 'gs://b/dir/:', '-', ], stdin=mock.ANY), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'objects', 'get-iam-policy', 'gs://b/dir2/o', '--format=json' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'objects', 'set-iam-policy', 'gs://b/dir2/o', '-', ], stdin=mock.ANY) ]) def test_iam_ch_raises_ls_error(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): get_account_process = subprocess.CompletedProcess( args=[], returncode=0, stdout='fake_account@gmail.com') ls_process = subprocess.CompletedProcess(args=[], returncode=1, stderr='An error.') self._mock_subprocess_run.side_effect = [get_account_process, ls_process] with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with self.assertRaisesRegex(CommandException, 'An error.'): self.RunCommand( 'iam', ['ch', 'allAuthenticatedUsers:modified-role', 'gs://b/o']) self.assertEqual(self._mock_subprocess_run.call_count, 2) def test_iam_ch_raises_get_error(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): get_account_process = subprocess.CompletedProcess( args=[], returncode=0, stdout='fake_account@gmail.com') ls_process = subprocess.CompletedProcess(args=[], returncode=0, stdout=json.dumps([{ 'url': 'gs://b/o', 'type': 'cloud_object' }])) get_process = subprocess.CompletedProcess(args=[], returncode=1, stderr='An error.') self._mock_subprocess_run.side_effect = [ get_account_process, ls_process, get_process ] with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with self.assertRaisesRegex(CommandException, 'An error.'): self.RunCommand( 'iam', ['ch', 'allAuthenticatedUsers:modified-role', 'gs://b/o']) self.assertEqual(self._mock_subprocess_run.call_count, 3) def test_iam_ch_raises_set_error(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): get_account_process = subprocess.CompletedProcess( args=[], returncode=0, stdout='fake_account@gmail.com') ls_process = subprocess.CompletedProcess(args=[], returncode=0, stdout=json.dumps([{ 'url': 'gs://b/o', 'type': 'cloud_object' }])) get_process = subprocess.CompletedProcess(args=[], returncode=0, stdout='{"bindings": []}') set_process = subprocess.CompletedProcess(args=[], returncode=1, stderr='An error.') self._mock_subprocess_run.side_effect = [ get_account_process, ls_process, get_process, set_process ] with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with self.assertRaisesRegex(CommandException, 'An error.'): self.RunCommand( 'iam', ['ch', 'allAuthenticatedUsers:modified-role', 'gs://b/o']) self.assertEqual(self._mock_subprocess_run.call_count, 4) def test_iam_ch_continues_on_ls_error(self): original_policy = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['allUsers'], }] } new_policy = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['allAuthenticatedUsers', 'allUsers'], }] } with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): ls_process = subprocess.CompletedProcess(args=[], returncode=1, stderr='An error.') ls_process2 = subprocess.CompletedProcess(args=[], returncode=1, stderr='Another error.') self._mock_subprocess_run.side_effect = [ self._FAKE_CONFIG_GET_ACCOUNT_PROCESS, ls_process, ls_process2 ] with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('iam', [ 'ch', '-f', 'allAuthenticatedUsers:modified-role', 'gs://b/o1', 'gs://b/o2', ], debug=1, return_log_handler=True) self.assertEqual(self._mock_subprocess_run.call_args_list, [ self._MOCK_CONFIG_GET_ACCOUNT_CALL, self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'ls', '--json', 'gs://b/o1' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'ls', '--json', 'gs://b/o2' ]), ]) error_lines = '\n'.join(mock_log_handler.messages['error']) self.assertIn('An error.', error_lines) self.assertIn('Another error.', error_lines) def test_iam_ch_continues_on_get_error(self): original_policy = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['allUsers'], }] } new_policy = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['allAuthenticatedUsers', 'allUsers'], }] } with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): ls_process = subprocess.CompletedProcess(args=[], returncode=0, stdout=json.dumps([{ 'url': 'gs://b/o1', 'type': 'cloud_object' }])) get_process = subprocess.CompletedProcess(args=[], returncode=1, stderr='An error.') ls_process2 = subprocess.CompletedProcess(args=[], returncode=1, stderr='Another error.') self._mock_subprocess_run.side_effect = [ self._FAKE_CONFIG_GET_ACCOUNT_PROCESS, ls_process, get_process, ls_process2 ] with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('iam', [ 'ch', '-f', 'allAuthenticatedUsers:modified-role', 'gs://b/o1', 'gs://b/o2', ], debug=1, return_log_handler=True) self.assertEqual(self._mock_subprocess_run.call_args_list, [ self._MOCK_CONFIG_GET_ACCOUNT_CALL, self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'ls', '--json', 'gs://b/o1' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'objects', 'get-iam-policy', 'gs://b/o1', '--format=json' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'ls', '--json', 'gs://b/o2' ]), ]) error_lines = '\n'.join(mock_log_handler.messages['error']) self.assertIn('An error.', error_lines) self.assertIn('Another error.', error_lines) def test_iam_ch_continues_on_set_error(self): original_policy = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['allUsers'], }] } new_policy = { 'bindings': [{ 'role': 'roles/storage.modified-role', 'members': ['allAuthenticatedUsers', 'allUsers'], }] } with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True')]): ls_process = subprocess.CompletedProcess(args=[], returncode=0, stdout=json.dumps([{ 'url': 'gs://b/o1', 'type': 'cloud_object' }])) get_process = subprocess.CompletedProcess( args=[], returncode=0, stdout=json.dumps(original_policy)) set_process = subprocess.CompletedProcess(args=[], returncode=1, stderr='An error.') ls_process2 = subprocess.CompletedProcess(args=[], returncode=1, stderr='Another error.') self._mock_subprocess_run.side_effect = [ self._FAKE_CONFIG_GET_ACCOUNT_PROCESS, ls_process, get_process, set_process, ls_process2 ] with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('iam', [ 'ch', '-f', 'allAuthenticatedUsers:modified-role', 'gs://b/o1', 'gs://b/o2' ], debug=1, return_log_handler=True) self.assertEqual(self._mock_subprocess_run.call_args_list, [ self._MOCK_CONFIG_GET_ACCOUNT_CALL, self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'ls', '--json', 'gs://b/o1' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'objects', 'get-iam-policy', 'gs://b/o1', '--format=json' ]), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'objects', 'set-iam-policy', 'gs://b/o1', '-', ], stdin=json.dumps(new_policy, sort_keys=True)), self._get_run_call([ shim_util._get_gcloud_binary_path('fake_dir'), 'storage', 'ls', '--json', 'gs://b/o2' ]), ]) error_lines = '\n'.join(mock_log_handler.messages['error']) self.assertIn('An error.', error_lines) self.assertIn('Another error.', error_lines) tests/test_file_part.py 0000644 00000006605 15025013616 0011272 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for FilePart class.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from gslib.file_part import FilePart import gslib.tests.testcase as testcase # pylint: disable=protected-access class TestFilePart(testcase.GsUtilUnitTestCase): """Unit tests for FilePart class.""" def test_tell(self): filename = 'test_tell' contents = 100 * b'x' fpath = self.CreateTempFile(file_name=filename, contents=contents) part_length = 23 start_pos = 50 fp = FilePart(fpath, start_pos, part_length) self.assertEqual(start_pos, fp._fp.tell()) self.assertEqual(0, fp.tell()) def test_seek(self): """Tests seeking in a FilePart.""" filename = 'test_seek' contents = 100 * b'x' part_length = 23 start_pos = 50 fpath = self.CreateTempFile(file_name=filename, contents=contents) fp = FilePart(fpath, start_pos, part_length) offset = 10 # Absolute positioning. fp.seek(offset) self.assertEqual(start_pos + offset, fp._fp.tell()) self.assertEqual(offset, fp.tell()) # Relative positioning. fp.seek(offset, whence=os.SEEK_CUR) self.assertEqual(start_pos + 2 * offset, fp._fp.tell()) self.assertEqual(2 * offset, fp.tell()) # Absolute positioning from EOF. fp.seek(-offset, whence=os.SEEK_END) self.assertEqual(start_pos + part_length - offset, fp._fp.tell()) self.assertEqual(part_length - offset, fp.tell()) # Seek past EOF. fp.seek(1, whence=os.SEEK_END) self.assertEqual(start_pos + part_length + 1, fp._fp.tell()) self.assertEqual(part_length + 1, fp.tell()) def test_read(self): """Tests various reaad operations with FilePart.""" filename = 'test_read' contents = bytearray(range(256)) part_length = 23 start_pos = 50 fpath = self.CreateTempFile(file_name=filename, contents=contents) # Read in the whole file. fp = FilePart(fpath, start_pos, part_length) whole_file = fp.read() self.assertEqual(contents[start_pos:(start_pos + part_length)], whole_file) # Read in a piece of the file from the beginning. fp.seek(0) offset = 10 partial_file = fp.read(offset) self.assertEqual(contents[start_pos:(start_pos + offset)], partial_file) # Read in the rest of the file. remaining_file = fp.read(part_length - offset) self.assertEqual(contents[(start_pos + offset):(start_pos + part_length)], remaining_file) self.assertEqual(contents[start_pos:(start_pos + part_length)], partial_file + remaining_file) # Try to read after reaching EOF. empty_file = fp.read(100) self.assertEqual(b'', empty_file) empty_file = fp.read() self.assertEqual(b'', empty_file) tests/test_defstorageclass.py 0000644 00000012673 15025013616 0012500 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for defstorageclass command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re from unittest import skipIf import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import ObjectToURI as suri @SkipForS3('S3 does not support storage class at bucket level.') class TestDefStorageClass(testcase.GsUtilIntegrationTestCase): """Integration tests for defstorageclass command.""" _set_dsc_cmd = ['defstorageclass', 'set'] _get_dsc_cmd = ['defstorageclass', 'get'] def test_set_and_get_for_one_bucket(self): bucket_uri = self.CreateBucket() # Set the storage class to nearline. new_storage_class = 'nearline' stderr = self.RunGsUtil( self._set_dsc_cmd + [new_storage_class, suri(bucket_uri)], return_stderr=True) if not self._use_gcloud_storage: self.assertRegexpMatchesWithFlags( stderr, r'Setting default storage class to "%s" for bucket %s' % (new_storage_class, suri(bucket_uri)), flags=re.IGNORECASE) # Make sure the storage class shows up as nearline from defstorageclass get. stdout = self.RunGsUtil(self._get_dsc_cmd + [suri(bucket_uri)], return_stdout=True) self.assertRegexpMatchesWithFlags(stdout, r'%s:\s+%s' % (suri(bucket_uri), new_storage_class), flags=re.IGNORECASE) def test_set_and_get_for_multiple_buckets(self): bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() # Set the storage class to nearline. new_storage_class = 'nearline' stderr = self.RunGsUtil( self._set_dsc_cmd + [new_storage_class, suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) for bucket_uri in (suri(bucket1_uri), suri(bucket2_uri)): if not self._use_gcloud_storage: self.assertRegexpMatchesWithFlags( stderr, r'Setting default storage class to "%s" for bucket %s' % (new_storage_class, bucket_uri), flags=re.IGNORECASE) # Make sure the storage class shows up as nearline from defstorageclass get. stdout = self.RunGsUtil( self._get_dsc_cmd + [suri(bucket1_uri), suri(bucket2_uri)], return_stdout=True) for bucket_uri in (suri(bucket1_uri), suri(bucket2_uri)): self.assertRegexpMatchesWithFlags(stdout, r'%s:\s+%s' % (bucket_uri, new_storage_class), flags=re.IGNORECASE) def test_set_invalid_storage_class_fails(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil(self._set_dsc_cmd + ['invalidclass', suri(bucket_uri)], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('Invalid storage class', stderr) else: self.assertIn('BadRequestException: 400', stderr) def test_too_few_arguments_fails(self): # No arguments for set, but valid subcommand. stderr = self.RunGsUtil(self._set_dsc_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Argument given for set, but no buckets listed. if self._use_gcloud_storage: expected_status = 2 expected_error_string = 'argument URL [URL ...]: Must be specified' else: expected_status = 1 expected_error_string = 'command requires at least' stderr = self.RunGsUtil(self._set_dsc_cmd + ['std'], return_stderr=True, expected_status=expected_status) self.assertIn(expected_error_string, stderr) # No arguments for get, but valid subcommand. stderr = self.RunGsUtil(self._get_dsc_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) def test_helpful_failure_with_s3_urls(self): s3_bucket_url = 's3://somebucket' failure_msg = 'does not support the URL "%s"' % s3_bucket_url stderr = self.RunGsUtil(self._get_dsc_cmd + [s3_bucket_url], return_stderr=True, expected_status=1) self.assertIn(failure_msg, stderr) stderr = self.RunGsUtil(self._set_dsc_cmd + ['ClassFoo', s3_bucket_url], return_stderr=True, expected_status=1) self.assertIn(failure_msg, stderr) tests/mock_logging_handler.py 0000644 00000002434 15025013616 0012416 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Mock logging handler to check for expected logs.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging class MockLoggingHandler(logging.Handler): """Mock logging handler to check for expected logs.""" def __init__(self, *args, **kwargs): self.reset() logging.Handler.__init__(self, *args, **kwargs) def emit(self, record): self.messages[record.levelname.lower()].append(record.getMessage()) def reset(self): self.messages = { 'debug': [], 'info': [], 'warning': [], 'error': [], 'critical': [], } tests/test_pap.py 0000644 00000013644 15025013616 0010106 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for pap command.""" from __future__ import absolute_import import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForGS from gslib.tests.testcase.integration_testcase import SkipForJSON from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest class TestPublicAccessPrevention(testcase.GsUtilIntegrationTestCase): """Integration tests for pap command.""" _set_pap_cmd = ['pap', 'set'] _get_pap_cmd = ['pap', 'get'] @SkipForXML('Public access prevention only runs on GCS JSON API') def test_off_on_default_buckets(self): bucket_uri = self.CreateBucket() self.VerifyPublicAccessPreventionValue(bucket_uri, 'inherited') @SkipForXML('Public access prevention only runs on GCS JSON API') def test_turning_off_on_enabled_buckets(self): bucket_uri = self.CreateBucket(public_access_prevention='enforced', prefer_json_api=True) self.VerifyPublicAccessPreventionValue(bucket_uri, 'enforced') self.RunGsUtil(self._set_pap_cmd + ['inherited', suri(bucket_uri)]) self.VerifyPublicAccessPreventionValue(bucket_uri, 'inherited') @SkipForXML('Public access prevention only runs on GCS JSON API') def test_turning_on(self): bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_pap_cmd + ['enforced', suri(bucket_uri)]) self.VerifyPublicAccessPreventionValue(bucket_uri, 'enforced') @SkipForXML('Public access prevention only runs on GCS JSON API') def test_turning_on_and_off(self): bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_pap_cmd + ['enforced', suri(bucket_uri)]) self.VerifyPublicAccessPreventionValue(bucket_uri, 'enforced') self.RunGsUtil(self._set_pap_cmd + ['inherited', suri(bucket_uri)]) self.VerifyPublicAccessPreventionValue(bucket_uri, 'inherited') @SkipForXML('Public access prevention only runs on GCS JSON API') def test_multiple_buckets(self): bucket_uri1 = self.CreateBucket() bucket_uri2 = self.CreateBucket() stdout = self.RunGsUtil( self._get_pap_cmd + [suri(bucket_uri1), suri(bucket_uri2)], return_stdout=True) self.assertRegex(stdout, r'%s:\s+inherited' % suri(bucket_uri1)) self.assertRegex(stdout, r'%s:\s+inherited' % suri(bucket_uri2)) @SkipForJSON('Testing XML only behavior') def test_xml_fails(self): # use HMAC for force XML API boto_config_hmac_auth_only = [ # Overwrite other credential types. ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ('Credentials', 'gs_service_key_file_password', None), # Add hmac credentials. ('Credentials', 'gs_access_key_id', 'dummykey'), ('Credentials', 'gs_secret_access_key', 'dummysecret'), ] with SetBotoConfigForTest(boto_config_hmac_auth_only): bucket_uri = 'gs://any-bucket-name' stderr = self.RunGsUtil(self._set_pap_cmd + ['inherited', bucket_uri], return_stderr=True, expected_status=1) self.assertIn('command can only be with the Cloud Storage JSON API', stderr) stderr = self.RunGsUtil(self._get_pap_cmd + [bucket_uri], return_stderr=True, expected_status=1) self.assertIn('command can only be with the Cloud Storage JSON API', stderr) @SkipForGS('Testing S3 only behavior') def test_s3_fails(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil(self._set_pap_cmd + ['inherited', suri(bucket_uri)], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('Flags disallowed for S3', stderr) else: self.assertIn('command can only be used for GCS Buckets', stderr) if not self._use_gcloud_storage: # gcloud storage uses a generic buckets describe command for this, and it # would not print a result instead of erroring. stderr = self.RunGsUtil(self._get_pap_cmd + [suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('command can only be used for GCS Buckets', stderr) def test_set_too_few_arguments_fails(self): stderr = self.RunGsUtil(self._set_pap_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) def test_get_too_few_arguments_fails(self): stderr = self.RunGsUtil(self._get_pap_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) def test_no_subcommand_fails(self): stderr = self.RunGsUtil(['pap'], return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) def test_invalid_subcommand_fails(self): stderr = self.RunGsUtil(['pap', 'fakecommand', 'test'], return_stderr=True, expected_status=1) self.assertIn('Invalid subcommand', stderr) tests/test_stat.py 0000644 00000026611 15025013616 0010277 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for stat command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.cs_api_map import ApiSelector from gslib.exception import NO_URLS_MATCHED_TARGET import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import GenerationFromURI as urigen from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import TEST_ENCRYPTION_CONTENT1 from gslib.tests.util import TEST_ENCRYPTION_CONTENT1_CRC32C from gslib.tests.util import TEST_ENCRYPTION_CONTENT1_MD5 from gslib.tests.util import TEST_ENCRYPTION_CONTENT2 from gslib.tests.util import TEST_ENCRYPTION_CONTENT2_CRC32C from gslib.tests.util import TEST_ENCRYPTION_CONTENT2_MD5 from gslib.tests.util import TEST_ENCRYPTION_CONTENT3 from gslib.tests.util import TEST_ENCRYPTION_CONTENT3_CRC32C from gslib.tests.util import TEST_ENCRYPTION_CONTENT3_MD5 from gslib.tests.util import TEST_ENCRYPTION_KEY1 from gslib.tests.util import TEST_ENCRYPTION_KEY1_SHA256_B64 from gslib.tests.util import TEST_ENCRYPTION_KEY2 from gslib.tests.util import TEST_ENCRYPTION_KEY2_SHA256_B64 from gslib.tests.util import unittest from gslib.utils.retry_util import Retry class TestStat(testcase.GsUtilIntegrationTestCase): """Integration tests for stat command.""" @SkipForS3('\'Archived time\' is a GS-specific response field.') @SkipForXML( 'XML API only supports \'DeletedTime\' response field when making a ' 'GET Bucket request to list all objects, which is heavy overhead when ' 'the real intent is just a HEAD Object call.') def test_versioned_stat_output(self): """Tests stat output of an outdated object under version control.""" bucket_uri = self.CreateVersionedBucket() old_object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'z') # Update object self.CreateObject(bucket_uri=bucket_uri, object_name=old_object_uri.object_name, contents=b'z', gs_idempotent_generation=urigen(old_object_uri)) stdout = self.RunGsUtil(['stat', old_object_uri.version_specific_uri], return_stdout=True) self.assertIn('Noncurrent time', stdout) def test_stat_output(self): """Tests stat output of a single object.""" object_uri = self.CreateObject(contents=b'z') stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True) self.assertIn(object_uri.uri, stdout) self.assertIn('Creation time:', stdout) # Cache-Control and Content-Encoding can be different depending on # whether the JSON or XML API is used. For JSON, only max-age and # no-cache are respected. Although the object field will be populated # with whatever we set, the actual header returned from the JSON API # may differ from it (and differ from the XML response for the same object). # # Likewise, with contentEncoding, the field value and the header value # are not guaranteed to match or be the same across APIs. # # JSON will not return a Cache-control or content-encoding with the # current test object creation, so check these only for the XML API. if self.default_provider == 'gs': if self.test_api == ApiSelector.XML: self.assertIn('Cache-Control:', stdout) self.assertIn('Content-Encoding:', stdout) # TODO: Remove JSON check after adding storage class parsing in Boto. elif self.test_api == ApiSelector.JSON: self.assertIn('Storage class:', stdout) self.assertIn('Generation:', stdout) self.assertIn('Metageneration:', stdout) self.assertIn('Hash (crc32c):', stdout) self.assertIn('Hash (md5):', stdout) self.assertNotIn('Archived time', stdout) # object is not archived self.assertIn('Content-Length:', stdout) self.assertIn('Content-Type:', stdout) self.assertIn('ETag:', stdout) def test_minus_q_stat(self): object_uri = self.CreateObject(contents=b'z') stdout = self.RunGsUtil(['-q', 'stat', suri(object_uri)], return_stdout=True) self.assertEqual(0, len(stdout)) stdout = self.RunGsUtil( ['-q', 'stat', suri(object_uri, 'junk')], return_stdout=True, expected_status=1) self.assertEqual(0, len(stdout)) def test_stat_of_non_object_uri(self): self.RunGsUtil(['-q', 'stat', 'gs://'], expected_status=1) self.RunGsUtil(['-q', 'stat', 'gs://bucket/object'], expected_status=1) self.RunGsUtil(['-q', 'stat', 'file://tmp/abc'], expected_status=1) def test_stat_one_missing(self): bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='notmissing', contents=b'z') stdout, stderr = self.RunGsUtil( ['stat', suri(bucket_uri, 'missing'), suri(bucket_uri, 'notmissing')], expected_status=1, return_stdout=True, return_stderr=True) self.assertIn(NO_URLS_MATCHED_TARGET % suri(bucket_uri, 'missing'), stderr) self.assertIn('%s:' % suri(bucket_uri, 'notmissing'), stdout) def test_stat_one_missing_wildcard(self): bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='notmissing', contents=b'z') stdout, stderr = self.RunGsUtil( ['stat', suri(bucket_uri, 'missin*'), suri(bucket_uri, 'notmissin*')], expected_status=1, return_stdout=True, return_stderr=True) self.assertIn(NO_URLS_MATCHED_TARGET % suri(bucket_uri, 'missin*'), stderr) self.assertIn('%s:' % suri(bucket_uri, 'notmissing'), stdout) def test_stat_bucket_wildcard(self): # Prefix is a workaround for XML API limitation, see PR 766 for details bucket_uri = self.CreateBucket(bucket_name_prefix='aaa-') self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'z') stat_string = suri(bucket_uri)[:-1] + '?/foo' self.RunGsUtil(['stat', stat_string]) stat_string2 = suri(bucket_uri)[:-1] + '*/foo' self.RunGsUtil(['stat', stat_string2]) def test_stat_object_wildcard(self): bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo1', contents=b'z') object2_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo2', contents=b'z') stat_string = suri(object1_uri)[:-2] + '*' # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['stat', stat_string], return_stdout=True) self.assertIn(suri(object1_uri), stdout) self.assertIn(suri(object2_uri), stdout) _Check1() @SkipForS3('S3 customer-supplied encryption keys are not supported.') def test_stat_encrypted_object(self): """Tests stat command with an encrypted object.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=TEST_ENCRYPTION_CONTENT1, encryption_key=TEST_ENCRYPTION_KEY1) # Stat object with key should return unencrypted hashes. with SetBotoConfigForTest([('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]): stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True) self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64.decode('ascii'), stdout) # Stat object without key should return encrypted hashes. stdout = self.RunGsUtil(['stat', suri(object_uri)], return_stdout=True) self.assertNotIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn('encrypted', stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64.decode('ascii'), stdout) def test_stat_encrypted_object_wildcard(self): """Tests stat command with a mix of encrypted and unencrypted objects.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo1', contents=TEST_ENCRYPTION_CONTENT1, encryption_key=TEST_ENCRYPTION_KEY1) object2_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo2', contents=TEST_ENCRYPTION_CONTENT2, encryption_key=TEST_ENCRYPTION_KEY2) object3_uri = self.CreateObject(bucket_uri=bucket_uri, object_name='foo3', contents=TEST_ENCRYPTION_CONTENT3) stat_string = suri(object1_uri)[:-2] + '*' # Stat 3 objects, two encrypted each with a different key, and one # unencrypted. Should result in two unencrypted listing and one encrypted # listing. with SetBotoConfigForTest([('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]): # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _StatExpectMixed(): """Runs stat and validates output.""" stdout, stderr = self.RunGsUtil(['stat', stat_string], return_stdout=True, return_stderr=True) self.assertIn(suri(object1_uri), stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT1_CRC32C, stdout) self.assertIn(TEST_ENCRYPTION_KEY1_SHA256_B64.decode('ascii'), stdout) self.assertIn(suri(object2_uri), stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT2_MD5, stdout) self.assertNotIn(TEST_ENCRYPTION_CONTENT2_CRC32C, stdout) self.assertIn('encrypted', stdout) self.assertIn(TEST_ENCRYPTION_KEY2_SHA256_B64.decode('ascii'), stdout) self.assertIn(suri(object3_uri), stdout) self.assertIn(TEST_ENCRYPTION_CONTENT3_MD5, stdout) self.assertIn(TEST_ENCRYPTION_CONTENT3_CRC32C, stdout) _StatExpectMixed() tests/test_requester_pays.py 0000644 00000027317 15025013616 0012403 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for notification command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import gslib.tests.testcase as testcase from gslib.project_id import PopulateProjectId from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import ObjectToURI as suri from gslib.utils.retry_util import Retry from gslib.utils.constants import UTF8 OBJECT_CONTENTS = b'innards' @SkipForS3('gsutil doesn\'t support S3 Requester Pays.') @SkipForXML('Requester Pays is not supported for the XML API.') class TestRequesterPays(testcase.GsUtilIntegrationTestCase): """Integration tests for Requester Pays. Passing in a user project should succeed for operations on Requester Pays buckets, and with the GA release will also succeed for non-Requester Pays buckets. """ _set_rp_cmd = ['requesterpays', 'set'] _get_rp_cmd = ['requesterpays', 'get'] def setUp(self): super(TestRequesterPays, self).setUp() self.non_requester_pays_bucket_uri = self.CreateBucket() self.requester_pays_bucket_uri = self.CreateBucket() self._set_requester_pays(self.requester_pays_bucket_uri) self.non_requester_pays_object_uri = self.CreateObject( bucket_uri=self.non_requester_pays_bucket_uri, contents=OBJECT_CONTENTS) self.requester_pays_object_uri = self.CreateObject( bucket_uri=self.requester_pays_bucket_uri, contents=OBJECT_CONTENTS) self.user_project_flag = ['-u', PopulateProjectId()] def _set_requester_pays(self, bucket_uri): self.RunGsUtil(['requesterpays', 'set', 'on', suri(bucket_uri)]) def _run_requester_pays_test(self, command_list, regex=None): """Test a command with a user project. Run a command with a user project on a Requester Pays bucket. The command is expected to pass because the source bucket is Requester Pays. If a regex pattern is supplied, also assert that stdout of the command matches it. """ stdout = self.RunGsUtil(self.user_project_flag + command_list, return_stdout=True) if regex: if isinstance(regex, bytes): regex = regex.decode(UTF8) self.assertRegexpMatchesWithFlags(stdout, regex, flags=re.IGNORECASE) def _run_non_requester_pays_test(self, command_list): """Test a command with a user project on a non-Requester Pays bucket. Run a command with a user project on a non-Requester Pays bucket. The command will still succeed, because with GA user project is accepted for all requests. """ stdout = self.RunGsUtil(self.user_project_flag + command_list, return_stdout=True) def test_off_default(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil(self._get_rp_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout.strip(), '%s: Disabled' % suri(bucket_uri)) def test_turning_on(self): bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_rp_cmd + ['on', suri(bucket_uri)]) def _Check1(): stdout = self.RunGsUtil(self._get_rp_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout.strip(), '%s: Enabled' % suri(bucket_uri)) _Check1() def test_turning_off(self): bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_rp_cmd + ['on', suri(bucket_uri)]) def _Check1(): stdout = self.RunGsUtil(self._get_rp_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout.strip(), '%s: Enabled' % suri(bucket_uri)) _Check1() self.RunGsUtil(self._set_rp_cmd + ['off', suri(bucket_uri)]) def _Check2(): stdout = self.RunGsUtil(self._get_rp_cmd + [suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout.strip(), '%s: Disabled' % suri(bucket_uri)) _Check2() def testTooFewArgumentsFails(self): """Ensures requesterpays commands fail with too few arguments.""" # No arguments for set, but valid subcommand. stderr = self.RunGsUtil(self._set_rp_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for get, but valid subcommand. stderr = self.RunGsUtil(self._get_rp_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Neither arguments nor subcommand. stderr = self.RunGsUtil(['requesterpays'], return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) def test_acl(self): requester_pays_bucket_uri = self.CreateBucket() self._set_requester_pays(requester_pays_bucket_uri) self._run_requester_pays_test( ['acl', 'set', 'public-read', suri(requester_pays_bucket_uri)]) self._run_requester_pays_test( ['acl', 'get', suri(requester_pays_bucket_uri)]) non_requester_pays_bucket_uri = self.CreateBucket() self._run_non_requester_pays_test( ['acl', 'set', 'public-read', suri(non_requester_pays_bucket_uri)]) self._run_non_requester_pays_test( ['acl', 'get', suri(non_requester_pays_bucket_uri)]) def test_ls(self): self._run_requester_pays_test(['ls', suri(self.requester_pays_bucket_uri)]) self._run_non_requester_pays_test( ['ls', suri(self.non_requester_pays_bucket_uri)]) def test_rb(self): rp_bucket_uri = self.CreateBucket() self._set_requester_pays(rp_bucket_uri) self._run_requester_pays_test(['rb', suri(rp_bucket_uri)]) non_rp_bucket_uri = self.CreateBucket() self._run_non_requester_pays_test(['rb', suri(non_rp_bucket_uri)]) def test_copy(self): dest_bucket_uri = self.CreateBucket() self._run_requester_pays_test( ['cp', suri(self.requester_pays_object_uri), suri(dest_bucket_uri)]) self._run_non_requester_pays_test( ['cp', suri(self.non_requester_pays_object_uri), suri(dest_bucket_uri)]) def test_compose(self): data_list = [b'apple', b'orange', b'banana'] bucket_uri = self.CreateBucket() components = [ self.CreateObject(bucket_uri=bucket_uri, contents=data).uri for data in data_list ] composite = self.StorageUriCloneReplaceName(bucket_uri, self.MakeTempName('obj')) self._run_non_requester_pays_test(['compose'] + components + [composite.uri]) rp_bucket_uri = self.CreateBucket() self._set_requester_pays(rp_bucket_uri) rp_components = [ self.CreateObject(bucket_uri=rp_bucket_uri, contents=data).uri for data in data_list ] rp_composite = suri(rp_bucket_uri) + '/composite.txt' self._run_requester_pays_test(['compose'] + rp_components + [rp_composite]) def test_cat(self): self._run_requester_pays_test( ['cat', suri(self.requester_pays_object_uri)], regex=OBJECT_CONTENTS) self._run_non_requester_pays_test( ['cat', suri(self.non_requester_pays_object_uri)]) def test_du_obj(self): @Retry(AssertionError, tries=3, timeout_secs=1) # Use @Retry as hedge against bucket listing eventual consistency. def _check(): self._run_requester_pays_test( ['du', suri(self.requester_pays_object_uri)]) self._run_non_requester_pays_test( ['du', suri(self.non_requester_pays_object_uri)]) _check() def test_hash(self): self._run_requester_pays_test( ['hash', '-c', suri(self.requester_pays_object_uri)], regex=r'Hash \(crc32c\)') self._run_non_requester_pays_test( ['hash', '-c', suri(self.non_requester_pays_object_uri)]) def test_iam(self): self._run_requester_pays_test( ['iam', 'get', str(self.requester_pays_object_uri)]) self._run_non_requester_pays_test( ['iam', 'get', str(self.non_requester_pays_object_uri)]) def test_mv(self): requester_pays_bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=requester_pays_bucket_uri, contents=b'foo') object2_uri = self.CreateObject(bucket_uri=requester_pays_bucket_uri, contents=b'oOOo') self.AssertNObjectsInBucket(requester_pays_bucket_uri, 2) self._set_requester_pays(requester_pays_bucket_uri) dest_bucket_uri = self.CreateBucket() # Move two objects from bucket1 to Requester Pays bucket. for obj in [object1_uri, object2_uri]: self._run_requester_pays_test(['mv', suri(obj), suri(dest_bucket_uri)]) self.AssertNObjectsInBucket(requester_pays_bucket_uri, 0) bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'bar') object2_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'baz') self.AssertNObjectsInBucket(bucket_uri, 2) for obj in [object1_uri, object2_uri]: self._run_non_requester_pays_test( ['mv', suri(obj), suri(dest_bucket_uri)]) self.AssertNObjectsInBucket(bucket_uri, 0) def test_rewrite(self): object_uri = self.CreateObject(contents=b'bar') storage_class = 'nearline' if self._use_gcloud_storage else 'dra' self._run_non_requester_pays_test( ['rewrite', '-s', storage_class, suri(object_uri)]) req_pays_bucket_uri = self.CreateBucket() self._set_requester_pays(req_pays_bucket_uri) req_pays_obj_uri = self.CreateObject(bucket_uri=req_pays_bucket_uri, contents=b'baz') self._run_requester_pays_test( ['rewrite', '-s', storage_class, suri(req_pays_obj_uri)]) def test_rsync(self): req_pays_bucket_uri = self.CreateBucket(test_objects=2) self._set_requester_pays(req_pays_bucket_uri) bucket_uri = self.CreateBucket(test_objects=1) self._run_requester_pays_test([ 'rsync', '-d', suri(req_pays_bucket_uri), suri(self.requester_pays_bucket_uri) ]) bucket_uri1 = self.CreateBucket(test_objects=2) bucket_uri2 = self.CreateBucket(test_objects=1) self._run_non_requester_pays_test( ['rsync', '-d', suri(bucket_uri1), suri(bucket_uri2)]) def test_setmeta(self): req_pays_obj_uri = self.CreateObject( bucket_uri=self.requester_pays_bucket_uri, contents=b'<html><body>text</body></html>') self._run_requester_pays_test( ['setmeta', '-h', 'content-type:text/html', suri(req_pays_obj_uri)]) obj_uri = self.CreateObject(bucket_uri=self.non_requester_pays_bucket_uri, contents=b'<html><body>text</body></html>') self._run_non_requester_pays_test( ['setmeta', '-h', 'content-type:text/html', suri(obj_uri)]) def test_stat(self): self._run_requester_pays_test( ['stat', suri(self.requester_pays_object_uri)]) self._run_non_requester_pays_test( ['stat', suri(self.non_requester_pays_object_uri)]) tests/test_command.py 0000644 00000006620 15025013616 0010740 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for command.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from unittest import mock from gslib import command from gslib.tests import testcase from gslib.utils import constants class FakeGsutilCommand(command.Command): """Implementation of a fake gsutil command.""" command_spec = command.Command.CreateCommandSpec('fake_gsutil', min_args=1, max_args=constants.NO_MAX, supported_sub_args='rz:', file_url_ok=True) help_spec = command.Command.HelpSpec( help_name='fake_gsutil', help_name_aliases=[], help_type='command_help', help_one_line_summary='Fake one line summary for the command.', help_text='Help text for fake command.', subcommand_help_text={}, ) class TestParseSubOpts(testcase.GsUtilUnitTestCase): """Test Command.ParseSubOpts method. Only testing cases which are not tested indirectly by other tests. """ def setUp(self): super().setUp() self._fake_command = FakeGsutilCommand( command_runner=mock.ANY, args=['-z', 'opt1', '-r', 'arg1', 'arg2'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) def test_raises_error_with_check_args_set_and_update_sub_opts_and_args_unset( self): with self.assertRaisesRegex( TypeError, 'Requested to check arguments but sub_opts and args have' ' not been updated.'): self._fake_command.ParseSubOpts(check_args=True, should_update_sub_opts_and_args=False) def test_uses_self_args_if_args_passed_is_None(self): args_list = ['fake', 'args'] self._fake_command.args = args_list _, parsed_args = self._fake_command.ParseSubOpts( should_update_sub_opts_and_args=False) self.assertEqual(parsed_args, args_list) @mock.patch.object(command, 'CreateOrGetGsutilLogger', autospec=True) def test_quiet_mode_gets_set(self, mock_logger): mock_logger.return_value.isEnabledFor.return_value = False self._fake_command = FakeGsutilCommand( command_runner=mock.ANY, args=['-z', 'opt1', '-r', 'arg1', 'arg2'], headers={}, debug=mock.ANY, trace_token=mock.ANY, parallel_operations=mock.ANY, bucket_storage_uri_class=mock.ANY, gsutil_api_class_map_factory=mock.MagicMock()) self.assertTrue(self._fake_command.quiet_mode) mock_logger.assert_called_once_with('fake_gsutil') tests/test_stet_util.py 0000644 00000016610 15025013616 0011336 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for stet_util.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import shutil from gslib import storage_url from gslib.tests import testcase from gslib.tests import util from gslib.tests.util import unittest from gslib.utils import execution_util from gslib.utils import stet_util from unittest import mock class TestStetUtil(testcase.GsUtilUnitTestCase): """Test STET utils.""" @mock.patch.object(execution_util, 'ExecuteExternalCommand') def test_stet_upload_uses_binary_and_config_from_boto( self, mock_execute_external_command): fake_config_path = self.CreateTempFile() mock_execute_external_command.return_value = ('stdout', 'stderr') mock_logger = mock.Mock() source_url = storage_url.StorageUrlFromString('in') destination_url = storage_url.StorageUrlFromString('gs://bucket/obj') with util.SetBotoConfigForTest([ ('GSUtil', 'stet_binary_path', 'fake_binary_path'), ('GSUtil', 'stet_config_path', fake_config_path), ]): out_file_url = stet_util.encrypt_upload(source_url, destination_url, mock_logger) self.assertEqual(out_file_url, storage_url.StorageUrlFromString('in_.stet_tmp')) mock_execute_external_command.assert_called_once_with([ 'fake_binary_path', 'encrypt', '--config-file={}'.format(fake_config_path), '--blob-id=gs://bucket/obj', 'in', 'in_.stet_tmp', ]) mock_logger.debug.assert_called_once_with('stderr') @mock.patch.object(execution_util, 'ExecuteExternalCommand') def test_stet_upload_runs_with_binary_from_path_with_correct_settings( self, mock_execute_external_command): fake_config_path = self.CreateTempFile() temporary_path_directory = self.CreateTempDir() fake_stet_binary_path = self.CreateTempFile(tmpdir=temporary_path_directory, file_name='stet') previous_path = os.getenv('PATH') os.environ['PATH'] += os.path.pathsep + temporary_path_directory mock_execute_external_command.return_value = ('stdout', 'stderr') mock_logger = mock.Mock() source_url = storage_url.StorageUrlFromString('in') destination_url = storage_url.StorageUrlFromString('gs://bucket/obj') with util.SetBotoConfigForTest([ ('GSUtil', 'stet_binary_path', None), ('GSUtil', 'stet_config_path', fake_config_path), ]): out_file_url = stet_util.encrypt_upload(source_url, destination_url, mock_logger) self.assertEqual(out_file_url, storage_url.StorageUrlFromString('in_.stet_tmp')) mock_execute_external_command.assert_called_once_with([ fake_stet_binary_path, 'encrypt', '--config-file={}'.format(fake_config_path), '--blob-id=gs://bucket/obj', 'in', 'in_.stet_tmp', ]) mock_logger.debug.assert_called_once_with('stderr') os.environ['PATH'] = previous_path @mock.patch.object(execution_util, 'ExecuteExternalCommand') def test_stet_upload_uses_no_config_if_not_provided( self, mock_execute_external_command): mock_execute_external_command.return_value = ('stdout', 'stderr') mock_logger = mock.Mock() source_url = storage_url.StorageUrlFromString('in') destination_url = storage_url.StorageUrlFromString('gs://bucket/obj') with util.SetBotoConfigForTest([ ('GSUtil', 'stet_binary_path', 'fake_binary_path'), ('GSUtil', 'stet_config_path', None), ]): with mock.patch.object(os.path, 'exists', new=mock.Mock(return_value=True)): out_file_url = stet_util.encrypt_upload(source_url, destination_url, mock_logger) self.assertEqual(out_file_url, storage_url.StorageUrlFromString('in_.stet_tmp')) mock_execute_external_command.assert_called_once_with([ 'fake_binary_path', 'encrypt', '--blob-id=gs://bucket/obj', 'in', 'in_.stet_tmp', ]) mock_logger.debug.assert_called_once_with('stderr') @mock.patch.object(shutil, 'move') @mock.patch.object(execution_util, 'ExecuteExternalCommand') def test_stet_download_runs_binary_and_replaces_temp_file( self, mock_execute_external_command, mock_move): fake_config_path = self.CreateTempFile() mock_execute_external_command.return_value = ('stdout', 'stderr') mock_logger = mock.Mock() source_url = storage_url.StorageUrlFromString('gs://bucket/obj') destination_url = storage_url.StorageUrlFromString('out') temporary_file_name = 'out_.gstmp' with util.SetBotoConfigForTest([ ('GSUtil', 'stet_binary_path', 'fake_binary_path'), ('GSUtil', 'stet_config_path', fake_config_path), ]): stet_util.decrypt_download(source_url, destination_url, temporary_file_name, mock_logger) mock_execute_external_command.assert_called_once_with([ 'fake_binary_path', 'decrypt', '--config-file={}'.format(fake_config_path), '--blob-id=gs://bucket/obj', 'out_.gstmp', 'out_.stet_tmp' ]) mock_logger.debug.assert_called_once_with('stderr') mock_move.assert_called_once_with('out_.stet_tmp', 'out_.gstmp') @mock.patch.object(stet_util, '_get_stet_binary_from_path', new=mock.Mock(return_value=None)) def test_stet_util_errors_if_no_binary(self): fake_config_path = self.CreateTempFile() source_url = storage_url.StorageUrlFromString('in') destination_url = storage_url.StorageUrlFromString('gs://bucket/obj') with util.SetBotoConfigForTest([ ('GSUtil', 'stet_binary_path', None), ('GSUtil', 'stet_config_path', fake_config_path), ]): with self.assertRaises(KeyError): stet_util.encrypt_upload(source_url, destination_url, None) @mock.patch.object(os.path, 'expanduser', autospec=True) @mock.patch.object(execution_util, 'ExecuteExternalCommand', new=mock.Mock(return_value=('stdout', 'stderr'))) def test_stet_util_expands_home_directory_symbol(self, mock_expanduser): fake_config_path = self.CreateTempFile() source_url = storage_url.StorageUrlFromString('in') destination_url = storage_url.StorageUrlFromString('gs://bucket/obj') with util.SetBotoConfigForTest([ ('GSUtil', 'stet_binary_path', 'fake_binary_path'), ('GSUtil', 'stet_config_path', fake_config_path), ]): stet_util.encrypt_upload(source_url, destination_url, mock.Mock()) mock_expanduser.assert_has_calls( [mock.call('fake_binary_path'), mock.call(fake_config_path)]) tests/test_cat.py 0000644 00000027070 15025013616 0010073 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for cat command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import sys from gslib.cs_api_map import ApiSelector from gslib.exception import NO_URLS_MATCHED_TARGET import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import GenerationFromURI as urigen from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import RUN_S3_TESTS from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import TEST_ENCRYPTION_KEY1 from gslib.tests.util import unittest from gslib.utils import cat_helper from gslib.utils import shim_util from unittest import mock class TestCat(testcase.GsUtilIntegrationTestCase): """Integration tests for cat command.""" def test_cat_range(self): """Tests cat command with various range arguments.""" key_uri = self.CreateObject(contents=b'0123456789') # Test various invalid ranges. stderr = self.RunGsUtil( ['cat', '-r a-b', suri(key_uri)], return_stderr=True, expected_status=2 if self._use_gcloud_storage else 1) if self._use_gcloud_storage: self.assertIn( 'Expected a non-negative integer value or a range of such values instead of', stderr) else: self.assertIn('Invalid range', stderr) stderr = self.RunGsUtil( ['cat', '-r 1-2-3', suri(key_uri)], return_stderr=True, expected_status=2 if self._use_gcloud_storage else 1) if self._use_gcloud_storage: self.assertIn( 'Expected a non-negative integer value or a range of such values instead of', stderr) else: self.assertIn('Invalid range', stderr) stderr = self.RunGsUtil( ['cat', '-r 1.7-3', suri(key_uri)], return_stderr=True, expected_status=2 if self._use_gcloud_storage else 1) if self._use_gcloud_storage: self.assertIn( 'Expected a non-negative integer value or a range of such values instead of', stderr) else: self.assertIn('Invalid range', stderr) # Test various valid ranges. stdout = self.RunGsUtil(['cat', '-r', '-', suri(key_uri)], return_stdout=True) self.assertEqual('0123456789', stdout) stdout = self.RunGsUtil( ['cat', '-r', '1000-3000', suri(key_uri)], return_stdout=True) self.assertEqual('', stdout) stdout = self.RunGsUtil( ['cat', '-r', '1000-', suri(key_uri)], return_stdout=True) self.assertEqual('', stdout) stdout = self.RunGsUtil( ['cat', '-r', '1-3', suri(key_uri)], return_stdout=True) self.assertEqual('123', stdout) stdout = self.RunGsUtil( ['cat', '-r', '8-', suri(key_uri)], return_stdout=True) self.assertEqual('89', stdout) stdout = self.RunGsUtil( ['cat', '-r', '0-0', suri(key_uri)], return_stdout=True) self.assertEqual('0', stdout) stdout = self.RunGsUtil( ['cat', '-r', '-3', suri(key_uri)], return_stdout=True) self.assertEqual('789', stdout) def test_cat_version(self): """Tests cat command on versioned objects.""" bucket_uri = self.CreateVersionedBucket() # Create 2 versions of an object. uri1 = self.CreateObject(bucket_uri=bucket_uri, contents=b'data1', gs_idempotent_generation=0) uri2 = self.CreateObject(bucket_uri=bucket_uri, object_name=uri1.object_name, contents=b'data2', gs_idempotent_generation=urigen(uri1)) stdout = self.RunGsUtil(['cat', suri(uri1)], return_stdout=True) # Last version written should be live. self.assertEqual('data2', stdout) # Using either version-specific URI should work. stdout = self.RunGsUtil(['cat', uri1.version_specific_uri], return_stdout=True) self.assertEqual('data1', stdout) stdout = self.RunGsUtil(['cat', uri2.version_specific_uri], return_stdout=True) self.assertEqual('data2', stdout) if RUN_S3_TESTS: # S3 GETs of invalid versions return 400s. # Also, appending between 1 and 3 characters to the version_id can # result in a success (200) response from the server. stderr = self.RunGsUtil(['cat', uri2.version_specific_uri + '23456'], return_stderr=True, expected_status=1) self.assertIn('BadRequestException: 400', stderr) else: # Attempting to cat invalid version should result in an error. stderr = self.RunGsUtil(['cat', uri2.version_specific_uri + '23'], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn( 'The following URLs matched no objects or files:\n{}23\n'.format( uri2.version_specific_uri), stderr) else: self.assertIn(NO_URLS_MATCHED_TARGET % uri2.version_specific_uri + '23', stderr) def test_cat_multi_arg(self): """Tests cat command with multiple arguments.""" bucket_uri = self.CreateBucket() data1 = b'0123456789' data2 = b'abcdefghij' obj_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents=data1) obj_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents=data2) stdout, stderr = self.RunGsUtil( ['cat', suri(obj_uri1), suri(bucket_uri) + 'nonexistent'], return_stdout=True, return_stderr=True, expected_status=1) # First object should print, second should produce an exception. self.assertIn(data1.decode('ascii'), stdout) if self._use_gcloud_storage: self.assertIn('The following URLs matched no objects or files', stderr) else: self.assertIn('NotFoundException', stderr) stdout, stderr = self.RunGsUtil( ['cat', suri(bucket_uri) + 'nonexistent', suri(obj_uri1)], return_stdout=True, return_stderr=True, expected_status=1) decoded_data1 = data1.decode('ascii') if self._use_gcloud_storage: self.assertIn(decoded_data1, stdout) self.assertIn('The following URLs matched no objects or files', stderr) else: # If first object is invalid, exception should halt output immediately. self.assertNotIn(decoded_data1, stdout) self.assertIn('NotFoundException', stderr) # Two valid objects should both print successfully. stdout = self.RunGsUtil( ['cat', suri(obj_uri1), suri(obj_uri2)], return_stdout=True) self.assertIn(decoded_data1 + data2.decode('ascii'), stdout) @SkipForS3('S3 customer-supplied encryption keys are not supported.') def test_cat_encrypted_object(self): if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') object_contents = b'0123456789' object_uri = self.CreateObject(object_name='foo', contents=object_contents, encryption_key=TEST_ENCRYPTION_KEY1) stderr = self.RunGsUtil(['cat', suri(object_uri)], expected_status=1, return_stderr=True) self.assertIn('No decryption key matches object', stderr) boto_config_for_test = [('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)] with SetBotoConfigForTest(boto_config_for_test): stdout = self.RunGsUtil(['cat', suri(object_uri)], return_stdout=True) self.assertEqual(stdout.encode('ascii'), object_contents) stdout = self.RunGsUtil( ['cat', '-r', '1-3', suri(object_uri)], return_stdout=True) self.assertEqual(stdout, '123') class TestShimCatFlags(testcase.ShimUnitTestBase): """Unit tests for shimming cat flags""" def test_shim_translates_flags(self): object_uri = self.CreateObject(contents='0123456789') with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'cat', ['-h', '-r', '2-4', suri(object_uri)], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage cat' ' -d -r 2-4 {}'.format( shim_util._get_gcloud_binary_path('fake_dir'), suri(object_uri)), info_lines) class TestCatHelper(testcase.GsUtilUnitTestCase): """Unit tests for cat helper.""" def test_cat_helper_runs_flush(self): cat_command_mock = mock.Mock() cat_helper_mock = cat_helper.CatHelper(command_obj=cat_command_mock) object_contents = '0123456789' bucket_uri = self.CreateBucket(bucket_name='bucket', provider=self.default_provider) obj = self.CreateObject(bucket_uri=bucket_uri, object_name='foo1', contents=object_contents) obj1 = self.CreateObject(bucket_uri=bucket_uri, object_name='foo2', contents=object_contents) cat_command_mock.WildcardIterator.return_value = self._test_wildcard_iterator( 'gs://bucket/foo*') stdout_mock = mock.mock_open()() # Mocks two functions because we need to record the order of the # function calls (write, flush, write, flush). write_flush_collector_mock = mock.Mock() cat_command_mock.gsutil_api.GetObjectMedia = write_flush_collector_mock stdout_mock.flush = write_flush_collector_mock cat_helper_mock.CatUrlStrings(url_strings=['url'], cat_out_fd=stdout_mock) mock_part_one = [ mock.call('bucket', 'foo1', stdout_mock, compressed_encoding=None, start_byte=0, end_byte=None, object_size=10, generation=None, decryption_tuple=None, provider='gs'), mock.call() ] mock_part_two = [ mock.call('bucket', 'foo2', stdout_mock, compressed_encoding=None, start_byte=0, end_byte=None, object_size=10, generation=None, decryption_tuple=None, provider='gs'), mock.call() ] # Needed to do these two checks because the object # ordering varies if run on Windows with Python 3.5. self.assertIn(write_flush_collector_mock.call_args_list[0:2], [mock_part_one, mock_part_two]) self.assertIn(write_flush_collector_mock.call_args_list[2:4], [mock_part_one, mock_part_two]) tests/test_gsutil.py 0000644 00000011267 15025013616 0010634 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for top-level gsutil command.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals import importlib import unittest from unittest import mock from google.auth import exceptions as google_auth_exceptions from gslib.command_runner import CommandRunner from gslib.utils import system_util import gslib import gslib.tests.testcase as testcase try: from gsutil import _fix_google_module # pylint: disable=g-import-not-at-top FIX_GOOGLE_MODULE_FUNCTION_AVAILABLE = True except ImportError: FIX_GOOGLE_MODULE_FUNCTION_AVAILABLE = False class TestGsUtil(testcase.GsUtilIntegrationTestCase): """Integration tests for top-level gsutil command.""" def test_long_version_arg(self): stdout = self.RunGsUtil(['--version'], return_stdout=True) self.assertEqual('gsutil version: %s\n' % gslib.VERSION, stdout) def test_version_command(self): stdout = self.RunGsUtil(['version'], return_stdout=True) self.assertEqual('gsutil version: %s\n' % gslib.VERSION, stdout) def test_version_long(self): stdout = self.RunGsUtil(['version', '-l'], return_stdout=True) self.assertIn('gsutil version: %s\n' % gslib.VERSION, stdout) self.assertIn('boto version', stdout) self.assertIn('checksum', stdout) self.assertIn('config path', stdout) self.assertIn('gsutil path', stdout) class TestGsUtilUnit(testcase.GsUtilUnitTestCase): """Unit tests for top-level gsutil command.""" @unittest.skipUnless( FIX_GOOGLE_MODULE_FUNCTION_AVAILABLE, 'The gsutil.py file is not available for certain installations like pip.') @mock.patch.object(importlib, 'reload', autospec=True) def test_fix_google_module(self, mock_reload): with mock.patch.dict('sys.modules', {'google': 'google'}): _fix_google_module() mock_reload.assert_called_once_with('google') @unittest.skipUnless( FIX_GOOGLE_MODULE_FUNCTION_AVAILABLE, 'The gsutil.py file is not available for certain installations like pip.') @mock.patch.object(importlib, 'reload', autospec=True) def test_fix_google_module_does_not_reload_if_module_missing( self, mock_reload): with mock.patch.dict('sys.modules', {}, clear=True): _fix_google_module() self.assertFalse(mock_reload.called) @mock.patch.object(system_util, 'InvokedViaCloudSdk', autospec=True) @mock.patch.object(gslib.__main__, '_OutputAndExit', autospec=True) def test_translates_oauth_error_cloudsdk(self, mock_output_and_exit, mock_invoke_via_cloud_sdk): mock_invoke_via_cloud_sdk.return_value = True command_runner = CommandRunner() with mock.patch.object(command_runner, 'RunNamedCommand') as mock_run: fake_error = google_auth_exceptions.OAuthError('fake error message') mock_run.side_effect = fake_error gslib.__main__._RunNamedCommandAndHandleExceptions(command_runner, command_name='fake') mock_output_and_exit.assert_called_once_with( 'Your credentials are invalid. Please run\n$ gcloud auth login', fake_error) @mock.patch.object(system_util, 'InvokedViaCloudSdk', autospec=True) @mock.patch.object(gslib.__main__, '_OutputAndExit', autospec=True) def test_translates_oauth_error_standalone(self, mock_output_and_exit, mock_invoke_via_cloud_sdk): mock_invoke_via_cloud_sdk.return_value = False command_runner = CommandRunner() with mock.patch.object(command_runner, 'RunNamedCommand') as mock_run: fake_error = google_auth_exceptions.OAuthError('fake error message') mock_run.side_effect = fake_error gslib.__main__._RunNamedCommandAndHandleExceptions(command_runner, command_name='fake') mock_output_and_exit.assert_called_once_with( 'Your credentials are invalid. For more help, see ' '"gsutil help creds", or re-run the gsutil config command (see ' '"gsutil help config").', fake_error) tests/test_mtls.py 0000644 00000004015 15025013616 0010275 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2020 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for mTLS authentication.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from boto import config from gslib.tests import testcase from gslib.tests.testcase import integration_testcase from gslib.tests.util import unittest MTLS_AVAILABILITY_MESSAGE = ( 'mTLS/DCA authentication is only available for the GCS JSON API.') class TestMtls(testcase.GsUtilIntegrationTestCase): """Integration tests for mTLS authentication.""" @unittest.skipIf( not config.getbool('Credentials', 'use_client_certificate'), 'mTLS requires "use_client_certificate" to be "True" in .boto config.') @integration_testcase.SkipForXML(MTLS_AVAILABILITY_MESSAGE) @integration_testcase.SkipForS3(MTLS_AVAILABILITY_MESSAGE) def test_can_list_bucket_with_mtls_authentication(self): # Cannot use self.CreateBucket because testing framework's authentication # doesn't work with mTLS. bucket_uri = 'gs://{}'.format(self.MakeTempName('bucket')) self.RunGsUtil(['mb', bucket_uri]) stdout = self.RunGsUtil(['-D', 'ls'], return_stdout=True) self.RunGsUtil(['rb', bucket_uri]) # # Check if mTLS API endpoint was hit in debug output. self.assertIn('storage.mtls.googleapis.com', stdout) # # If bucket was successfully listed, it implies successful authentication. self.assertIn(bucket_uri, stdout) tests/testcase/base.py 0000644 00000023205 15025013616 0011006 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Base test case class for unit and integration tests.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from functools import wraps import os.path import random import re import shutil import tempfile import six import boto import gslib.tests.util as util from gslib.tests.util import unittest from gslib.utils.constants import UTF8 from gslib.utils.posix_util import NA_ID from gslib.utils.posix_util import NA_MODE MAX_BUCKET_LENGTH = 63 def NotParallelizable(test): """Wrapper for cases that are not parallelizable.""" setattr(test, 'is_parallelizable', False) return test def RequiresIsolation(func): """Wrapper function for cases that require running in a separate process.""" @wraps(func) def RequiresIsolationFunc(*args, **kwargs): return func(*args, **kwargs) RequiresIsolationFunc.requires_isolation = True return RequiresIsolationFunc class GsUtilTestCase(unittest.TestCase): """Base test case class for unit and integration tests.""" def setUp(self): if six.PY2: self.assertRegex = self.assertRegexpMatches self.assertNotRegex = self.assertNotRegexpMatches if util.RUN_S3_TESTS: self.test_api = 'XML' self.default_provider = 's3' self.provider_custom_meta = 'amz' else: self.test_api = boto.config.get('GSUtil', 'prefer_api', 'JSON').upper() self.default_provider = 'gs' self.provider_custom_meta = 'goog' self.tempdirs = [] def tearDown(self): while self.tempdirs: tmpdir = self.tempdirs.pop() shutil.rmtree(tmpdir, ignore_errors=True) def assertNumLines(self, text, numlines): self.assertEqual(text.count('\n'), numlines) def GetTestMethodName(self): return self._testMethodName def MakeRandomTestString(self): """Creates a random string of hex characters 8 characters long.""" return '%08x' % random.randrange(256**4) def MakeTempName(self, kind, prefix='', suffix=''): """Creates a temporary name that is most-likely unique. Args: kind (str): A string indicating what kind of test name this is. prefix (str): Prefix prepended to the temporary name. suffix (str): Suffix string appended to end of temporary name. Returns: (str) The temporary name. If `kind` was "bucket", the temporary name may have coerced this string, including the supplied `prefix`, such that it contains only characters that are valid across all supported storage providers (e.g. replacing "_" with "-", converting uppercase letters to lowercase, etc.). """ name = '{prefix}gsutil-test-{method}-{kind}'.format( prefix=prefix, method=self.GetTestMethodName(), kind=kind) name = name[:MAX_BUCKET_LENGTH - 13] name = '{name}-{rand}'.format(name=name, rand=self.MakeRandomTestString()) total_name_len = len(name) + len(suffix) if suffix: if kind == 'bucket' and total_name_len > MAX_BUCKET_LENGTH: self.fail( 'Tried to create a psuedo-random bucket name with a specific ' 'suffix, but the generated name was too long and there was not ' 'enough room for the suffix. Please use shorter strings or perform ' 'name randomization manually.\nRequested name: ' + name + suffix) name += suffix if kind == 'bucket': name = util.MakeBucketNameValid(name) return name # TODO: Convert tests to use this for object names. def MakeTempUnicodeName(self, kind, prefix=''): return self.MakeTempName(kind, prefix=prefix) + '材' def CreateTempDir(self, test_files=0, contents=None): """Creates a temporary directory on disk. The directory and all of its contents will be deleted after the test. Args: test_files: The number of test files to place in the directory or a list of test file names. contents: The contents for each generated test file. Returns: The path to the new temporary directory. """ tmpdir = tempfile.mkdtemp(prefix=self.MakeTempName('directory')) self.tempdirs.append(tmpdir) try: iter(test_files) except TypeError: test_files = [self.MakeTempName('file') for _ in range(test_files)] for i, name in enumerate(test_files): contents_file = contents if contents_file is None: contents_file = ('test %d' % i).encode('ascii') self.CreateTempFile(tmpdir=tmpdir, file_name=name, contents=contents_file) return tmpdir def CreateTempFifo(self, tmpdir=None, file_name=None): """Creates a temporary fifo file on disk. Should not be used on Windows. Args: tmpdir: The temporary directory to place the file in. If not specified, a new temporary directory is created. file_name: The name to use for the file. If not specified, a temporary test file name is constructed. This can also be a tuple, where ('dir', 'foo') means to create a file named 'foo' inside a subdirectory named 'dir'. Returns: The path to the new temporary fifo. """ tmpdir = tmpdir or self.CreateTempDir() file_name = file_name or self.MakeTempName('fifo') if isinstance(file_name, six.string_types): fpath = os.path.join(tmpdir, file_name) else: fpath = os.path.join(tmpdir, *file_name) os.mkfifo(fpath) return fpath def CreateTempFile(self, tmpdir=None, contents=None, file_name=None, mtime=None, mode=NA_MODE, uid=NA_ID, gid=NA_ID): """Creates a temporary file on disk. Note: if mode, uid, or gid are present, they must be validated by ValidateFilePermissionAccess and ValidatePOSIXMode before calling this function. Args: tmpdir: The temporary directory to place the file in. If not specified, a new temporary directory is created. contents: The contents to write to the file. If not specified, a test string is constructed and written to the file. Since the file is opened 'wb', the contents must be bytes. file_name: The name to use for the file. If not specified, a temporary test file name is constructed. This can also be a tuple, where ('dir', 'foo') means to create a file named 'foo' inside a subdirectory named 'dir'. mtime: The modification time of the file in POSIX time (seconds since UTC 1970-01-01). If not specified, this defaults to the current system time. mode: The POSIX mode for the file. Must be a base-8 3-digit integer represented as a string. uid: A POSIX user ID. gid: A POSIX group ID. Returns: The path to the new temporary file. """ tmpdir = six.ensure_str(tmpdir or self.CreateTempDir()) file_name = file_name or self.MakeTempName(str('file')) if isinstance(file_name, (six.text_type, six.binary_type)): fpath = os.path.join(tmpdir, six.ensure_str(file_name)) else: file_name = map(six.ensure_str, file_name) fpath = os.path.join(tmpdir, *file_name) if not os.path.isdir(os.path.dirname(fpath)): os.makedirs(os.path.dirname(fpath)) if isinstance(fpath, six.binary_type): fpath = fpath.decode(UTF8) with open(fpath, 'wb') as f: contents = (contents if contents is not None else self.MakeTempName( str('contents'))) if isinstance(contents, bytearray): contents = bytes(contents) else: contents = six.ensure_binary(contents) f.write(contents) if mtime is not None: # Set the atime and mtime to be the same. os.utime(fpath, (mtime, mtime)) if uid != NA_ID or int(gid) != NA_ID: os.chown(fpath, uid, int(gid)) if int(mode) != NA_MODE: os.chmod(fpath, int(mode, 8)) return fpath def assertRegexpMatchesWithFlags(self, text, pattern, msg=None, flags=0): """Like assertRegexpMatches, but allows specifying additional re flags. Args: text: The text in which to search for pattern. pattern: The pattern to search for; should be either a string or compiled regex returned from re.compile(). msg: The message to be displayed if pattern is not found in text. The values for pattern and text will be included after this message. flags: Additional flags from the re module to be included when compiling pattern. If pattern is a regex that was compiled with existing flags, these, flags will be added via a bitwise-or. """ if isinstance(pattern, six.string_types): pattern = re.compile(pattern, flags=flags) else: # It's most likely an already-compiled pattern. pattern = re.compile(pattern.pattern, flags=pattern.flags | flags) if not pattern.search(text): failure_msg = msg or 'Regex didn\'t match' failure_msg = '%s: %r not found in %r' % (failure_msg, pattern.pattern, text) raise self.failureException(failure_msg) tests/testcase/shim_unit_test_base.py 0000644 00000003446 15025013616 0014131 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2023 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains base class to be used for shim unit tests.""" import subprocess from unittest import mock from gslib.tests.testcase.unit_testcase import GsUtilUnitTestCase class ShimUnitTestBase(GsUtilUnitTestCase): """Base class for unit testing shim behavior. This class mocks the `subprocess.run()` method because it gets called for all shim operations to check if there is an active account configured for gcloud. """ def setUp(self): super().setUp() # Translator calls `gcloud config get account` to check active account # using subprocess.run(). # We don't care about this call for most of the tests below, so we are # simply patching this here. # There are separate tests to check this call is being made. self._subprocess_run_patcher = mock.patch.object(subprocess, 'run', autospec=True) self._mock_subprocess_run = self._subprocess_run_patcher.start() self._mock_subprocess_run.return_value.returncode = 0 def tearDown(self): if self._subprocess_run_patcher is not None: self._subprocess_run_patcher.stop() super().tearDown() tests/testcase/unit_testcase.py 0000644 00000042344 15025013616 0012753 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains gsutil base unit test case class.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import sys import tempfile import six import boto from boto.utils import get_utf8able_str from gslib import project_id from gslib import wildcard_iterator from gslib.boto_translation import BotoTranslation from gslib.cloud_api_delegator import CloudApiDelegator from gslib.command_runner import CommandRunner from gslib.cs_api_map import ApiMapConstants from gslib.cs_api_map import ApiSelector from gslib.discard_messages_queue import DiscardMessagesQueue from gslib.gcs_json_api import GcsJsonApi from gslib.tests.mock_logging_handler import MockLoggingHandler from gslib.tests.testcase import base import gslib.tests.util as util from gslib.tests.util import unittest from gslib.tests.util import WorkingDirectory from gslib.utils.constants import UTF8 from gslib.utils.text_util import print_to_fd def _AttemptToCloseSysFd(fd): """Suppress IOError when closing sys.stdout or sys.stderr in tearDown.""" # In PY2, if another sibling thread/process tried closing it at the same # time we did, it succeeded either way, so we just continue. This approach # was taken from https://github.com/pytest-dev/pytest/pull/3305. if not six.PY2: # This doesn't happen in PY3, AFAICT. fd.close() return try: fd.close() except IOError: pass class GsutilApiUnitTestClassMapFactory(object): """Class map factory for use in unit tests. BotoTranslation is used for all cases so that GSMockBucketStorageUri can be used to communicate with the mock XML service. """ @classmethod def GetClassMap(cls): """Returns a class map for use in unit tests.""" gs_class_map = { ApiSelector.XML: BotoTranslation, # TODO: This should be replaced with 'ApiSelector.JSON: GcsJsonApi'. # Refer Issue https://github.com/GoogleCloudPlatform/gsutil/issues/970 ApiSelector.JSON: BotoTranslation } s3_class_map = {ApiSelector.XML: BotoTranslation} class_map = {'gs': gs_class_map, 's3': s3_class_map} return class_map @unittest.skipUnless(util.RUN_UNIT_TESTS, 'Not running integration tests.') class GsUtilUnitTestCase(base.GsUtilTestCase): """Base class for gsutil unit tests.""" @classmethod def setUpClass(cls): base.GsUtilTestCase.setUpClass() cls.mock_bucket_storage_uri = util.GSMockBucketStorageUri cls.mock_gsutil_api_class_map_factory = GsutilApiUnitTestClassMapFactory cls.logger = logging.getLogger() cls.command_runner = CommandRunner( bucket_storage_uri_class=cls.mock_bucket_storage_uri, gsutil_api_class_map_factory=cls.mock_gsutil_api_class_map_factory) # Ensure unit tests don't fail if no default_project_id is defined in the # boto config file. project_id.UNIT_TEST_PROJECT_ID = 'mock-project-id-for-unit-tests' def setUp(self): super(GsUtilUnitTestCase, self).setUp() self.bucket_uris = [] self.stdout_save = sys.stdout self.stderr_save = sys.stderr fd, self.stdout_file = tempfile.mkstemp() # Specify the encoding explicitly to ensure Windows uses 'utf-8' instead of # the default of 'cp1252'. if six.PY2: sys.stdout = os.fdopen(fd, 'w+') else: sys.stdout = os.fdopen(fd, 'w+', encoding='utf-8') fd, self.stderr_file = tempfile.mkstemp() # do not set sys.stderr to be 'wb+' - it will blow up the logger if six.PY2: sys.stderr = os.fdopen(fd, 'w+') else: sys.stderr = os.fdopen(fd, 'w+', encoding='utf-8') self.accumulated_stdout = [] self.accumulated_stderr = [] self.root_logger = logging.getLogger() self.is_debugging = self.root_logger.isEnabledFor(logging.DEBUG) self.log_handlers_save = self.root_logger.handlers fd, self.log_handler_file = tempfile.mkstemp() self.log_handler_stream = os.fdopen(fd, 'w+') self.temp_log_handler = logging.StreamHandler(self.log_handler_stream) self.root_logger.handlers = [self.temp_log_handler] def tearDown(self): super(GsUtilUnitTestCase, self).tearDown() self.root_logger.handlers = self.log_handlers_save self.temp_log_handler.flush() self.temp_log_handler.close() self.log_handler_stream.seek(0) log_output = self.log_handler_stream.read() self.log_handler_stream.close() os.unlink(self.log_handler_file) sys.stdout.seek(0) sys.stderr.seek(0) if six.PY2: stdout = sys.stdout.read() stderr = sys.stderr.read() else: try: stdout = sys.stdout.read() stderr = sys.stderr.read() except UnicodeDecodeError: sys.stdout.seek(0) sys.stderr.seek(0) stdout = sys.stdout.buffer.read() stderr = sys.stderr.buffer.read() [six.ensure_text(string) for string in self.accumulated_stderr] [six.ensure_text(string) for string in self.accumulated_stdout] stdout = six.ensure_text(get_utf8able_str(stdout)) stderr = six.ensure_text(get_utf8able_str(stderr)) stdout += ''.join(self.accumulated_stdout) stderr += ''.join(self.accumulated_stderr) _AttemptToCloseSysFd(sys.stdout) _AttemptToCloseSysFd(sys.stderr) sys.stdout = self.stdout_save sys.stderr = self.stderr_save os.unlink(self.stdout_file) os.unlink(self.stderr_file) _id = six.ensure_text(self.id()) if self.is_debugging and stdout: print_to_fd('==== stdout {} ====\n'.format(_id), file=sys.stderr) print_to_fd(stdout, file=sys.stderr) print_to_fd('==== end stdout ====\n', file=sys.stderr) if self.is_debugging and stderr: print_to_fd('==== stderr {} ====\n'.format(_id), file=sys.stderr) print_to_fd(stderr, file=sys.stderr) print_to_fd('==== end stderr ====\n', file=sys.stderr) if self.is_debugging and log_output: print_to_fd('==== log output {} ====\n'.format(_id), file=sys.stderr) print_to_fd(log_output, file=sys.stderr) print_to_fd('==== end log output ====\n', file=sys.stderr) def RunCommand(self, command_name, args=None, headers=None, debug=0, return_stdout=False, return_stderr=False, return_log_handler=False, cwd=None): """Method for calling gslib.command_runner.CommandRunner. Passes parallel_operations=False for all tests, optionally saving/returning stdout output. We run all tests multi-threaded, to exercise those more complicated code paths. TODO: Change to run with parallel_operations=True for all tests. At present when you do this it causes many test failures. Args: command_name: The name of the command being run. args: Command-line args (arg0 = actual arg, not command name ala bash). headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). return_stdout: If True, will save and return stdout produced by command. return_stderr: If True, will save and return stderr produced by command. return_log_handler: If True, will return a MockLoggingHandler instance that was attached to the command's logger while running. cwd: The working directory that should be switched to before running the command. The working directory will be reset back to its original value after running the command. If not specified, the working directory is left unchanged. Returns: One or a tuple of requested return values, depending on whether return_stdout, return_stderr, and/or return_log_handler were specified. Return Types: stdout - str (binary in Py2, text in Py3) stderr - str (binary in Py2, text in Py3) log_handler - MockLoggingHandler """ args = args or [] command_line = six.ensure_text(' '.join([command_name] + args)) if self.is_debugging: print_to_fd('\nRunCommand of {}\n'.format(command_line), file=self.stderr_save) # Save and truncate stdout and stderr for the lifetime of RunCommand. This # way, we can return just the stdout and stderr that was output during the # RunNamedCommand call below. sys.stdout.seek(0) sys.stderr.seek(0) stdout = sys.stdout.read() stderr = sys.stderr.read() if stdout: self.accumulated_stdout.append(stdout) if stderr: self.accumulated_stderr.append(stderr) sys.stdout.seek(0) sys.stderr.seek(0) sys.stdout.truncate() sys.stderr.truncate() mock_log_handler = MockLoggingHandler() logging.getLogger(command_name).addHandler(mock_log_handler) if debug: logging.getLogger(command_name).setLevel(logging.DEBUG) try: with WorkingDirectory(cwd): self.command_runner.RunNamedCommand(command_name, args=args, headers=headers, debug=debug, parallel_operations=False, do_shutdown=False) finally: sys.stdout.seek(0) sys.stderr.seek(0) if six.PY2: stdout = sys.stdout.read() stderr = sys.stderr.read() else: try: stdout = sys.stdout.read() stderr = sys.stderr.read() except UnicodeDecodeError: sys.stdout.seek(0) sys.stderr.seek(0) stdout = sys.stdout.buffer.read() stderr = sys.stderr.buffer.read() logging.getLogger(command_name).removeHandler(mock_log_handler) mock_log_handler.close() log_output = '\n'.join( '%s:\n ' % level + '\n '.join(records) for level, records in six.iteritems(mock_log_handler.messages) if records) _id = six.ensure_text(self.id()) if self.is_debugging and log_output: print_to_fd('==== logging RunCommand {} {} ====\n'.format( _id, command_line), file=self.stderr_save) print_to_fd(log_output, file=self.stderr_save) print_to_fd('\n==== end logging ====\n', file=self.stderr_save) if self.is_debugging and stdout: print_to_fd('==== stdout RunCommand {} {} ====\n'.format( _id, command_line), file=self.stderr_save) print_to_fd(stdout, file=self.stderr_save) print_to_fd('==== end stdout ====\n', file=self.stderr_save) if self.is_debugging and stderr: print_to_fd('==== stderr RunCommand {} {} ====\n'.format( _id, command_line), file=self.stderr_save) print_to_fd(stderr, file=self.stderr_save) print_to_fd('==== end stderr ====\n', file=self.stderr_save) # Reset stdout and stderr files, so that we won't print them out again # in tearDown if debugging is enabled. sys.stdout.seek(0) sys.stderr.seek(0) sys.stdout.truncate() sys.stderr.truncate() to_return = [] if return_stdout: to_return.append(stdout) if return_stderr: to_return.append(stderr) if return_log_handler: to_return.append(mock_log_handler) if len(to_return) == 1: return to_return[0] return tuple(to_return) @classmethod def MakeGsUtilApi(cls, debug=0): gsutil_api_map = { ApiMapConstants.API_MAP: (cls.mock_gsutil_api_class_map_factory.GetClassMap()), ApiMapConstants.SUPPORT_MAP: { 'gs': [ApiSelector.XML, ApiSelector.JSON], 's3': [ApiSelector.XML] }, ApiMapConstants.DEFAULT_MAP: { 'gs': ApiSelector.JSON, 's3': ApiSelector.XML } } return CloudApiDelegator(cls.mock_bucket_storage_uri, gsutil_api_map, cls.logger, DiscardMessagesQueue(), debug=debug) @classmethod def _test_wildcard_iterator(cls, uri_or_str, exclude_tuple=None, debug=0): """Convenience method for instantiating a test instance of WildcardIterator. This makes it unnecessary to specify all the params of that class (like bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri). Also, naming the factory method this way makes it clearer in the test code that WildcardIterator needs to be set up for testing. Args are same as for wildcard_iterator.wildcard_iterator(), except there are no class args for bucket_storage_uri_class or gsutil_api_class. Args: uri_or_str: StorageUri or string representing the wildcard string. exclude_tuple: (base_url, exclude_pattern), where base_url is top-level URL to list; exclude_pattern is a regex of paths to ignore during iteration. debug: debug level to pass to the underlying connection (0..3) Returns: WildcardIterator, over which caller can iterate. """ # TODO: Remove when tests no longer pass StorageUri arguments. uri_string = uri_or_str if hasattr(uri_or_str, 'uri'): uri_string = uri_or_str.uri return wildcard_iterator.CreateWildcardIterator(uri_string, cls.MakeGsUtilApi(debug), exclude_tuple=exclude_tuple) @staticmethod def _test_storage_uri(uri_str, default_scheme='file', debug=0, validate=True): """Convenience method for instantiating a testing instance of StorageUri. This makes it unnecessary to specify bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri. Also naming the factory method this way makes it clearer in the test code that StorageUri needs to be set up for testing. Args, Returns, and Raises are same as for boto.storage_uri(), except there's no bucket_storage_uri_class arg. Args: uri_str: Uri string to create StorageUri for. default_scheme: Default scheme for the StorageUri debug: debug level to pass to the underlying connection (0..3) validate: If True, validate the resource that the StorageUri refers to. Returns: StorageUri based on the arguments. """ return boto.storage_uri(uri_str, default_scheme, debug, validate, util.GSMockBucketStorageUri) def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None, provider='gs'): """Creates a test bucket. The bucket and all of its contents will be deleted after the test. Args: bucket_name: Create the bucket with this name. If not provided, a temporary test bucket name is constructed. test_objects: The number of objects that should be placed in the bucket or a list of object names to place in the bucket. Defaults to 0. storage_class: storage class to use. If not provided we us standard. provider: string provider to use, default gs. Returns: StorageUri for the created bucket. """ bucket_name = bucket_name or self.MakeTempName('bucket') bucket_uri = boto.storage_uri( '%s://%s' % (provider, bucket_name.lower()), suppress_consec_slashes=False, bucket_storage_uri_class=util.GSMockBucketStorageUri) bucket_uri.create_bucket(storage_class=storage_class) self.bucket_uris.append(bucket_uri) try: iter(test_objects) except TypeError: test_objects = [self.MakeTempName('obj') for _ in range(test_objects)] for i, name in enumerate(test_objects): self.CreateObject(bucket_uri=bucket_uri, object_name=name, contents='test {}'.format(i).encode(UTF8)) return bucket_uri def CreateObject(self, bucket_uri=None, object_name=None, contents=None): """Creates a test object. Args: bucket_uri: The URI of the bucket to place the object in. If not specified, a new temporary bucket is created. object_name: The name to use for the object. If not specified, a temporary test object name is constructed. contents: The contents to write to the object. If not specified, the key is not written to, which means that it isn't actually created yet on the server. Returns: A StorageUri for the created object. """ bucket_uri = bucket_uri or self.CreateBucket(provider=self.default_provider) object_name = object_name or self.MakeTempName('obj') key_uri = bucket_uri.clone_replace_name(object_name) if contents is not None: key_uri.set_contents_from_string(contents) return key_uri tests/testcase/__init__.py 0000644 00000002104 15025013616 0011626 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains test case base classes for gsutil tests.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.tests.testcase.integration_testcase import GsUtilIntegrationTestCase from gslib.tests.util import KmsTestingResources from gslib.tests.testcase.unit_testcase import GsUtilUnitTestCase from gslib.tests.testcase.shim_unit_test_base import ShimUnitTestBase tests/testcase/integration_testcase.py 0000644 00000151120 15025013616 0014310 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains gsutil base integration test case class.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals import contextlib import datetime import locale import logging import os import re import signal import subprocess import sys import tempfile import threading import time import boto from boto import config from boto.exception import StorageResponseError from boto.s3.deletemarker import DeleteMarker from boto.storage_uri import BucketStorageUri import gslib from gslib.boto_translation import BotoTranslation from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.discard_messages_queue import DiscardMessagesQueue from gslib.exception import CommandException from gslib.gcs_json_api import GcsJsonApi from gslib.kms_api import KmsApi from gslib.project_id import GOOG_PROJ_ID_HDR from gslib.project_id import PopulateProjectId from gslib.tests.testcase import base import gslib.tests.util as util from gslib.tests.util import InvokedFromParFile from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import RUN_S3_TESTS from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import unittest from gslib.tests.util import USING_JSON_API import gslib.third_party.storage_apitools.storage_v1_messages as apitools_messages from gslib.utils.constants import UTF8 from gslib.utils.encryption_helper import Base64Sha256FromBase64EncryptionKey from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.hashing_helper import Base64ToHexHash from gslib.utils.metadata_util import CreateCustomMetadata from gslib.utils.metadata_util import GetValueFromObjectCustomMetadata from gslib.utils.posix_util import ATIME_ATTR from gslib.utils.posix_util import GID_ATTR from gslib.utils.posix_util import MODE_ATTR from gslib.utils.posix_util import MTIME_ATTR from gslib.utils.posix_util import UID_ATTR from gslib.utils.retry_util import Retry import six from six.moves import range LOGGER = logging.getLogger('integration-test') # TODO: Replace tests which looks for test_api == ApiSelector.(XML|JSON) with # these decorators. def SkipForXML(reason): """Skips the test if running S3 tests, or if prefer_api isn't set to json.""" if not USING_JSON_API or RUN_S3_TESTS: return unittest.skip(reason) else: return lambda func: func def SkipForJSON(reason): if USING_JSON_API: return unittest.skip(reason) else: return lambda func: func def SkipForGS(reason): if not RUN_S3_TESTS: return unittest.skip(reason) else: return lambda func: func def SkipForS3(reason): if RUN_S3_TESTS: return unittest.skip(reason) else: return lambda func: func # TODO: Right now, most tests use the XML API. Instead, they should respect # prefer_api in the same way that commands do. @unittest.skipUnless(util.RUN_INTEGRATION_TESTS, 'Not running integration tests.') class GsUtilIntegrationTestCase(base.GsUtilTestCase): """Base class for gsutil integration tests.""" GROUP_TEST_ADDRESS = 'gs-discussion@googlegroups.com' GROUP_TEST_ID = ( '00b4903a97d097895ab58ef505d535916a712215b79c3e54932c2eb502ad97f5') USER_TEST_ADDRESS = 'gsutiltesting123@gmail.com' # This is the legacy CanonicalID for the above email. USER_TEST_ID = ( '00b4903a97f0baa2680740f5adb90b2dcf9c8b878abd84ba1bdba653de949619') DOMAIN_TEST = 'google.com' # No one can create this bucket without owning the gmail.com domain, and we # won't create this bucket, so it shouldn't exist. # It would be nice to use google.com here but JSON API disallows # 'google' in resource IDs. nonexistent_bucket_name = 'nonexistent-bucket-foobar.gmail.com' def setUp(self): """Creates base configuration for integration tests.""" super(GsUtilIntegrationTestCase, self).setUp() self.bucket_uris = [] # Set up API version and project ID handler. self.api_version = boto.config.get_value('GSUtil', 'default_api_version', '1') # Instantiate a JSON API for use by the current integration test. self.json_api = GcsJsonApi(BucketStorageUri, logging.getLogger(), DiscardMessagesQueue(), 'gs') self.xml_api = BotoTranslation(BucketStorageUri, logging.getLogger(), DiscardMessagesQueue, self.default_provider) self.kms_api = KmsApi(logging.getLogger()) self.multiregional_buckets = util.USE_MULTIREGIONAL_BUCKETS self._use_gcloud_storage = config.getbool('GSUtil', 'use_gcloud_storage', False) if util.RUN_S3_TESTS: self.nonexistent_bucket_name = ( 'nonexistentbucket-asf801rj3r9as90mfnnkjxpo02') # Retry with an exponential backoff if a server error is received. This # ensures that we try *really* hard to clean up after ourselves. # TODO: As long as we're still using boto to do the teardown, # we decorate with boto exceptions. Eventually this should be migrated # to CloudApi exceptions. @Retry(StorageResponseError, tries=7, timeout_secs=1) def tearDown(self): super(GsUtilIntegrationTestCase, self).tearDown() while self.bucket_uris: bucket_uri = self.bucket_uris[-1] try: bucket_list = self._ListBucket(bucket_uri) except StorageResponseError as e: # This can happen for tests of rm -r command, which for bucket-only # URIs delete the bucket at the end. if e.status == 404: self.bucket_uris.pop() continue else: raise while bucket_list: error = None for k in bucket_list: try: if isinstance(k, DeleteMarker): bucket_uri.get_bucket().delete_key(k.name, version_id=k.version_id) else: k.delete() except StorageResponseError as e: # This could happen if objects that have already been deleted are # still showing up in the listing due to eventual consistency. In # that case, we continue on until we've tried to deleted every # object in the listing before raising the error on which to retry. if e.status == 404: # This could happen if objects that have already been deleted are # still showing up in the listing due to eventual consistency. In # that case, we continue on until we've tried to deleted every # obj in the listing before raising the error on which to retry. error = e elif e.status == 403 and (e.error_code == 'ObjectUnderActiveHold' or e.error_code == 'RetentionPolicyNotMet'): # Object deletion fails if they are under active Temporary Hold, # Event-Based hold or still under retention. # # We purposefully do not raise error in order to allow teardown # to process all the objects in a bucket first. The retry logic on # the teardown method will kick in when bucket deletion fails (due # to bucket being non-empty) and retry deleting these objects # and their associated buckets. self._ClearHoldsOnObjectAndWaitForRetentionDuration( bucket_uri, k.name) else: raise if error: raise error # pylint: disable=raising-bad-type bucket_list = self._ListBucket(bucket_uri) bucket_uri.delete_bucket() self.bucket_uris.pop() def _ClearHoldsOnObjectAndWaitForRetentionDuration(self, bucket_uri, object_name): """Removes Holds on test objects and waits till retention duration is over. This method makes sure that object is not under active Temporary Hold or Release Hold. It also waits (up to 1 minute) till retention duration for the object is over. This is necessary for cleanup, otherwise such test objects cannot be deleted. It's worth noting that tests should do their best to remove holds and wait for objects' retention period on their own and this is just a fallback. Additionally, Tests should not use retention duration longer than 1 minute, preferably only few seconds in order to avoid lengthening test execution time unnecessarily. Args: bucket_uri: bucket's uri. object_name: object's name. """ object_metadata = self.json_api.GetObjectMetadata( bucket_uri.bucket_name, object_name, fields=['timeCreated', 'temporaryHold', 'eventBasedHold']) object_uri = '{}{}'.format(bucket_uri, object_name) if object_metadata.temporaryHold: self.RunGsUtil(['retention', 'temp', 'release', object_uri]) if object_metadata.eventBasedHold: self.RunGsUtil(['retention', 'event', 'release', object_uri]) retention_policy = self.json_api.GetBucket(bucket_uri.bucket_name, fields=['retentionPolicy' ]).retentionPolicy retention_period = (retention_policy.retentionPeriod if retention_policy is not None else 0) # throwing exceptions for Retention durations larger than 60 seconds. if retention_period <= 60: time.sleep(retention_period) else: raise CommandException(('Retention duration is too large for bucket "{}".' ' Use shorter durations for Retention duration in' ' tests').format(bucket_uri)) def _SetObjectCustomMetadataAttribute(self, provider, bucket_name, object_name, attr_name, attr_value): """Sets a custom metadata attribute for an object. Args: provider: Provider string for the bucket, ex. 'gs' or 's3. bucket_name: The name of the bucket the object is in. object_name: The name of the object itself. attr_name: The name of the custom metadata attribute to set. attr_value: The value of the custom metadata attribute to set. Returns: None """ obj_metadata = apitools_messages.Object() obj_metadata.metadata = CreateCustomMetadata({attr_name: attr_value}) if provider == 'gs': self.json_api.PatchObjectMetadata(bucket_name, object_name, obj_metadata, provider=provider) else: self.xml_api.PatchObjectMetadata(bucket_name, object_name, obj_metadata, provider=provider) def SetPOSIXMetadata(self, provider, bucket_name, object_name, atime=None, mtime=None, uid=None, gid=None, mode=None): """Sets POSIX metadata for the object.""" obj_metadata = apitools_messages.Object() obj_metadata.metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) if atime is not None: CreateCustomMetadata(entries={ATIME_ATTR: atime}, custom_metadata=obj_metadata.metadata) if mode is not None: CreateCustomMetadata(entries={MODE_ATTR: mode}, custom_metadata=obj_metadata.metadata) if mtime is not None: CreateCustomMetadata(entries={MTIME_ATTR: mtime}, custom_metadata=obj_metadata.metadata) if uid is not None: CreateCustomMetadata(entries={UID_ATTR: uid}, custom_metadata=obj_metadata.metadata) if gid is not None: CreateCustomMetadata(entries={GID_ATTR: gid}, custom_metadata=obj_metadata.metadata) if provider == 'gs': self.json_api.PatchObjectMetadata(bucket_name, object_name, obj_metadata, provider=provider) else: self.xml_api.PatchObjectMetadata(bucket_name, object_name, obj_metadata, provider=provider) def ClearPOSIXMetadata(self, obj): """Uses the setmeta command to clear POSIX attributes from user metadata. Args: obj: The object to clear POSIX metadata for. """ provider_meta_string = 'goog' if obj.scheme == 'gs' else 'amz' self.RunGsUtil([ 'setmeta', '-h', 'x-%s-meta-%s' % (provider_meta_string, ATIME_ATTR), '-h', 'x-%s-meta-%s' % (provider_meta_string, MTIME_ATTR), '-h', 'x-%s-meta-%s' % (provider_meta_string, UID_ATTR), '-h', 'x-%s-meta-%s' % (provider_meta_string, GID_ATTR), '-h', 'x-%s-meta-%s' % (provider_meta_string, MODE_ATTR), suri(obj) ]) def _ServiceAccountCredentialsPresent(self): # TODO: Currently, service accounts cannot be project owners (unless # they are exempted for legacy reasons). Unfortunately, setting a canned ACL # other than project-private, the ACL that buckets get by default, removes # project-editors access from the bucket ACL. So any canned ACL that would # actually represent a change the bucket would also orphan the service # account's access to the bucket. If service accounts can be owners # in the future, remove this function and update all callers. return (config.has_option('Credentials', 'gs_service_key_file') or config.has_option('GoogleCompute', 'service_account')) def _ListBucket(self, bucket_uri): if bucket_uri.scheme == 's3': # storage_uri will omit delete markers from bucket listings, but # these must be deleted before we can remove an S3 bucket. return list(v for v in bucket_uri.get_bucket().list_versions()) return list(bucket_uri.list_bucket(all_versions=True)) def AssertNObjectsInBucket(self, bucket_uri, num_objects, versioned=False): """Checks (with retries) that 'ls bucket_uri/**' returns num_objects. This is a common test pattern to deal with eventual listing consistency for tests that rely on a set of objects to be listed. Args: bucket_uri: storage_uri for the bucket. num_objects: number of objects expected in the bucket. versioned: If True, perform a versioned listing. Raises: AssertionError if number of objects does not match expected value. Returns: Listing split across lines. """ def _CheckBucket(): command = ['ls', '-a'] if versioned else ['ls'] b_uri = [suri(bucket_uri) + '/**'] if num_objects else [suri(bucket_uri)] listing = self.RunGsUtil(command + b_uri, return_stdout=True).split('\n') # num_objects + one trailing newline. self.assertEqual(len(listing), num_objects + 1) return listing if self.multiregional_buckets: # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=5, timeout_secs=1) def _Check1(): return _CheckBucket() return _Check1() else: return _CheckBucket() def AssertObjectUsesCSEK(self, object_uri_str, encryption_key): """Strongly consistent check that the correct CSEK encryption key is used. This check forces use of the JSON API, as encryption information is not returned in object metadata via the XML API. Args: object_uri_str: uri for the object. encryption_key: expected CSEK key. """ with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): stdout = self.RunGsUtil(['stat', object_uri_str], return_stdout=True, force_gsutil=True) self.assertIn( Base64Sha256FromBase64EncryptionKey(encryption_key).decode('ascii'), stdout, 'Object %s did not use expected encryption key with hash %s. ' 'Actual object: %s' % (object_uri_str, Base64Sha256FromBase64EncryptionKey(encryption_key), stdout)) def AssertObjectUsesCMEK(self, object_uri_str, encryption_key): """Strongly consistent check that the correct KMS encryption key is used. This check forces use of the JSON API, as encryption information is not returned in object metadata via the XML API. Args: object_uri_str: uri for the object. encryption_key: expected CMEK key. """ with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): stdout = self.RunGsUtil(['stat', object_uri_str], return_stdout=True, force_gsutil=True) self.assertRegex(stdout, r'KMS key:\s+%s' % encryption_key) def AssertObjectUnencrypted(self, object_uri_str): """Checks that no CSEK or CMEK attributes appear in `stat` output. This check forces use of the JSON API, as encryption information is not returned in object metadata via the XML API. Args: object_uri_str: uri for the object. """ with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): stdout = self.RunGsUtil(['stat', object_uri_str], return_stdout=True, force_gsutil=True) self.assertNotIn('Encryption key SHA256', stdout) self.assertNotIn('KMS key', stdout) def CreateBucketWithRetentionPolicy(self, retention_period_in_seconds, is_locked=None, bucket_name=None): """Creates a test bucket with Retention Policy. The bucket and all of its contents will be deleted after the test. Args: retention_period_in_seconds: Retention duration in seconds is_locked: Indicates whether Retention Policy should be locked on the bucket or not. bucket_name: Create the bucket with this name. If not provided, a temporary test bucket name is constructed. Returns: StorageUri for the created bucket. """ # Creating bucket with Retention Policy. retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=retention_period_in_seconds)) bucket_uri = self.CreateBucket(bucket_name=bucket_name, retention_policy=retention_policy, prefer_json_api=True) if is_locked: # Locking Retention Policy self.RunGsUtil(['retention', 'lock', suri(bucket_uri)], stdin='y') # Verifying Retention Policy on the bucket. self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=retention_period_in_seconds, expected_is_locked=is_locked) return bucket_uri def VerifyRetentionPolicy(self, bucket_uri, expected_retention_period_in_seconds=None, expected_is_locked=None): """Verifies the Retention Policy on a bucket. Args: bucket_uri: Specifies the bucket. expected_retention_period_in_seconds: Specifies the expected Retention Period of the Retention Policy on the bucket. Setting this field to None, implies that no Retention Policy should be present. expected_is_locked: Indicates whether the Retention Policy should be locked or not. """ actual_retention_policy = self.json_api.GetBucket( bucket_uri.bucket_name, fields=['retentionPolicy']).retentionPolicy if expected_retention_period_in_seconds is None: self.assertEqual(actual_retention_policy, None) else: self.assertEqual(actual_retention_policy.retentionPeriod, expected_retention_period_in_seconds) self.assertEqual(actual_retention_policy.isLocked, expected_is_locked) # Verifying the effectiveTime of the Retention Policy: # since this is integration test and we don't have exact time of the # server. We just verify that the effective time is a timestamp within # last minute. effective_time_in_seconds = self.DateTimeToSeconds( actual_retention_policy.effectiveTime) current_time_in_seconds = self.DateTimeToSeconds(datetime.datetime.now()) self.assertGreater(effective_time_in_seconds, current_time_in_seconds - 60) def DateTimeToSeconds(self, datetime_obj): return int(time.mktime(datetime_obj.timetuple())) def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None, retention_policy=None, provider=None, prefer_json_api=False, versioning_enabled=False, bucket_policy_only=False, bucket_name_prefix='', bucket_name_suffix='', location=None, public_access_prevention=None): """Creates a test bucket. The bucket and all of its contents will be deleted after the test. Args: bucket_name: Create the bucket with this name. If not provided, a temporary test bucket name is constructed. test_objects: The number of objects that should be placed in the bucket. Defaults to 0. storage_class: Storage class to use. If not provided we us standard. retention_policy: Retention policy to be used on the bucket. provider: Provider to use - either "gs" (the default) or "s3". prefer_json_api: If True, use the JSON creation functions where possible. versioning_enabled: If True, set the bucket's versioning attribute to True. bucket_policy_only: If True, set the bucket's iamConfiguration's bucketPolicyOnly attribute to True. bucket_name_prefix: Unicode string to be prepended to bucket_name bucket_name_suffix: Unicode string to be appended to bucket_name location: The location/region in which the bucket should be created. public_access_prevention: String value of public access prevention. Valid values are "enforced" and "unspecified". Returns: StorageUri for the created bucket. """ if not provider: provider = self.default_provider # Location is controlled by the -b test flag. if location is None: if self.multiregional_buckets or provider == 's3': location = None else: # We default to the "us-central1" location for regional buckets, # but allow overriding this value in the Boto config. location = boto.config.get('GSUtil', 'test_cmd_regional_bucket_location', 'us-central1') bucket_name_prefix = six.ensure_text(bucket_name_prefix) bucket_name_suffix = six.ensure_text(bucket_name_suffix) if bucket_name: bucket_name = ''.join( [bucket_name_prefix, bucket_name, bucket_name_suffix]) bucket_name = util.MakeBucketNameValid(bucket_name) else: bucket_name = self.MakeTempName('bucket', prefix=bucket_name_prefix, suffix=bucket_name_suffix) if prefer_json_api and provider == 'gs': json_bucket = self.CreateBucketJson( bucket_name=bucket_name, test_objects=test_objects, storage_class=storage_class, location=location, versioning_enabled=versioning_enabled, retention_policy=retention_policy, bucket_policy_only=bucket_policy_only, public_access_prevention=public_access_prevention) bucket_uri = boto.storage_uri('gs://%s' % json_bucket.name.lower(), suppress_consec_slashes=False) return bucket_uri bucket_uri = boto.storage_uri('%s://%s' % (provider, bucket_name.lower()), suppress_consec_slashes=False) if provider == 'gs': # Apply API version and project ID headers if necessary. headers = { 'x-goog-api-version': self.api_version, GOOG_PROJ_ID_HDR: PopulateProjectId() } else: headers = {} if not bucket_policy_only: # S3 test account settings disable ACLs by default, # but they should be re-enabled if requested. headers['x-amz-object-ownership'] = 'ObjectWriter' @Retry(StorageResponseError, tries=7, timeout_secs=1) def _CreateBucketWithExponentialBackoff(): """Creates a bucket, retrying with exponential backoff on error. Parallel tests can easily run into bucket creation quotas. Retry with exponential backoff so that we create them as fast as we reasonably can. Returns: StorageUri for the created bucket """ try: bucket_uri.create_bucket(storage_class=storage_class, location=location or '', headers=headers) except StorageResponseError as e: # If the service returns a transient error or a connection breaks, # it's possible the request succeeded. If that happens, the service # will return 409s for all future calls even though our intent # succeeded. If the error message says we already own the bucket, # assume success to reduce test flakiness. This depends on # randomness of test naming buckets to prevent name collisions for # test buckets created concurrently in the same project, which is # acceptable because this is far less likely than service errors. if e.status == 409 and e.body and 'already own' in e.body: pass else: raise _CreateBucketWithExponentialBackoff() self.bucket_uris.append(bucket_uri) if versioning_enabled: bucket_uri.configure_versioning(True) if provider != 'gs' and not public_access_prevention: # S3 test account settings enable public access prevention # by default, so we should disable it if requested. xml_body = ('<?xml version="1.0" encoding="UTF-8"?>' '<PublicAccessBlockConfiguration>' '<BlockPublicAcls>False</BlockPublicAcls>' '</PublicAccessBlockConfiguration>') bucket_uri.set_subresource('publicAccessBlock', xml_body) for i in range(test_objects): self.CreateObject(bucket_uri=bucket_uri, object_name=self.MakeTempName('obj'), contents='test {:d}'.format(i).encode('ascii')) return bucket_uri def CreateVersionedBucket(self, bucket_name=None, test_objects=0): """Creates a versioned test bucket. The bucket and all of its contents will be deleted after the test. Args: bucket_name: Create the bucket with this name. If not provided, a temporary test bucket name is constructed. test_objects: The number of objects that should be placed in the bucket. Defaults to 0. Returns: StorageUri for the created bucket with versioning enabled. """ # Note that we prefer the JSON API so that we don't require two separate # steps to create and then set versioning on the bucket (as versioning # propagation on an existing bucket is subject to eventual consistency). bucket_uri = self.CreateBucket(bucket_name=bucket_name, test_objects=test_objects, prefer_json_api=True, versioning_enabled=True) return bucket_uri def CreateObject(self, bucket_uri=None, object_name=None, contents=None, prefer_json_api=False, encryption_key=None, mode=None, mtime=None, uid=None, gid=None, storage_class=None, gs_idempotent_generation=0, kms_key_name=None): """Creates a test object. Args: bucket_uri: The URI of the bucket to place the object in. If not specified, a new temporary bucket is created. object_name: The name to use for the object. If not specified, a temporary test object name is constructed. contents: The contents to write to the object. If not specified, the key is not written to, which means that it isn't actually created yet on the server. prefer_json_api: If true, use the JSON creation functions where possible. encryption_key: AES256 encryption key to use when creating the object, if any. mode: The POSIX mode for the object. Must be a base-8 3-digit integer represented as a string. mtime: The modification time of the file in POSIX time (seconds since UTC 1970-01-01). If not specified, this defaults to the current system time. uid: A POSIX user ID. gid: A POSIX group ID. storage_class: String representing the storage class to use for the object. gs_idempotent_generation: For use when overwriting an object for which you know the previously uploaded generation. Create GCS object idempotently by supplying this generation number as a precondition and assuming the current object is correct on precondition failure. Defaults to 0 (new object); to disable, set to None. kms_key_name: Fully-qualified name of the KMS key that should be used to encrypt the object. Note that this is currently only valid for 'gs' objects. Returns: A StorageUri for the created object. """ bucket_uri = bucket_uri or self.CreateBucket() # checking for valid types - None or unicode/binary text if contents is not None: if not isinstance(contents, (six.binary_type, six.text_type)): raise TypeError('contents must be either none or bytes, not {}'.format( type(contents))) contents = six.ensure_binary(contents) if (contents and bucket_uri.scheme == 'gs' and (prefer_json_api or encryption_key or kms_key_name)): object_name = object_name or self.MakeTempName('obj') json_object = self.CreateObjectJson( contents=contents, bucket_name=bucket_uri.bucket_name, object_name=object_name, encryption_key=encryption_key, mtime=mtime, storage_class=storage_class, gs_idempotent_generation=gs_idempotent_generation, kms_key_name=kms_key_name) object_uri = bucket_uri.clone_replace_name(object_name) # pylint: disable=protected-access # Need to update the StorageUri with the correct values while # avoiding creating a versioned string. md5 = (Base64ToHexHash(json_object.md5Hash), json_object.md5Hash.strip('\n"\'')) object_uri._update_from_values(None, json_object.generation, True, md5=md5) # pylint: enable=protected-access return object_uri bucket_uri = bucket_uri or self.CreateBucket() object_name = object_name or self.MakeTempName('obj') key_uri = bucket_uri.clone_replace_name(object_name) if contents is not None: if bucket_uri.scheme == 'gs' and gs_idempotent_generation is not None: try: key_uri.set_contents_from_string(contents, headers={ 'x-goog-if-generation-match': str(gs_idempotent_generation) }) except StorageResponseError as e: if e.status == 412: pass else: raise else: key_uri.set_contents_from_string(contents) custom_metadata_present = (mode is not None or mtime is not None or uid is not None or gid is not None) if custom_metadata_present: self.SetPOSIXMetadata(bucket_uri.scheme, bucket_uri.bucket_name, object_name, atime=None, mtime=mtime, uid=uid, gid=gid, mode=mode) return key_uri def CreateBucketJson(self, bucket_name=None, test_objects=0, storage_class=None, location=None, versioning_enabled=False, retention_policy=None, bucket_policy_only=False, public_access_prevention=None): """Creates a test bucket using the JSON API. The bucket and all of its contents will be deleted after the test. Args: bucket_name: Create the bucket with this name. If not provided, a temporary test bucket name is constructed. test_objects: The number of objects that should be placed in the bucket. Defaults to 0. storage_class: Storage class to use. If not provided we use standard. location: Location to use. versioning_enabled: If True, set the bucket's versioning attribute to True. retention_policy: Retention policy to be used on the bucket. bucket_policy_only: If True, set the bucket's iamConfiguration's bucketPolicyOnly attribute to True. public_access_prevention: String value of public access prevention. Valid values are "enforced" and "unspecified". Returns: Apitools Bucket for the created bucket. """ bucket_name = util.MakeBucketNameValid(bucket_name or self.MakeTempName('bucket')) bucket_metadata = apitools_messages.Bucket(name=bucket_name.lower()) if storage_class: bucket_metadata.storageClass = storage_class if location: bucket_metadata.location = location if versioning_enabled: bucket_metadata.versioning = (apitools_messages.Bucket.VersioningValue( enabled=True)) if retention_policy: bucket_metadata.retentionPolicy = retention_policy if bucket_policy_only or public_access_prevention: iam_config = apitools_messages.Bucket.IamConfigurationValue() if bucket_policy_only: iam_config.bucketPolicyOnly = iam_config.BucketPolicyOnlyValue() iam_config.bucketPolicyOnly.enabled = True if public_access_prevention: iam_config.publicAccessPrevention = public_access_prevention bucket_metadata.iamConfiguration = iam_config # TODO: Add retry and exponential backoff. bucket = self.json_api.CreateBucket(bucket_name, metadata=bucket_metadata) # Add bucket to list of buckets to be cleaned up. # TODO: Clean up JSON buckets using JSON API. self.bucket_uris.append( boto.storage_uri('gs://%s' % bucket_name, suppress_consec_slashes=False)) for i in range(test_objects): self.CreateObjectJson(bucket_name=bucket_name, object_name=self.MakeTempName('obj'), contents='test {:d}'.format(i).encode('ascii')) return bucket def CreateObjectJson(self, contents, bucket_name=None, object_name=None, encryption_key=None, mtime=None, storage_class=None, gs_idempotent_generation=None, kms_key_name=None): """Creates a test object (GCS provider only) using the JSON API. Args: contents: The contents to write to the object. bucket_name: Name of bucket to place the object in. If not specified, a new temporary bucket is created. Assumes the given bucket name is valid. object_name: The name to use for the object. If not specified, a temporary test object name is constructed. encryption_key: AES256 encryption key to use when creating the object, if any. mtime: The modification time of the file in POSIX time (seconds since UTC 1970-01-01). If not specified, this defaults to the current system time. storage_class: String representing the storage class to use for the object. gs_idempotent_generation: For use when overwriting an object for which you know the previously uploaded generation. Create GCS object idempotently by supplying this generation number as a precondition and assuming the current object is correct on precondition failure. Defaults to 0 (new object); to disable, set to None. kms_key_name: Fully-qualified name of the KMS key that should be used to encrypt the object. Note that this is currently only valid for 'gs' objects. Returns: An apitools Object for the created object. """ bucket_name = bucket_name or self.CreateBucketJson().name object_name = object_name or self.MakeTempName('obj') preconditions = Preconditions(gen_match=gs_idempotent_generation) custom_metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) if mtime is not None: CreateCustomMetadata({MTIME_ATTR: mtime}, custom_metadata) object_metadata = apitools_messages.Object( name=object_name, metadata=custom_metadata, bucket=bucket_name, contentType='application/octet-stream', storageClass=storage_class, kmsKeyName=kms_key_name) encryption_keywrapper = CryptoKeyWrapperFromKey(encryption_key) try: return self.json_api.UploadObject(six.BytesIO(contents), object_metadata, provider='gs', encryption_tuple=encryption_keywrapper, preconditions=preconditions) except PreconditionException: if gs_idempotent_generation is None: raise with SetBotoConfigForTest([('GSUtil', 'decryption_key1', encryption_key) ]): return self.json_api.GetObjectMetadata(bucket_name, object_name) def GetObjectMetadataWithFields(self, bucket_name, object_name, fields): """Retrieves and verifies an object's metadata attribute. Args: bucket_name: The name of the bucket the object is in. object_name: The name of the object itself. fields: List of attributes strings. Custom attributes begin "metadata/". Returns: Apitools object. """ gsutil_api = (self.json_api if self.default_provider == 'gs' else self.xml_api) return gsutil_api.GetObjectMetadata(bucket_name, object_name, provider=self.default_provider, fields=fields) def VerifyObjectCustomAttribute(self, bucket_name, object_name, attr_name, expected_value, expected_present=True): """Retrieves and verifies an object's custom metadata attribute. Args: bucket_name: The name of the bucket the object is in. object_name: The name of the object itself. attr_name: The name of the custom metadata attribute. expected_value: The expected retrieved value for the attribute. expected_present: True if the attribute must be present in the object metadata, False if it must not be present. Returns: None """ metadata = self.GetObjectMetadataWithFields( bucket_name, object_name, fields=['metadata/%s' % attr_name]) attr_present, value = GetValueFromObjectCustomMetadata( metadata, attr_name, default_value=expected_value) self.assertEqual(expected_present, attr_present) self.assertEqual(expected_value, value) def VerifyPublicAccessPreventionValue(self, bucket_uri, value): # TODO: Delete this method in favor of VerifyCommandGet stdout = self.RunGsUtil(['publicaccessprevention', 'get', suri(bucket_uri)], return_stdout=True) public_access_prevention_re = re.compile(r':\s+(?P<pap_val>.+)$') public_access_prevention_match = re.search(public_access_prevention_re, stdout) public_access_prevention_val = public_access_prevention_match.group( 'pap_val') self.assertEqual(str(value), public_access_prevention_val) def VerifyCommandGet(self, bucket_uri, command, expected): """Verifies if <command> get returns the expected value.""" stdout = self.RunGsUtil([command, 'get', suri(bucket_uri)], return_stdout=True) output_regex = re.compile('{}: (?P<actual>.+)$'.format(suri(bucket_uri))) output_match = re.search(output_regex, stdout) actual = output_match.group('actual') self.assertEqual(actual, expected) def RunGsUtil(self, cmd, return_status=False, return_stdout=False, return_stderr=False, expected_status=0, stdin=None, env_vars=None, force_gsutil=False): """Runs the gsutil command. Args: cmd: The command to run, as a list, e.g. ['cp', 'foo', 'bar'] return_status: If True, the exit status code is returned. return_stdout: If True, the standard output of the command is returned. return_stderr: If True, the standard error of the command is returned. expected_status: The expected return code. If not specified, defaults to 0. If the return code is a different value, an exception is raised. stdin: A string of data to pipe to the process as standard input. env_vars: A dictionary of variables to extend the subprocess's os.environ with. force_gsutil: If True, will always run the command using gsutil, irrespective of the value provided for use_gcloud_storage. Returns: If multiple return_* values were specified, this method returns a tuple containing the desired return values specified by the return_* arguments (in the order those parameters are specified in the method definition). If only one return_* value was specified, that value is returned directly rather than being returned within a 1-tuple. """ full_gsutil_command = util.GetGsutilCommand(cmd, force_gsutil=force_gsutil) process = util.GetGsutilSubprocess(full_gsutil_command, env_vars=env_vars) c_out = util.CommunicateWithTimeout(process, stdin=stdin) stdout = c_out[0].replace(os.linesep, '\n') stderr = c_out[1].replace(os.linesep, '\n') status = process.returncode if expected_status is not None: cmd = map(six.ensure_text, cmd) self.assertEqual( int(status), int(expected_status), msg='Expected status {}, got {}.\nCommand:\n{}\n\nstderr:\n{}'.format( expected_status, status, ' '.join(cmd), stderr)) toreturn = [] if return_status: toreturn.append(status) if return_stdout: toreturn.append(stdout) if return_stderr: toreturn.append(stderr) if len(toreturn) == 1: return toreturn[0] elif toreturn: return tuple(toreturn) def RunGsUtilTabCompletion(self, cmd, expected_results=None): """Runs the gsutil command in tab completion mode. Args: cmd: The command to run, as a list, e.g. ['cp', 'foo', 'bar'] expected_results: The expected tab completion results for the given input. """ cmd = [gslib.GSUTIL_PATH] + ['--testexceptiontraces'] + cmd if InvokedFromParFile(): argcomplete_start_idx = 1 else: argcomplete_start_idx = 2 # Prepend the interpreter path; ensures we use the same interpreter that # was used to invoke the integration tests. In practice, this only differs # when you're running the tests using a different interpreter than # whatever `/usr/bin/env python` resolves to. cmd = [str(sys.executable)] + cmd cmd_str = ' '.join(cmd) @Retry(AssertionError, tries=5, timeout_secs=1) def _RunTabCompletion(): """Runs the tab completion operation with retries.""" # Set this to True if the argcomplete tests start failing and you want to # see any output you can get. I've had to do this so many times that I'm # just going to leave this in the code for convenience ¯\_(ツ)_/¯ # # If set, this will print out extra info from the argcomplete subprocess. # You'll probably want to find one test that's failing and run it # individually, e.g.: # python3 ./gsutil test tabcomplete.TestTabComplete.test_single_object # so that only one subprocess is run, thus routing the output to your # local terminal rather than swallowing it. hacky_debugging = False results_string = None with tempfile.NamedTemporaryFile( delete=False) as tab_complete_result_file: if hacky_debugging: # These redirectons are valuable for debugging purposes. 1 and 2 are, # obviously, stdout and stderr of the subprocess. 9 is the fd for # argparse debug stream. cmd_str_with_result_redirect = ( '{cs} 1>{fn} 2>{fn} 8>{fn} 9>{fn}'.format( cs=cmd_str, fn=tab_complete_result_file.name)) else: # argcomplete returns results via the '8' file descriptor, so we # redirect to a file so we can capture the completion results. cmd_str_with_result_redirect = '{cs} 8>{fn}'.format( cs=cmd_str, fn=tab_complete_result_file.name) env = os.environ.copy() env['_ARGCOMPLETE'] = str(argcomplete_start_idx) # Use a sane default for COMP_WORDBREAKS. env['_ARGCOMPLETE_COMP_WORDBREAKS'] = '''"'@><=;|&(:''' if 'COMP_WORDBREAKS' in env: env['_ARGCOMPLETE_COMP_WORDBREAKS'] = env['COMP_WORDBREAKS'] env['COMP_LINE'] = cmd_str env['COMP_POINT'] = str(len(cmd_str)) subprocess.call(cmd_str_with_result_redirect, env=env, shell=True) results_string = tab_complete_result_file.read().decode( locale.getpreferredencoding()) if results_string: if hacky_debugging: print('---------------------------------------') print(results_string) print('---------------------------------------') results = results_string.split('\013') else: results = [] self.assertEqual(results, expected_results) # When tests are run in parallel, tab completion could take a long time, # so choose a long timeout value. with SetBotoConfigForTest([('GSUtil', 'tab_completion_timeout', '120')]): _RunTabCompletion() @contextlib.contextmanager def SetAnonymousBotoCreds(self): # Tell gsutil not to override the real error message with a warning about # anonymous access if no credentials are provided in the config file. boto_config_for_test = [('Tests', 'bypass_anonymous_access_warning', 'True') ] # Also, maintain any custom host/port/API configuration, since we'll need # to contact the same host when operating in a development environment. for creds_config_key in ('gs_host', 'gs_json_host', 'gs_json_host_header', 'gs_post', 'gs_json_port'): boto_config_for_test.append(('Credentials', creds_config_key, boto.config.get('Credentials', creds_config_key, None))) boto_config_for_test.append( ('Boto', 'https_validate_certificates', boto.config.get('Boto', 'https_validate_certificates', None))) for api_config_key in ('json_api_version', 'prefer_api'): boto_config_for_test.append(('GSUtil', api_config_key, boto.config.get('GSUtil', api_config_key, None))) with SetBotoConfigForTest(boto_config_for_test, use_existing_config=False): # Make sure to reset Developer Shell credential port so that the child # gsutil process is really anonymous. Also, revent Boto from falling back # on credentials from other files like ~/.aws/credentials or environment # variables. with SetEnvironmentForTest({ 'DEVSHELL_CLIENT_PORT': None, 'AWS_SECRET_ACCESS_KEY': '_', 'AWS_ACCESS_KEY_ID': '_', # If shim is used, gcloud might attempt to load credentials. 'CLOUDSDK_AUTH_DISABLE_CREDENTIALS': 'True', }): yield def _VerifyLocalMode(self, path, expected_mode): """Verifies the mode of the file specified at path. Args: path: The path of the file on the local file system. expected_mode: The expected mode as a 3-digit base-8 number. Returns: None """ self.assertEqual(expected_mode, int(oct(os.stat(path).st_mode)[-3:], 8)) def _VerifyLocalUid(self, path, expected_uid): """Verifies the uid of the file specified at path. Args: path: The path of the file on the local file system. expected_uid: The expected uid of the file. Returns: None """ self.assertEqual(expected_uid, os.stat(path).st_uid) def _VerifyLocalGid(self, path, expected_gid): """Verifies the gid of the file specified at path. Args: path: The path of the file on the local file system. expected_gid: The expected gid of the file. Returns: None """ self.assertEqual(expected_gid, os.stat(path).st_gid) def VerifyLocalPOSIXPermissions(self, path, gid=None, uid=None, mode=None): """Verifies the uid, gid, and mode of the file specified at path. Will only check the attribute if the corresponding method parameter is not None. Args: path: The path of the file on the local file system. gid: The expected gid of the file. uid: The expected uid of the file. mode: The expected mode of the file. Returns: None """ if gid is not None: self._VerifyLocalGid(path, gid) if uid is not None: self._VerifyLocalUid(path, uid) if mode is not None: self._VerifyLocalMode(path, mode) def FlatListDir(self, directory): """Perform a flat listing over directory. Args: directory: The directory to list Returns: Listings with path separators canonicalized to '/', to make assertions easier for Linux vs Windows. """ result = [] for dirpath, _, filenames in os.walk(directory): for f in filenames: result.append(os.path.join(dirpath, f)) return '\n'.join(result).replace('\\', '/') def FlatListBucket(self, bucket_url_string): """Perform a flat listing over bucket_url_string.""" return self.RunGsUtil(['ls', suri(bucket_url_string, '**')], return_stdout=True) def StorageUriCloneReplaceKey(self, storage_uri, key): """Wrapper for StorageUri.clone_replace_key(). Args: storage_uri: URI representing the object to be cloned key: key for the new StorageUri to represent """ return storage_uri.clone_replace_key(key) def StorageUriCloneReplaceName(self, storage_uri, name): """Wrapper for StorageUri.clone_replace_name(). Args: storage_uri: URI representing the object to be cloned key: new object name """ return storage_uri.clone_replace_name(name) def StorageUriSetContentsFromString(self, storage_uri, contents): """Wrapper for StorageUri.set_contents_from_string(). Args: storage_uri: URI representing the object contents: String of the new contents of the object """ return storage_uri.set_contents_from_string(contents) tests/test_compose.py 0000644 00000034167 15025013616 0010776 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for compose command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from six.moves import range from six.moves import range from gslib.commands.compose import MAX_COMPOSE_ARITY from gslib.cs_api_map import ApiSelector import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import AuthorizeProjectToUseTestingKmsKey from gslib.tests.util import GenerationFromURI as urigen from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import TEST_ENCRYPTION_KEY1 from gslib.tests.util import TEST_ENCRYPTION_KEY2 from gslib.tests.util import unittest from gslib.project_id import PopulateProjectId @SkipForS3('S3 does not support object composition.') class TestCompose(testcase.GsUtilIntegrationTestCase): """Integration tests for compose command.""" def check_n_ary_compose(self, num_components): """Tests composing num_components object.""" bucket_uri = self.CreateBucket() data_list = [ ('data-%d,' % i).encode('ascii') for i in range(num_components) ] components = [ self.CreateObject(bucket_uri=bucket_uri, contents=data).uri for data in data_list ] composite = self.StorageUriCloneReplaceName(bucket_uri, self.MakeTempName('obj')) self.RunGsUtil(['compose'] + components + [composite.uri]) self.assertEqual(composite.get_contents_as_string(), b''.join(data_list)) def test_compose_too_many_fails(self): components = ['gs://b/component-obj'] * (MAX_COMPOSE_ARITY + 1) stderr = self.RunGsUtil(['compose'] + components + ['gs://b/composite-obj'], expected_status=1, return_stderr=True) self.assertIn('command accepts at most', stderr) def test_compose_too_few_fails(self): stderr = self.RunGsUtil( ['compose', 'gs://b/composite-obj'], expected_status=2 if self._use_gcloud_storage else 1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('argument DESTINATION: Must be specified.', stderr) else: self.assertIn( 'CommandException: "compose" requires at least 1 component object.\n', stderr) def test_compose_between_buckets_fails(self): bucket_uri_1 = self.CreateBucket() bucket_uri_2 = self.CreateBucket() object_uri1 = self.CreateObject(bucket_uri=bucket_uri_1, contents=b'1') object_uri2 = self.CreateObject(bucket_uri=bucket_uri_2, contents=b'2') composite_object_uri = self.StorageUriCloneReplaceName( bucket_uri_1, self.MakeTempName('obj')) stderr = self.RunGsUtil([ 'compose', suri(object_uri1), suri(object_uri2), suri(composite_object_uri) ], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('Inter-bucket composing not supported\n', stderr) else: self.assertIn( 'CommandException: GCS does ' 'not support inter-bucket composing.\n', stderr) def test_versioned_target_disallowed(self): stderr = self.RunGsUtil( ['compose', 'gs://b/o1', 'gs://b/o2', 'gs://b/o3#1234'], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn( 'Verison-specific URLs are not valid destinations because composing always results in creating an object with the latest generation.', stderr) else: self.assertIn( 'CommandException: A version-specific URL (%s) ' 'cannot be the destination for gsutil compose - abort.' % 'gs://b/o3#1234', stderr) def test_simple_compose(self): self.check_n_ary_compose(1) self.check_n_ary_compose(2) def test_compose_copies_type_and_encoding_from_first_object(self): bucket_uri = self.CreateBucket() object_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents=b'1') object_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents=b'2') composite = self.StorageUriCloneReplaceName(bucket_uri, self.MakeTempName('obj')) self.RunGsUtil([ 'setmeta', '-h', 'Content-Type:python-x', '-h', 'Content-Encoding:gzip', suri(object_uri1) ]) self.RunGsUtil( ['compose', suri(object_uri1), suri(object_uri2), suri(composite)]) stdout = self.RunGsUtil(['stat', suri(composite)], return_stdout=True) self.assertRegex(stdout, r'Content-Type:\s+python-x') self.assertRegex(stdout, r'Content-Encoding:\s+gzip') def test_maximal_compose(self): self.check_n_ary_compose(MAX_COMPOSE_ARITY) def test_compose_with_wildcard(self): """Tests composing objects with a wildcarded URI.""" bucket_uri = self.CreateBucket() component1 = self.CreateObject(bucket_uri=bucket_uri, contents=b'hello ', object_name='component1') component2 = self.CreateObject(bucket_uri=bucket_uri, contents=b'world!', object_name='component2') composite = self.StorageUriCloneReplaceName(bucket_uri, self.MakeTempName('obj')) self.RunGsUtil(['compose', component1.uri, component2.uri, composite.uri]) self.assertEqual(composite.get_contents_as_string(), b'hello world!') def test_compose_with_precondition(self): """Tests composing objects with a destination precondition.""" # Tests that cp -v option handles the if-generation-match header correctly. bucket_uri = self.CreateVersionedBucket() k1_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'data1') k2_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'data2') g1 = k1_uri.generation gen_match_header = 'x-goog-if-generation-match:%s' % g1 # Append object 1 and 2 self.RunGsUtil([ '-h', gen_match_header, 'compose', suri(k1_uri), suri(k2_uri), suri(k1_uri) ]) # Second compose should fail the precondition. stderr = self.RunGsUtil([ '-h', gen_match_header, 'compose', suri(k1_uri), suri(k2_uri), suri(k1_uri) ], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn( 'At least one of the pre-conditions you specified did not hold', stderr) else: self.assertIn('PreconditionException', stderr) def test_compose_with_encryption(self): """Tests composing encrypted objects.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo', encryption_key=TEST_ENCRYPTION_KEY1) object_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY1) # Compose without correct key should fail. stderr = self.RunGsUtil([ 'compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj') ], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('Missing decryption key', stderr) else: self.assertIn('is encrypted by a customer-supplied encryption key', stderr) # Compose with different encryption key should fail; source and destination # encryption keys must match. with SetBotoConfigForTest([ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY2), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY1) ]): stderr = self.RunGsUtil([ 'compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj') ], expected_status=1, return_stderr=True) self.assertIn('provided encryption key is incorrect', stderr) with SetBotoConfigForTest([('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]): self.RunGsUtil([ 'compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj') ]) @SkipForS3('Test uses gs-specific KMS encryption') def test_compose_with_kms_encryption(self): """Tests composing encrypted objects.""" if self.test_api == ApiSelector.XML: return unittest.skip( 'gsutil does not support encryption with the XML API') bucket_uri = self.CreateBucket() object_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') object_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents=b'bar') obj_suri = suri(bucket_uri, 'composed') key_fqn = AuthorizeProjectToUseTestingKmsKey() with SetBotoConfigForTest([('GSUtil', 'encryption_key', key_fqn)]): self.RunGsUtil([ 'compose', suri(object_uri1), suri(object_uri2), obj_suri, ]) # verify composed object uses CMEK. with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): self.AssertObjectUsesCMEK(obj_suri, key_fqn) def test_compose_different_encryption_keys(self): """Tests composing encrypted objects with different encryption keys.""" bucket_uri = self.CreateBucket() object_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo', encryption_key=TEST_ENCRYPTION_KEY1) object_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents=b'bar', encryption_key=TEST_ENCRYPTION_KEY2) with SetBotoConfigForTest([ ('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1), ('GSUtil', 'decryption_key1', TEST_ENCRYPTION_KEY2) ]): stderr = self.RunGsUtil([ 'compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj') ], expected_status=1, return_stderr=True) self.assertIn('The provided encryption key is incorrect', stderr) # Should also fail if we don't have the second key. with SetBotoConfigForTest([('GSUtil', 'encryption_key', TEST_ENCRYPTION_KEY1)]): stderr = self.RunGsUtil([ 'compose', suri(object_uri1), suri(object_uri2), suri(bucket_uri, 'obj') ], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('Missing decryption key', stderr) else: self.assertIn('The provided encryption key is incorrect', stderr) def test_compose_missing_second_source_object(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') # Compose with missing source object stderr = self.RunGsUtil([ 'compose', suri(object_uri), suri(bucket_uri, 'nonexistent-obj'), suri(bucket_uri, 'valid-destination') ], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('The following URLs matched no objects or files:', stderr) else: self.assertIn('NotFoundException', stderr) if self.test_api == ApiSelector.JSON: self.assertIn('One of the source objects does not exist', stderr) def test_compose_with_generations(self): """Tests composing objects with generations.""" bucket_uri = self.CreateBucket() components = [] data_list = [b'1', b'2', b'3'] for data in data_list: object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=data) components.append(object_uri.version_specific_uri) composite = self.StorageUriCloneReplaceName(bucket_uri, self.MakeTempName('obj')) self.RunGsUtil(['compose'] + components + [composite.uri]) self.assertEqual(composite.get_contents_as_string(), b''.join(data_list)) class TestCompatibleCompose(testcase.GsUtilIntegrationTestCase): def test_compose_non_gcs_target(self): stderr = self.RunGsUtil(['compose', 'gs://b/o1', 'gs://b/o2', 's3://b/o3'], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('Composing across providers is not supported.', stderr) else: expected_msg = ('CommandException: "compose" called on URL with ' 'unsupported provider (%s).\n' % 's3://b/o3') self.assertIn(expected_msg, stderr) def test_compose_non_gcs_component(self): stderr = self.RunGsUtil(['compose', 'gs://b/o1', 's3://b/o2', 'gs://b/o3'], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('Composing across providers is not supported.', stderr) else: expected_msg = ('CommandException: "compose" called on URL with ' 'unsupported provider (%s).\n' % 's3://b/o2') self.assertIn(expected_msg, stderr) tests/test_plurality_checkable_iterator.py 0000644 00000017040 15025013616 0015237 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Unit tests for PluralityCheckableIterator.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys import six from six.moves import range from gslib.plurality_checkable_iterator import PluralityCheckableIterator import gslib.tests.testcase as testcase class CustomTestException(Exception): pass class PluralityCheckableIteratorTests(testcase.GsUtilUnitTestCase): """Unit tests for PluralityCheckableIterator.""" def testPluralityCheckableIteratorWith0Elems(self): """Tests empty PluralityCheckableIterator.""" input_list = list(range(0)) it = iter(input_list) pcit = PluralityCheckableIterator(it) self.assertTrue(pcit.IsEmpty()) self.assertFalse(pcit.HasPlurality()) output_list = list(pcit) self.assertEqual(input_list, output_list) def testPluralityCheckableIteratorWith1Elem(self): """Tests PluralityCheckableIterator with 1 element.""" input_list = list(range(1)) it = iter(input_list) pcit = PluralityCheckableIterator(it) self.assertFalse(pcit.IsEmpty()) self.assertFalse(pcit.HasPlurality()) output_list = list(pcit) self.assertEqual(input_list, output_list) def testPluralityCheckableIteratorWith2Elems(self): """Tests PluralityCheckableIterator with 2 elements.""" input_list = list(range(2)) it = iter(input_list) pcit = PluralityCheckableIterator(it) self.assertFalse(pcit.IsEmpty()) self.assertTrue(pcit.HasPlurality()) output_list = list(pcit) self.assertEqual(input_list, output_list) def testPluralityCheckableIteratorWith3Elems(self): """Tests PluralityCheckableIterator with 3 elements.""" input_list = list(range(3)) it = iter(input_list) pcit = PluralityCheckableIterator(it) self.assertFalse(pcit.IsEmpty()) self.assertTrue(pcit.HasPlurality()) output_list = list(pcit) self.assertEqual(input_list, output_list) def testPluralityCheckableIteratorWith1Elem1Exception(self): """Tests PluralityCheckableIterator with 2 elements. The second element raises an exception. """ class IterTest(six.Iterator): def __init__(self): self.position = 0 def __iter__(self): return self def __next__(self): if self.position == 0: self.position += 1 return 1 elif self.position == 1: self.position += 1 raise CustomTestException('Test exception') else: raise StopIteration() pcit = PluralityCheckableIterator(IterTest()) self.assertFalse(pcit.IsEmpty()) self.assertTrue(pcit.HasPlurality()) iterated_value = None try: for value in pcit: iterated_value = value self.fail('Expected exception from iterator') except CustomTestException: pass self.assertEqual(iterated_value, 1) def testPluralityCheckableIteratorWith2Exceptions(self): """Tests PluralityCheckableIterator with 2 elements that both raise.""" class IterTest(six.Iterator): def __init__(self): self.position = 0 def __iter__(self): return self def __next__(self): if self.position < 2: self.position += 1 raise CustomTestException('Test exception %s' % self.position) else: raise StopIteration() pcit = PluralityCheckableIterator(IterTest()) try: pcit.PeekException() self.fail('Expected exception 1 from PeekException') except CustomTestException as e: self.assertIn(str(e), 'Test exception 1') try: for _ in pcit: pass self.fail('Expected exception 1 from iterator') except CustomTestException as e: self.assertIn(str(e), 'Test exception 1') try: pcit.PeekException() self.fail('Expected exception 2 from PeekException') except CustomTestException as e: self.assertIn(str(e), 'Test exception 2') try: for _ in pcit: pass self.fail('Expected exception 2 from iterator') except CustomTestException as e: self.assertIn(str(e), 'Test exception 2') for _ in pcit: self.fail('Expected StopIteration') def testPluralityCheckableIteratorWithYieldedException(self): """Tests PCI with an iterator that yields an exception. The yielded exception is in the form of a tuple and must also contain a stack trace. """ class IterTest(six.Iterator): def __init__(self): self.position = 0 def __iter__(self): return self def __next__(self): if self.position == 0: try: self.position += 1 raise CustomTestException('Test exception 0') except CustomTestException as e: return (e, sys.exc_info()[2]) elif self.position == 1: self.position += 1 return 1 else: raise StopIteration() pcit = PluralityCheckableIterator(IterTest()) iterated_value = None try: for _ in pcit: pass self.fail('Expected exception 0 from iterator') except CustomTestException as e: self.assertIn(str(e), 'Test exception 0') for value in pcit: iterated_value = value self.assertEqual(iterated_value, 1) def testPluralityCheckableIteratorReadsAheadAsNeeded(self): """Tests that the PCI does not unnecessarily read new elements.""" class IterTest(six.Iterator): def __init__(self): self.position = 0 def __iter__(self): return self def __next__(self): if self.position == 3: raise StopIteration() self.position += 1 # IsEmpty and PeekException should retrieve only 1 element from the # underlying iterator. pcit = PluralityCheckableIterator(IterTest()) pcit.IsEmpty() pcit.PeekException() self.assertEqual(pcit.orig_iterator.position, 1) # HasPlurality requires populating 2 elements into the iterator. pcit.HasPlurality() self.assertEqual(pcit.orig_iterator.position, 2) # next should yield already-populated elements without advancing the # iterator. next(pcit) # Yields element 1 self.assertEqual(pcit.orig_iterator.position, 2) next(pcit) # Yields element 2 self.assertEqual(pcit.orig_iterator.position, 2) next(pcit) # Yields element 3 self.assertEqual(pcit.orig_iterator.position, 3) try: next(pcit) # Underlying iterator is empty self.fail('Expected StopIteration') except StopIteration: pass tests/test_hash.py 0000644 00000023127 15025013616 0010246 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for hash command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from gslib.commands import hash from gslib.exception import CommandException import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.utils import shim_util from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock _TEST_FILE_CONTENTS = b'123456\n' _TEST_FILE_B64_CRC = 'nYmSiA==' _TEST_FILE_B64_MD5 = '9EeyCn/L9TpdW+AT6gsVrw==' _TEST_FILE_HEX_CRC = '9D899288' _TEST_FILE_HEX_MD5 = 'f447b20a7fcbf53a5d5be013ea0b15af' _TEST_COMPOSITE_ADDED_CONTENTS = b'tmp' _TEST_COMPOSITE_B64_CRC = 'M3DYBg==' _TEST_COMPOSITE_HEX_CRC = '3370D806' class TestHashUnit(testcase.GsUtilUnitTestCase): """Unit tests for hash command.""" def testHashContents(self): tmp_file = self.CreateTempFile(contents=_TEST_FILE_CONTENTS) stdout = self.RunCommand('hash', args=[tmp_file], return_stdout=True) self.assertIn('Hashes [base64]', stdout) self.assertIn(('\tHash (crc32c):\t\t%s' % _TEST_FILE_B64_CRC), stdout) self.assertIn(('\tHash (md5):\t\t%s' % _TEST_FILE_B64_MD5), stdout) def testHashNoMatch(self): try: self.RunCommand('hash', args=['non-existent-file']) self.fail('Did not get expected CommandException') except CommandException as e: self.assertIn('No files matched', e.reason) def testHashHexFormat(self): tmp_file = self.CreateTempFile(contents=_TEST_FILE_CONTENTS) stdout = self.RunCommand('hash', args=['-h', tmp_file], return_stdout=True) self.assertIn('Hashes [hex]', stdout) self.assertIn(('\tHash (crc32c):\t\t%s' % _TEST_FILE_HEX_CRC), stdout) self.assertIn(('\tHash (md5):\t\t%s' % _TEST_FILE_HEX_MD5), stdout) def testHashWildcard(self): num_test_files = 2 tmp_dir = self.CreateTempDir(test_files=num_test_files) stdout = self.RunCommand('hash', args=[os.path.join(tmp_dir, '*')], return_stdout=True) # One summary line and two hash lines per file. num_expected_lines = num_test_files * (1 + 2) self.assertEqual(len(stdout.splitlines()), num_expected_lines) def testHashSelectAlg(self): tmp_file = self.CreateTempFile(contents=_TEST_FILE_CONTENTS) stdout_crc = self.RunCommand('hash', args=['-c', tmp_file], return_stdout=True) stdout_md5 = self.RunCommand('hash', args=['-m', tmp_file], return_stdout=True) stdout_both = self.RunCommand('hash', args=['-c', '-m', tmp_file], return_stdout=True) for stdout in (stdout_crc, stdout_both): self.assertIn(('\tHash (crc32c):\t\t%s' % _TEST_FILE_B64_CRC), stdout) for stdout in (stdout_md5, stdout_both): self.assertIn(('\tHash (md5):\t\t%s' % _TEST_FILE_B64_MD5), stdout) self.assertNotIn('md5', stdout_crc) self.assertNotIn('crc32c', stdout_md5) class TestHash(testcase.GsUtilIntegrationTestCase): """Integration tests for hash command.""" def testHashCloudObject(self): """Test hash command on a cloud object.""" obj1 = self.CreateObject(object_name='obj1', contents=_TEST_FILE_CONTENTS) # Tests cloud object with -h. stdout = self.RunGsUtil(['hash', '-h', suri(obj1)], return_stdout=True) self.assertIn('Hashes [hex]', stdout) if self.default_provider == 'gs': # Hex hashes for cloud objects get converted to lowercase but their # meaning is the same. self.assertIn(('\tHash (crc32c):\t\t%s' % _TEST_FILE_HEX_CRC.lower()), stdout) self.assertIn(('\tHash (md5):\t\t%s' % _TEST_FILE_HEX_MD5), stdout) # Tests cloud object as base64. stdout = self.RunGsUtil(['hash', suri(obj1)], return_stdout=True) self.assertIn('Hashes [base64]', stdout) if self.default_provider == 'gs': self.assertIn(('\tHash (crc32c):\t\t%s' % _TEST_FILE_B64_CRC), stdout) self.assertIn(('\tHash (md5):\t\t%s' % _TEST_FILE_B64_MD5), stdout) @SkipForS3('No composite object or crc32c support for S3.') def testHashCompositeObject(self): """Test hash command on a composite object (which only has crc32c).""" bucket = self.CreateBucket() obj1 = self.CreateObject(bucket_uri=bucket, object_name='obj1', contents=_TEST_FILE_CONTENTS) obj2 = self.CreateObject(bucket_uri=bucket, object_name='tmp', contents=_TEST_COMPOSITE_ADDED_CONTENTS) self.RunGsUtil(['compose', suri(obj1), suri(obj2), suri(obj1)]) stdout = self.RunGsUtil(['hash', '-h', suri(obj1)], return_stdout=True) self.assertIn('Hashes [hex]', stdout) # Hex hashes for cloud objects get converted to lowercase but their # meaning is the same. self.assertIn(('\tHash (crc32c):\t\t%s' % _TEST_COMPOSITE_HEX_CRC.lower()), stdout) stdout = self.RunGsUtil(['hash', suri(obj1)], return_stdout=True) self.assertIn('Hashes [base64]', stdout) self.assertIn(('\tHash (crc32c):\t\t%s' % _TEST_COMPOSITE_B64_CRC), stdout) class TestHashShim(testcase.ShimUnitTestBase): @mock.patch.object(hash.HashCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_basic_hash_command(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('hash', ['gs://b/o1', 'gs://b/o2'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage hash {}' ' gs://b/o1 gs://b/o2').format( shim_util._get_gcloud_binary_path('fake_dir'), hash._GCLOUD_FORMAT_STRING), info_lines) @mock.patch.object(hash.HashCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_both_crc32c_and_md5_to_skip_nothing_flag(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('hash', ['-c', '-m', 'gs://b/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage hash {}' ' gs://b/o').format(shim_util._get_gcloud_binary_path('fake_dir'), hash._GCLOUD_FORMAT_STRING), info_lines) @mock.patch.object(hash.HashCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_md5_flag_to_skip_crc32c(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('hash', ['-m', 'gs://b/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage hash {}' ' --skip-crc32c gs://b/o').format( shim_util._get_gcloud_binary_path('fake_dir'), hash._GCLOUD_FORMAT_STRING), info_lines) @mock.patch.object(hash.HashCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_crc32c_flag_to_skip_md5(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('hash', ['-c', 'gs://b/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage hash {}' ' --skip-md5 gs://b/o').format( shim_util._get_gcloud_binary_path('fake_dir'), hash._GCLOUD_FORMAT_STRING), info_lines) tests/test_wrapped_credentials.py 0000644 00000033714 15025013616 0013345 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2022 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for wrapped_credentials.py.""" import datetime import json import httplib2 from google.auth import aws from google.auth import external_account from google.auth import external_account_authorized_user from google.auth import identity_pool from google.auth import pluggable from gslib.tests import testcase from gslib.utils.wrapped_credentials import WrappedCredentials import oauth2client from six import add_move, MovedModule add_move(MovedModule("mock", "mock", "unittest.mock")) from six.moves import mock ACCESS_TOKEN = "foo" CONTENT = "content" RESPONSE = httplib2.Response({ "content-type": "text/plain", "status": "200", "content-length": len(CONTENT), }) class MockCredentials(external_account.Credentials): def __init__(self, token=None, expiry=None, *args, **kwargs): super().__init__(*args, **kwargs) self._audience = None self.expiry = expiry self.token = None def side_effect(*args, **kwargs): del args, kwargs # Unused. self.token = token self.refresh = mock.Mock(side_effect=side_effect) def retrieve_subject_token(): pass class HeadersWithAuth(dict): """A utility class to use to make sure a set of headers includes specific authentication""" def __init__(self, token): self.token = token or "" def __eq__(self, headers): return headers[b"Authorization"] == bytes("Bearer " + self.token, "utf-8") class TestWrappedCredentials(testcase.GsUtilUnitTestCase): """Test logic for interacting with Wrapped Credentials the way we intend to use them.""" @mock.patch.object(httplib2, "Http", autospec=True) def testWrappedCredentialUsage(self, http): http.return_value.request.return_value = (RESPONSE, CONTENT) req = http.return_value.request creds = WrappedCredentials( MockCredentials(token=ACCESS_TOKEN, audience="foo", subject_token_type="bar", token_url="https://sts.googleapis.com", credential_source="qux")) http = oauth2client.transport.get_http_object() creds.authorize(http) _, content = http.request(uri="google.com") self.assertEqual(content, CONTENT) creds._base.refresh.assert_called_once_with(mock.ANY) # Make sure the default request gets called with the correct token. req.assert_called_once_with("google.com", method="GET", headers=HeadersWithAuth(ACCESS_TOKEN), body=None, connection_type=mock.ANY, redirections=mock.ANY) def testWrappedCredentialSerialization(self): """Test logic for converting Wrapped Credentials to and from JSON for serialization.""" creds = WrappedCredentials( identity_pool.Credentials(audience="foo", subject_token_type="bar", token_url="https://sts.googleapis.com", credential_source={"url": "google.com"})) creds.access_token = ACCESS_TOKEN creds.token_expiry = datetime.datetime(2001, 12, 5, 0, 0) creds_json = creds.to_json() json_values = json.loads(creds_json) self.assertEqual(json_values["client_id"], "foo") self.assertEqual(json_values['access_token'], ACCESS_TOKEN) self.assertEqual(json_values['token_expiry'], "2001-12-05T00:00:00Z") self.assertEqual(json_values["_base"]["audience"], "foo") self.assertEqual(json_values["_base"]["subject_token_type"], "bar") self.assertEqual(json_values["_base"]["token_url"], "https://sts.googleapis.com") self.assertEqual(json_values["_base"]["credential_source"]["url"], "google.com") creds2 = WrappedCredentials.from_json(creds_json) self.assertIsInstance(creds2, WrappedCredentials) self.assertIsInstance(creds2._base, identity_pool.Credentials) self.assertEqual(creds2.client_id, "foo") self.assertEqual(creds2.access_token, ACCESS_TOKEN) self.assertEqual(creds2.token_expiry, creds.token_expiry) def testWrappedCredentialSerializationMissingKeywords(self): """Test logic for creating a Wrapped Credentials using keywords that exist in IdentityPool but not AWS.""" creds = WrappedCredentials.from_json( json.dumps({ "client_id": "foo", "access_token": ACCESS_TOKEN, "token_expiry": "2001-12-05T00:00:00Z", "_base": { "type": "external_account", "audience": "foo", "subject_token_type": "bar", "token_url": "https://sts.googleapis.com", "credential_source": { "url": "google.com", "workforce_pool_user_project": "1234567890" } } })) self.assertIsInstance(creds, WrappedCredentials) self.assertIsInstance(creds._base, identity_pool.Credentials) @mock.patch.object(httplib2, "Http", autospec=True) def testWrappedCredentialUsageExternalAccountAuthorizedUser(self, http): http.return_value.request.return_value = (RESPONSE, CONTENT) req = http.return_value.request creds = WrappedCredentials( external_account_authorized_user.Credentials( audience= "//iam.googleapis.com/locations/global/workforcePools/$WORKFORCE_POOL_ID/providers/$PROVIDER_ID", refresh_token="refreshToken", token_url="https://sts.googleapis.com/v1/oauth/token", token_info_url="https://sts.googleapis.com/v1/instrospect", client_id="clientId", client_secret="clientSecret")) def _refresh_token_side_effect(*args, **kwargs): del args, kwargs # Unused. creds._base.token = ACCESS_TOKEN creds._base.refresh = mock.Mock(side_effect=_refresh_token_side_effect) http = oauth2client.transport.get_http_object() creds.authorize(http) _, content = http.request(uri="google.com") self.assertEqual(content, CONTENT) creds._base.refresh.assert_called_once_with(mock.ANY) # Make sure the default request gets called with the correct token. req.assert_called_once_with("google.com", method="GET", headers=HeadersWithAuth(ACCESS_TOKEN), body=None, connection_type=mock.ANY, redirections=mock.ANY) def testWrappedCredentialSerializationExternalAccountAuthorizedUser(self): """Test logic for converting Wrapped Credentials to and from JSON for serialization.""" creds = WrappedCredentials( external_account_authorized_user.Credentials( audience= "//iam.googleapis.com/locations/global/workforcePools/$WORKFORCE_POOL_ID/providers/$PROVIDER_ID", refresh_token="refreshToken", token_url="https://sts.googleapis.com/v1/oauth/token", token_info_url="https://sts.googleapis.com/v1/instrospect", client_id="clientId", client_secret="clientSecret")) creds.access_token = ACCESS_TOKEN creds.token_expiry = datetime.datetime(2001, 12, 5, 0, 0) creds_json = creds.to_json() json_values = json.loads(creds_json) expected_json_values = { "_class": "WrappedCredentials", "_module": "gslib.utils.wrapped_credentials", "client_id": "clientId", "access_token": ACCESS_TOKEN, "token_expiry": "2001-12-05T00:00:00Z", "client_secret": "clientSecret", "refresh_token": "refreshToken", "id_token": None, "id_token_jwt": None, "invalid": False, "revoke_uri": None, "scopes": [], "token_info_uri": None, "token_response": None, "token_uri": None, "user_agent": None, "_base": { "type": "external_account_authorized_user", "audience": "//iam.googleapis.com/locations/global/workforcePools/$WORKFORCE_POOL_ID/providers/$PROVIDER_ID", "token": ACCESS_TOKEN, "expiry": "2001-12-05T00:00:00Z", "token_url": "https://sts.googleapis.com/v1/oauth/token", "token_info_url": "https://sts.googleapis.com/v1/instrospect", "refresh_token": "refreshToken", "client_id": "clientId", "client_secret": "clientSecret", } } self.assertEqual(json_values, expected_json_values) creds2 = WrappedCredentials.from_json(creds_json) self.assertIsInstance(creds2, WrappedCredentials) self.assertIsInstance(creds2._base, external_account_authorized_user.Credentials) self.assertEqual(creds2.client_id, "clientId") def testFromJsonAWSCredentials(self): creds = WrappedCredentials.from_json( json.dumps({ "_base": { "audience": "//iam.googleapis.com/projects/123456/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID", "credential_source": { "environment_id": "aws1", "region_url": "http://169.254.169.254/latest/meta-data/placement/availability-zone", "regional_cred_verification_url": "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15", "url": "http://169.254.169.254/latest/meta-data/iam/security-credentials" }, "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/service-1234@service-name.iam.gserviceaccount.com:generateAccessToken", "subject_token_type": "urn:ietf:params:aws:token-type:aws4_request", "token_url": "https://sts.googleapis.com/v1/token", "type": "external_account" } })) self.assertIsInstance(creds, WrappedCredentials) self.assertIsInstance(creds._base, external_account.Credentials) self.assertIsInstance(creds._base, aws.Credentials) def testFromJsonFileBasedCredentials(self): creds = WrappedCredentials.from_json( json.dumps({ "_base": { "audience": "//iam.googleapis.com/projects/123456/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID", "credential_source": { "file": "/var/run/secrets/goog.id/token" }, "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/service-1234@service-name.iam.gserviceaccount.com:generateAccessToken", "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": "https://sts.googleapis.com/v1/token", "type": "external_account" } })) self.assertIsInstance(creds, WrappedCredentials) self.assertIsInstance(creds._base, external_account.Credentials) self.assertIsInstance(creds._base, identity_pool.Credentials) def testFromJsonPluggableCredentials(self): creds = WrappedCredentials.from_json( json.dumps({ "_base": { "audience": "//iam.googleapis.com/projects/123456/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID", "credential_source": { "executable": { "command": "/path/to/command.sh" } }, "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/service-1234@service-name.iam.gserviceaccount.com:generateAccessToken", "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": "https://sts.googleapis.com/v1/token", "type": "external_account" } })) self.assertIsInstance(creds, WrappedCredentials) self.assertIsInstance(creds._base, external_account.Credentials) self.assertIsInstance(creds._base, pluggable.Credentials) def testFromJsonExternalAccountAuthorizedUserCredentials(self): creds = WrappedCredentials.from_json( json.dumps({ "_base": { "type": "external_account_authorized_user", "audience": "//iam.googleapis.com/locations/global/workforcePools/$WORKFORCE_POOL_ID/providers/$PROVIDER_ID", "refresh_token": "refreshToken", "token_url": "https://sts.googleapis.com/v1/oauth/token", "token_info_url": "https://sts.googleapis.com/v1/instrospect", "client_id": "clientId", "client_secret": "clientSecret", } })) self.assertIsInstance(creds, WrappedCredentials) self.assertIsInstance(creds._base, external_account_authorized_user.Credentials) tests/test_notification.py 0000644 00000013632 15025013616 0012011 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for notification command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import time import uuid import boto from gslib.cloud_api_delegator import CloudApiDelegator import gslib.tests.testcase as testcase from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import unittest from gslib.utils.retry_util import Retry from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock def _LoadNotificationUrl(): return boto.config.get_value('GSUtil', 'test_notification_url') NOTIFICATION_URL = _LoadNotificationUrl() class TestNotificationUnit(testcase.GsUtilUnitTestCase): @mock.patch.object(CloudApiDelegator, 'CreateNotificationConfig', autospec=True) def test_notification_splits_dash_m_value_correctly(self, mock_create_notification): bucket_uri = self.CreateBucket(bucket_name='foo_notification') stdout = self.RunCommand( 'notification', ['create', '-f', 'none', '-s', '-m', 'foo:bar:baz', suri(bucket_uri)], return_stdout=True) mock_create_notification.assert_called_once_with( mock.ANY, # Client instance. 'foo_notification', pubsub_topic=mock.ANY, payload_format=mock.ANY, custom_attributes={'foo': 'bar:baz'}, event_types=None, object_name_prefix=mock.ANY, provider=mock.ANY) class TestNotification(testcase.GsUtilIntegrationTestCase): """Integration tests for notification command.""" @unittest.skipUnless(NOTIFICATION_URL, 'Test requires notification URL configuration.') def test_watch_bucket(self): """Tests creating a notification channel on a bucket.""" bucket_uri = self.CreateBucket() self.RunGsUtil( ['notification', 'watchbucket', NOTIFICATION_URL, suri(bucket_uri)]) identifier = str(uuid.uuid4()) token = str(uuid.uuid4()) stderr = self.RunGsUtil([ 'notification', 'watchbucket', '-i', identifier, '-t', token, NOTIFICATION_URL, suri(bucket_uri) ], return_stderr=True) self.assertIn('token: %s' % token, stderr) self.assertIn('identifier: %s' % identifier, stderr) @unittest.skipUnless(NOTIFICATION_URL, 'Test requires notification URL configuration.') def test_stop_channel(self): """Tests stopping a notification channel on a bucket.""" bucket_uri = self.CreateBucket() stderr = self.RunGsUtil( ['notification', 'watchbucket', NOTIFICATION_URL, suri(bucket_uri)], return_stderr=True) channel_id = re.findall(r'channel identifier: (?P<id>.*)', stderr) self.assertEqual(len(channel_id), 1) resource_id = re.findall(r'resource identifier: (?P<id>.*)', stderr) self.assertEqual(len(resource_id), 1) channel_id = channel_id[0] resource_id = resource_id[0] self.RunGsUtil(['notification', 'stopchannel', channel_id, resource_id]) @unittest.skipUnless(NOTIFICATION_URL, 'Test requires notification URL configuration.') def test_list_one_channel(self): """Tests listing notification channel on a bucket.""" # TODO(b/132277269): Re-enable these once the service-side bug is fixed. return unittest.skip('Functionality has been disabled due to b/132277269') bucket_uri = self.CreateBucket() # Set up an OCN (object change notification) on the newly created bucket. self.RunGsUtil( ['notification', 'watchbucket', NOTIFICATION_URL, suri(bucket_uri)], return_stderr=False) # The OCN listing in the service is eventually consistent. In initial # tests, it almost never was ready immediately after calling WatchBucket # above, so we A) sleep for a few seconds before the first OCN listing # attempt, and B) wrap the OCN listing attempt in retry logic in case # it raises a BucketNotFoundException (note that RunGsUtil will raise this # as an AssertionError due to the exit status not being 0). @Retry(AssertionError, tries=3, timeout_secs=5) def _ListObjectChangeNotifications(): stderr = self.RunGsUtil(['notification', 'list', '-o', suri(bucket_uri)], return_stderr=True) return stderr time.sleep(5) stderr = _ListObjectChangeNotifications() channel_id = re.findall(r'Channel identifier: (?P<id>.*)', stderr) self.assertEqual(len(channel_id), 1) resource_id = re.findall(r'Resource identifier: (?P<id>.*)', stderr) self.assertEqual(len(resource_id), 1) push_url = re.findall(r'Application URL: (?P<id>.*)', stderr) self.assertEqual(len(push_url), 1) subscriber_email = re.findall(r'Created by: (?P<id>.*)', stderr) self.assertEqual(len(subscriber_email), 1) creation_time = re.findall(r'Creation time: (?P<id>.*)', stderr) self.assertEqual(len(creation_time), 1) def test_invalid_subcommand(self): stderr = self.RunGsUtil(['notification', 'foo', 'bar', 'baz'], return_stderr=True, expected_status=1) self.assertIn('Invalid subcommand', stderr) tests/test_rb.py 0000644 00000005664 15025013616 0007734 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for rb command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import gslib.tests.testcase as testcase from gslib.tests.util import ObjectToURI as suri class TestRb(testcase.GsUtilIntegrationTestCase): """Integration tests for rb command.""" def test_rb_bucket_works(self): bucket_uri = self.CreateBucket() self.RunGsUtil(['rb', suri(bucket_uri)]) stderr = self.RunGsUtil( ['ls', '-Lb', 'gs://%s' % self.nonexistent_bucket_name], return_stderr=True, expected_status=1) self.assertIn('404', stderr) def test_rb_bucket_not_empty(self): bucket_uri = self.CreateBucket(test_objects=1) stderr = self.RunGsUtil(['rb', suri(bucket_uri)], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('Bucket is not empty', stderr) else: self.assertIn('BucketNotEmpty', stderr) def test_rb_versioned_bucket_not_empty(self): bucket_uri = self.CreateVersionedBucket(test_objects=1) stderr = self.RunGsUtil(['rb', suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn('Bucket is not empty. Note: this is a versioned bucket', stderr) def test_rb_nonexistent_bucket(self): stderr = self.RunGsUtil( ['rb', 'gs://%s' % self.nonexistent_bucket_name], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('not found', stderr) else: self.assertIn('does not exist.', stderr) def test_rb_minus_f(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil([ 'rb', '-f', 'gs://%s' % self.nonexistent_bucket_name, suri(bucket_uri) ], return_stderr=True, expected_status=1) # There should be no error output, and existing bucket named after # non-existent bucket should be gone. self.assertNotIn('bucket does not exist.', stderr) stderr = self.RunGsUtil(['ls', '-Lb', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('404', stderr) tests/test_gcs_json_credentials.py 0000644 00000023552 15025013616 0013507 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2022 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for gcs_json_credentials.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from apitools.base.py import GceAssertionCredentials from google_reauth import reauth_creds from gslib import gcs_json_api from gslib import gcs_json_credentials from gslib.cred_types import CredTypes from gslib.exception import CommandException from gslib.tests import testcase from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import unittest from gslib.utils.wrapped_credentials import WrappedCredentials import logging from oauth2client.service_account import ServiceAccountCredentials import pkgutil from six import add_move, MovedModule add_move(MovedModule("mock", "mock", "unittest.mock")) from six.moves import mock try: from OpenSSL.crypto import load_pkcs12 HAS_OPENSSL = True except ImportError: HAS_OPENSSL = False try: import cryptography HAS_CRYPTO = True except ImportError: HAS_CRYPTO = False ERROR_MESSAGE = "This is the error message" def getBotoCredentialsConfig( service_account_creds=None, user_account_creds=None, gce_creds=None, external_account_creds=None, external_account_authorized_user_creds=None, ): config = [] if service_account_creds: config.append(("Credentials", "gs_service_key_file", service_account_creds["keyfile"])) config.append(("Credentials", "gs_service_client_id", service_account_creds["client_id"])) else: config.append(("Credentials", "gs_service_key_file", None)) config.extend([("Credentials", "gs_oauth2_refresh_token", user_account_creds), ("GoogleCompute", "service_account", gce_creds), ("Credentials", "gs_external_account_file", external_account_creds), ("Credentials", "gs_external_account_authorized_user_file", external_account_authorized_user_creds)]) return config class TestGcsJsonCredentials(testcase.GsUtilUnitTestCase): """Test logic for interacting with GCS JSON Credentials.""" @mock.patch.object(gcs_json_credentials.P12Credentials, "from_service_account_pkcs12_keystring", return_value=gcs_json_credentials.P12Credentials(mock.Mock(), token_uri='123', service_account_email='123', scopes=['a', 'b'])) def testOauth2ServiceAccountCredential(self, _): contents = pkgutil.get_data("gslib", "tests/test_data/test.p12") tmpfile = self.CreateTempFile(contents=contents) with SetBotoConfigForTest( getBotoCredentialsConfig(service_account_creds={ "keyfile": tmpfile, "client_id": "?", })): self.assertTrue(gcs_json_credentials._HasOauth2ServiceAccountCreds()) client = gcs_json_api.GcsJsonApi(None, None, None, None) self.assertEqual(client.credentials.service_account_email, '123') self.assertIsInstance(client.credentials, gcs_json_credentials.P12Credentials) def testP12CredentialsthrowsErrorIfProvidedWithMissingFields(self): contents = pkgutil.get_data("gslib", "tests/test_data/test.p12") tmpfile = self.CreateTempFile(contents=contents) with self.assertRaises(Exception) as exc: gcs_json_credentials.CreateP12ServiceAccount(tmpfile) @unittest.skipUnless(HAS_CRYPTO, 'p12credentials requires cryptography.') @mock.patch.object(gcs_json_credentials.P12Credentials, "__init__", side_effect=ValueError(ERROR_MESSAGE)) def testOauth2ServiceAccountFailure(self, _): contents = pkgutil.get_data("gslib", "tests/test_data/test.p12") tmpfile = self.CreateTempFile(contents=contents) with SetBotoConfigForTest( getBotoCredentialsConfig(service_account_creds={ "keyfile": tmpfile, "client_id": "?", })): with self.assertLogs() as logger: with self.assertRaises(Exception) as exc: gcs_json_api.GcsJsonApi(None, logging.getLogger(), None, None) self.assertIn(ERROR_MESSAGE, str(exc.exception)) self.assertIn(CredTypes.OAUTH2_SERVICE_ACCOUNT, logger.output[0]) def testOauth2UserCredential(self): with SetBotoConfigForTest(getBotoCredentialsConfig(user_account_creds="?")): self.assertTrue(gcs_json_credentials._HasOauth2UserAccountCreds()) client = gcs_json_api.GcsJsonApi(None, None, None, None) self.assertIsInstance(client.credentials, reauth_creds.Oauth2WithReauthCredentials) @mock.patch.object(reauth_creds.Oauth2WithReauthCredentials, "__init__", side_effect=ValueError(ERROR_MESSAGE)) def testOauth2UserFailure(self, _): with SetBotoConfigForTest(getBotoCredentialsConfig(user_account_creds="?")): with self.assertLogs() as logger: with self.assertRaises(Exception) as exc: gcs_json_api.GcsJsonApi(None, logging.getLogger(), None, None) self.assertIn(ERROR_MESSAGE, str(exc.exception)) self.assertIn(CredTypes.OAUTH2_USER_ACCOUNT, logger.output[0]) @mock.patch.object(gcs_json_credentials.credentials_lib, 'GceAssertionCredentials', autospec=True) def testGCECredential(self, mock_credentials): def set_store(store): mock_credentials.return_value.store = store mock_credentials.return_value.client_id = None mock_credentials.return_value.refresh_token = "rEfrEshtOkEn" mock_credentials.return_value.set_store = set_store with SetBotoConfigForTest(getBotoCredentialsConfig(gce_creds="?")): self.assertTrue(gcs_json_credentials._HasGceCreds()) client = gcs_json_api.GcsJsonApi(None, None, None, None) self.assertIsInstance(client.credentials, GceAssertionCredentials) self.assertEqual(client.credentials.refresh_token, "rEfrEshtOkEn") self.assertIs(client.credentials.client_id, None) @mock.patch.object(GceAssertionCredentials, "__init__", side_effect=ValueError(ERROR_MESSAGE)) def testGCECredentialFailure(self, _): with SetBotoConfigForTest(getBotoCredentialsConfig(gce_creds="?")): with self.assertLogs() as logger: with self.assertRaises(Exception) as exc: gcs_json_api.GcsJsonApi(None, logging.getLogger(), None, None) self.assertIn(ERROR_MESSAGE, str(exc.exception)) self.assertIn(CredTypes.GCE, logger.output[0]) def testExternalAccountCredential(self): contents = pkgutil.get_data( "gslib", "tests/test_data/test_external_account_credentials.json") tmpfile = self.CreateTempFile(contents=contents) with SetBotoConfigForTest( getBotoCredentialsConfig(external_account_creds=tmpfile)): client = gcs_json_api.GcsJsonApi(None, None, None, None) self.assertIsInstance(client.credentials, WrappedCredentials) @mock.patch.object(WrappedCredentials, "__init__", side_effect=ValueError(ERROR_MESSAGE)) def testExternalAccountFailure(self, _): contents = pkgutil.get_data( "gslib", "tests/test_data/test_external_account_credentials.json") tmpfile = self.CreateTempFile(contents=contents) with SetBotoConfigForTest( getBotoCredentialsConfig(external_account_creds=tmpfile)): with self.assertLogs() as logger: with self.assertRaises(Exception) as exc: gcs_json_api.GcsJsonApi(None, logging.getLogger(), None, None) self.assertIn(ERROR_MESSAGE, str(exc.exception)) self.assertIn(CredTypes.EXTERNAL_ACCOUNT, logger.output[0]) def testExternalAccountAuthorizedUserCredential(self): contents = pkgutil.get_data( "gslib", "tests/test_data/test_external_account_authorized_user_credentials.json" ) tmpfile = self.CreateTempFile(contents=contents) with SetBotoConfigForTest( getBotoCredentialsConfig( external_account_authorized_user_creds=tmpfile)): client = gcs_json_api.GcsJsonApi(None, None, None, None) self.assertIsInstance(client.credentials, WrappedCredentials) @mock.patch.object(WrappedCredentials, "__init__", side_effect=ValueError(ERROR_MESSAGE)) def testExternalAccountAuthorizedUserFailure(self, _): contents = pkgutil.get_data( "gslib", "tests/test_data/test_external_account_authorized_user_credentials.json" ) tmpfile = self.CreateTempFile(contents=contents) with SetBotoConfigForTest( getBotoCredentialsConfig( external_account_authorized_user_creds=tmpfile)): with self.assertLogs() as logger: with self.assertRaises(Exception) as exc: gcs_json_api.GcsJsonApi(None, logging.getLogger(), None, None) self.assertIn(ERROR_MESSAGE, str(exc.exception)) self.assertIn(CredTypes.EXTERNAL_ACCOUNT_AUTHORIZED_USER, logger.output[0]) def testOauth2ServiceAccountAndOauth2UserCredential(self): with SetBotoConfigForTest( getBotoCredentialsConfig(user_account_creds="?", service_account_creds={ "keyfile": "?", "client_id": "?", })): with self.assertRaises(CommandException): gcs_json_api.GcsJsonApi(None, None, None, None) tests/test_cloud_api_delegator.py 0000644 00000004003 15025013616 0013300 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2020 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for cloud_api_delegator.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import cloud_api from gslib import cloud_api_delegator from gslib import context_config from gslib import cs_api_map from gslib.tests import testcase from gslib.tests.testcase import base from gslib.tests.util import unittest from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock class TestCloudApiDelegator(testcase.GsUtilUnitTestCase): """Test delegator class for cloud provider API clients.""" @mock.patch.object(context_config, 'get_context_config') def testRaisesErrorIfMtlsUsedWithXml(self, mock_get_context_config): mock_context_config = mock.Mock() mock_context_config.use_client_certificate = True mock_get_context_config.return_value = mock_context_config # api_map setup from command_runner.py. api_map = cs_api_map.GsutilApiMapFactory.GetApiMap( gsutil_api_class_map_factory=cs_api_map.GsutilApiClassMapFactory, support_map={'s3': [cs_api_map.ApiSelector.XML]}, default_map={'s3': cs_api_map.ApiSelector.XML}) delegator = cloud_api_delegator.CloudApiDelegator(None, api_map, None, None) with self.assertRaises(cloud_api.ArgumentException): delegator.GetApiSelector(provider='s3') tests/test_parallel_cp.py 0000644 00000024227 15025013616 0011603 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2010 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Tests for parallel uploads ported from gsutil naming tests. Currently, the mock storage service is not thread-safe and therefore not suitable for multiprocess/multithreaded testing. Since parallel composite uploads necessarily create at least one worker thread outside of main, these tests are present in this file as temporary (slower) integration tests to provide validation for parallel composite uploads until a thread-safe mock storage service rewrite. Tests for relative paths are not included as integration_testcase does not support modifying the current working directory. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SequentialAndParallelTransfer from gslib.utils.retry_util import Retry class TestParallelCp(testcase.GsUtilIntegrationTestCase): """Unit tests for gsutil naming logic.""" @SequentialAndParallelTransfer def testCopyingTopLevelFileToBucket(self): """Tests copying one top-level file to a bucket.""" src_file = self.CreateTempFile(file_name='f0') dst_bucket_uri = self.CreateBucket() self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri)]) lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) self.assertEqual(suri(dst_bucket_uri, 'f0'), lines[0]) @SequentialAndParallelTransfer def testCopyingMultipleFilesToBucket(self): """Tests copying multiple files to a bucket.""" src_file0 = self.CreateTempFile(file_name='f0') src_file1 = self.CreateTempFile(file_name='f1') dst_bucket_uri = self.CreateBucket() self.RunGsUtil(['cp', src_file0, src_file1, suri(dst_bucket_uri)]) lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2) self.assertEqual(suri(dst_bucket_uri, 'f0'), lines[0]) self.assertEqual(suri(dst_bucket_uri, 'f1'), lines[1]) @SequentialAndParallelTransfer def testCopyingNestedFileToBucketSubdir(self): """Tests copying a nested file to a bucket subdir. Tests that we correctly translate local FS-specific delimiters ('\' on Windows) to bucket delimiter (/). """ tmpdir = self.CreateTempDir() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) src_file = self.CreateTempFile(tmpdir=tmpdir, file_name='obj', contents=b'') dst_bucket_uri = self.CreateBucket() # Make an object under subdir so next copy will treat subdir as a subdir. self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri, 'subdir/a')]) self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri, 'subdir')]) lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2) self.assertEqual(suri(dst_bucket_uri, 'subdir/a'), lines[0]) self.assertEqual(suri(dst_bucket_uri, 'subdir/obj'), lines[1]) @SequentialAndParallelTransfer def testCopyingAbsolutePathDirToBucket(self): """Tests recursively copying absolute path directory to a bucket.""" dst_bucket_uri = self.CreateBucket() src_dir_root = self.CreateTempDir( test_files=['f0', 'f1', 'f2.txt', ('dir0', 'dir1', 'nested')]) self.RunGsUtil(['cp', '-R', src_dir_root, suri(dst_bucket_uri)]) src_tmpdir = os.path.split(src_dir_root)[1] lines = self.AssertNObjectsInBucket(dst_bucket_uri, 4) self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'dir0', 'dir1', 'nested'), lines[0]) self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f0'), lines[1]) self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f1'), lines[2]) self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f2.txt'), lines[3]) @SequentialAndParallelTransfer def testCopyingDirContainingOneFileToBucket(self): """Tests copying a directory containing 1 file to a bucket. We test this case to ensure that correct bucket handling isn't dependent on the copy being treated as a multi-source copy. """ dst_bucket_uri = self.CreateBucket() src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'foo')]) self.RunGsUtil([ 'cp', '-R', os.path.join(src_dir, 'dir0', 'dir1'), suri(dst_bucket_uri) ]) lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) self.assertEqual(suri(dst_bucket_uri, 'dir1', 'foo'), lines[0]) @SkipForS3('The boto lib used for S3 does not handle objects ' 'starting with slashes if we use V4 signature') @SequentialAndParallelTransfer def testCopyingFileToObjectWithConsecutiveSlashes(self): """Tests copying a file to an object containing consecutive slashes.""" src_file = self.CreateTempFile(file_name='f0') dst_bucket_uri = self.CreateBucket() self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri) + '//obj']) lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) self.assertEqual(suri(dst_bucket_uri) + '//obj', lines[0]) @SequentialAndParallelTransfer def testCopyingObjsAndFilesToBucket(self): """Tests copying objects and files to a bucket.""" src_bucket_uri = self.CreateBucket() self.CreateObject(src_bucket_uri, object_name='f1', contents=b'foo') src_dir = self.CreateTempDir(test_files=['f2']) dst_bucket_uri = self.CreateBucket() self.RunGsUtil([ 'cp', '-R', suri(src_bucket_uri, '**'), '%s%s**' % (src_dir, os.sep), suri(dst_bucket_uri) ]) lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2) self.assertEqual(suri(dst_bucket_uri, 'f1'), lines[0]) self.assertEqual(suri(dst_bucket_uri, 'f2'), lines[1]) @SequentialAndParallelTransfer def testCopyingSubdirRecursiveToNonexistentSubdir(self): """Tests copying a directory with a single file recursively to a bucket. The file should end up in a new bucket subdirectory with the file's directory structure starting below the recursive copy point, as in Unix cp. Example: filepath: dir1/dir2/foo cp -r dir1 dir3 Results in dir3/dir2/foo being created. """ src_dir = self.CreateTempDir() self.CreateTempFile(tmpdir=src_dir + '/dir1/dir2', file_name='foo') dst_bucket_uri = self.CreateBucket() self.RunGsUtil( ['cp', '-R', src_dir + '/dir1', suri(dst_bucket_uri, 'dir3')]) lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1) self.assertEqual(suri(dst_bucket_uri, 'dir3/dir2/foo'), lines[0]) @SequentialAndParallelTransfer def testCopyingWildcardedFilesToBucketSubDir(self): """Tests copying wildcarded files to a bucket subdir.""" # Test with and without final slash on dest subdir. for final_dst_char in ('', '/'): dst_bucket_uri = self.CreateBucket() self.CreateObject(dst_bucket_uri, object_name='subdir0/existing', contents=b'foo') self.CreateObject(dst_bucket_uri, object_name='subdir1/existing', contents=b'foo') src_dir = self.CreateTempDir(test_files=['f0', 'f1', 'f2']) for i in range(2): self.RunGsUtil([ 'cp', os.path.join(src_dir, 'f?'), suri(dst_bucket_uri, 'subdir%d' % i) + final_dst_char ]) @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Validate files were copied to the correct destinations.""" stdout = self.RunGsUtil( ['ls', suri(dst_bucket_uri, 'subdir%d' % i, '**')], return_stdout=True) lines = stdout.split('\n') self.assertEqual(5, len(lines)) self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'existing'), lines[0]) self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f0'), lines[1]) self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f1'), lines[2]) self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f2'), lines[3]) _Check1() @SequentialAndParallelTransfer def testCopyingOneNestedFileToBucketSubDir(self): """Tests copying one nested file to a bucket subdir.""" # Test with and without final slash on dest subdir. for final_dst_char in ('', '/'): dst_bucket_uri = self.CreateBucket() self.CreateObject(dst_bucket_uri, object_name='d0/placeholder', contents=b'foo') self.CreateObject(dst_bucket_uri, object_name='d1/placeholder', contents=b'foo') for i in range(2): src_dir = self.CreateTempDir(test_files=[('d3', 'd4', 'nested', 'f1')]) self.RunGsUtil([ 'cp', '-r', suri(src_dir, 'd3'), suri(dst_bucket_uri, 'd%d' % i) + final_dst_char ]) lines = self.AssertNObjectsInBucket(dst_bucket_uri, 4) self.assertEqual(suri(dst_bucket_uri, 'd0', 'd3', 'd4', 'nested', 'f1'), lines[0]) self.assertEqual(suri(dst_bucket_uri, 'd0', 'placeholder'), lines[1]) self.assertEqual(suri(dst_bucket_uri, 'd1', 'd3', 'd4', 'nested', 'f1'), lines[2]) self.assertEqual(suri(dst_bucket_uri, 'd1', 'placeholder'), lines[3]) tests/test_execution_util.py 0000644 00000007605 15025013616 0012366 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for execution_util.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import subprocess from unittest import mock from gslib import exception from gslib.tests import testcase from gslib.utils import execution_util class TestExecutionUtil(testcase.GsUtilUnitTestCase): """Test execution utils.""" @mock.patch.object(subprocess, 'Popen') def testExternalCommandReturnsNoOutput(self, mock_Popen): mock_command_process = mock.Mock() mock_command_process.returncode = 0 mock_command_process.communicate.return_value = (None, None) mock_Popen.return_value = mock_command_process stdout, stderr = execution_util.ExecuteExternalCommand(['fake-command']) self.assertIsNone(stdout) self.assertIsNone(stderr) mock_Popen.assert_called_once_with(['fake-command'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @mock.patch.object(subprocess, 'Popen') def testExternalCommandReturnsStringOutput(self, mock_Popen): mock_command_process = mock.Mock() mock_command_process.returncode = 0 mock_command_process.communicate.return_value = ('a', 'b') mock_Popen.return_value = mock_command_process stdout, stderr = execution_util.ExecuteExternalCommand(['fake-command']) self.assertEqual(stdout, 'a') self.assertEqual(stderr, 'b') mock_Popen.assert_called_once_with(['fake-command'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @mock.patch.object(subprocess, 'Popen') def testExternalCommandReturnsBytesOutput(self, mock_Popen): mock_command_process = mock.Mock() mock_command_process.returncode = 0 mock_command_process.communicate.return_value = (b'a', b'b') mock_Popen.return_value = mock_command_process stdout, stderr = execution_util.ExecuteExternalCommand(['fake-command']) self.assertEqual(stdout, 'a') self.assertEqual(stderr, 'b') mock_Popen.assert_called_once_with(['fake-command'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @mock.patch.object(subprocess, 'Popen') def testExternalCommandReturnsNoOutput(self, mock_Popen): mock_command_process = mock.Mock() mock_command_process.returncode = 1 mock_command_process.communicate.return_value = (None, b'error') mock_Popen.return_value = mock_command_process with self.assertRaises(exception.ExternalBinaryError): execution_util.ExecuteExternalCommand(['fake-command']) mock_Popen.assert_called_once_with(['fake-command'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) @mock.patch.object(subprocess, 'Popen') def testExternalCommandRaisesFormattedStderr(self, mock_Popen): mock_command_process = mock.Mock() mock_command_process.returncode = 1 mock_command_process.communicate.return_value = (None, b'error.\n') mock_Popen.return_value = mock_command_process with self.assertRaisesRegex(exception.ExternalBinaryError, 'error'): execution_util.ExecuteExternalCommand(['fake-command']) tests/test_bucketconfig.py 0000644 00000011756 15025013616 0011773 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for multiple bucket configuration commands.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.utils.constants import UTF8 class TestBucketConfig(testcase.GsUtilIntegrationTestCase): """Integration tests for multiple bucket configuration commands.""" _set_cors_command = ['cors', 'set'] _get_cors_command = ['cors', 'get'] empty_cors = '[]' cors_doc = ( '[{"origin": ["http://origin1.example.com", ' '"http://origin2.example.com"], ' '"responseHeader": ["foo", "bar"], "method": ["GET", "PUT", "POST"], ' '"maxAgeSeconds": 3600},' '{"origin": ["http://origin3.example.com"], ' '"responseHeader": ["foo2", "bar2"], "method": ["GET", "DELETE"]}]\n') cors_json_obj = json.loads(cors_doc) _set_lifecycle_command = ['lifecycle', 'set'] _get_lifecycle_command = ['lifecycle', 'get'] empty_lifecycle = '{}' lifecycle_doc = ( '{"rule": [{"action": {"type": "Delete"}, "condition": {"age": 365}}]}\n') lifecycle_json_obj = json.loads(lifecycle_doc) _set_acl_command = ['acl', 'set'] _get_acl_command = ['acl', 'get'] _set_defacl_command = ['defacl', 'set'] _get_defacl_command = ['defacl', 'get'] @SkipForS3('A number of configs in this test are not supported by S3') def test_set_multi_config(self): """Tests that bucket config patching affects only the desired config.""" bucket_uri = self.CreateBucket() lifecycle_path = self.CreateTempFile( contents=self.lifecycle_doc.encode(UTF8)) cors_path = self.CreateTempFile(contents=self.cors_doc.encode(UTF8)) self.RunGsUtil(self._set_cors_command + [cors_path, suri(bucket_uri)]) cors_out = self.RunGsUtil(self._get_cors_command + [suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(cors_out), self.cors_json_obj) self.RunGsUtil(self._set_lifecycle_command + [lifecycle_path, suri(bucket_uri)]) cors_out = self.RunGsUtil(self._get_cors_command + [suri(bucket_uri)], return_stdout=True) lifecycle_out = self.RunGsUtil(self._get_lifecycle_command + [suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(cors_out), self.cors_json_obj) self.assertEqual(json.loads(lifecycle_out), self.lifecycle_json_obj) if not self._ServiceAccountCredentialsPresent(): # See comments in _ServiceAccountCredentialsPresent self.RunGsUtil( self._set_acl_command + ['authenticated-read', suri(bucket_uri)]) cors_out = self.RunGsUtil(self._get_cors_command + [suri(bucket_uri)], return_stdout=True) lifecycle_out = self.RunGsUtil(self._get_lifecycle_command + [suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(cors_out), self.cors_json_obj) self.assertEqual(json.loads(lifecycle_out), self.lifecycle_json_obj) if not self._ServiceAccountCredentialsPresent(): acl_out = self.RunGsUtil(self._get_acl_command + [suri(bucket_uri)], return_stdout=True) self.assertIn('allAuthenticatedUsers', acl_out) self.RunGsUtil(self._set_defacl_command + ['public-read', suri(bucket_uri)]) cors_out = self.RunGsUtil(self._get_cors_command + [suri(bucket_uri)], return_stdout=True) lifecycle_out = self.RunGsUtil(self._get_lifecycle_command + [suri(bucket_uri)], return_stdout=True) def_acl_out = self.RunGsUtil(self._get_defacl_command + [suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(cors_out), self.cors_json_obj) self.assertEqual(json.loads(lifecycle_out), self.lifecycle_json_obj) self.assertIn('allUsers', def_acl_out) if not self._ServiceAccountCredentialsPresent(): acl_out = self.RunGsUtil(self._get_acl_command + [suri(bucket_uri)], return_stdout=True) self.assertIn('allAuthenticatedUsers', acl_out) tests/test_stet_cp.py 0000644 00000013425 15025013616 0010764 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for cp STET binary integration. Could go with cp tests but that file is bulky.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals import os import stat from gslib import storage_url from gslib.tests import testcase from gslib.tests import util from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import unittest from gslib.utils import system_util from gslib.utils import temporary_file_util FAKE_STET_BINARY = """\ #!/bin/sh if [ "$#" -ne 5 ]; then echo "Expected 5 arguments. Got $#." exit 1 fi echo "subcommand: $1 config file: $2 blob id: $3 in file: $4 out file: $5" > $5 """ @unittest.skipIf(not system_util.IS_LINUX, 'STET binary supports only Linux.') class TestStetCp(testcase.GsUtilIntegrationTestCase): """Integration tests for cp integration with STET binary.""" def setUp(self): super(TestStetCp, self).setUp() # Substitute for actual binary. self.stet_binary_path = self.CreateTempFile(contents=FAKE_STET_BINARY) # Make shell script executable in addition to current permissions. current_stat = os.stat(self.stet_binary_path) os.chmod(self.stet_binary_path, current_stat.st_mode | stat.S_IEXEC) # Fake config. self.stet_config_path = self.CreateTempFile() def test_encrypts_upload_if_stet_is_enabled(self): object_uri = self.CreateObject() test_file = self.CreateTempFile(contents='will be rewritten') stderr = self.RunGsUtil([ '-o', 'GSUtil:stet_binary_path={}'.format(self.stet_binary_path), '-o', 'GSUtil:stet_config_path={}'.format( self.stet_config_path), 'cp', '--stet', test_file, suri(object_uri) ], return_stderr=True) # Progress indicator should show transformed file size (variable based on # random string generation above). 4.0 B is the pre-transform size. self.assertNotIn('/4.0 B]', stderr) stdout = self.RunGsUtil(['cat', suri(object_uri)], return_stdout=True) self.assertIn('subcommand: encrypt', stdout) self.assertIn('config file: --config-file={}'.format(self.stet_config_path), stdout) self.assertIn('blob id: --blob-id={}'.format(suri(object_uri)), stdout) self.assertIn('in file: {}'.format(test_file), stdout) self.assertIn('out file: {}_.stet_tmp'.format(test_file), stdout) self.assertFalse( os.path.exists( temporary_file_util.GetStetTempFileName( storage_url.StorageUrlFromString(test_file)))) def test_decrypts_download_if_stet_is_enabled(self): object_uri = self.CreateObject(contents='abc') test_file = self.CreateTempFile() stderr = self.RunGsUtil([ '-o', 'GSUtil:stet_binary_path={}'.format(self.stet_binary_path), '-o', 'GSUtil:stet_config_path={}'.format( self.stet_config_path), 'cp', '--stet', suri(object_uri), test_file ], return_stderr=True) # Progress indicator should show transformed file size (variable based on # random string generation above). 4.0 B is the pre-transform size. self.assertNotIn('/4.0 B]', stderr) with open(test_file) as file_reader: downloaded_text = file_reader.read() self.assertIn('subcommand: decrypt', downloaded_text) self.assertIn('config file: --config-file={}'.format(self.stet_config_path), downloaded_text) self.assertIn('blob id: --blob-id={}'.format(suri(object_uri)), downloaded_text) self.assertIn('in file: {}'.format(test_file), downloaded_text) self.assertIn('out file: {}_.stet_tmp'.format(test_file), downloaded_text) self.assertFalse( os.path.exists( temporary_file_util.GetStetTempFileName( storage_url.StorageUrlFromString(test_file)))) def test_does_not_seek_ahead_for_bytes_if_stet_transform(self): """Tests that cp does not seek-ahead for bytes if file size will change.""" tmpdir = self.CreateTempDir() for _ in range(3): self.CreateTempFile(tmpdir=tmpdir, contents=b'123456') bucket_uri = self.CreateBucket() with util.SetBotoConfigForTest([ ('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True') ]): stderr = self.RunGsUtil([ '-m', '-o', 'GSUtil:stet_binary_path={}'.format( self.stet_binary_path), '-o', 'GSUtil:stet_config_path={}'.format( self.stet_config_path), 'cp', '-r', '--stet', tmpdir, suri(bucket_uri) ], return_stderr=True) # Check the denominator of the progress indicator to see if the # correct total file size is shown. This is a proxy for if the seek-ahead # thread ran because if seak-ahead runs before STET encryption, an # incorrect, smaller file size will show. The full progress string looks # something like: "[3 files][ 2.0 KiB/ 2.0 KiB]" # Pre-encryption total file size: self.assertNotIn('18.0 B]', stderr) # Post-encryption total file size: # +-0.1 KiB b/c of rounding and different platforms. self.assertRegex(stderr, r'2\.\d KiB]') tests/test_tabcomplete.py 0000644 00000034473 15025013616 0011630 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for tab completion.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import time import sys from gslib.command import CreateOrGetGsutilLogger from gslib.tab_complete import CloudObjectCompleter from gslib.tab_complete import TAB_COMPLETE_CACHE_TTL from gslib.tab_complete import TabCompletionCache import gslib.tests.testcase as testcase from gslib.tests.util import ARGCOMPLETE_AVAILABLE from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import unittest from gslib.tests.util import WorkingDirectory from gslib.utils.boto_util import GetTabCompletionCacheFilename @unittest.skipUnless(ARGCOMPLETE_AVAILABLE, 'Tab completion requires argcomplete') @unittest.skipUnless(sys.version_info < (3, 11, 9), 'Tab completion is only supported on Python < 3.11.9') class TestTabComplete(testcase.GsUtilIntegrationTestCase): """Integration tests for tab completion.""" def setUp(self): super(TestTabComplete, self).setUp() self.logger = CreateOrGetGsutilLogger('tab_complete') def test_single_bucket(self): """Tests tab completion matching a single bucket.""" # Prefix is a workaround for XML API limitation, see PR 766 for details bucket_name = self.MakeTempName('bucket', prefix='aaa-') self.CreateBucket(bucket_name) request = '%s://%s' % (self.default_provider, bucket_name[:-2]) expected_result = '//%s/' % bucket_name self.RunGsUtilTabCompletion(['ls', request], expected_results=[expected_result]) def test_bucket_only_single_bucket(self): """Tests bucket-only tab completion matching a single bucket.""" bucket_name = self.MakeTempName('bucket', prefix='aaa-') # Workaround for XML API limitation, see PR 766 for details self.CreateBucket(bucket_name) request = '%s://%s' % (self.default_provider, bucket_name[:-2]) expected_result = '//%s ' % bucket_name self.RunGsUtilTabCompletion(['rb', request], expected_results=[expected_result]) def test_bucket_only_no_objects(self): """Tests that bucket-only tab completion doesn't match objects.""" object_name = self.MakeTempName('obj') object_uri = self.CreateObject(object_name=object_name, contents=b'data') request = '%s://%s/%s' % (self.default_provider, object_uri.bucket_name, object_name[:-2]) self.RunGsUtilTabCompletion(['rb', request], expected_results=[]) def test_single_subdirectory(self): """Tests tab completion matching a single subdirectory.""" object_base_name = self.MakeTempName('obj') object_name = object_base_name + '/subobj' object_uri = self.CreateObject(object_name=object_name, contents=b'data') request = '%s://%s/' % (self.default_provider, object_uri.bucket_name) expected_result = '//%s/%s/' % (object_uri.bucket_name, object_base_name) self.RunGsUtilTabCompletion(['ls', request], expected_results=[expected_result]) def test_multiple_buckets(self): """Tests tab completion matching multiple buckets.""" base_name = self.MakeTempName('bucket') # Workaround for XML API limitation, see PR 766 for details prefix = 'aaa-' self.CreateBucket(base_name, bucket_name_prefix=prefix, bucket_name_suffix='1') self.CreateBucket(base_name, bucket_name_prefix=prefix, bucket_name_suffix='2') request = '%s://%s' % (self.default_provider, ''.join([prefix, base_name])) expected_result1 = '//%s/' % ''.join([prefix, base_name, '1']) expected_result2 = '//%s/' % ''.join([prefix, base_name, '2']) self.RunGsUtilTabCompletion( ['ls', request], expected_results=[expected_result1, expected_result2]) def test_single_object(self): """Tests tab completion matching a single object.""" object_name = self.MakeTempName('obj') object_uri = self.CreateObject(object_name=object_name, contents=b'data') request = '%s://%s/%s' % (self.default_provider, object_uri.bucket_name, object_name[:-2]) expected_result = '//%s/%s ' % (object_uri.bucket_name, object_name) self.RunGsUtilTabCompletion(['ls', request], expected_results=[expected_result]) def test_multiple_objects(self): """Tests tab completion matching multiple objects.""" bucket_uri = self.CreateBucket() object_base_name = self.MakeTempName('obj') object1_name = object_base_name + '-suffix1' self.CreateObject(bucket_uri=bucket_uri, object_name=object1_name, contents=b'data') object2_name = object_base_name + '-suffix2' self.CreateObject(bucket_uri=bucket_uri, object_name=object2_name, contents=b'data') request = '%s://%s/%s' % (self.default_provider, bucket_uri.bucket_name, object_base_name) expected_result1 = '//%s/%s' % (bucket_uri.bucket_name, object1_name) expected_result2 = '//%s/%s' % (bucket_uri.bucket_name, object2_name) self.RunGsUtilTabCompletion( ['ls', request], expected_results=[expected_result1, expected_result2]) def test_subcommands(self): """Tests tab completion for commands with subcommands.""" bucket_name = self.MakeTempName('bucket', prefix='aaa-') self.CreateBucket(bucket_name) bucket_request = '%s://%s' % (self.default_provider, bucket_name[:-2]) expected_bucket_result = '//%s ' % bucket_name local_file = 'a_local_file' local_dir = self.CreateTempDir(test_files=[local_file]) local_file_request = '%s%s' % (local_dir, os.sep) expected_local_file_result = '%s ' % os.path.join(local_dir, local_file) # Should invoke Cloud bucket URL completer. self.RunGsUtilTabCompletion(['cors', 'get', bucket_request], expected_results=[expected_bucket_result]) # Should invoke File URL completer which should match the local file. self.RunGsUtilTabCompletion(['cors', 'set', local_file_request], expected_results=[expected_local_file_result]) # Should invoke Cloud bucket URL completer. self.RunGsUtilTabCompletion(['cors', 'set', 'some_file', bucket_request], expected_results=[expected_bucket_result]) def test_invalid_partial_bucket_name(self): """Tests tab completion with a partial URL that by itself is not valid. The bucket name in a Cloud URL cannot end in a dash, but a partial URL during tab completion may end in a dash and completion should still work. """ bucket_base_name = self.MakeTempName('bucket', prefix='aaa-') bucket_name = bucket_base_name + '-s' self.CreateBucket(bucket_name) request = '%s://%s-' % (self.default_provider, bucket_base_name) expected_result = '//%s/' % bucket_name self.RunGsUtilTabCompletion(['ls', request], expected_results=[expected_result]) def test_acl_argument(self): """Tests tab completion for ACL arguments.""" local_file = 'a_local_file' local_dir = self.CreateTempDir(test_files=[local_file]) local_file_request = '%s%s' % (local_dir, os.sep) expected_local_file_result = '%s ' % os.path.join(local_dir, local_file) # Should invoke File URL completer which should match the local file. self.RunGsUtilTabCompletion(['acl', 'set', local_file_request], expected_results=[expected_local_file_result]) # Should match canned ACL name. self.RunGsUtilTabCompletion(['acl', 'set', 'priv'], expected_results=['private ']) local_file = 'priv_file' local_dir = self.CreateTempDir(test_files=[local_file]) with WorkingDirectory(local_dir): # Should match both a file and a canned ACL since argument takes # either one. self.RunGsUtilTabCompletion(['acl', 'set', 'priv'], expected_results=[local_file, 'private']) def _WriteTabCompletionCache(prefix, results, timestamp=None, partial_results=False): if timestamp is None: timestamp = time.time() cache = TabCompletionCache(prefix, results, timestamp, partial_results) cache.WriteToFile(GetTabCompletionCacheFilename()) @unittest.skipUnless(ARGCOMPLETE_AVAILABLE, 'Tab completion requires argcomplete') class TestTabCompleteUnitTests(testcase.unit_testcase.GsUtilUnitTestCase): """Unit tests for tab completion.""" def test_cached_results(self): """Tests tab completion results returned from cache.""" with SetBotoConfigForTest([('GSUtil', 'state_dir', self.CreateTempDir())]): request = 'gs://prefix' cached_results = ['gs://prefix1', 'gs://prefix2'] _WriteTabCompletionCache(request, cached_results) completer = CloudObjectCompleter(self.MakeGsUtilApi()) results = completer(request) self.assertEqual(cached_results, results) def test_expired_cached_results(self): """Tests tab completion results not returned from cache when too old.""" with SetBotoConfigForTest([('GSUtil', 'state_dir', self.CreateTempDir())]): bucket_base_name = self.MakeTempName('bucket') bucket_name = bucket_base_name + '-suffix' self.CreateBucket(bucket_name) request = '%s://%s' % (self.default_provider, bucket_base_name) expected_result = '%s://%s/' % (self.default_provider, bucket_name) cached_results = ['//%s1' % bucket_name, '//%s2' % bucket_name] _WriteTabCompletionCache(request, cached_results, time.time() - TAB_COMPLETE_CACHE_TTL) completer = CloudObjectCompleter(self.MakeGsUtilApi()) results = completer(request) self.assertEqual([expected_result], results) def test_prefix_caching(self): """Tests tab completion results returned from cache with prefix match. If the tab completion prefix is an extension of the cached prefix, tab completion should return results from the cache that start with the prefix. """ with SetBotoConfigForTest([('GSUtil', 'state_dir', self.CreateTempDir())]): cached_prefix = 'gs://prefix' cached_results = ['gs://prefix-first', 'gs://prefix-second'] _WriteTabCompletionCache(cached_prefix, cached_results) request = 'gs://prefix-f' completer = CloudObjectCompleter(self.MakeGsUtilApi()) results = completer(request) self.assertEqual(['gs://prefix-first'], results) def test_prefix_caching_boundary(self): """Tests tab completion prefix caching not spanning directory boundaries. If the tab completion prefix is an extension of the cached prefix, but is not within the same bucket/sub-directory then the cached results should not be used. """ with SetBotoConfigForTest([('GSUtil', 'state_dir', self.CreateTempDir())]): object_uri = self.CreateObject(object_name='subdir/subobj', contents=b'test data') cached_prefix = '%s://%s/' % (self.default_provider, object_uri.bucket_name) cached_results = [ '%s://%s/subdir' % (self.default_provider, object_uri.bucket_name) ] _WriteTabCompletionCache(cached_prefix, cached_results) request = '%s://%s/subdir/' % (self.default_provider, object_uri.bucket_name) expected_result = '%s://%s/subdir/subobj' % (self.default_provider, object_uri.bucket_name) completer = CloudObjectCompleter(self.MakeGsUtilApi()) results = completer(request) self.assertEqual([expected_result], results) def test_prefix_caching_no_results(self): """Tests tab completion returning empty result set using cached prefix. If the tab completion prefix is an extension of the cached prefix, but does not match any of the cached results then no remote request should be made and an empty result set should be returned. """ with SetBotoConfigForTest([('GSUtil', 'state_dir', self.CreateTempDir())]): object_uri = self.CreateObject(object_name='obj', contents=b'test data') cached_prefix = '%s://%s/' % (self.default_provider, object_uri.bucket_name) cached_results = [] _WriteTabCompletionCache(cached_prefix, cached_results) request = '%s://%s/o' % (self.default_provider, object_uri.bucket_name) completer = CloudObjectCompleter(self.MakeGsUtilApi()) results = completer(request) self.assertEqual([], results) def test_prefix_caching_partial_results(self): """Tests tab completion prefix matching ignoring partial cached results. If the tab completion prefix is an extension of the cached prefix, but the cached result set is partial, the cached results should not be used because the matching results for the prefix may be incomplete. """ with SetBotoConfigForTest([('GSUtil', 'state_dir', self.CreateTempDir())]): object_uri = self.CreateObject(object_name='obj', contents=b'test data') cached_prefix = '%s://%s/' % (self.default_provider, object_uri.bucket_name) cached_results = [] _WriteTabCompletionCache(cached_prefix, cached_results, partial_results=True) request = '%s://%s/o' % (self.default_provider, object_uri.bucket_name) completer = CloudObjectCompleter(self.MakeGsUtilApi()) results = completer(request) self.assertEqual([str(object_uri)], results) tests/test_mv.py 0000644 00000032052 15025013616 0007742 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for mv command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from gslib.cs_api_map import ApiSelector from gslib.tests.test_cp import TestCpMvPOSIXBucketToLocalErrors from gslib.tests.test_cp import TestCpMvPOSIXBucketToLocalNoErrors from gslib.tests.test_cp import TestCpMvPOSIXLocalToBucketNoErrors import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SequentialAndParallelTransfer from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import unittest from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.retry_util import Retry from gslib.utils.system_util import IS_WINDOWS from gslib.utils import shim_util class TestMvUnitTests(testcase.GsUtilUnitTestCase): """Unit tests for mv command.""" def test_move_bucket_objects_with_duplicate_names_inter_bucket(self): """Tests moving multiple top-level items between buckets.""" bucket1_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='dir1/file.txt', contents=b'data') self.CreateObject(bucket_uri=bucket1_uri, object_name='dir2/file.txt', contents=b'data') bucket2_uri = self.CreateBucket() self.RunCommand('mv', [suri(bucket1_uri, '*'), suri(bucket2_uri)]) actual = set( str(u) for u in self._test_wildcard_iterator(suri(bucket2_uri, '**')).IterAll( expand_top_level_buckets=True)) expected = set([ suri(bucket2_uri, 'dir1', 'file.txt'), suri(bucket2_uri, 'dir2', 'file.txt'), ]) self.assertEqual(actual, expected) def test_move_bucket_objects_with_duplicate_names_to_bucket_subdir(self): """Tests moving multiple top-level items between buckets.""" bucket1_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='dir1/file.txt', contents=b'data') self.CreateObject(bucket_uri=bucket1_uri, object_name='dir2/file.txt', contents=b'data') bucket2_uri = self.CreateBucket() self.RunCommand('mv', [suri(bucket1_uri, '*'), suri(bucket2_uri, 'dir')]) actual = set( str(u) for u in self._test_wildcard_iterator(suri(bucket2_uri, '**')).IterAll( expand_top_level_buckets=True)) expected = set([ suri(bucket2_uri, 'dir', 'dir1', 'file.txt'), suri(bucket2_uri, 'dir', 'dir2', 'file.txt'), ]) self.assertEqual(actual, expected) def test_move_dirs_with_duplicate_file_names_to_bucket(self): """Tests moving multiple top-level items to a bucket.""" bucket_uri = self.CreateBucket() dir_path = self.CreateTempDir(test_files=[ ('dir1', 'file.txt'), ('dir2', 'file.txt'), ]) self.RunCommand('mv', [dir_path + '/*', suri(bucket_uri)]) actual = set( str(u) for u in self._test_wildcard_iterator(suri(bucket_uri, '**')).IterAll( expand_top_level_buckets=True)) expected = set([ suri(bucket_uri, 'dir1', 'file.txt'), suri(bucket_uri, 'dir2', 'file.txt'), ]) self.assertEqual(actual, expected) class TestMvUnitTestsWithShim(testcase.ShimUnitTestBase): """Unit tests for mv command with shim.""" def test_shim_translates_flags(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=b'abcd') with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'mv', ['-a', 'public-read', fpath, suri(bucket_uri)], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage mv' ' --predefined-acl publicRead {} {}'.format( shim_util._get_gcloud_binary_path('fake_dir'), fpath, suri(bucket_uri)), info_lines) class TestMvE2ETests(testcase.GsUtilIntegrationTestCase): """Integration tests for mv command.""" def test_moving(self): """Tests moving two buckets, one with 2 objects and one with 0 objects.""" bucket1_uri = self.CreateBucket(test_objects=2) self.AssertNObjectsInBucket(bucket1_uri, 2) bucket2_uri = self.CreateBucket() self.AssertNObjectsInBucket(bucket2_uri, 0) # Move two objects from bucket1 to bucket2. objs = [ self.StorageUriCloneReplaceKey(bucket1_uri, key).versionless_uri for key in bucket1_uri.list_bucket() ] cmd = (['-m', 'mv'] + objs + [suri(bucket2_uri)]) stderr = self.RunGsUtil(cmd, return_stderr=True) # Rewrite API may output an additional 'Copying' progress notification. self.assertGreaterEqual( stderr.count('Copying'), 2, 'stderr did not contain 2 "Copying" lines:\n%s' % stderr) self.assertLessEqual( stderr.count('Copying'), 4, 'stderr did not contain <= 4 "Copying" lines:\n%s' % stderr) self.assertEqual( stderr.count('Copying') % 2, 0, 'stderr did not contain even number of "Copying" lines:\n%s' % stderr) self.assertEqual(stderr.count('Removing'), 2, 'stderr did not contain 2 "Removing" lines:\n%s' % stderr) self.AssertNObjectsInBucket(bucket1_uri, 0) self.AssertNObjectsInBucket(bucket2_uri, 2) # Remove one of the objects. objs = [ self.StorageUriCloneReplaceKey(bucket2_uri, key).versionless_uri for key in bucket2_uri.list_bucket() ] obj1 = objs[0] self.RunGsUtil(['rm', obj1]) self.AssertNObjectsInBucket(bucket1_uri, 0) self.AssertNObjectsInBucket(bucket2_uri, 1) # Move the 1 remaining object back. objs = [ suri(self.StorageUriCloneReplaceKey(bucket2_uri, key)) for key in bucket2_uri.list_bucket() ] cmd = (['-m', 'mv'] + objs + [suri(bucket1_uri)]) stderr = self.RunGsUtil(cmd, return_stderr=True) # Rewrite API may output an additional 'Copying' progress notification. self.assertGreaterEqual( stderr.count('Copying'), 1, 'stderr did not contain >= 1 "Copying" lines:\n%s' % stderr) self.assertLessEqual( stderr.count('Copying'), 2, 'stderr did not contain <= 2 "Copying" lines:\n%s' % stderr) self.assertEqual(stderr.count('Removing'), 1) self.AssertNObjectsInBucket(bucket1_uri, 1) self.AssertNObjectsInBucket(bucket2_uri, 0) def test_move_bucket_to_dir(self): """Tests moving a local directory to a bucket.""" bucket_uri = self.CreateBucket(test_objects=2) self.AssertNObjectsInBucket(bucket_uri, 2) tmpdir = self.CreateTempDir() self.RunGsUtil(['mv', suri(bucket_uri, '*'), tmpdir]) dir_list = [] for dirname, _, filenames in os.walk(tmpdir): for filename in filenames: dir_list.append(os.path.join(dirname, filename)) self.assertEqual(len(dir_list), 2) self.AssertNObjectsInBucket(bucket_uri, 0) def test_move_dir_to_bucket(self): """Tests moving a local directory to a bucket.""" bucket_uri = self.CreateBucket() dir_to_move = self.CreateTempDir(test_files=2) self.RunGsUtil(['mv', dir_to_move, suri(bucket_uri)]) self.AssertNObjectsInBucket(bucket_uri, 2) @SequentialAndParallelTransfer def test_stdin_args(self): """Tests mv with the -I option.""" tmpdir = self.CreateTempDir() fpath1 = self.CreateTempFile(tmpdir=tmpdir, contents=b'data1') fpath2 = self.CreateTempFile(tmpdir=tmpdir, contents=b'data2') bucket_uri = self.CreateBucket() self.RunGsUtil(['mv', '-I', suri(bucket_uri)], stdin='\n'.join((fpath1, fpath2))) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['ls', suri(bucket_uri)], return_stdout=True) self.assertIn(os.path.basename(fpath1), stdout) self.assertIn(os.path.basename(fpath2), stdout) self.assertNumLines(stdout, 2) _Check1() def test_mv_no_clobber(self): """Tests mv with the -n option.""" fpath1 = self.CreateTempFile(contents=b'data1') bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'data2') stderr = self.RunGsUtil( ['mv', '-n', fpath1, suri(object_uri)], return_stderr=True) # Copy should be skipped and source file should not be removed. if self._use_gcloud_storage: self.assertIn( 'Skipping existing destination item (no-clobber): %s' % suri(object_uri), stderr) else: self.assertIn('Skipping existing item: %s' % suri(object_uri), stderr) self.assertNotIn('Removing %s' % suri(fpath1), stderr) # Object content should be unchanged. contents = self.RunGsUtil(['cat', suri(object_uri)], return_stdout=True) self.assertEqual(contents, 'data2') @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_mv_preserve_posix_bucket_to_dir_no_errors(self): """Tests use of the -P flag with mv from a bucket to a local dir. Specifically tests combinations of POSIX attributes in metadata that will pass validation. """ bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() TestCpMvPOSIXBucketToLocalNoErrors(self, bucket_uri, tmpdir, is_cp=False) @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') def test_mv_preserve_posix_bucket_to_dir_errors(self): """Tests use of the -P flag with mv from a bucket to a local dir. Specifically, combinations of POSIX attributes in metadata that will fail validation. """ bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() obj = self.CreateObject(bucket_uri=bucket_uri, object_name='obj', contents=b'obj') TestCpMvPOSIXBucketToLocalErrors(self, bucket_uri, obj, tmpdir, is_cp=False) @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') def test_mv_preseve_posix_dir_to_bucket_no_errors(self): """Tests use of the -P flag with mv from a local dir to a bucket.""" bucket_uri = self.CreateBucket() TestCpMvPOSIXLocalToBucketNoErrors(self, bucket_uri, is_cp=False) @SkipForS3('Test is only relevant for gs storage classes.') def test_mv_early_deletion_warning(self): """Tests that mv on a recent nearline object warns about early deletion.""" if self.test_api == ApiSelector.XML: return unittest.skip('boto does not return object storage class') bucket_uri = self.CreateBucket(storage_class='NEARLINE') object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'obj') stderr = self.RunGsUtil( ['mv', suri(object_uri), suri(bucket_uri, 'foo')], return_stderr=True) self.assertIn( 'Warning: moving nearline object %s may incur an early deletion ' 'charge, because the original object is less than 30 days old ' 'according to the local system time.' % suri(object_uri), stderr) def test_move_bucket_objects_with_duplicate_names_to_dir(self): """Tests moving multiple top-level items to a bucket.""" bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='dir1/file.txt', contents=b'data') self.CreateObject(bucket_uri=bucket_uri, object_name='dir2/file.txt', contents=b'data') self.AssertNObjectsInBucket(bucket_uri, 2) tmpdir = self.CreateTempDir() self.RunGsUtil(['mv', suri(bucket_uri, '*'), tmpdir]) file_list = [] for dirname, _, filenames in os.walk(tmpdir): for filename in filenames: file_list.append(os.path.join(dirname, filename)) self.assertEqual(len(file_list), 2) self.assertIn('{}{}dir1{}file.txt'.format(tmpdir, os.sep, os.sep), file_list) self.assertIn('{}{}dir2{}file.txt'.format(tmpdir, os.sep, os.sep), file_list) self.AssertNObjectsInBucket(bucket_uri, 0) tests/test_acl.py 0000644 00000157767 15025013616 0010104 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for the acl command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import re from gslib.commands import acl from gslib.command import CreateOrGetGsutilLogger from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import StorageUrlFromString import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForGS from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import GenerationFromURI as urigen from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import unittest from gslib.utils import acl_helper from gslib.utils.constants import UTF8 from gslib.utils.retry_util import Retry from gslib.utils.translation_helper import AclTranslation from gslib.utils import shim_util from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock PUBLIC_READ_JSON_ACL_TEXT = '"entity":"allUsers","role":"READER"' class TestAclBase(testcase.GsUtilIntegrationTestCase): """Integration test case base class for acl command.""" _set_acl_prefix = ['acl', 'set'] _get_acl_prefix = ['acl', 'get'] _set_defacl_prefix = ['defacl', 'set'] _ch_acl_prefix = ['acl', 'ch'] _project_team = 'viewers' @SkipForS3('Tests use GS ACL model.') class TestAcl(TestAclBase): """Integration tests for acl command.""" def setUp(self): super(TestAcl, self).setUp() self.sample_uri = self.CreateBucket() self.sample_url = StorageUrlFromString(str(self.sample_uri)) self.logger = CreateOrGetGsutilLogger('acl') # Argument to acl ch -p must be the project number, not a name; create a # bucket to perform translation. self._project_number = self.json_api.GetBucket( self.CreateBucket().bucket_name, fields=['projectNumber']).projectNumber self._project_test_acl = '%s-%s' % (self._project_team, self._project_number) def test_set_invalid_acl_object(self): """Ensures that invalid content returns a bad request error.""" obj_uri = suri(self.CreateObject(contents=b'foo')) inpath = self.CreateTempFile(contents=b'badAcl') stderr = self.RunGsUtil(self._set_acl_prefix + [inpath, obj_uri], return_stderr=True, expected_status=1) if self._use_gcloud_storage: error_text = 'Found invalid JSON/YAML' else: error_text = 'ArgumentException' self.assertIn(error_text, stderr) def test_set_invalid_acl_bucket(self): """Ensures that invalid content returns a bad request error.""" bucket_uri = suri(self.CreateBucket()) inpath = self.CreateTempFile(contents=b'badAcl') stderr = self.RunGsUtil(self._set_acl_prefix + [inpath, bucket_uri], return_stderr=True, expected_status=1) if self._use_gcloud_storage: error_text = 'Found invalid JSON/YAML' else: error_text = 'ArgumentException' self.assertIn(error_text, stderr) def test_set_xml_acl_json_api_object(self): """Ensures XML content returns a bad request error and migration warning.""" obj_uri = suri(self.CreateObject(contents=b'foo')) inpath = self.CreateTempFile(contents=b'<ValidXml></ValidXml>') stderr = self.RunGsUtil(self._set_acl_prefix + [inpath, obj_uri], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('Found invalid JSON/YAML', stderr) # XML not currently supported in gcloud storage. else: self.assertIn('ArgumentException', stderr) self.assertIn('XML ACL data provided', stderr) def test_set_xml_acl_json_api_bucket(self): """Ensures XML content returns a bad request error and migration warning.""" bucket_uri = suri(self.CreateBucket()) inpath = self.CreateTempFile(contents=b'<ValidXml></ValidXml>') stderr = self.RunGsUtil(self._set_acl_prefix + [inpath, bucket_uri], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('Found invalid JSON/YAML', stderr) # XML not currently supported in gcloud storage. else: self.assertIn('ArgumentException', stderr) self.assertIn('XML ACL data provided', stderr) def test_set_valid_acl_object(self): """Tests setting a valid ACL on an object.""" obj_uri = suri(self.CreateObject(contents=b'foo')) acl_string = self.RunGsUtil(self._get_acl_prefix + [obj_uri], return_stdout=True) inpath = self.CreateTempFile(contents=acl_string.encode(UTF8)) self.RunGsUtil(self._set_acl_prefix + ['public-read', obj_uri]) acl_string2 = self.RunGsUtil(self._get_acl_prefix + [obj_uri], return_stdout=True) self.RunGsUtil(self._set_acl_prefix + [inpath, obj_uri]) acl_string3 = self.RunGsUtil(self._get_acl_prefix + [obj_uri], return_stdout=True) self.assertNotEqual(acl_string, acl_string2) self.assertEqual(acl_string, acl_string3) def test_set_valid_permission_whitespace_object(self): """Ensures that whitespace is allowed in role and entity elements.""" obj_uri = suri(self.CreateObject(contents=b'foo')) acl_string = self.RunGsUtil(self._get_acl_prefix + [obj_uri], return_stdout=True) acl_string = re.sub(r'"role"', r'"role" \n', acl_string) acl_string = re.sub(r'"entity"', r'\n "entity"', acl_string) inpath = self.CreateTempFile(contents=acl_string.encode(UTF8)) self.RunGsUtil(self._set_acl_prefix + [inpath, obj_uri]) def test_set_valid_acl_bucket(self): """Ensures that valid canned and XML ACLs work with get/set.""" if self._ServiceAccountCredentialsPresent(): # See comments in _ServiceAccountCredentialsPresent return unittest.skip('Canned ACLs orphan service account permissions.') bucket_uri = suri(self.CreateBucket()) acl_string = self.RunGsUtil(self._get_acl_prefix + [bucket_uri], return_stdout=True) inpath = self.CreateTempFile(contents=acl_string.encode(UTF8)) self.RunGsUtil(self._set_acl_prefix + ['public-read', bucket_uri]) acl_string2 = self.RunGsUtil(self._get_acl_prefix + [bucket_uri], return_stdout=True) self.RunGsUtil(self._set_acl_prefix + [inpath, bucket_uri]) acl_string3 = self.RunGsUtil(self._get_acl_prefix + [bucket_uri], return_stdout=True) self.assertNotEqual(acl_string, acl_string2) self.assertEqual(acl_string, acl_string3) def test_invalid_canned_acl_object(self): """Ensures that an invalid canned ACL returns a CommandException.""" obj_uri = suri(self.CreateObject(contents=b'foo')) stderr = self.RunGsUtil(self._set_acl_prefix + ['not-a-canned-acl', obj_uri], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('AttributeError', stderr) else: self.assertIn('CommandException', stderr) self.assertIn('Invalid canned ACL', stderr) def test_set_valid_def_acl_bucket(self): """Ensures that valid default canned and XML ACLs works with get/set.""" bucket_uri = self.CreateBucket() # Default ACL is project private. obj_uri1 = suri(self.CreateObject(bucket_uri=bucket_uri, contents=b'foo')) acl_string = self.RunGsUtil(self._get_acl_prefix + [obj_uri1], return_stdout=True) # Change it to authenticated-read. self.RunGsUtil( self._set_defacl_prefix + ['authenticated-read', suri(bucket_uri)]) # Default object ACL may take some time to propagate. @Retry(AssertionError, tries=5, timeout_secs=1) def _Check1(): obj_uri2 = suri(self.CreateObject(bucket_uri=bucket_uri, contents=b'foo2')) acl_string2 = self.RunGsUtil(self._get_acl_prefix + [obj_uri2], return_stdout=True) self.assertNotEqual(acl_string, acl_string2) self.assertIn('allAuthenticatedUsers', acl_string2) _Check1() # Now change it back to the default via XML. inpath = self.CreateTempFile(contents=acl_string.encode(UTF8)) self.RunGsUtil(self._set_defacl_prefix + [inpath, suri(bucket_uri)]) # Default object ACL may take some time to propagate. @Retry(AssertionError, tries=5, timeout_secs=1) def _Check2(): obj_uri3 = suri(self.CreateObject(bucket_uri=bucket_uri, contents=b'foo3')) acl_string3 = self.RunGsUtil(self._get_acl_prefix + [obj_uri3], return_stdout=True) self.assertEqual(acl_string, acl_string3) _Check2() def test_acl_set_version_specific_uri(self): """Tests setting an ACL on a specific version of an object.""" bucket_uri = self.CreateVersionedBucket() # Create initial object version. uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'data') # Create a second object version. inpath = self.CreateTempFile(contents=b'def') self.RunGsUtil(['cp', inpath, uri.uri]) # Find out the two object version IDs. lines = self.AssertNObjectsInBucket(bucket_uri, 2, versioned=True) v0_uri_str, v1_uri_str = lines[0], lines[1] # Check that neither version currently has public-read permission # (default ACL is project-private). orig_acls = [] for uri_str in (v0_uri_str, v1_uri_str): acl = self.RunGsUtil(self._get_acl_prefix + [uri_str], return_stdout=True) self.assertNotIn(PUBLIC_READ_JSON_ACL_TEXT, self._strip_json_whitespace(acl)) orig_acls.append(acl) # Set the ACL for the older version of the object to public-read. self.RunGsUtil(self._set_acl_prefix + ['public-read', v0_uri_str]) # Check that the older version's ACL is public-read, but newer version # is not. acl = self.RunGsUtil(self._get_acl_prefix + [v0_uri_str], return_stdout=True) self.assertIn(PUBLIC_READ_JSON_ACL_TEXT, self._strip_json_whitespace(acl)) acl = self.RunGsUtil(self._get_acl_prefix + [v1_uri_str], return_stdout=True) self.assertNotIn(PUBLIC_READ_JSON_ACL_TEXT, self._strip_json_whitespace(acl)) # Check that reading the ACL with the version-less URI returns the # original ACL (since the version-less URI means the current version). acl = self.RunGsUtil(self._get_acl_prefix + [uri.uri], return_stdout=True) self.assertEqual(acl, orig_acls[0]) def _strip_json_whitespace(self, json_text): return re.sub(r'\s*', '', json_text) def _MakeScopeRegex(self, role, entity_type, email_address): template_regex = (r'\{.*"entity":\s*"%s-%s".*"role":\s*"%s".*\}' % (entity_type, email_address, role)) return re.compile(template_regex, flags=re.DOTALL) def _MakeProjectScopeRegex(self, role, project_team, project_number): template_regex = ( r'\{.*"entity":\s*"project-%s-%s",\s*"projectTeam":\s*\{\s*"' r'projectNumber":\s*"%s",\s*"team":\s*"%s"\s*\},\s*"role":\s*"%s".*\}' % (project_team, project_number, project_number, project_team, role)) return re.compile(template_regex, flags=re.DOTALL) def testAclChangeWithUserId(self): test_regex = self._MakeScopeRegex('READER', 'user', self.USER_TEST_ADDRESS) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-u', self.USER_TEST_ID + ':r', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-d', self.USER_TEST_ADDRESS, suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) def testAclChangeWithGroupId(self): test_regex = self._MakeScopeRegex('READER', 'group', self.GROUP_TEST_ADDRESS) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-g', self.GROUP_TEST_ID + ':r', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-d', self.GROUP_TEST_ADDRESS, suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) def testAclChangeWithUserEmail(self): test_regex = self._MakeScopeRegex('READER', 'user', self.USER_TEST_ADDRESS) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-u', self.USER_TEST_ADDRESS + ':r', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-d', self.USER_TEST_ADDRESS, suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) def testAclChangeWithGroupEmail(self): test_regex = self._MakeScopeRegex('OWNER', 'group', self.GROUP_TEST_ADDRESS) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil( self._ch_acl_prefix + ['-g', self.GROUP_TEST_ADDRESS + ':fc', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-d', self.GROUP_TEST_ADDRESS, suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) def testAclChangeWithDomain(self): test_regex = self._MakeScopeRegex('READER', 'domain', self.DOMAIN_TEST) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-g', self.DOMAIN_TEST + ':r', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text, test_regex) self.RunGsUtil( self._ch_acl_prefix + ['-d', self.DOMAIN_TEST, suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) @SkipForXML('XML API does not support project scopes.') def testAclChangeWithProjectOwners(self): test_regex = self._MakeProjectScopeRegex('WRITER', self._project_team, self._project_number) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-p', self._project_test_acl + ':w', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text, test_regex) def testAclChangeWithAllUsers(self): test_regex = re.compile( r'\{.*"entity":\s*"allUsers".*"role":\s*"WRITER".*\}', flags=re.DOTALL) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-g', 'allusers' + ':w', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-d', 'allusers', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) def testAclChangeWithAllAuthUsers(self): test_regex = re.compile( r'\{.*"entity":\s*"allAuthenticatedUsers".*"role":\s*"READER".*\}', flags=re.DOTALL) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil( self._ch_acl_prefix + ['-g', 'allauthenticatedusers' + ':r', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-d', 'allauthenticatedusers', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) def testBucketAclChange(self): """Tests acl change on a bucket.""" test_regex = self._MakeScopeRegex('OWNER', 'user', self.USER_TEST_ADDRESS) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil( self._ch_acl_prefix + ['-u', self.USER_TEST_ADDRESS + ':fc', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text, test_regex) test_regex2 = self._MakeScopeRegex('WRITER', 'user', self.USER_TEST_ADDRESS) s1, s2 = self.RunGsUtil( self._ch_acl_prefix + ['-u', self.USER_TEST_ADDRESS + ':w', suri(self.sample_uri)], return_stderr=True, return_stdout=True) json_text2 = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text2, test_regex2) self.RunGsUtil(self._ch_acl_prefix + ['-d', self.USER_TEST_ADDRESS, suri(self.sample_uri)]) json_text3 = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text3, test_regex) def testProjectAclChangesOnBucket(self): """Tests project entity acl changes on a bucket.""" if self.test_api == ApiSelector.XML: stderr = self.RunGsUtil( self._ch_acl_prefix + ['-p', self._project_test_acl + ':w', suri(self.sample_uri)], expected_status=1, return_stderr=True) self.assertIn(('CommandException: XML API does not support project' ' scopes, cannot translate ACL.'), stderr) else: test_regex = self._MakeProjectScopeRegex('WRITER', self._project_team, self._project_number) self.RunGsUtil( self._ch_acl_prefix + ['-p', self._project_test_acl + ':w', suri(self.sample_uri)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-d', self._project_test_acl, suri(self.sample_uri)]) json_text2 = self.RunGsUtil(self._get_acl_prefix + [suri(self.sample_uri)], return_stdout=True) self.assertNotRegex(json_text2, test_regex) def testObjectAclChange(self): """Tests acl change on an object.""" obj = self.CreateObject(bucket_uri=self.sample_uri, contents=b'something') self.AssertNObjectsInBucket(self.sample_uri, 1) test_regex = self._MakeScopeRegex('READER', 'group', self.GROUP_TEST_ADDRESS) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-g', self.GROUP_TEST_ADDRESS + ':READ', suri(obj)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertRegex(json_text, test_regex) test_regex2 = self._MakeScopeRegex('OWNER', 'group', self.GROUP_TEST_ADDRESS) self.RunGsUtil(self._ch_acl_prefix + ['-g', self.GROUP_TEST_ADDRESS + ':OWNER', suri(obj)]) json_text2 = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertRegex(json_text2, test_regex2) self.RunGsUtil(self._ch_acl_prefix + ['-d', self.GROUP_TEST_ADDRESS, suri(obj)]) json_text3 = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertNotRegex(json_text3, test_regex2) all_auth_regex = re.compile( r'\{.*"entity":\s*"allAuthenticatedUsers".*"role":\s*"OWNER".*\}', flags=re.DOTALL) self.RunGsUtil(self._ch_acl_prefix + ['-g', 'AllAuth:O', suri(obj)]) json_text4 = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertRegex(json_text4, all_auth_regex) def testObjectAclChangeAllUsers(self): """Tests acl ch AllUsers:R on an object.""" obj = self.CreateObject(bucket_uri=self.sample_uri, contents=b'something') self.AssertNObjectsInBucket(self.sample_uri, 1) all_users_regex = re.compile( r'\{.*"entity":\s*"allUsers".*"role":\s*"READER".*\}', flags=re.DOTALL) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertNotRegex(json_text, all_users_regex) self.RunGsUtil(self._ch_acl_prefix + ['-g', 'AllUsers:R', suri(obj)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertRegex(json_text, all_users_regex) def testSeekAheadAcl(self): """Tests seek-ahead iterator with ACL sub-commands.""" object_uri = self.CreateObject(contents=b'foo') # Get the object's current ACL for application via set. current_acl = self.RunGsUtil(['acl', 'get', suri(object_uri)], return_stdout=True) current_acl_file = self.CreateTempFile(contents=current_acl.encode(UTF8)) with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil( ['-m', 'acl', 'ch', '-u', 'AllUsers:R', suri(object_uri)], return_stderr=True) self.assertIn('Estimated work for this command: objects: 1\n', stderr) stderr = self.RunGsUtil( ['-m', 'acl', 'set', current_acl_file, suri(object_uri)], return_stderr=True) self.assertIn('Estimated work for this command: objects: 1\n', stderr) with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '0'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil( ['-m', 'acl', 'ch', '-u', 'AllUsers:R', suri(object_uri)], return_stderr=True) self.assertNotIn('Estimated work', stderr) def testMultithreadedAclChange(self, count=10): """Tests multi-threaded acl changing on several objects.""" objects = [] for i in range(count): objects.append( self.CreateObject(bucket_uri=self.sample_uri, contents='something {0}'.format(i).encode('ascii'))) self.AssertNObjectsInBucket(self.sample_uri, count) test_regex = self._MakeScopeRegex('READER', 'group', self.GROUP_TEST_ADDRESS) json_texts = [] for obj in objects: json_texts.append( self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True)) for json_text in json_texts: self.assertNotRegex(json_text, test_regex) uris = [suri(obj) for obj in objects] self.RunGsUtil(['-m', '-DD'] + self._ch_acl_prefix + ['-g', self.GROUP_TEST_ADDRESS + ':READ'] + uris) json_texts = [] for obj in objects: json_texts.append( self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True)) for json_text in json_texts: self.assertRegex(json_text, test_regex) def testRecursiveChangeAcl(self): """Tests recursively changing ACLs on nested objects.""" obj = self.CreateObject(bucket_uri=self.sample_uri, object_name='foo/bar', contents=b'something') self.AssertNObjectsInBucket(self.sample_uri, 1) test_regex = self._MakeScopeRegex('READER', 'group', self.GROUP_TEST_ADDRESS) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertNotRegex(json_text, test_regex) @Retry(AssertionError, tries=5, timeout_secs=1) def _AddAcl(): self.RunGsUtil( self._ch_acl_prefix + ['-R', '-g', self.GROUP_TEST_ADDRESS + ':READ', suri(obj)[:-3]]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertRegex(json_text, test_regex) _AddAcl() @Retry(AssertionError, tries=5, timeout_secs=1) def _DeleteAcl(): # Make sure we treat grant addresses case insensitively. delete_grant = self.GROUP_TEST_ADDRESS.upper() self.RunGsUtil(self._ch_acl_prefix + ['-d', delete_grant, suri(obj)]) json_text = self.RunGsUtil(self._get_acl_prefix + [suri(obj)], return_stdout=True) self.assertNotRegex(json_text, test_regex) _DeleteAcl() def testMultiVersionSupport(self): """Tests changing ACLs on multiple object versions.""" bucket = self.CreateVersionedBucket() object_name = self.MakeTempName('obj') obj1_uri = self.CreateObject(bucket_uri=bucket, object_name=object_name, contents=b'One thing') # Create another on the same URI, giving us a second version. self.CreateObject(bucket_uri=bucket, object_name=object_name, contents=b'Another thing', gs_idempotent_generation=urigen(obj1_uri)) lines = self.AssertNObjectsInBucket(bucket, 2, versioned=True) obj_v1, obj_v2 = lines[0], lines[1] test_regex = self._MakeScopeRegex('READER', 'group', self.GROUP_TEST_ADDRESS) json_text = self.RunGsUtil(self._get_acl_prefix + [obj_v1], return_stdout=True) self.assertNotRegex(json_text, test_regex) self.RunGsUtil(self._ch_acl_prefix + ['-g', self.GROUP_TEST_ADDRESS + ':READ', obj_v1]) json_text = self.RunGsUtil(self._get_acl_prefix + [obj_v1], return_stdout=True) self.assertRegex(json_text, test_regex) json_text = self.RunGsUtil(self._get_acl_prefix + [obj_v2], return_stdout=True) self.assertNotRegex(json_text, test_regex) def testBadRequestAclChange(self): stdout, stderr = self.RunGsUtil( self._ch_acl_prefix + ['-u', 'invalid_$$@hello.com:R', suri(self.sample_uri)], return_stdout=True, return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('HTTPError', stderr) else: self.assertIn('BadRequestException', stderr) self.assertNotIn('Retrying', stdout) self.assertNotIn('Retrying', stderr) def testAclGetWithoutFullControl(self): object_uri = self.CreateObject(contents=b'foo') expected_error_regex = r'Anonymous \S+ do(es)? not have' with self.SetAnonymousBotoCreds(): stderr = self.RunGsUtil(self._get_acl_prefix + [suri(object_uri)], return_stderr=True, expected_status=1) self.assertRegex(stderr, expected_error_regex) def testTooFewArgumentsFails(self): """Tests calling ACL commands with insufficient number of arguments.""" # No arguments for get, but valid subcommand. stderr = self.RunGsUtil(self._get_acl_prefix, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for set, but valid subcommand. stderr = self.RunGsUtil(self._set_acl_prefix, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for ch, but valid subcommand. stderr = self.RunGsUtil(self._ch_acl_prefix, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Neither arguments nor subcommand. stderr = self.RunGsUtil(['acl'], return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) def testMinusF(self): """Tests -f option to continue after failure.""" bucket_uri = self.CreateBucket() obj_uri = suri( self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'foo')) acl_string = self.RunGsUtil(self._get_acl_prefix + [obj_uri], return_stdout=True) self.RunGsUtil(['-d'] + self._set_acl_prefix + ['-f', 'public-read', obj_uri + 'foo2', obj_uri], expected_status=1) acl_string2 = self.RunGsUtil(self._get_acl_prefix + [obj_uri], return_stdout=True) self.assertNotEqual(acl_string, acl_string2) class TestS3CompatibleAcl(TestAclBase): """ACL integration tests that work for s3 and gs URLs.""" def testAclObjectGetSet(self): bucket_uri = self.CreateBucket() obj_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') self.AssertNObjectsInBucket(bucket_uri, 1) stdout = self.RunGsUtil(self._get_acl_prefix + [suri(obj_uri)], return_stdout=True) set_contents = self.CreateTempFile(contents=stdout.encode(UTF8)) self.RunGsUtil(self._set_acl_prefix + [set_contents, suri(obj_uri)]) def testAclBucketGetSet(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil(self._get_acl_prefix + [suri(bucket_uri)], return_stdout=True) set_contents = self.CreateTempFile(contents=stdout.encode(UTF8)) self.RunGsUtil(self._set_acl_prefix + [set_contents, suri(bucket_uri)]) @SkipForGS('S3 ACLs accept XML and should not cause an XML warning.') class TestS3OnlyAcl(TestAclBase): """ACL integration tests that work only for s3 URLs.""" # TODO: Format all test case names consistently. def test_set_xml_acl(self): """Ensures XML content does not return an XML warning for S3.""" obj_uri = suri(self.CreateObject(contents=b'foo')) inpath = self.CreateTempFile(contents=b'<ValidXml></ValidXml>') stderr = self.RunGsUtil(self._set_acl_prefix + [inpath, obj_uri], return_stderr=True, expected_status=1) self.assertIn('BadRequestException', stderr) self.assertNotIn('XML ACL data provided', stderr) def test_set_xml_acl_bucket(self): """Ensures XML content does not return an XML warning for S3.""" bucket_uri = suri(self.CreateBucket()) inpath = self.CreateTempFile(contents=b'<ValidXml></ValidXml>') stderr = self.RunGsUtil(self._set_acl_prefix + [inpath, bucket_uri], return_stderr=True, expected_status=1) self.assertIn('BadRequestException', stderr) self.assertNotIn('XML ACL data provided', stderr) class TestAclOldAlias(TestAcl): _set_acl_prefix = ['setacl'] _get_acl_prefix = ['getacl'] _set_defacl_prefix = ['setdefacl'] _ch_acl_prefix = ['chacl'] class TestAclShim(testcase.ShimUnitTestBase): @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_acl_get_object(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('acl', ['get', 'gs://bucket/object'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage objects describe' ' --format=multi(acl:format=json)' ' gs://bucket/object').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_acl_get_bucket(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('acl', ['get', 'gs://bucket'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage buckets describe' ' --format=multi(acl:format=json)' ' gs://bucket').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_acl_set_object(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'acl', ['set', inpath, 'gs://bucket/object'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage objects update' ' --acl-file={}').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_acl_set_bucket(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('acl', ['set', inpath, 'gs://bucket'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage buckets update' ' --acl-file={} gs://bucket').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_predefined_acl_set_object(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'acl', ['set', 'private', 'gs://bucket/object'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage objects update' ' --predefined-acl=private gs://bucket/object'.format( shim_util._get_gcloud_binary_path('fake_dir'))), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_predefined_acl_set_bucket(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('acl', ['set', 'private', 'gs://bucket'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage buckets update' ' --predefined-acl=private gs://bucket').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_xml_predefined_acl_for_set(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'acl', ['set', 'public-read', 'gs://bucket'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage buckets update' ' --predefined-acl=publicRead gs://bucket').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_acl_set_multiple_buckets_urls(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('acl', [ 'set', '-f', inpath, 'gs://bucket', 'gs://bucket1', 'gs://bucket2' ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage buckets update' ' --acl-file={} --continue-on-error' ' gs://bucket gs://bucket1 gs://bucket2').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_acl_set_multiple_objects_urls(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('acl', [ 'set', '-f', inpath, 'gs://bucket/object', 'gs://bucket/object1', 'gs://bucket/object2' ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage objects update' ' --acl-file={} --continue-on-error gs://bucket/object' ' gs://bucket/object1 gs://bucket/object2').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_acl_set_multiple_buckets_urls_recursive_all_versions( self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('acl', [ 'set', '-r', '-a', inpath, 'gs://bucket', 'gs://bucket1/o', 'gs://bucket2' ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage objects update' ' --acl-file={} --recursive --all-versions gs://bucket' ' gs://bucket1/o gs://bucket2').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_acl_set_mix_buckets_and_objects_raises_error(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with self.assertRaisesRegex( CommandException, 'Cannot operate on a mix of buckets and objects.'): self.RunCommand( 'acl', ['set', 'acl-file', 'gs://bucket', 'gs://bucket1/object']) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_changes_bucket_acls_for_user(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'acl', ['ch', '-u', 'user@example.com:R', 'gs://bucket1', 'gs://bucket2'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage buckets update' ' --add-acl-grant entity=user-user@example.com,role=READER' ' gs://bucket1 gs://bucket2').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_changes_object_acls_for_user(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('acl', [ 'ch', '-u', 'user@example.com:R', 'gs://bucket1/o', 'gs://bucket2/o' ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage objects update' ' --add-acl-grant entity=user-user@example.com,role=READER' ' gs://bucket1/o gs://bucket2/o').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_raises_error_for_mix_of_objects_and_buckets(self): with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): with self.assertRaisesRegex( CommandException, 'Cannot operate on a mix of buckets and objects.'): self.RunCommand('acl', ['ch', 'gs://bucket', 'gs://bucket1/object']) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_changes_acls_for_group(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'acl', ['ch', '-g', 'group@example.com:W', 'gs://bucket1/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage objects update' ' --add-acl-grant entity=group-group@example.com,role=WRITER' ' gs://bucket1/o').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_changes_acls_for_domain(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'acl', ['ch', '-g', 'example.com:O', 'gs://bucket1/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage objects update' ' --add-acl-grant entity=domain-example.com,role=OWNER' ' gs://bucket1/o').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_changes_acls_for_project(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'acl', ['ch', '-p', 'owners-example:O', 'gs://bucket1/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage objects update' ' --add-acl-grant entity=project-owners-example,role=OWNER' ' gs://bucket1/o').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_changes_acls_for_all_users(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): # Non-exhaustive set of strings allowed by gsutil's regex. for identifier in ['all', 'allUsers', 'AllUsers']: mock_log_handler = self.RunCommand( 'acl', ['ch', '-g', identifier + ':O', 'gs://bucket1/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage objects update' ' --add-acl-grant entity=allUsers,role=OWNER' ' gs://bucket1/o').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_changes_acls_for_all_authenticated_users(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): # Non-exhaustive set of strings allowed by gsutil's regex. for identifier in [ 'allauth', 'allAuthenticatedUsers', 'AllAuthenticatedUsers' ]: mock_log_handler = self.RunCommand( 'acl', ['ch', '-g', identifier + ':O', 'gs://bucket1/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage objects update' ' --add-acl-grant entity=allAuthenticatedUsers,role=OWNER' ' gs://bucket1/o').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_deletes_acls(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): # Non-exhaustive set of strings allowed by gsutil's regex. mock_log_handler = self.RunCommand('acl', [ 'ch', '-d', 'user@example.com', '-d', 'user1@example.com', 'gs://bucket1/o' ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage objects update' ' --remove-acl-grant user@example.com' ' --remove-acl-grant user1@example.com' ' gs://bucket1/o').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_removes_acls_for_all_users(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): # Non-exhaustive set of strings allowed by gsutil's regex. for identifier in ['all', 'allUsers', 'AllUsers']: mock_log_handler = self.RunCommand( 'acl', ['ch', '-d', identifier, 'gs://bucket1/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage objects update' ' --remove-acl-grant AllUsers' ' gs://bucket1/o').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) @mock.patch.object(acl.AclCommand, 'RunCommand', new=mock.Mock()) def test_shim_removes_acls_for_all_authenticated_users(self): inpath = self.CreateTempFile() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): # Non-exhaustive set of strings allowed by gsutil's regex. for identifier in [ 'allauth', 'allAuthenticatedUsers', 'AllAuthenticatedUsers' ]: mock_log_handler = self.RunCommand( 'acl', ['ch', '-d', identifier, 'gs://bucket1/o'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage objects update' ' --remove-acl-grant AllAuthenticatedUsers' ' gs://bucket1/o').format( shim_util._get_gcloud_binary_path('fake_dir'), inpath), info_lines) tests/test_cors.py 0000644 00000030721 15025013616 0010267 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for cors command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import posixpath from xml.dom.minidom import parseString import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.utils.constants import UTF8 from gslib.utils.retry_util import Retry from gslib.utils.translation_helper import CorsTranslation @SkipForS3('CORS command is only supported for gs:// URLs') class TestCors(testcase.GsUtilIntegrationTestCase): """Integration tests for cors command.""" _set_cmd_prefix = ['cors', 'set'] _get_cmd_prefix = ['cors', 'get'] empty_doc1 = '[]' empty_doc2 = '[ {} ]' cors_bad = ( '[{"origin": ["http://origin1.example.com", ' '"http://origin2.example.com"], ' '"responseHeader": ["foo", "bar"], "badmethod": ["GET", "PUT", "POST"], ' '"maxAgeSeconds": 3600},' '{"origin": ["http://origin3.example.com"], ' '"responseHeader": ["foo2", "bar2"], "method": ["GET", "DELETE"]}])') xml_cors_doc = parseString( '<CorsConfig><Cors><Origins>' '<Origin>http://origin1.example.com</Origin>' '<Origin>http://origin2.example.com</Origin>' '</Origins><Methods><Method>GET</Method>' '<Method>PUT</Method><Method>POST</Method></Methods>' '<ResponseHeaders><ResponseHeader>foo</ResponseHeader>' '<ResponseHeader>bar</ResponseHeader></ResponseHeaders>' '<MaxAgeSec>3600</MaxAgeSec></Cors>' '<Cors><Origins><Origin>http://origin3.example.com</Origin></Origins>' '<Methods><Method>GET</Method><Method>DELETE</Method></Methods>' '<ResponseHeaders><ResponseHeader>foo2</ResponseHeader>' '<ResponseHeader>bar2</ResponseHeader></ResponseHeaders>' '</Cors></CorsConfig>').toprettyxml(indent=' ') cors_doc = ( '[{"origin": ["http://origin1.example.com", ' '"http://origin2.example.com"], ' '"responseHeader": ["foo", "bar"], "method": ["GET", "PUT", "POST"], ' '"maxAgeSeconds": 3600},' '{"origin": ["http://origin3.example.com"], ' '"responseHeader": ["foo2", "bar2"], "method": ["GET", "DELETE"]}]\n') cors_json_obj = json.loads(cors_doc) cors_doc_not_nested_in_list = ( '{"origin": ["http://origin.example.com", "http://origin2.example.com"], ' '"responseHeader": ["foo", "bar"], ' '"method": ["GET", "PUT", "POST"], ' '"maxAgeSeconds": 3600}') cors_doc2 = ( '[{"origin": ["http://origin1.example.com", ' '"http://origin2.example.com"], ' '"responseHeader": ["foo", "bar"], "method": ["GET", "PUT", "POST"]}]\n') cors_json_obj2 = json.loads(cors_doc2) def setUp(self): super(TestCors, self).setUp() self.no_cors = 'has no CORS configuration' def test_cors_translation(self): """Tests cors translation for various formats.""" json_text = self.cors_doc entries_list = CorsTranslation.JsonCorsToMessageEntries(json_text) boto_cors = CorsTranslation.BotoCorsFromMessage(entries_list) converted_entries_list = CorsTranslation.BotoCorsToMessage(boto_cors) converted_json_text = CorsTranslation.MessageEntriesToJson( converted_entries_list) self.assertEqual(json.loads(json_text), json.loads(converted_json_text)) def test_default_cors(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket_uri)], return_stdout=True) self.assertIn(self.no_cors, stdout) def test_set_empty_cors1(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.empty_doc1.encode(UTF8)) self.RunGsUtil(self._set_cmd_prefix + [fpath, suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket_uri)], return_stdout=True) self.assertIn(self.no_cors, stdout) def test_set_empty_cors2(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.empty_doc2.encode(UTF8)) self.RunGsUtil(self._set_cmd_prefix + [fpath, suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket_uri)], return_stdout=True) self.assertIn(self.no_cors, stdout) def test_non_null_cors(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.cors_doc.encode(UTF8)) self.RunGsUtil(self._set_cmd_prefix + [fpath, suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.cors_json_obj) def test_bad_cors_xml(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.xml_cors_doc.encode(UTF8)) stderr = self.RunGsUtil(self._set_cmd_prefix + [fpath, suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn('XML CORS data provided', stderr) def test_bad_cors(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.cors_bad.encode(UTF8)) stderr = self.RunGsUtil(self._set_cmd_prefix + [fpath, suri(bucket_uri)], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('Found invalid JSON/YAML', stderr) else: self.assertNotIn('XML CORS data provided', stderr) def test_cors_doc_not_wrapped_in_json_list(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempFile( contents=self.cors_doc_not_nested_in_list.encode(UTF8)) stderr = self.RunGsUtil(self._set_cmd_prefix + [fpath, suri(bucket_uri)], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn("'str' object has no attribute 'items'", stderr) else: self.assertIn('should be formatted as a list', stderr) def test_set_cors_and_reset(self): """Tests setting CORS then removing it.""" bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() fpath = self.CreateTempFile(tmpdir=tmpdir, contents=self.cors_doc) self.RunGsUtil(self._set_cmd_prefix + [fpath, suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.cors_json_obj) fpath = self.CreateTempFile(tmpdir=tmpdir, contents=self.empty_doc1) self.RunGsUtil(self._set_cmd_prefix + [fpath, suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket_uri)], return_stdout=True) self.assertIn(self.no_cors, stdout) def test_set_partial_cors_and_reset(self): """Tests setting CORS without maxAgeSeconds, then removing it.""" bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() fpath = self.CreateTempFile(tmpdir=tmpdir, contents=self.cors_doc2) self.RunGsUtil(self._set_cmd_prefix + [fpath, suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.cors_json_obj2) fpath = self.CreateTempFile(tmpdir=tmpdir, contents=self.empty_doc1) self.RunGsUtil(self._set_cmd_prefix + [fpath, suri(bucket_uri)]) stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket_uri)], return_stdout=True) self.assertIn(self.no_cors, stdout) def test_set_multi_non_null_cors(self): """Tests setting different CORS configurations.""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() fpath = self.CreateTempFile(contents=self.cors_doc) self.RunGsUtil( self._set_cmd_prefix + [fpath, suri(bucket1_uri), suri(bucket2_uri)]) stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket1_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.cors_json_obj) stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket2_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.cors_json_obj) def test_set_wildcard_non_null_cors(self): """Tests setting CORS on a wildcarded bucket URI.""" random_prefix = self.MakeRandomTestString() bucket1_name = self.MakeTempName('bucket', prefix=random_prefix) bucket2_name = self.MakeTempName('bucket', prefix=random_prefix) bucket1_uri = self.CreateBucket(bucket_name=bucket1_name) bucket2_uri = self.CreateBucket(bucket_name=bucket2_name) # This just double checks that the common prefix of the two buckets is what # we think it should be (based on implementation detail of CreateBucket). # We want to be careful when setting a wildcard on buckets to make sure we # don't step outside the test buckets to affect other buckets. common_prefix = posixpath.commonprefix( [suri(bucket1_uri), suri(bucket2_uri)]) self.assertTrue( common_prefix.startswith('gs://%sgsutil-test-test-set-wildcard-non' % random_prefix)) wildcard = '%s*' % common_prefix fpath = self.CreateTempFile(contents=self.cors_doc.encode(UTF8)) # Use @Retry as hedge against bucket listing eventual consistency. if self._use_gcloud_storage: expected = set([ 'Updating %s' % suri(bucket1_uri), 'Updating %s' % suri(bucket2_uri) ]) else: expected = set([ 'Setting CORS on %s/...' % suri(bucket1_uri), 'Setting CORS on %s/...' % suri(bucket2_uri) ]) actual = set() @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Ensures expect set lines are present in command output.""" stderr = self.RunGsUtil(self._set_cmd_prefix + [fpath, wildcard], return_stderr=True) outlines = stderr.splitlines() for line in outlines: # Ignore the deprecation warnings from running the old cors command. if ('You are using a deprecated alias' in line or 'gsutil help cors' in line or 'Please use "cors" with the appropriate sub-command' in line): continue actual.add(line) for line in expected: if self._use_gcloud_storage: # Not exact match because gcloud may print progress or other logs. self.assertIn(line, stderr) else: self.assertIn(line, actual) self.assertEqual(stderr.count('Setting CORS'), 2) _Check1() stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket1_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.cors_json_obj) stdout = self.RunGsUtil(self._get_cmd_prefix + [suri(bucket2_uri)], return_stdout=True) self.assertEqual(json.loads(stdout), self.cors_json_obj) def testTooFewArgumentsFails(self): """Ensures CORS commands fail with too few arguments.""" # No arguments for get, but valid subcommand. stderr = self.RunGsUtil(self._get_cmd_prefix, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for set, but valid subcommand. stderr = self.RunGsUtil(self._set_cmd_prefix, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Neither arguments nor subcommand. stderr = self.RunGsUtil(['cors'], return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) class TestCorsOldAlias(TestCors): _set_cmd_prefix = ['setcors'] _get_cmd_prefix = ['getcors'] tests/test_tracker_file.py 0000644 00000023634 15025013616 0011760 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for tracker_file and parallel_tracker_file.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import stat from gslib.exception import CommandException from gslib.parallel_tracker_file import ObjectFromTracker from gslib.parallel_tracker_file import ReadParallelUploadTrackerFile from gslib.parallel_tracker_file import ValidateParallelCompositeTrackerData from gslib.parallel_tracker_file import WriteComponentToParallelUploadTrackerFile from gslib.parallel_tracker_file import WriteParallelUploadTrackerFile from gslib.storage_url import StorageUrlFromString from gslib.tests.testcase.unit_testcase import GsUtilUnitTestCase from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.tracker_file import _HashFilename from gslib.tracker_file import DeleteTrackerFile from gslib.tracker_file import GetRewriteTrackerFilePath from gslib.tracker_file import HashRewriteParameters from gslib.tracker_file import ReadRewriteTrackerFile from gslib.tracker_file import WriteRewriteTrackerFile from gslib.utils import parallelism_framework_util from gslib.utils.constants import UTF8 class TestTrackerFile(GsUtilUnitTestCase): """Unit tests for parallel upload functions in cp command.""" def test_HashFilename(self): # Tests that _HashFilename function works for both string and unicode # filenames (without raising any Unicode encode/decode errors). _HashFilename(b'file1') _HashFilename('file1') def test_RewriteTrackerFile(self): """Tests Rewrite tracker file functions.""" tracker_file_name = GetRewriteTrackerFilePath('bk1', 'obj1', 'bk2', 'obj2', self.test_api) # Should succeed regardless of whether it exists. DeleteTrackerFile(tracker_file_name) src_obj_metadata = apitools_messages.Object(bucket='bk1', name='obj1', etag='etag1', md5Hash='12345') src_obj2_metadata = apitools_messages.Object(bucket='bk1', name='obj1', etag='etag2', md5Hash='67890') dst_obj_metadata = apitools_messages.Object(bucket='bk2', name='obj2') rewrite_token = 'token1' self.assertIsNone( ReadRewriteTrackerFile(tracker_file_name, src_obj_metadata)) rewrite_params_hash = HashRewriteParameters(src_obj_metadata, dst_obj_metadata, 'full') WriteRewriteTrackerFile(tracker_file_name, rewrite_params_hash, rewrite_token) self.assertEqual( ReadRewriteTrackerFile(tracker_file_name, rewrite_params_hash), rewrite_token) # Tracker file for an updated source object (with non-matching etag/md5) # should return None. rewrite_params_hash2 = HashRewriteParameters(src_obj2_metadata, dst_obj_metadata, 'full') self.assertIsNone( ReadRewriteTrackerFile(tracker_file_name, rewrite_params_hash2)) DeleteTrackerFile(tracker_file_name) def testReadGsutil416ParallelUploadTrackerFile(self): """Tests the parallel upload tracker file format prior to gsutil 4.17.""" random_prefix = '123' objects = ['obj1', '42', 'obj2', '314159'] contents = '\n'.join([random_prefix] + objects) + '\n' fpath = self.CreateTempFile(file_name='foo', contents=contents.encode(UTF8)) expected_objects = [ ObjectFromTracker(objects[2 * i], objects[2 * i + 1]) for i in range(0, len(objects) // 2) ] (_, actual_prefix, actual_objects) = ReadParallelUploadTrackerFile(fpath, self.logger) self.assertEqual(random_prefix, actual_prefix) self.assertEqual(expected_objects, actual_objects) def testReadEmptyGsutil416ParallelUploadTrackerFile(self): """Tests reading an empty pre-gsutil 4.17 parallel upload tracker file.""" fpath = self.CreateTempFile(file_name='foo', contents=b'') (_, actual_prefix, actual_objects) = ReadParallelUploadTrackerFile(fpath, self.logger) self.assertEqual(None, actual_prefix) self.assertEqual([], actual_objects) def testParallelUploadTrackerFileNoEncryption(self): fpath = self.CreateTempFile(file_name='foo') random_prefix = '123' objects = [ ObjectFromTracker('obj1', '42'), ObjectFromTracker('obj2', '314159') ] WriteParallelUploadTrackerFile(fpath, random_prefix, objects) (enc_key, actual_prefix, actual_objects) = ReadParallelUploadTrackerFile(fpath, self.logger) self.assertEqual(random_prefix, actual_prefix) self.assertEqual(None, enc_key) self.assertEqual(objects, actual_objects) def testParallelUploadTrackerFileWithEncryption(self): fpath = self.CreateTempFile(file_name='foo') random_prefix = '123' enc_key = '456' objects = [ ObjectFromTracker('obj1', '42'), ObjectFromTracker('obj2', '314159') ] WriteParallelUploadTrackerFile(fpath, random_prefix, objects, encryption_key_sha256=enc_key) (actual_key, actual_prefix, actual_objects) = ReadParallelUploadTrackerFile(fpath, self.logger) self.assertEqual(enc_key, actual_key) self.assertEqual(random_prefix, actual_prefix) self.assertEqual(objects, actual_objects) def testWriteComponentToParallelUploadTrackerFile(self): tracker_file_lock = parallelism_framework_util.CreateLock() fpath = self.CreateTempFile(file_name='foo') random_prefix = '123' enc_key = '456' objects = [ ObjectFromTracker('obj1', '42'), ObjectFromTracker('obj2', '314159') ] WriteParallelUploadTrackerFile(fpath, random_prefix, objects, encryption_key_sha256=enc_key) new_object = ObjectFromTracker('obj3', '43') try: WriteComponentToParallelUploadTrackerFile(fpath, tracker_file_lock, new_object, self.logger, encryption_key_sha256=None) self.fail('Expected CommandException due to different encryption key') except CommandException as e: self.assertIn('does not match encryption key', str(e)) WriteComponentToParallelUploadTrackerFile(fpath, tracker_file_lock, new_object, self.logger, encryption_key_sha256='456') (actual_key, actual_prefix, actual_objects) = ReadParallelUploadTrackerFile(fpath, self.logger) self.assertEqual(enc_key, actual_key) self.assertEqual(random_prefix, actual_prefix) self.assertEqual(objects + [new_object], actual_objects) def testValidateParallelCompositeTrackerData(self): tempdir = self.CreateTempDir() fpath = os.path.join(tempdir, 'foo') random_prefix = '123' old_enc_key = '456' bucket_url = StorageUrlFromString('gs://foo') objects = [ ObjectFromTracker('obj1', '42'), ObjectFromTracker('obj2', '314159') ] WriteParallelUploadTrackerFile(fpath, random_prefix, objects, encryption_key_sha256=old_enc_key) # Test the permissions if os.name == 'posix': mode = oct(stat.S_IMODE(os.stat(fpath).st_mode)) # Assert that only user has read/write permission self.assertEqual(oct(0o600), mode) # Mock command object since Valdiate will call Apply() to delete the # existing components. class MockCommandObject(object): delete_called = False # We call Apply with parallel_operations_override, which expects this enum # class to exist. class ParallelOverrideReason(object): SPEED = 'speed' def Apply(self, *unused_args, **unused_kwargs): self.delete_called = True def MockDeleteFunc(): pass def MockDeleteExceptionHandler(): pass command_obj = MockCommandObject() # Validate with correct key should succeed. (actual_prefix, actual_objects) = ValidateParallelCompositeTrackerData( fpath, old_enc_key, random_prefix, objects, old_enc_key, bucket_url, command_obj, self.logger, MockDeleteFunc, MockDeleteExceptionHandler) self.assertEqual(False, command_obj.delete_called) self.assertEqual(random_prefix, actual_prefix) self.assertEqual(objects, actual_objects) new_enc_key = '789' command_obj = MockCommandObject() (actual_prefix, actual_objects) = ValidateParallelCompositeTrackerData( fpath, old_enc_key, random_prefix, objects, new_enc_key, bucket_url, command_obj, self.logger, MockDeleteFunc, MockDeleteExceptionHandler) self.assertEqual(True, command_obj.delete_called) self.assertEqual(None, actual_prefix) self.assertEqual([], actual_objects) tests/test_signurl.py 0000644 00000061304 15025013616 0011005 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for signurl command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from datetime import datetime from datetime import timedelta import os import pkgutil import boto import gslib.commands.signurl from gslib.commands.signurl import HAVE_OPENSSL from gslib.commands.signurl import HAVE_CRYPTO from gslib.exception import CommandException from gslib.gcs_json_api import GcsJsonApi from gslib.iamcredentials_api import IamcredentailsApi from gslib.impersonation_credentials import ImpersonationCredentials import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import (SkipForS3, SkipForXML) from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import unittest import gslib.tests.signurl_signatures as sigs from oauth2client import client from oauth2client.service_account import ServiceAccountCredentials from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock SERVICE_ACCOUNT = boto.config.get_value('GSUtil', 'test_impersonate_service_account') TEST_EMAIL = 'test%40developer.gserviceaccount.com' # pylint: disable=protected-access @unittest.skipUnless(HAVE_OPENSSL, 'signurl requires pyopenssl.') @SkipForS3('Signed URLs are only supported for gs:// URLs.') class TestSignUrl(testcase.GsUtilIntegrationTestCase): """Integration tests for signurl command.""" def _GetJSONKsFile(self): if not hasattr(self, 'json_ks_file'): # Dummy json keystore constructed from test.p12. contents = pkgutil.get_data('gslib', 'tests/test_data/test.json') self.json_ks_file = self.CreateTempFile(contents=contents) return self.json_ks_file def _GetKsFile(self): if not hasattr(self, 'ks_file'): # Dummy pkcs12 keystore generated with the command # openssl req -new -passout pass:notasecret -batch \ # -x509 -keyout signed_url_test.key -out signed_url_test.pem \ # -subj '/CN=test.apps.googleusercontent.com' # && # openssl pkcs12 -export -passin pass:notasecret \ # -passout pass:notasecret -inkey signed_url_test.key \ # -in signed_url_test.pem -out test.p12 # && # rm signed_url_test.key signed_url_test.pem contents = pkgutil.get_data('gslib', 'tests/test_data/test.p12') self.ks_file = self.CreateTempFile(contents=contents) return self.ks_file def testSignUrlInvalidDuration(self): """Tests signurl fails with out of bounds value for valid duration.""" if self._use_gcloud_storage: expected_status = 2 else: expected_status = 1 stderr = self.RunGsUtil(['signurl', '-d', '123d', 'ks_file', 'gs://uri'], return_stderr=True, expected_status=expected_status) if self._use_gcloud_storage: self.assertIn('value must be less than or equal to 7d', stderr) else: self.assertIn('CommandException: Max valid duration allowed is 7 days', stderr) def testSignUrlInvalidDurationWithUseServiceAccount(self): """Tests signurl with -u flag fails duration > 12 hours.""" stderr = self.RunGsUtil(['signurl', '-d', '13h', '-u', 'gs://uri'], return_stderr=True, expected_status=1) self.assertIn('CommandException: Max valid duration allowed is 12:00:00', stderr) @unittest.skipUnless(not HAVE_CRYPTO, 'signurl requires crypto to decode .p12 keys') def testSignUrlRaiseErrorForP12KeysWithoutCrypto(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'z') cmd = [ 'signurl', '-p', 'notasecret', '-m', 'PUT', self._GetKsFile(), suri(object_uri) ] stderr = self.RunGsUtil(cmd, return_stderr=1, expected_status=1) self.assertIn('CommandException: pyca/cryptography is not available. Either install it, or please consider using the .json keyfile', stderr) @unittest.skipUnless(HAVE_CRYPTO, 'signurl requires crypto to decode .p12 keys.') def testSignUrlOutputP12(self): """Tests signurl output of a sample object with pkcs12 keystore.""" bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'z') cmd = [ 'signurl', '-p', 'notasecret', '-m', 'PUT', self._GetKsFile(), suri(object_uri) ] stdout = self.RunGsUtil(cmd, return_stdout=True) self.assertIn('x-goog-credential=test.apps.googleusercontent.com', stdout) self.assertIn('x-goog-expires=3600', stdout) self.assertIn('%2Fus-central1%2F', stdout) self.assertIn('\tPUT\t', stdout) def testSignUrlOutputJSON(self): """Tests signurl output of a sample object with JSON keystore.""" bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'z') cmd = ['signurl', '-m', 'PUT', self._GetJSONKsFile(), suri(object_uri)] stdout = self.RunGsUtil(cmd, return_stdout=True) self.assertIn('x-goog-credential=' + TEST_EMAIL, stdout) self.assertIn('x-goog-expires=3600', stdout) self.assertIn('%2Fus-central1%2F', stdout) self.assertIn('\tPUT\t', stdout) def testSignUrlWithJSONKeyFileAndObjectGeneration(self): """Tests signurl output of a sample object version with JSON keystore.""" bucket_uri = self.CreateBucket(versioning_enabled=True) object_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'z') cmd = ['signurl', self._GetJSONKsFile(), object_uri.version_specific_uri] stdout = self.RunGsUtil(cmd, return_stdout=True) self.assertIn('x-goog-credential=' + TEST_EMAIL, stdout) self.assertIn('generation=' + object_uri.generation, stdout) def testSignUrlWithURLEncodeRequiredChars(self): objs = [ 'gs://example.org/test 1', 'gs://example.org/test/test 2', 'gs://example.org/Аудиоарi хив' ] expected_partial_urls = [ 'https://storage.googleapis.com/example.org/test%201?x-goog-signature=', ('https://storage.googleapis.com/example.org/test/test%202' '?x-goog-signature='), ('https://storage.googleapis.com/example.org/%D0%90%D1%83%D0%B4%D0%B8%D' '0%BE%D0%B0%D1%80i%20%D1%85%D0%B8%D0%B2?x-goog-signature=') ] self.assertEqual(len(objs), len(expected_partial_urls)) cmd_args = [ 'signurl', '-m', 'PUT', '-r', 'us', self._GetJSONKsFile() ] cmd_args.extend(objs) stdout = self.RunGsUtil(cmd_args, return_stdout=True) lines = stdout.split('\n') # Header, signed urls, trailing newline. self.assertEqual(len(lines), len(objs) + 2) # Strip the header line to make the indices line up. lines = lines[1:] for obj, line, partial_url in zip(objs, lines, expected_partial_urls): self.assertIn(obj, line) self.assertIn(partial_url, line) self.assertIn('x-goog-credential='+TEST_EMAIL, line) self.assertIn('%2Fus%2F', stdout) def testSignUrlWithWildcard(self): objs = ['test1', 'test2', 'test3'] obj_urls = [] bucket = self.CreateBucket() for obj_name in objs: obj_urls.append( self.CreateObject(bucket_uri=bucket, object_name=obj_name, contents=b'')) stdout = self.RunGsUtil( ['signurl', self._GetJSONKsFile(), suri(bucket) + '/*'], return_stdout=True) # Header, 3 signed urls, trailing newline self.assertEqual(len(stdout.split('\n')), 5) for obj_url in obj_urls: self.assertIn(suri(obj_url), stdout) @unittest.skipUnless(SERVICE_ACCOUNT, 'Test requires test_impersonate_service_account.') @SkipForS3('Tests only uses gs credentials.') @SkipForXML('Tests only run on JSON API.') def testSignUrlWithServiceAccount(self): with SetBotoConfigForTest([('Credentials', 'gs_impersonate_service_account', SERVICE_ACCOUNT)]): stdout, stderr = self.RunGsUtil( ['signurl', '-r', 'us-east1', '-u', 'gs://pub'], return_stdout=True, return_stderr=True) # The signed url returned in stdout relies on current time. # We are not able to mock the datetime here because RunGsUtil creates # a separate process and runs the command. self.assertIn('https://storage.googleapis.com/pub', stdout) self.assertIn('All API calls will be executed as [%s]' % SERVICE_ACCOUNT, stderr) def testSignUrlOfNonObjectUrl(self): """Tests the signurl output of a non-existent file.""" self.RunGsUtil(['signurl', self._GetJSONKsFile(), 'gs://'], expected_status=1) self.RunGsUtil(['signurl', 'file://tmp/abc', 'gs://bucket'], expected_status=1) @unittest.skipUnless(HAVE_OPENSSL, 'signurl requires pyopenssl.') class UnitTestSignUrl(testcase.GsUtilUnitTestCase): """Unit tests for the signurl command.""" # Helpful for comparing mismatched signed URLs that would be truncated. # https://stackoverflow.com/questions/14493670/how-to-set-self-maxdiff-in-nose-to-get-full-diff-output maxDiff = None def setUp(self): super(UnitTestSignUrl, self).setUp() ks_contents = pkgutil.get_data('gslib', 'tests/test_data/test.p12') self.key, self.client_email = gslib.commands.signurl._ReadKeystore( ks_contents, 'notasecret') def fake_now(): return datetime(1900, 1, 1, 0, 5, 55) gslib.utils.signurl_helper._NowUTC = fake_now def _get_mock_api_delegator(self): mock_api_delegator = self.MakeGsUtilApi() # The MAkeGsUtilAPi maps apiclass.gs.JSON to BotoTranslation # instead of GcsJsonApi # Issue https://github.com/GoogleCloudPlatform/gsutil/issues/970 # SignUrl relies on the GcsJsonApi so we are replacing the mapping here. mock_api_delegator.api_map['apiclass']['gs']['JSON'] = GcsJsonApi return mock_api_delegator def testDurationSpec(self): tests = [ ('1h', timedelta(hours=1)), ('2d', timedelta(days=2)), ('5D', timedelta(days=5)), ('35s', timedelta(seconds=35)), ('1h', timedelta(hours=1)), ('33', timedelta(hours=33)), ('22m', timedelta(minutes=22)), ('3.7', None), ('27Z', None), ] for inp, expected in tests: try: td = gslib.commands.signurl._DurationToTimeDelta(inp) self.assertEqual(td, expected) except CommandException: if expected is not None: self.fail('{0} failed to parse') def testSignPutUsingKeyFile(self): """Tests the _GenSignedUrl function with a PUT method using Key file.""" expected = sigs.TEST_SIGN_PUT_SIG duration = timedelta(seconds=3600) with SetBotoConfigForTest([('Credentials', 'gs_host', 'storage.googleapis.com')]): signed_url = gslib.commands.signurl._GenSignedUrl( self.key, api=None, use_service_account=False, provider='gs', client_id=self.client_email, method='RESUMABLE', gcs_path='test/test.txt', duration=duration, logger=self.logger, region='us-east', content_type='') self.assertEqual(expected, signed_url) @SkipForS3('Tests only uses gs credentials.') @SkipForXML('Tests only run on JSON API.') def testSignPutUsingServiceAccount(self): """Tests the _GenSignedUrl function PUT method with service account.""" expected = sigs.TEST_SIGN_URL_PUT_WITH_SERVICE_ACCOUNT duration = timedelta(seconds=3600) mock_api_delegator = self._get_mock_api_delegator() json_api = mock_api_delegator._GetApi('gs') # patch a service account credentials mock_credentials = mock.Mock(spec=ServiceAccountCredentials) mock_credentials.service_account_email = 'fake_service_account_email' mock_credentials.sign_blob.return_value = ('fake_key', b'fake_signature') json_api.credentials = mock_credentials with SetBotoConfigForTest([('Credentials', 'gs_host', 'storage.googleapis.com')]): signed_url = gslib.commands.signurl._GenSignedUrl( None, api=mock_api_delegator, use_service_account=True, provider='gs', client_id=self.client_email, method='PUT', gcs_path='test/test.txt', duration=duration, logger=self.logger, region='us-east1', content_type='') self.assertEqual(expected, signed_url) mock_credentials.sign_blob.assert_called_once_with( b'GOOG4-RSA-SHA256\n19000101T000555Z\n19000101/us-east1/storage/' b'goog4_request\n7f110b30eeca7fdd8846e876bceee85384d8e4c7388b359' b'6544b1b503f9e2320') @SkipForS3('Tests only uses gs credentials.') @SkipForXML('Tests only run on JSON API.') def testSignUrlWithIncorrectAccountType(self): """Tests the _GenSignedUrl with incorrect account type. Test that GenSignedUrl function with 'use_service_account' set to True and a service account not used for credentials raises an error. """ expected = sigs.TEST_SIGN_URL_PUT_WITH_SERVICE_ACCOUNT duration = timedelta(seconds=3600) mock_api_delegator = self._get_mock_api_delegator() json_api = mock_api_delegator._GetApi('gs') # patch a service account credentials mock_credentials = mock.Mock(spec=client.OAuth2Credentials) mock_credentials.service_account_email = 'fake_service_account_email' json_api.credentials = mock_credentials with SetBotoConfigForTest([('Credentials', 'gs_host', 'storage.googleapis.com')]): self.assertRaises(CommandException, gslib.commands.signurl._GenSignedUrl, None, api=mock_api_delegator, use_service_account=True, provider='gs', client_id=self.client_email, method='PUT', gcs_path='test/test.txt', duration=duration, logger=self.logger, region='us-east1', content_type='') @SkipForS3('Tests only uses gs credentials.') @SkipForXML('Tests only run on JSON API.') @mock.patch('gslib.iamcredentials_api.apitools_client') @mock.patch('gslib.iamcredentials_api.apitools_messages') def testSignPutUsingImersonatedServiceAccount(self, mock_api_messages, mock_apiclient): """Tests the _GenSignedUrl function PUT method with impersonation. Test _GenSignedUrl function using an impersonated service account. """ expected = sigs.TEST_SIGN_URL_PUT_WITH_SERVICE_ACCOUNT duration = timedelta(seconds=3600) mock_api_delegator = self._get_mock_api_delegator() json_api = mock_api_delegator._GetApi('gs') # A mock object of type ImpersonationCredentials. mock_credentials = mock.Mock(spec=ImpersonationCredentials) api_client_obj = mock.Mock() mock_apiclient.IamcredentialsV1.return_value = api_client_obj # The api_client.IamcredntialsV1 get's in IamCredentialsApi's init mock_iam_cred_api = IamcredentailsApi(credentials=mock.Mock()) mock_credentials.api = mock_iam_cred_api mock_credentials.service_account_id = 'fake_service_account_email' # Mock the response and assign it as a return value for the SignBlob func. mock_resp = mock.Mock() mock_resp.signedBlob = b'fake_signature' api_client_obj.projects_serviceAccounts.SignBlob.return_value = mock_resp json_api.credentials = mock_credentials with SetBotoConfigForTest([('Credentials', 'gs_host', 'storage.googleapis.com')]): signed_url = gslib.commands.signurl._GenSignedUrl( None, api=mock_api_delegator, use_service_account=True, provider='gs', client_id=self.client_email, method='PUT', gcs_path='test/test.txt', duration=duration, logger=self.logger, region='us-east1', content_type='') self.assertEqual(expected, signed_url) mock_api_messages.SignBlobRequest.assert_called_once_with( payload=b'GOOG4-RSA-SHA256\n19000101T000555Z\n19000101/us-east1' b'/storage/goog4_request\n7f110b30eeca7fdd8846e876bceee' b'85384d8e4c7388b3596544b1b503f9e2320') def testSignResumableWithKeyFile(self): """Tests _GenSignedUrl using key file with a RESUMABLE method.""" expected = sigs.TEST_SIGN_RESUMABLE class MockLogger(object): def __init__(self): self.warning_issued = False def warn(self, unused_msg): self.warning_issued = True mock_logger = MockLogger() duration = timedelta(seconds=3600) with SetBotoConfigForTest([('Credentials', 'gs_host', 'storage.googleapis.com')]): signed_url = gslib.commands.signurl._GenSignedUrl( self.key, api=None, use_service_account=False, provider='gs', client_id=self.client_email, method='RESUMABLE', gcs_path='test/test.txt', duration=duration, logger=mock_logger, region='us-east', content_type='') self.assertEqual(expected, signed_url) # Resumable uploads with no content-type should issue a warning. self.assertTrue(mock_logger.warning_issued) mock_logger2 = MockLogger() with SetBotoConfigForTest([('Credentials', 'gs_host', 'storage.googleapis.com')]): signed_url = gslib.commands.signurl._GenSignedUrl( self.key, api=None, use_service_account=False, provider='gs', client_id=self.client_email, method='RESUMABLE', gcs_path='test/test.txt', duration=duration, logger=mock_logger2, region='us-east', content_type='image/jpeg') # No warning, since content type was included. self.assertFalse(mock_logger2.warning_issued) def testSignurlPutContentypeUsingKeyFile(self): """Tests _GenSignedUrl using key file with a PUT method and content type.""" expected = sigs.TEST_SIGN_URL_PUT_CONTENT duration = timedelta(seconds=3600) with SetBotoConfigForTest([('Credentials', 'gs_host', 'storage.googleapis.com')]): signed_url = gslib.commands.signurl._GenSignedUrl( self.key, api=None, use_service_account=False, provider='gs', client_id=self.client_email, method='PUT', gcs_path='test/test.txt', duration=duration, logger=self.logger, region='eu', content_type='text/plain') self.assertEqual(expected, signed_url) def testSignurlGetUsingKeyFile(self): """Tests the _GenSignedUrl function using key file with a GET method.""" expected = sigs.TEST_SIGN_URL_GET duration = timedelta(seconds=0) with SetBotoConfigForTest([('Credentials', 'gs_host', 'storage.googleapis.com')]): signed_url = gslib.commands.signurl._GenSignedUrl( self.key, api=None, use_service_account=False, provider='gs', client_id=self.client_email, method='GET', gcs_path='test/test.txt', duration=duration, logger=self.logger, region='asia', content_type='') self.assertEqual(expected, signed_url) def testSignurlGetWithJSONKeyUsingKeyFile(self): """Tests _GenSignedUrl with a GET method and the test JSON private key.""" expected = sigs.TEST_SIGN_URL_GET_WITH_JSON_KEY json_contents = pkgutil.get_data('gslib', 'tests/test_data/test.json').decode() key, client_email = gslib.commands.signurl._ReadJSONKeystore(json_contents) duration = timedelta(seconds=0) with SetBotoConfigForTest([('Credentials', 'gs_host', 'storage.googleapis.com')]): signed_url = gslib.commands.signurl._GenSignedUrl( key, api=None, use_service_account=False, provider='gs', client_id=client_email, method='GET', gcs_path='test/test.txt', duration=duration, logger=self.logger, region='asia', content_type='') self.assertEqual(expected, signed_url) def testSignurlGetWithUserProject(self): """Tests the _GenSignedUrl function with a userproject.""" expected = sigs.TEST_SIGN_URL_GET_USERPROJECT duration = timedelta(seconds=0) with SetBotoConfigForTest([('Credentials', 'gs_host', 'storage.googleapis.com')]): signed_url = gslib.commands.signurl._GenSignedUrl( self.key, api=None, use_service_account=False, provider='gs', client_id=self.client_email, method='GET', gcs_path='test/test.txt', duration=duration, logger=self.logger, region='asia', content_type='', billing_project='myproject') self.assertEqual(expected, signed_url) @unittest.skipUnless(HAVE_OPENSSL, 'signurl requires pyopenssl.') class UnitTestSignUrlWithShim(testcase.ShimUnitTestBase): def testShimTranslatesFlags(self): key_contents = pkgutil.get_data('gslib', 'tests/test_data/test.json') key_path = self.CreateTempFile(contents=key_contents) with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('signurl', [ '-d', '2m', '-m', 'RESUMABLE', '-r', 'US', '-b', 'project', '-c', 'application/octet-stream', key_path, 'gs://bucket/object' ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'storage sign-url' ' --format=csv[separator="\\t"](resource:label="URL", http_verb:label="HTTP Method", expiration:label="Expiration", signed_url:label="Signed URL")' ' --private-key-file={}' ' --headers=x-goog-resumable=start' ' --duration 120s' ' --http-verb POST' ' --region US' ' --query-params userProject=project' ' --headers content-type=application/octet-stream' ' gs://bucket/object'.format(key_path), info_lines) def testShimTranslatesFlagsWithP12Key(self): key_contents = pkgutil.get_data('gslib', 'tests/test_data/test.p12') key_path = self.CreateTempFile(contents=key_contents) key_password = 'notasecret' with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('signurl', [ '-d', '2m', '-m', 'RESUMABLE', '-p', key_password, '-r', 'US', '-b', 'project', '-c', 'application/octet-stream', key_path, 'gs://bucket/object' ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'storage sign-url' ' --format=csv[separator="\\t"](resource:label="URL", http_verb:label="HTTP Method", expiration:label="Expiration", signed_url:label="Signed URL")' ' --private-key-file={}' ' --headers=x-goog-resumable=start' ' --duration 120s' ' --http-verb POST' ' --private-key-password {}' ' --region US' ' --query-params userProject=project' ' --headers content-type=application/octet-stream' ' gs://bucket/object'.format(key_path, key_password), info_lines) tests/test_rpo.py 0000644 00000024337 15025013616 0010127 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for rpo command.""" from __future__ import absolute_import import os import textwrap from gslib.commands.rpo import RpoCommand from gslib.exception import CommandException from gslib.gcs_json_api import GcsJsonApi from gslib.storage_url import StorageUrlFromString import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForGS from gslib.tests.testcase.integration_testcase import SkipForJSON from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.utils import shim_util from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock class TestRpoUnit(testcase.GsUtilUnitTestCase): def test_get_for_multiple_bucket_calls_api(self): bucket_uri1 = self.CreateBucket(bucket_name='rpofoo') bucket_uri2 = self.CreateBucket(bucket_name='rpobar') stdout = self.RunCommand( 'rpo', ['get', suri(bucket_uri1), suri(bucket_uri2)], return_stdout=True) expected_string = textwrap.dedent("""\ gs://rpofoo: None gs://rpobar: None """) self.assertEqual(expected_string, stdout) def test_get_with_wildcard(self): self.CreateBucket(bucket_name='boo1') self.CreateBucket(bucket_name='boo2') stdout = self.RunCommand('rpo', ['get', 'gs://boo*'], return_stdout=True) actual = '\n'.join(sorted(stdout.strip().split('\n'))) expected_string = textwrap.dedent("""\ gs://boo1: None gs://boo2: None""") self.assertEqual(actual, expected_string) def test_get_with_wrong_url_raises_error(self): with self.assertRaisesRegex(CommandException, 'No URLs matched'): self.RunCommand('rpo', ['get', 'gs://invalid*']) def test_set_called_with_incorrect_value_raises_error(self): with self.assertRaisesRegex( CommandException, r'Invalid value for rpo set. Should be one of \(ASYNC_TURBO\|DEFAULT\)' ): self.RunCommand('rpo', ['set', 'random', 'gs://boo*']) def test_set_called_with_lower_case_value_raises_error(self): with self.assertRaisesRegex( CommandException, r'Invalid value for rpo set. Should be one of \(ASYNC_TURBO\|DEFAULT\)' ): self.RunCommand('rpo', ['set', 'async_turbo', 'gs://boo*']) def test_invalid_subcommand_raises_error(self): with self.assertRaisesRegex( CommandException, 'Invalid subcommand "blah", use get|set instead'): self.RunCommand('rpo', ['blah', 'DEFAULT', 'gs://boo*']) class TestRpoUnitWithShim(testcase.ShimUnitTestBase): def test_shim_translates_recovery_point_objective_get_command(self): fake_cloudsdk_dir = 'fake_dir' with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': fake_cloudsdk_dir, }): self.CreateBucket(bucket_name='fake-bucket-get-rpo-1') mock_log_handler = self.RunCommand( 'rpo', args=['get', 'gs://fake-bucket-get-rpo-1'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage' ' buckets list --format=value[separator=": "]' '(format("gs://{}", name),rpo.yesno(no="None"))' ' --raw').format(shim_util._get_gcloud_binary_path('fake_dir'), r'{}'), info_lines) def test_shim_translates_recovery_point_objective_set_command(self): fake_cloudsdk_dir = 'fake_dir' with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': fake_cloudsdk_dir, }): self.CreateBucket(bucket_name='fake-bucket-set-rpo') mock_log_handler = self.RunCommand( 'rpo', args=['set', 'DEFAULT', 'gs://fake-bucket-set-rpo'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage' ' buckets update --recovery-point-objective DEFAULT').format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) class TestRpoE2E(testcase.GsUtilIntegrationTestCase): """Integration tests for rpo command.""" # TODO: Delete this method once rpo get results are consistent # from the backend, and replace this call with # VerifyCommandGet(bucket_uri, 'rpo', 'DEFAULT'). # Currently, the rpo results are inconsistent # and None is a valid default value for all the buckets. # See b/197251750#comment19 def _verify_get_returns_default_or_none(self, bucket_uri): """Checks if the rpo get command returns default.""" try: self.VerifyCommandGet(bucket_uri, 'rpo', 'DEFAULT') except AssertionError: self.VerifyCommandGet(bucket_uri, 'rpo', 'None') @SkipForXML('RPO only runs on GCS JSON API.') def test_get_returns_default_for_dual_region_bucket(self): bucket_uri = self.CreateBucket(location='nam4') self._verify_get_returns_default_or_none(bucket_uri) @SkipForXML('RPO only runs on GCS JSON API.') def test_get_returns_none_for_regional_bucket(self): bucket_uri = self.CreateBucket(location='us-central1') self.VerifyCommandGet(bucket_uri, 'rpo', 'None') @SkipForXML('RPO only runs on GCS JSON API.') def test_set_and_get_async_turbo(self): bucket_uri = self.CreateBucket(location='nam4') self._verify_get_returns_default_or_none(bucket_uri) self.RunGsUtil(['rpo', 'set', 'ASYNC_TURBO', suri(bucket_uri)]) self.VerifyCommandGet(bucket_uri, 'rpo', 'ASYNC_TURBO') @SkipForXML('RPO only runs on GCS JSON API.') def test_set_default(self): bucket_uri = self.CreateBucket(location='nam4') self.RunGsUtil(['rpo', 'set', 'ASYNC_TURBO', suri(bucket_uri)]) self.VerifyCommandGet(bucket_uri, 'rpo', 'ASYNC_TURBO') self.RunGsUtil(['rpo', 'set', 'DEFAULT', suri(bucket_uri)]) self._verify_get_returns_default_or_none(bucket_uri) @SkipForXML('RPO only runs on GCS JSON API.') def test_set_async_turbo_fails_for_regional_buckets(self): bucket_uri = self.CreateBucket(location='us-central1') stderr = self.RunGsUtil(['rpo', 'set', 'ASYNC_TURBO', suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertIn('ASYNC_TURBO cannot be enabled on REGION bucket', stderr) @SkipForJSON('Testing XML only behavior.') def test_xml_fails_for_set(self): # Use HMAC for force XML API. boto_config_hmac_auth_only = [ # Overwrite other credential types. ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ('Credentials', 'gs_service_key_file_password', None), # Add hmac credentials. ('Credentials', 'gs_access_key_id', 'dummykey'), ('Credentials', 'gs_secret_access_key', 'dummysecret'), ] with SetBotoConfigForTest(boto_config_hmac_auth_only): bucket_uri = 'gs://any-bucket-name' stderr = self.RunGsUtil(['rpo', 'set', 'default', bucket_uri], return_stderr=True, expected_status=1) self.assertIn('command can only be with the Cloud Storage JSON API', stderr) @SkipForJSON('Testing XML only behavior.') def test_xml_fails_for_get(self): # Use HMAC for force XML API. boto_config_hmac_auth_only = [ # Overwrite other credential types. ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ('Credentials', 'gs_service_key_file_password', None), # Add HMAC credentials. ('Credentials', 'gs_access_key_id', 'dummykey'), ('Credentials', 'gs_secret_access_key', 'dummysecret'), ] with SetBotoConfigForTest(boto_config_hmac_auth_only): bucket_uri = 'gs://any-bucket-name' stderr = self.RunGsUtil(['rpo', 'get', bucket_uri], return_stderr=True, expected_status=1) self.assertIn('command can only be with the Cloud Storage JSON API', stderr) @SkipForGS('Testing S3 only behavior.') def test_s3_fails_for_set(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil( ['rpo', 'set', 'DEFAULT', suri(bucket_uri)], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn( 'Features disallowed for S3: Setting Recovery Point Objective', stderr) else: self.assertIn('command can only be used for GCS buckets', stderr) @SkipForGS('Testing S3 only behavior.') def test_s3_fails_for_get(self): bucket_uri = self.CreateBucket() expected_status = 0 if self._use_gcloud_storage else 1 stdout, stderr = self.RunGsUtil( ['rpo', 'get', suri(bucket_uri)], return_stderr=True, return_stdout=True, expected_status=expected_status) if self._use_gcloud_storage: # TODO(b/265304295) self.assertIn('gs://None: None', stdout) else: self.assertIn('command can only be used for GCS buckets', stderr) tests/test_gcs_json_api.py 0000644 00000005631 15025013616 0011761 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2020 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for gcs_json_api.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import cloud_api from gslib import gcs_json_api from gslib import context_config from gslib.tests import testcase from gslib.tests.testcase import base from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import unittest from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock class TestGcsJsonApi(testcase.GsUtilUnitTestCase): """Test logic for interacting with GCS JSON API.""" @mock.patch.object(context_config, 'get_context_config') def testSetsHostBaseToMtlsIfClientCertificateEnabled(self, mock_get_context_config): mock_context_config = mock.Mock() mock_context_config.use_client_certificate = True mock_get_context_config.return_value = mock_context_config with SetBotoConfigForTest([('Credentials', 'gs_json_host', None), ('Credentials', 'gs_host', None)]): client = gcs_json_api.GcsJsonApi(None, None, None, None) self.assertEqual(client.host_base, gcs_json_api.MTLS_HOST) @mock.patch.object(context_config, 'get_context_config') def testRaisesErrorIfConflictingJsonAndMtlsHost(self, mock_get_context_config): mock_context_config = mock.Mock() mock_context_config.use_client_certificate = True mock_get_context_config.return_value = mock_context_config with SetBotoConfigForTest([('Credentials', 'gs_json_host', 'host')]): with self.assertRaises(cloud_api.ArgumentException): gcs_json_api.GcsJsonApi(None, None, None, None) def testSetsCustomJsonHost(self): with SetBotoConfigForTest([('Credentials', 'gs_json_host', 'host')]): client = gcs_json_api.GcsJsonApi(None, None, None, None) self.assertEqual(client.host_base, 'host') def testSetsDefaultHost(self): with SetBotoConfigForTest([('Credentials', 'gs_json_host', None), ('Credentials', 'gs_host', None)]): client = gcs_json_api.GcsJsonApi(None, None, None, None) self.assertEqual(client.host_base, gcs_json_api.DEFAULT_HOST) tests/test_notification_pubsub.py 0000644 00000012731 15025013616 0013370 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for notification command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import unittest from gslib.cs_api_map import ApiSelector from gslib.project_id import PopulateProjectId from gslib.pubsub_api import PubsubApi import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri @SkipForS3('gsutil doesn\'t support S3 notifications') class TestNotificationPubSub(testcase.GsUtilIntegrationTestCase): """Integration tests for notification command (the Cloud Pub/Sub parts).""" def setUp(self): super(TestNotificationPubSub, self).setUp() self.pubsub_api = PubsubApi(logger=logging.getLogger()) self.created_topic = None def tearDown(self): super(TestNotificationPubSub, self).tearDown() # Cleanup any created topics. if self.created_topic: self.pubsub_api.DeleteTopic(self.created_topic) self.created_topic = None def _RegisterDefaultTopicCreation(self, bucket_name): """Records the name of a topic we expect to create, for cleanup.""" if self.test_api == ApiSelector.XML: return unittest.skip('Notifications only work with the JSON API.') expected_topic_name = 'projects/%s/topics/%s' % (PopulateProjectId(None), bucket_name) self.created_topic = expected_topic_name return expected_topic_name def test_list_new_bucket(self): """Tests listing notification configs on a new bucket.""" if self.test_api == ApiSelector.XML: return unittest.skip('Notifications only work with the JSON API.') bucket_uri = self.CreateBucket() stdout = self.RunGsUtil( ['notification', 'list', suri(bucket_uri)], return_stdout=True) self.assertFalse(stdout) def test_delete_with_no_notifications(self): """Tests deleting all notification configs when there are none.""" if self.test_api == ApiSelector.XML: return unittest.skip('Notifications only work with the JSON API.') bucket_uri = self.CreateBucket() stdout = self.RunGsUtil( ['notification', 'delete', suri(bucket_uri)], return_stdout=True) self.assertFalse(stdout) def test_create_basic(self): """Tests the create command succeeds in normal circumstances.""" if self.test_api == ApiSelector.XML: return unittest.skip('Notifications only work with the JSON API.') bucket_uri = self.CreateBucket() topic_name = self._RegisterDefaultTopicCreation(bucket_uri.bucket_name) stdout, stderr = self.RunGsUtil( ['notification', 'create', '-f', 'json', suri(bucket_uri)], return_stdout=True, return_stderr=True) if self._use_gcloud_storage: self.assertIn('kind: storage#notification', stdout) self.assertIn(topic_name, stdout) else: self.assertIn('Created notification', stderr) self.assertIn(topic_name, stderr) def test_list_one_entry(self): """Tests notification config listing with one entry.""" if self.test_api == ApiSelector.XML: return unittest.skip('Notifications only work with the JSON API.') bucket_uri = self.CreateBucket() bucket_name = bucket_uri.bucket_name topic_name = self._RegisterDefaultTopicCreation(bucket_uri.bucket_name) self.RunGsUtil([ 'notification', 'create', '-f', 'json', '-e', 'OBJECT_FINALIZE', '-e', 'OBJECT_DELETE', '-m', 'someKey:someValue', '-p', 'somePrefix', suri(bucket_uri) ], return_stderr=True) stdout = self.RunGsUtil( ['notification', 'list', suri(bucket_uri)], return_stdout=True) if self._use_gcloud_storage: # gcloud and gsutil look the same in the terminal, but, for some reason, # the shim test script detects an extra newline. trailing_space = '\n' else: trailing_space = '' self.assertEqual( stdout, ('projects/_/buckets/{bucket_name}/notificationConfigs/1\n' '\tCloud Pub/Sub topic: {topic_name}\n' '\tCustom attributes:\n' '\t\tsomeKey: someValue\n' '\tFilters:\n' '\t\tEvent Types: OBJECT_FINALIZE, OBJECT_DELETE\n' '\t\tObject name prefix: \'somePrefix\'\n{trailing_space}'.format( bucket_name=bucket_name, topic_name=topic_name, trailing_space=trailing_space))) def test_delete(self): """Tests the create command succeeds in normal circumstances.""" if self.test_api == ApiSelector.XML: return unittest.skip('Notifications only work with the JSON API.') bucket_uri = self.CreateBucket() self._RegisterDefaultTopicCreation(bucket_uri.bucket_name) self.RunGsUtil(['notification', 'create', '-f', 'json', suri(bucket_uri)]) self.RunGsUtil(['notification', 'delete', suri(bucket_uri)]) tests/test_retention.py 0000644 00000071631 15025013616 0011335 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for retention command.""" from __future__ import absolute_import import datetime import re import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import ObjectToURI as suri _SECONDS_IN_DAY = 24 * 60 * 60 _DAYS_IN_MONTH = 31 _SECONDS_IN_MONTH = _DAYS_IN_MONTH * _SECONDS_IN_DAY _DAYS_IN_YEAR = 365.25 _SECONDS_IN_YEAR = int(_DAYS_IN_YEAR * _SECONDS_IN_DAY) class TestRetention(testcase.GsUtilIntegrationTestCase): """Integration tests for retention command.""" @SkipForS3('Retention is not supported for s3 objects.') @SkipForXML('Retention is not supported for XML API.') def test_set_retention_seconds(self): bucket_uri = self.CreateBucket() self.RunGsUtil(['retention', 'set', '60s', suri(bucket_uri)]) self.VerifyRetentionPolicy(bucket_uri, expected_retention_period_in_seconds=60) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_set_retention_days(self): bucket_uri = self.CreateBucket() self.RunGsUtil(['retention', 'set', '1d', suri(bucket_uri)]) self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=_SECONDS_IN_DAY) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_set_retention_months(self): bucket_uri = self.CreateBucket() self.RunGsUtil(['retention', 'set', '1m', suri(bucket_uri)]) self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=_SECONDS_IN_MONTH) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_set_retention_years(self): bucket_uri = self.CreateBucket() self.RunGsUtil(['retention', 'set', '1y', suri(bucket_uri)]) self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=_SECONDS_IN_YEAR) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_set_retention_multiple_sequential(self): bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.RunGsUtil( ['retention', 'set', '1y', suri(bucket1_uri), suri(bucket2_uri)]) self.VerifyRetentionPolicy( bucket1_uri, expected_retention_period_in_seconds=_SECONDS_IN_YEAR) self.VerifyRetentionPolicy( bucket2_uri, expected_retention_period_in_seconds=_SECONDS_IN_YEAR) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_set_retention_multiple_parallel(self): bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.RunGsUtil( ['-m', 'retention', 'set', '1y', suri(bucket1_uri), suri(bucket2_uri)]) self.VerifyRetentionPolicy( bucket1_uri, expected_retention_period_in_seconds=_SECONDS_IN_YEAR) self.VerifyRetentionPolicy( bucket2_uri, expected_retention_period_in_seconds=_SECONDS_IN_YEAR) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_increase_retention_unlocked(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY) self.RunGsUtil(['retention', 'set', '1m', suri(bucket_uri)]) self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=_SECONDS_IN_MONTH) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_decrease_retention_unlocked(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_MONTH) self.RunGsUtil( ['retention', 'set', '{}s'.format(_SECONDS_IN_DAY), suri(bucket_uri)]) self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=_SECONDS_IN_DAY) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_clear_unlocked_retention(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY) self.RunGsUtil(['retention', 'clear', suri(bucket_uri)]) self.VerifyRetentionPolicy(bucket_uri, expected_retention_period_in_seconds=None) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_set_retention_unlocked_invalid_arg(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil( ['retention', 'set', '1a', suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertRegex(stderr, r'Incorrect retention period specified') @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_lock_retention_userConfirms(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY) self.RunGsUtil(['retention', 'lock', suri(bucket_uri)], stdin='y') self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=_SECONDS_IN_DAY, expected_is_locked=True) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_lock_retention_userDoesNotConfirm(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY) stderr = self.RunGsUtil( ['retention', 'lock', suri(bucket_uri)], stdin='n', return_stderr=True) self.assertRegex(stderr, 'Abort [Ll]ocking [Rr]etention [Pp]olicy on') self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=_SECONDS_IN_DAY) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_lock_with_no_retention_policy_invalid(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil( ['retention', 'lock', suri(bucket_uri)], stdin='y', expected_status=1, return_stderr=True) self.assertRegex(stderr, 'does not have a(n Unlocked)? [Rr]etention [Pp]olicy') self.VerifyRetentionPolicy(bucket_uri, expected_retention_period_in_seconds=None) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_lock_retention_with_invalid_arg(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil( ['retention', 'lock', '-a', suri(bucket_uri)], stdin='y', expected_status=1, return_stderr=True) self.assertRegex(stderr, r'Incorrect option\(s\) specified') @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_lock_retention_already_locked(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY, is_locked=True) stderr = self.RunGsUtil( ['retention', 'lock', suri(bucket_uri)], stdin='y', return_stderr=True) self.assertRegex(stderr, r'Retention [Pp]olicy on .* is already locked') @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_increase_retention_locked(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY, is_locked=True) self.RunGsUtil([ 'retention', 'set', '{}s'.format(_SECONDS_IN_DAY + 1), suri(bucket_uri) ]) self.VerifyRetentionPolicy( bucket_uri, expected_retention_period_in_seconds=_SECONDS_IN_DAY + 1, expected_is_locked=True) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_decrease_retention_locked(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY, is_locked=True) stderr = self.RunGsUtil([ 'retention', 'set', '{}s'.format(_SECONDS_IN_DAY - 1), suri(bucket_uri) ], expected_status=1, return_stderr=True) self.assertRegex( stderr, 'Cannot reduce retention duration of a ' 'locked Retention Policy for bucket') @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_clear_locked_retention(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY, is_locked=True) stderr = self.RunGsUtil( ['retention', 'clear', suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertRegex( stderr, r'Bucket .* has a locked Retention Policy which cannot be removed') @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_get_retention_locked(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY, is_locked=True) stdout = self.RunGsUtil( ['-DD', 'retention', 'get', suri(bucket_uri)], return_stdout=True) if self._use_gcloud_storage: self.assertRegex(stdout, r'isLocked\: true') self.assertRegex(stdout, r'retentionPeriod\: \'86400\'') self.assertRegex(stdout, r'effectiveTime\: \'.*\'') else: self.assertRegex(stdout, r'Retention Policy \(LOCKED\):') self.assertRegex(stdout, r'Duration: 1 Day\(s\)') self.assertRegex(stdout, r'Effective Time: .* GMT') actual_retention_policy = self.json_api.GetBucket( bucket_uri.bucket_name, fields=['retentionPolicy']).retentionPolicy if self._use_gcloud_storage: expected_effective_time = datetime.datetime.fromisoformat( re.search(r'effectiveTime\: \'(.*)\'', stdout).group(1)) actual_effective_time = actual_retention_policy.effectiveTime else: expected_effective_time = self._ConvertTimeStringToSeconds( re.search(r'(?<=Time: )[\w,: ]+', stdout).group()) actual_effective_time = self.DateTimeToSeconds( actual_retention_policy.effectiveTime.replace(tzinfo=None)) self.assertEqual(actual_effective_time, expected_effective_time) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_get_retention_unlocked(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY) stdout = self.RunGsUtil( ['retention', 'get', suri(bucket_uri)], return_stdout=True) if self._use_gcloud_storage: # Sometimes the field is absent if isLocked is false. self.assertNotRegex(stdout, r'isLocked \: true') self.assertRegex(stdout, r'retentionPeriod\: \'86400\'') self.assertRegex(stdout, r'effectiveTime\: \'.*\'') else: self.assertRegex(stdout, r'Retention Policy \(UNLOCKED\):') self.assertRegex(stdout, r'Duration: 1 Day\(s\)') self.assertRegex(stdout, r'Effective Time: .* GMT') actual_retention_policy = self.json_api.GetBucket( bucket_uri.bucket_name, fields=['retentionPolicy']).retentionPolicy if self._use_gcloud_storage: expected_effective_time = datetime.datetime.fromisoformat( re.search(r'effectiveTime\: \'(.*)\'', stdout).group(1)) actual_effective_time = actual_retention_policy.effectiveTime else: expected_effective_time = self._ConvertTimeStringToSeconds( re.search(r'(?<=Time: )[\w,: ]+', stdout).group()) actual_effective_time = self.DateTimeToSeconds( actual_retention_policy.effectiveTime.replace(tzinfo=None)) self.assertEqual(actual_effective_time, expected_effective_time) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_get_no_retention(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil( ['retention', 'get', suri(bucket_uri)], return_stdout=True) if self._use_gcloud_storage: self.assertRegex(stdout, 'null') else: self.assertRegex(stdout, 'has no Retention Policy') @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_get_invalid_args(self): bucket_uri = self.CreateBucketWithRetentionPolicy( retention_period_in_seconds=_SECONDS_IN_DAY) stderr = self.RunGsUtil( ['retention', 'get', '-a', suri(bucket_uri)], expected_status=1, return_stderr=True) self.assertRegex(stderr, r'Incorrect option\(s\) specified.') @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_set_temporary_hold_invalid_arg(self): object_uri = self.CreateObject() stderr = self.RunGsUtil(['retention', 'temp', 'held', suri(object_uri)], expected_status=1, return_stderr=True) self.assertRegex( stderr, r'Invalid subcommand ".*" for the "retention temp" command') @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_temporary_hold_bucket_with_no_retention(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, temporary_hold=None) self.RunGsUtil(['retention', 'temp', 'set', suri(object_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, temporary_hold=True) self.RunGsUtil(['retention', 'temp', 'release', suri(object_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, temporary_hold=False) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_temporary_hold_bucket_with_retention(self): retention_period = 1 bucket_uri = self.CreateBucketWithRetentionPolicy(retention_period) object_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, temporary_hold=None, retention_period=retention_period) self.RunGsUtil(['retention', 'temp', 'set', suri(object_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, temporary_hold=True, retention_period=retention_period) self.RunGsUtil(['retention', 'temp', 'release', suri(object_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, temporary_hold=False, retention_period=retention_period) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_temporary_hold_multiple_sequential(self): bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') object2_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') self.RunGsUtil( ['retention', 'temp', 'set', suri(object1_uri), suri(object2_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object1_uri, temporary_hold=True) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object2_uri, temporary_hold=True) self.RunGsUtil( ['retention', 'temp', 'release', suri(object1_uri), suri(object2_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object1_uri, temporary_hold=False) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object2_uri, temporary_hold=False) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_temporary_hold_multiple_parallel(self): bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') object2_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') self.RunGsUtil([ '-m', 'retention', 'temp', 'set', suri(object1_uri), suri(object2_uri) ]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object1_uri, temporary_hold=True) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object2_uri, temporary_hold=True) self.RunGsUtil([ '-m', 'retention', 'temp', 'release', suri(object1_uri), suri(object2_uri) ]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object1_uri, temporary_hold=False) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object2_uri, temporary_hold=False) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_set_event_based_hold_invalid_arg(self): object_uri = self.CreateObject() stderr = self.RunGsUtil(['retention', 'event', 'rel', suri(object_uri)], expected_status=1, return_stderr=True) self.assertRegex( stderr, r'Invalid subcommand ".*" for the "retention event" command') @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_event_based_hold_bucket_with_no_retention(self): bucket_uri = self.CreateBucket() object_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, event_based_hold=None) self.RunGsUtil(['retention', 'event', 'set', suri(object_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, event_based_hold=True) self.RunGsUtil(['retention', 'event', 'release', suri(object_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, event_based_hold=False) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_event_based_hold_bucket_with_retention(self): retention_period = 1 bucket_uri = self.CreateBucketWithRetentionPolicy(retention_period) object_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, event_based_hold=None, retention_period=retention_period) self.RunGsUtil(['retention', 'event', 'set', suri(object_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, event_based_hold=True, retention_period=None) self.RunGsUtil(['retention', 'event', 'release', suri(object_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object_uri, event_based_hold=False, retention_period=retention_period) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_event_based_hold_multiple_sequential(self): bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') object2_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') self.RunGsUtil( ['retention', 'event', 'set', suri(object1_uri), suri(object2_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object1_uri, event_based_hold=True) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object2_uri, event_based_hold=True) self.RunGsUtil( ['retention', 'event', 'release', suri(object1_uri), suri(object2_uri)]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object1_uri, event_based_hold=False) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object2_uri, event_based_hold=False) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_event_based_hold_multiple_parallel(self): bucket_uri = self.CreateBucket() object1_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') object2_uri = self.CreateObject(bucket_uri=bucket_uri, contents='content') self.RunGsUtil([ '-m', 'retention', 'event', 'set', suri(object1_uri), suri(object2_uri) ]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object1_uri, event_based_hold=True) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object2_uri, event_based_hold=True) self.RunGsUtil([ '-m', 'retention', 'event', 'release', suri(object1_uri), suri(object2_uri) ]) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object1_uri, event_based_hold=False) self._VerifyObjectHoldAndRetentionStatus(bucket_uri, object2_uri, event_based_hold=False) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_default_event_based_hold(self): bucket_uri = self.CreateBucket() self.RunGsUtil(['retention', 'event-default', 'set', suri(bucket_uri)]) self._VerifyDefaultEventBasedHold(bucket_uri, expected_default_event_based_hold=True) self.RunGsUtil(['retention', 'event-default', 'release', suri(bucket_uri)]) self._VerifyDefaultEventBasedHold(bucket_uri, expected_default_event_based_hold=False) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_default_event_based_hold_multiple_sequential(self): bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.RunGsUtil([ 'retention', 'event-default', 'set', suri(bucket1_uri), suri(bucket2_uri) ]) self._VerifyDefaultEventBasedHold(bucket1_uri, expected_default_event_based_hold=True) self._VerifyDefaultEventBasedHold(bucket2_uri, expected_default_event_based_hold=True) @SkipForS3('Retention is not supported for s3 objects') @SkipForXML('Retention is not supported for XML API') def test_default_event_based_hold_multiple_parallel(self): bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.RunGsUtil([ 'retention', 'event-default', 'set', suri(bucket1_uri), suri(bucket2_uri) ]) self._VerifyDefaultEventBasedHold(bucket1_uri, expected_default_event_based_hold=True) self._VerifyDefaultEventBasedHold(bucket2_uri, expected_default_event_based_hold=True) def _VerifyObjectHoldAndRetentionStatus(self, bucket_uri, object_uri, temporary_hold=None, event_based_hold=None, retention_period=None): object_metadata = self.json_api.GetObjectMetadata( bucket_uri.bucket_name, object_uri.object_name, fields=[ 'timeCreated', 'temporaryHold', 'eventBasedHold', 'retentionExpirationTime' ]) if temporary_hold is None: self.assertEqual(object_metadata.temporaryHold, None) else: self.assertEqual(object_metadata.temporaryHold, temporary_hold) if event_based_hold is None: self.assertEqual(object_metadata.eventBasedHold, None) else: self.assertEqual(object_metadata.eventBasedHold, event_based_hold) if retention_period is None: self.assertEqual(object_metadata.retentionExpirationTime, None) elif event_based_hold is False: retention_policy = self.json_api.GetBucket(bucket_uri.bucket_name, fields=['retentionPolicy' ]).retentionPolicy time_delta = datetime.timedelta(0, retention_policy.retentionPeriod) expected_expiration_time = object_metadata.timeCreated + time_delta if event_based_hold is None: self.assertEqual(object_metadata.retentionExpirationTime, expected_expiration_time) else: # since we don't expose the release time of event-based hold we can # only verify that expected_expiration_time is greater than # object-creation-time + retention period # that is because # eventBased-hold's release time > object-creation-time self.assertGreater(object_metadata.retentionExpirationTime, expected_expiration_time) def _VerifyDefaultEventBasedHold(self, bucket_uri, expected_default_event_based_hold=None): actual_default_event_based_hold = self.json_api.GetBucket( bucket_uri.bucket_name, fields=['defaultEventBasedHold']).defaultEventBasedHold if expected_default_event_based_hold is None: self.assertEqual(actual_default_event_based_hold, None) else: self.assertEqual(actual_default_event_based_hold, expected_default_event_based_hold) def _ConvertTimeStringToSeconds(self, time_string): """Converts time in following format to its equivalent timestamp in seconds. Format: '%a, %d %b %Y %H:%M:%S GMT' i.e.: 'Fri, 18 Aug 2017 23:31:39 GMT' Args: time_string: time in string format. Returns: returns equivalent timestamp in seconds of given time. """ converted_time = datetime.datetime.strptime(time_string, '%a, %d %b %Y %H:%M:%S GMT') return self.DateTimeToSeconds(converted_time) tests/test_autoclass.py 0000644 00000015547 15025013616 0011330 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for autoclass command.""" from __future__ import absolute_import import re import gslib.tests.testcase as testcase from gslib import exception from gslib.tests.testcase.integration_testcase import SkipForGS from gslib.tests.testcase.integration_testcase import SkipForJSON from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest class TestAutoclassUnit(testcase.GsUtilUnitTestCase): def test_set_too_few_arguments_fails(self): with self.assertRaisesRegex(exception.CommandException, 'command requires at least'): self.RunCommand('autoclass', ['set']) def test_get_too_few_arguments_fails(self): with self.assertRaisesRegex(exception.CommandException, 'command requires at least'): self.RunCommand('autoclass', ['get']) def test_no_subcommand_fails(self): with self.assertRaisesRegex(exception.CommandException, 'command requires at least'): self.RunCommand('autoclass', []) def test_invalid_subcommand_fails(self): with self.assertRaisesRegex(exception.CommandException, 'Invalid subcommand'): self.RunCommand('autoclass', ['fakecommand', 'test']) def test_gets_multiple_buckets_with_wildcard(self): bucket_uri1 = self.CreateBucket(bucket_name='bucket1') bucket_uri2 = self.CreateBucket(bucket_name='bucket2') stdout = self.RunCommand('autoclass', ['get', 'gs://bucket*'], return_stdout=True) self.assertIn(bucket_uri1.bucket_name, stdout) self.assertIn(bucket_uri2.bucket_name, stdout) class TestAutoclassE2E(testcase.GsUtilIntegrationTestCase): """E2E tests for autoclass command.""" _set_autoclass_cmd = ['autoclass', 'set'] _get_autoclass_cmd = ['autoclass', 'get'] @SkipForXML('Autoclass only runs on GCS JSON API.') def test_off_on_default_buckets(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil(self._get_autoclass_cmd + [suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'Enabled: False') self.assertRegex(stdout, r'Toggle Time: None') @SkipForXML('Autoclass only runs on GCS JSON API.') def test_turning_on_and_off(self): bucket_uri = self.CreateBucket() stdout, stderr = self.RunGsUtil(self._set_autoclass_cmd + ['on', suri(bucket_uri)], return_stdout=True, return_stderr=True) if self._use_gcloud_storage: self.assertRegex(stderr, re.escape('Updating {}'.format(str(bucket_uri)))) else: self.assertRegex( stdout, re.escape('Setting Autoclass on for {}\n'.format( str(bucket_uri).rstrip('/')))) stdout = self.RunGsUtil(self._get_autoclass_cmd + [suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'Enabled: True') self.assertRegex(stdout, r'Toggle Time: \d+') stdout, stderr = self.RunGsUtil(self._set_autoclass_cmd + ['off', suri(bucket_uri)], return_stdout=True, return_stderr=True) if self._use_gcloud_storage: self.assertRegex(stderr, re.escape('Updating {}'.format(str(bucket_uri)))) else: self.assertRegex( stdout, re.escape('Setting Autoclass off for {}\n'.format( str(bucket_uri).rstrip('/')))) stdout = self.RunGsUtil(self._get_autoclass_cmd + [suri(bucket_uri)], return_stdout=True) self.assertRegex(stdout, r'Enabled: False') self.assertRegex(stdout, r'Toggle Time: \d+') @SkipForXML('Autoclass only runs on GCS JSON API.') def test_multiple_buckets(self): bucket_uri1 = self.CreateBucket() bucket_uri2 = self.CreateBucket() stdout = self.RunGsUtil( self._get_autoclass_cmd + [suri(bucket_uri1), suri(bucket_uri2)], return_stdout=True) output_regex = (r'{}:\n' r' Enabled: False\n' r' Toggle Time: None\n' r'{}:\n' r' Enabled: False\n' r' Toggle Time: None'.format(suri(bucket_uri1), suri(bucket_uri2))) self.assertRegex(stdout, output_regex) @SkipForJSON('Testing XML only behavior.') def test_xml_fails(self): # Use HMAC for force XML API. boto_config_hmac_auth_only = [ # Overwrite other credential types. ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ('Credentials', 'gs_service_key_file_password', None), # Add HMAC credentials. ('Credentials', 'gs_access_key_id', 'dummykey'), ('Credentials', 'gs_secret_access_key', 'dummysecret'), ] with SetBotoConfigForTest(boto_config_hmac_auth_only): bucket_uri = 'gs://any-bucket-name' stderr = self.RunGsUtil(self._set_autoclass_cmd + ['on', bucket_uri], return_stderr=True, expected_status=1) self.assertIn('command can only be with the Cloud Storage JSON API', stderr) stderr = self.RunGsUtil(self._get_autoclass_cmd + [bucket_uri], return_stderr=True, expected_status=1) self.assertIn('command can only be with the Cloud Storage JSON API', stderr) @SkipForGS('Testing S3 only behavior') def test_s3_fails(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil(self._set_autoclass_cmd + ['on', suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('command can only be used for GCS Buckets', stderr) stderr = self.RunGsUtil(self._get_autoclass_cmd + [suri(bucket_uri)], return_stderr=True, expected_status=1) self.assertIn('command can only be used for GCS Buckets', stderr) tests/test_hmac.py 0000644 00000057633 15025013616 0010244 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for the hmac command.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals import boto import os import re from gslib.commands import hmac from gslib.project_id import PopulateProjectId import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import unittest from gslib.utils.retry_util import Retry from gslib.utils import shim_util from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock def _LoadServiceAccount(account_field): return boto.config.get_value('GSUtil', account_field) SERVICE_ACCOUNT = _LoadServiceAccount('test_hmac_service_account') ALT_SERVICE_ACCOUNT = _LoadServiceAccount('test_hmac_alt_service_account') LIST_SERVICE_ACCOUNT = _LoadServiceAccount('test_hmac_list_service_account') MAX_SA_HMAC_KEYS = 10 class KeyLimitError(Exception): pass @SkipForS3('S3 does not have an equivalent API') @SkipForXML('XML HMAC control is not supported.') class TestHmacIntegration(testcase.GsUtilIntegrationTestCase): """Hmac integration test cases. These tests rely on the presence of 3 service accounts specified in the BOTO config. test_hmac_service_account and test_hmac_alt_service_account should not have any undeleted keys and test_hmac_list_service_account should have only deleted and active keys. """ def ExtractAccessId(self, output_string): id_match = re.search(r'(GOOG[\S]*)', output_string) if not id_match: self.fail('Couldn\'t find Access Id in output string:\n"%s"' % output_string) return id_match.group(0) def ExtractEtag(self, output_string): etag_match = re.search(r'\sEtag:\s+([\S]+)$', output_string) if not etag_match: self.fail('Couldn\'t find Etag in output string:\n"%s"' % output_string) return etag_match.group(1) def AssertKeyMetadataMatches(self, output_string, access_id='GOOG.*', state='ACTIVE', service_account='.*', project='.*'): self.assertRegex(output_string, r'Access ID %s:' % access_id) self.assertRegex(output_string, r'\sState:\s+%s' % state) self.assertRegex(output_string, r'\s+Service Account:\s+%s\n' % service_account) self.assertRegex(output_string, r'\s+Project:\s+%s' % project) self.assertRegex(output_string, r'\s+Time Created:\s+.*') self.assertRegex(output_string, r'\s+Time Last Updated:\s+.*') def CleanupHelper(self, access_id): # Set the key to inactive if it isn't already. try: self.RunGsUtil(['hmac', 'update', '-s', 'INACTIVE', access_id]) except AssertionError as e: if 'Update must modify the credential' not in str(e): raise self.RunGsUtil(['hmac', 'delete', access_id]) @Retry(KeyLimitError, tries=5, timeout_secs=3) def _CreateWithRetry(self, service_account): """Retry creation on key limit failures.""" try: return self.RunGsUtil(['hmac', 'create', service_account], return_stdout=True) except AssertionError as e: if 'HMAC key limit reached' in str(e): raise KeyLimitError(str(e)) else: raise def CreateHelper(self, service_account): stdout = self._CreateWithRetry(service_account) return self.ExtractAccessId(stdout) def test_malformed_commands(self): params = [ ('hmac create', 'requires a service account', 'argument SERVICE_ACCOUNT: Must be specified'), ('hmac create -p proj', 'requires a service account', 'argument SERVICE_ACCOUNT: Must be specified'), ('hmac delete', 'requires an Access ID', 'argument ACCESS_ID: Must be specified'), ('hmac delete -p proj', 'requires an Access ID', 'argument ACCESS_ID: Must be specified'), ('hmac get', 'requires an Access ID', 'argument ACCESS_ID: Must be specified'), ('hmac get -p proj', 'requires an Access ID', 'argument ACCESS_ID: Must be specified'), ('hmac list account1', 'unexpected arguments', 'unrecognized arguments'), ('hmac update keyname', 'state flag must be supplied', 'Exactly one of (--activate | --deactivate) must be specified.'), ('hmac update -s INACTIVE', 'requires an Access ID', 'argument ACCESS_ID: Must be specified'), ('hmac update -s INACTIVE -p proj', 'requires an Access ID', 'argument ACCESS_ID: Must be specified'), ] for command, gsutil_error_substr, gcloud_error_substr in params: expected_status = 2 if self._use_gcloud_storage else 1 stderr = self.RunGsUtil(command.split(), return_stderr=True, expected_status=(expected_status)) if self._use_gcloud_storage: self.assertIn(gcloud_error_substr, stderr) else: self.assertIn(gsutil_error_substr, stderr) def test_malformed_commands_that_cannot_be_translated_using_the_shim(self): if self._use_gcloud_storage: raise unittest.SkipTest('These commands cannot be translated using the' ' shim.') params = [ ('hmac create -u email', 'requires a service account'), ('hmac update -s KENTUCKY', 'state flag value must be one of'), ] for command, gsutil_error_substr in params: stderr = self.RunGsUtil(command.split(), return_stderr=True, expected_status=1) self.assertIn(gsutil_error_substr, stderr) @unittest.skipUnless(SERVICE_ACCOUNT, 'Test requires service account configuration.') def test_create(self): stdout = self.RunGsUtil(['hmac', 'create', SERVICE_ACCOUNT], return_stdout=True) try: self.assertRegex(stdout, r'Access ID:\s+\S+') self.assertRegex(stdout, r'Secret:\s+\S+') finally: access_id = self.ExtractAccessId(stdout) self.CleanupHelper(access_id) def test_create_sa_not_found(self): stderr = self.RunGsUtil(['hmac', 'create', 'DNE@mail.com'], return_stderr=True, expected_status=1) self.assertIn('Service Account \'DNE@mail.com\' not found.', stderr) @unittest.skipUnless(ALT_SERVICE_ACCOUNT, 'Test requires service account configuration.') def test_delete(self): access_id = self.CreateHelper(ALT_SERVICE_ACCOUNT) self.RunGsUtil(['hmac', 'update', '-s', 'INACTIVE', access_id]) self.RunGsUtil(['hmac', 'delete', access_id]) stdout = self.RunGsUtil(['hmac', 'list', '-u', ALT_SERVICE_ACCOUNT], return_stdout=True) self.assertNotIn(access_id, stdout) @unittest.skipUnless(SERVICE_ACCOUNT, 'Test requires service account configuration.') def test_delete_active_key(self): access_id = self.CreateHelper(SERVICE_ACCOUNT) stderr = self.RunGsUtil(['hmac', 'delete', access_id], return_stderr=True, expected_status=1) try: if self._use_gcloud_storage: self.assertIn('HTTPError 400: Cannot delete keys in ACTIVE state.', stderr) else: self.assertIn('400 Cannot delete keys in ACTIVE state.', stderr) finally: self.CleanupHelper(access_id) def test_delete_key_not_found(self): stderr = self.RunGsUtil(['hmac', 'delete', 'GOOG1234DNE'], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('HTTPError 404: Access ID not found in project', stderr) else: self.assertIn('404 Access ID not found', stderr) @unittest.skipUnless(ALT_SERVICE_ACCOUNT, 'Test requires service account configuration.') def test_get(self): access_id = self.CreateHelper(ALT_SERVICE_ACCOUNT) stdout = self.RunGsUtil(['hmac', 'get', access_id], return_stdout=True) try: self.AssertKeyMetadataMatches( stdout, access_id=access_id, service_account=ALT_SERVICE_ACCOUNT, project=PopulateProjectId(None), ) finally: self.CleanupHelper(access_id) def test_get_not_found(self): stderr = self.RunGsUtil(['hmac', 'get', 'GOOG1234DNE'], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn('HTTPError 404: Access ID not found in project', stderr) else: self.assertIn('404 Access ID not found', stderr) def setUpListTest(self): for _ in range(MAX_SA_HMAC_KEYS): self.RunGsUtil(['hmac', 'create', LIST_SERVICE_ACCOUNT], expected_status=None) @unittest.skipUnless(LIST_SERVICE_ACCOUNT and SERVICE_ACCOUNT, 'Test requires service account configuration.') def test_list(self): # Ensure LIST_SERVICE_ACCOUNT has MAX_SA_HMAC_KEYS. These keys should all be # active otherwise the tests will fail. self.setUpListTest() alt_access_id = self.CreateHelper(SERVICE_ACCOUNT) self.RunGsUtil(['hmac', 'update', '-s', 'INACTIVE', alt_access_id]) # Test listing single service account only - non deleted keys stdout = self.RunGsUtil(['hmac', 'list', '-u', LIST_SERVICE_ACCOUNT], return_stdout=True) stdout = stdout.strip().split('\n') list_account_key_count = 0 for line in stdout: access_id, state, account = line.split() self.assertIn('GOOG', access_id) self.assertEqual(account, LIST_SERVICE_ACCOUNT) self.assertEqual(state, 'ACTIVE') list_account_key_count += 1 self.assertEqual(list_account_key_count, MAX_SA_HMAC_KEYS) # Test listing all project service account HMAC keys. stdout = self.RunGsUtil(['hmac', 'list'], return_stdout=True) stdout = stdout.strip().split('\n') project_key_count = 0 inactive_key_listed = False for line in stdout: _, state, account = line.split() project_key_count += 1 if account == SERVICE_ACCOUNT and state == 'INACTIVE': inactive_key_listed = True # Other tests generate keys for this project. Can't assert exact quantity. self.assertGreater(project_key_count, list_account_key_count) self.assertTrue(inactive_key_listed) # Test listing deleted keys. self.RunGsUtil(['hmac', 'delete', alt_access_id]) stdout = self.RunGsUtil(['hmac', 'list', '-a'], return_stdout=True) stdout = stdout.strip().split('\n') project_key_count = 0 deleted_key_listed = False for line in stdout: _, state, account = line.split() project_key_count += 1 if account == SERVICE_ACCOUNT and state == 'DELETED': deleted_key_listed = True self.assertTrue(deleted_key_listed) self.assertGreater(project_key_count, list_account_key_count) def ParseListOutput(self, stdout): current_key = '' for l in stdout.split('\n'): if current_key and l.startswith('Access ID'): yield current_key current_key = l else: current_key += l current_key += '\n' @unittest.skipUnless(LIST_SERVICE_ACCOUNT and ALT_SERVICE_ACCOUNT, 'Test requires service account configuration.') def test_list_long_format(self): # Ensure LIST_SERVICE_ACCOUNT has MAX_SA_HMAC_KEYS. These keys should all be # active otherwise the tests will fail. self.setUpListTest() alt_access_id = self.CreateHelper(ALT_SERVICE_ACCOUNT) self.RunGsUtil(['hmac', 'update', '-s', 'INACTIVE', alt_access_id]) stdout = self.RunGsUtil(['hmac', 'list', '-l'], return_stdout=True) try: self.assertIn(' ACTIVE', stdout) self.assertIn('INACTIVE', stdout) self.assertIn(ALT_SERVICE_ACCOUNT, stdout) self.assertIn(LIST_SERVICE_ACCOUNT, stdout) for key_metadata in self.ParseListOutput(stdout): self.AssertKeyMetadataMatches(key_metadata, state='.*') finally: self.CleanupHelper(alt_access_id) def test_list_service_account_not_found(self): service_account = 'service-account-DNE@gmail.com' stderr = self.RunGsUtil(['hmac', 'list', '-u', service_account], return_stderr=True, expected_status=1) if self._use_gcloud_storage: self.assertIn( 'HTTPError 404: Service Account \'{}\' not found.'.format( service_account), stderr) else: self.assertIn('Service Account \'%s\' not found.' % service_account, stderr) @unittest.skipUnless(ALT_SERVICE_ACCOUNT, 'Test requires service account configuration.') def test_update(self): access_id = self.CreateHelper(ALT_SERVICE_ACCOUNT) stdout = self.RunGsUtil(['hmac', 'get', access_id], return_stdout=True) etag = self.ExtractEtag(stdout) try: self.AssertKeyMetadataMatches(stdout, state='ACTIVE') stdout = self.RunGsUtil( ['hmac', 'update', '-s', 'INACTIVE', '-e', etag, access_id], return_stdout=True) self.AssertKeyMetadataMatches(stdout, state='INACTIVE') stdout = self.RunGsUtil(['hmac', 'get', access_id], return_stdout=True) self.AssertKeyMetadataMatches(stdout, state='INACTIVE') stdout = self.RunGsUtil(['hmac', 'update', '-s', 'ACTIVE', access_id], return_stdout=True) self.AssertKeyMetadataMatches(stdout, state='ACTIVE') stdout = self.RunGsUtil(['hmac', 'get', access_id], return_stdout=True) self.AssertKeyMetadataMatches(stdout, state='ACTIVE') stderr = self.RunGsUtil( ['hmac', 'update', '-s', 'INACTIVE', '-e', 'badEtag', access_id], return_stderr=True, expected_status=1) self.assertIn('Etag does not match expected value.', stderr) finally: self.CleanupHelper(access_id) @SkipForS3('S3 does not have an equivalent API') class TestHmacXmlIntegration(testcase.GsUtilIntegrationTestCase): """XML integration tests for the "hmac" command.""" boto_config_hmac_auth_only = [ # Overwrite other credential types. ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ('Credentials', 'gs_service_key_file_password', None), # Add hmac credentials. ('Credentials', 'gs_access_key_id', 'dummykey'), ('Credentials', 'gs_secret_access_key', 'dummysecret'), ] def test_hmac_fails_for_xml(self): with SetBotoConfigForTest(self.boto_config_hmac_auth_only): for subcommand in ['create', 'delete', 'get', 'list', 'update']: command = ['hmac', subcommand] stderr = self.RunGsUtil(command, expected_status=1, return_stderr=True) self.assertIn( 'The "hmac" command can only be used with the GCS JSON API', stderr) class TestHmacUnitShim(testcase.ShimUnitTestBase): @mock.patch.object(hmac.HmacCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_hmac_create_command(self): fake_cloudsdk_dir = 'fake_dir' service_account = ( 'test.service.account@test_project.iam.gserviceaccount.com') project = 'test_project' with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': fake_cloudsdk_dir, }): mock_log_handler = self.RunCommand( 'hmac', args=['create', '-p', project, service_account], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage' ' hmac create {} --project {} {}').format( shim_util._get_gcloud_binary_path('fake_dir'), hmac._CREATE_COMMAND_FORMAT, project, service_account), info_lines) @mock.patch.object(hmac.HmacCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_delete_command(self): fake_cloudsdk_dir = 'fake_dir' project = 'test-project' access_id = 'fake123456789' with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': fake_cloudsdk_dir, }): mock_log_handler = self.RunCommand( 'hmac', args=['delete', '-p', project, access_id], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage' ' hmac delete --project {} {}').format( shim_util._get_gcloud_binary_path('fake_dir'), project, access_id), info_lines) @mock.patch.object(hmac.HmacCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_get_commannd(self): fake_cloudsdk_dir = 'fake_dir' project = 'test-project' access_id = 'fake123456789' with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': fake_cloudsdk_dir, }): mock_log_handler = self.RunCommand( 'hmac', args=['get', '-p', project, access_id], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage' ' hmac describe {} --project {} {}').format( shim_util._get_gcloud_binary_path('fake_dir'), hmac._DESCRIBE_COMMAND_FORMAT, project, access_id), info_lines) @mock.patch.object(hmac.HmacCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_hmac_list_command_using_short_format(self): fake_cloudsdk_dir = 'fake_dir' project = 'test-project' service_account = ( 'test.service.account@test_project.iam.gserviceaccount.com') with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': fake_cloudsdk_dir, }): mock_log_handler = self.RunCommand( 'hmac', args=['list', '-a', '-u', service_account, '-p', project], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage' ' hmac list {} --all --service-account {} --project {}').format( shim_util._get_gcloud_binary_path('fake_dir'), hmac._LIST_COMMAND_SHORT_FORMAT, service_account, project), info_lines) @mock.patch.object(hmac.HmacCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_hmac_list_command_using_long_format(self): fake_cloudsdk_dir = 'fake_dir' project = 'test-project' service_account = ( 'test.service.account@test_project.iam.gserviceaccount.com') with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': fake_cloudsdk_dir, }): mock_log_handler = self.RunCommand( 'hmac', args=['list', '-a', '-u', service_account, '-l', '-p', project], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( ('Gcloud Storage Command: {} storage' ' hmac list {} --all --service-account {} --long --project {}' ).format(shim_util._get_gcloud_binary_path('fake_dir'), hmac._DESCRIBE_COMMAND_FORMAT, service_account, project), info_lines) @mock.patch.object(hmac.HmacCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_hmac_update_command_when_active_state_option_is_passed( self): fake_cloudsdk_dir = 'fake_dir' etag = 'ABCDEFGHIK=' project = 'test-project' access_id = 'fake123456789' with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': fake_cloudsdk_dir, }): mock_log_handler = self.RunCommand('hmac', args=[ 'update', '-e', etag, '-p', project, '-s', 'ACTIVE', access_id ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage' ' hmac update {} --etag {} --project {} --{} {}').format( shim_util._get_gcloud_binary_path('fake_dir'), hmac._DESCRIBE_COMMAND_FORMAT, etag, project, 'activate', access_id), info_lines) @mock.patch.object(hmac.HmacCommand, 'RunCommand', new=mock.Mock()) def test_shim_translates_hmac_update_command_when_inactive_state_option_is_passed( self): fake_cloudsdk_dir = 'fake_dir' etag = 'ABCDEFGHIK=' project = 'test-project' access_id = 'fake123456789' with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': fake_cloudsdk_dir, }): mock_log_handler = self.RunCommand('hmac', args=[ 'update', '-e', etag, '-p', project, '-s', 'INACTIVE', access_id ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn(('Gcloud Storage Command: {} storage' ' hmac update {} --etag {} --project {} --{} {}').format( shim_util._get_gcloud_binary_path('fake_dir'), hmac._DESCRIBE_COMMAND_FORMAT, etag, project, 'deactivate', access_id), info_lines) tests/test_boto_util.py 0000644 00000023231 15025013616 0011317 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2020 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for boto_util.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import boto.auth from gslib import cloud_api from gslib.utils import boto_util from gslib import context_config from gslib.tests import testcase from gslib.tests.testcase import base from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import unittest from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock class TestBotoUtil(testcase.GsUtilUnitTestCase): """Test utils that make use of the Boto dependency.""" @mock.patch.object(context_config, 'get_context_config') def testSetsHostBaseToMtlsIfClientCertificateEnabled(self, mock_get_context_config): mock_context_config = mock.Mock() mock_context_config.use_client_certificate = True mock_context_config.client_cert_path = 'path' mock_context_config.client_cert_password = 'password' mock_get_context_config.return_value = mock_context_config mock_http_class = mock.Mock(return_value=mock.Mock()) mock_http = boto_util.GetNewHttp(mock_http_class) mock_http.add_certificate.assert_called_once_with( key='path', cert='path', domain='', password='password', ) @mock.patch.object(boto.auth, 'get_auth_handler', return_value=None) def testHasConfiguredCredentialsNoCreds(self, _): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', None), ('Credentials', 'gs_secret_access_key', None), ('Credentials', 'aws_access_key_id', None), ('Credentials', 'aws_secret_access_key', None), ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_external_account_file', None), ('Credentials', 'gs_external_account_authorized_user_file', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertFalse(boto_util.HasConfiguredCredentials()) @mock.patch.object(boto.auth, 'get_auth_handler', return_value=None) def testHasConfiguredCredentialsGoogCreds(self, _): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', "?????"), ('Credentials', 'gs_secret_access_key', "?????"), ('Credentials', 'aws_access_key_id', None), ('Credentials', 'aws_secret_access_key', None), ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_external_account_file', None), ('Credentials', 'gs_external_account_authorized_user_file', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertTrue(boto_util.HasConfiguredCredentials()) @mock.patch.object(boto.auth, 'get_auth_handler', return_value=None) def testHasConfiguredCredentialsAmznCreds(self, _): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', None), ('Credentials', 'gs_secret_access_key', None), ('Credentials', 'aws_access_key_id', "?????"), ('Credentials', 'aws_secret_access_key', "?????"), ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_external_account_file', None), ('Credentials', 'gs_external_account_authorized_user_file', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertTrue(boto_util.HasConfiguredCredentials()) @mock.patch.object(boto.auth, 'get_auth_handler', return_value=None) def testHasConfiguredCredentialsOauthCreds(self, _): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', None), ('Credentials', 'gs_secret_access_key', None), ('Credentials', 'aws_access_key_id', None), ('Credentials', 'aws_secret_access_key', None), ('Credentials', 'gs_oauth2_refresh_token', "?????"), ('Credentials', 'gs_external_account_file', None), ('Credentials', 'gs_external_account_authorized_user_file', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertTrue(boto_util.HasConfiguredCredentials()) @mock.patch.object(boto.auth, 'get_auth_handler', return_value=None) def testHasConfiguredCredentialsExternalCreds(self, _): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', None), ('Credentials', 'gs_secret_access_key', None), ('Credentials', 'aws_access_key_id', None), ('Credentials', 'aws_secret_access_key', None), ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_external_account_file', "?????"), ('Credentials', 'gs_external_account_authorized_user_file', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertTrue(boto_util.HasConfiguredCredentials()) @mock.patch.object(boto.auth, 'get_auth_handler', return_value=None) def testHasConfiguredCredentialsExternalAuthorizedUserCreds(self, _): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', None), ('Credentials', 'gs_secret_access_key', None), ('Credentials', 'aws_access_key_id', None), ('Credentials', 'aws_secret_access_key', None), ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_external_account_file', None), ('Credentials', 'gs_external_account_authorized_user_file', "?????"), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertTrue(boto_util.HasConfiguredCredentials()) def testUsingGsHmacWithHmacAndServiceAccountCreds(self): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', "?????"), ('Credentials', 'gs_secret_access_key', "?????"), ('Credentials', 'gs_external_account_file', None), ('Credentials', 'gs_external_account_authorized_user_file', None), ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', "?????"), ('Credentials', 'gs_service_key_file', "?????"), ]): self.assertFalse(boto_util.UsingGsHmac()) def testUsingGsHmacWithHmacAndOauth2RefreshToken(self): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', "?????"), ('Credentials', 'gs_secret_access_key', "?????"), ('Credentials', 'gs_external_account_file', None), ('Credentials', 'gs_external_account_authorized_user_file', None), ('Credentials', 'gs_oauth2_refresh_token', "?????"), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertFalse(boto_util.UsingGsHmac()) def testUsingGsHmacWithIncompleteHmacOnly(self): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', "?????"), ('Credentials', 'gs_secret_access_key', None), ('Credentials', 'gs_external_account_file', None), ('Credentials', 'gs_external_account_authorized_user_file', None), ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertFalse(boto_util.UsingGsHmac()) def testUsingGsHmacWithHmacAndExternalAccountFile(self): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', "?????"), ('Credentials', 'gs_secret_access_key', "?????"), ('Credentials', 'gs_external_account_file', "?????"), ('Credentials', 'gs_external_account_authorized_user_file', None), ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertTrue(boto_util.UsingGsHmac()) def testUsingGsHmacWithHmacAndExternalAccountAuthorizedUserFile(self): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', "?????"), ('Credentials', 'gs_secret_access_key', "?????"), ('Credentials', 'gs_external_account_file', None), ('Credentials', 'gs_external_account_authorized_user_file', "?????"), ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertTrue(boto_util.UsingGsHmac()) def testUsingGsHmacWithHmacOnly(self): with SetBotoConfigForTest([ ('Credentials', 'gs_access_key_id', "?????"), ('Credentials', 'gs_secret_access_key', "?????"), ('Credentials', 'gs_external_account_file', None), ('Credentials', 'gs_external_account_authorized_user_file', None), ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ]): self.assertTrue(boto_util.UsingGsHmac()) tests/test_kms.py 0000644 00000041265 15025013616 0010120 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for kms command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from random import randint from unittest import mock from gslib.cloud_api import AccessDeniedException from gslib.project_id import PopulateProjectId import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForJSON from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.utils.retry_util import Retry from gslib.utils import shim_util _DUMMY_KEYNAME = ('projects/my-project/locations/us-central1/' 'keyRings/my-keyring/cryptoKeys/my-key') @SkipForS3('gsutil does not support KMS operations for S3 buckets.') @SkipForXML('gsutil does not support KMS operations for S3 buckets.') class TestKmsSuccessCases(testcase.GsUtilIntegrationTestCase): """Integration tests for the kms command.""" def setUp(self): super(TestKmsSuccessCases, self).setUp() # Make sure our keyRing exists (only needs to be done once, but subsequent # attempts will receive a 409 and be treated as a success). Save the fully # qualified name for use with creating keys later. self.keyring_fqn = self.kms_api.CreateKeyRing( PopulateProjectId(None), testcase.KmsTestingResources.KEYRING_NAME, location=testcase.KmsTestingResources.KEYRING_LOCATION) @Retry(AssertionError, tries=3, timeout_secs=1) def DoTestAuthorize(self, specified_project=None): # Randomly pick 1 of 1000 key names. key_name = testcase.KmsTestingResources.MUTABLE_KEY_NAME_TEMPLATE % ( randint(0, 9), randint(0, 9), randint(0, 9)) # Make sure the key with that name has been created. key_fqn = self.kms_api.CreateCryptoKey(self.keyring_fqn, key_name) # They key may have already been created and used in a previous test # invocation; make sure it doesn't contain the IAM policy binding that # allows our project to encrypt/decrypt with it. key_policy = self.kms_api.GetKeyIamPolicy(key_fqn) while key_policy.bindings: key_policy.bindings.pop() self.kms_api.SetKeyIamPolicy(key_fqn, key_policy) # Set up the authorize command tokens. authorize_cmd = ['kms', 'authorize', '-k', key_fqn] if specified_project: authorize_cmd.extend(['-p', specified_project]) stdout1 = self.RunGsUtil(authorize_cmd, return_stdout=True) stdout2 = self.RunGsUtil(authorize_cmd, return_stdout=True) self.assertIn( 'Authorized project %s to encrypt and decrypt with key:\n%s' % (PopulateProjectId(None), key_fqn), stdout1) self.assertIn( ('Project %s was already authorized to encrypt and decrypt with ' 'key:\n%s.' % (PopulateProjectId(None), key_fqn)), stdout2) def DoTestServiceaccount(self, specified_project=None): serviceaccount_cmd = ['kms', 'serviceaccount'] if specified_project: serviceaccount_cmd.extend(['-p', specified_project]) stdout = self.RunGsUtil(serviceaccount_cmd, return_stdout=True) self.assertRegex(stdout, r'[^@]+@gs-project-accounts\.iam\.gserviceaccount\.com') def testKmsAuthorizeWithoutProjectOption(self): self.DoTestAuthorize() def testKmsAuthorizeWithProjectOption(self): self.DoTestAuthorize(specified_project=PopulateProjectId(None)) def testKmsServiceaccountWithoutProjectOption(self): self.DoTestServiceaccount() def testKmsServiceaccountWithProjectOption(self): self.DoTestServiceaccount(specified_project=PopulateProjectId(None)) def testKmsEncryptionFlow(self): # Since we have to create a bucket and set a default KMS key to test most # of these behaviors, we just test them all in one flow to reduce the number # of API calls. bucket_uri = self.CreateBucket() # Make sure our key exists. key_fqn = self.kms_api.CreateCryptoKey( self.keyring_fqn, testcase.KmsTestingResources.CONSTANT_KEY_NAME) encryption_get_cmd = ['kms', 'encryption', suri(bucket_uri)] # Test output for bucket with no default KMS key set. stdout = self.RunGsUtil(encryption_get_cmd, return_stdout=True) self.assertIn('Bucket %s has no default encryption key' % suri(bucket_uri), stdout) # Test that setting a bucket's default KMS key works and shows up correctly # via a follow-up call to display it. stdout = self.RunGsUtil( ['kms', 'encryption', '-k', key_fqn, suri(bucket_uri)], return_stdout=True) self.assertIn('Setting default KMS key for bucket %s...' % suri(bucket_uri), stdout) stdout = self.RunGsUtil(encryption_get_cmd, return_stdout=True) self.assertIn( 'Default encryption key for %s:\n%s' % (suri(bucket_uri), key_fqn), stdout) # Finally, remove the bucket's default KMS key and make sure a follow-up # call to display it shows that no default key is set. stdout = self.RunGsUtil( ['kms', 'encryption', '-d', suri(bucket_uri)], return_stdout=True) self.assertIn( 'Clearing default encryption key for %s...' % suri(bucket_uri), stdout) stdout = self.RunGsUtil(encryption_get_cmd, return_stdout=True) self.assertIn('Bucket %s has no default encryption key' % suri(bucket_uri), stdout) @SkipForS3('gsutil does not support KMS operations for S3 buckets.') @SkipForJSON('These tests only check for failures when the XML API is forced.') class TestKmsSubcommandsFailWhenXmlForced(testcase.GsUtilIntegrationTestCase): """Tests that kms subcommands fail early when forced to use the XML API.""" boto_config_hmac_auth_only = [ # Overwrite other credential types. ('Credentials', 'gs_oauth2_refresh_token', None), ('Credentials', 'gs_service_client_id', None), ('Credentials', 'gs_service_key_file', None), ('Credentials', 'gs_service_key_file_password', None), # Add hmac credentials. ('Credentials', 'gs_access_key_id', 'dummykey'), ('Credentials', 'gs_secret_access_key', 'dummysecret'), ] _DUMMY_KEYNAME = ('projects/my-project/locations/us-central1/' 'keyRings/my-keyring/cryptoKeys/my-key') def DoTestSubcommandFailsWhenXmlForcedFromHmacInBotoConfig(self, subcommand): with SetBotoConfigForTest(self.boto_config_hmac_auth_only): stderr = self.RunGsUtil(subcommand, expected_status=1, return_stderr=True) self.assertIn('The "kms" command can only be used with', stderr) def testEncryptionFailsWhenXmlForcedFromHmacInBotoConfig(self): self.DoTestSubcommandFailsWhenXmlForcedFromHmacInBotoConfig( ['kms', 'encryption', 'gs://dummybucket']) def testEncryptionDashKFailsWhenXmlForcedFromHmacInBotoConfig(self): self.DoTestSubcommandFailsWhenXmlForcedFromHmacInBotoConfig( ['kms', 'encryption', '-k', _DUMMY_KEYNAME, 'gs://dummybucket']) def testEncryptionDashDFailsWhenXmlForcedFromHmacInBotoConfig(self): self.DoTestSubcommandFailsWhenXmlForcedFromHmacInBotoConfig( ['kms', 'encryption', '-d', 'gs://dummybucket']) def testServiceaccountFailsWhenXmlForcedFromHmacInBotoConfig(self): self.DoTestSubcommandFailsWhenXmlForcedFromHmacInBotoConfig( ['kms', 'serviceaccount', 'gs://dummybucket']) def testAuthorizeFailsWhenXmlForcedFromHmacInBotoConfig(self): self.DoTestSubcommandFailsWhenXmlForcedFromHmacInBotoConfig( ['kms', 'authorize', '-k', _DUMMY_KEYNAME, 'gs://dummybucket']) class TestKmsUnitTests(testcase.GsUtilUnitTestCase): """Unit tests for gsutil kms.""" @mock.patch( 'gslib.cloud_api_delegator.CloudApiDelegator.GetProjectServiceAccount') @mock.patch('gslib.cloud_api_delegator.CloudApiDelegator.PatchBucket') @mock.patch('gslib.kms_api.KmsApi.GetKeyIamPolicy') @mock.patch('gslib.kms_api.KmsApi.SetKeyIamPolicy') def testEncryptionSetKeySucceedsWhenUpdateKeyPolicySucceeds( self, mock_set_key_iam_policy, mock_get_key_iam_policy, mock_patch_bucket, mock_get_project_service_account): bucket_uri = self.CreateBucket() mock_get_key_iam_policy.return_value.bindings = [] mock_get_project_service_account.return_value.email_address = 'dummy@google.com' stdout = self.RunCommand( 'kms', ['encryption', '-k', _DUMMY_KEYNAME, suri(bucket_uri)], return_stdout=True) self.assertIn('Setting default KMS key for bucket', stdout) self.assertTrue(mock_patch_bucket.called) @mock.patch( 'gslib.cloud_api_delegator.CloudApiDelegator.GetProjectServiceAccount') @mock.patch('gslib.cloud_api_delegator.CloudApiDelegator.PatchBucket') @mock.patch('gslib.kms_api.KmsApi.GetKeyIamPolicy') @mock.patch('gslib.kms_api.KmsApi.SetKeyIamPolicy') def testEncryptionSetKeySucceedsWhenUpdateKeyPolicyFailsWithWarningFlag( self, mock_set_key_iam_policy, mock_get_key_iam_policy, mock_patch_bucket, mock_get_project_service_account): bucket_uri = self.CreateBucket() mock_get_key_iam_policy.side_effect = AccessDeniedException( 'Permission denied') mock_get_project_service_account.return_value.email_address = 'dummy@google.com' stdout = self.RunCommand( 'kms', ['encryption', '-k', _DUMMY_KEYNAME, '-w', suri(bucket_uri)], return_stdout=True) self.assertIn('Setting default KMS key for bucket', stdout) self.assertTrue(mock_patch_bucket.called) @mock.patch( 'gslib.cloud_api_delegator.CloudApiDelegator.GetProjectServiceAccount') @mock.patch('gslib.cloud_api_delegator.CloudApiDelegator.PatchBucket') @mock.patch('gslib.kms_api.KmsApi.GetKeyIamPolicy') @mock.patch('gslib.kms_api.KmsApi.SetKeyIamPolicy') def testEncryptionSetKeyFailsWhenUpdateKeyPolicyFailsWithoutWarningFlag( self, mock_set_key_iam_policy, mock_get_key_iam_policy, mock_patch_bucket, mock_get_project_service_account): bucket_uri = self.CreateBucket() mock_get_key_iam_policy.side_effect = AccessDeniedException( 'Permission denied') mock_get_project_service_account.return_value.email_address = 'dummy@google.com' try: stdout = self.RunCommand( 'kms', ['encryption', '-k', _DUMMY_KEYNAME, suri(bucket_uri)], return_stdout=True) self.fail('Did not get expected AccessDeniedException') except AccessDeniedException as e: self.assertIn('Permission denied', e.reason) class TestKmsUnitTestsWithShim(testcase.ShimUnitTestBase): """Unit tests for gsutil kms using shim.""" @mock.patch( 'gslib.cloud_api_delegator.CloudApiDelegator.GetProjectServiceAccount') @mock.patch('gslib.kms_api.KmsApi.GetKeyIamPolicy') @mock.patch('gslib.kms_api.KmsApi.SetKeyIamPolicy') def test_shim_translates_authorize_flags(self, mock_get_key_iam_policy, mock_set_key_iam_policy, mock_get_project_service_account): del mock_set_key_iam_policy mock_get_project_service_account.return_value.email_address = 'dummy@google.com' mock_get_key_iam_policy.return_value.bindings = [] with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('kms', [ 'authorize', '-p', 'foo', '-k', _DUMMY_KEYNAME, ], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage service-agent' ' --project foo --authorize-cmek {}'.format( shim_util._get_gcloud_binary_path('fake_dir'), _DUMMY_KEYNAME), info_lines) def test_shim_translates_clear_encryption_key(self): bucket_uri = self.CreateBucket() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'kms', ['encryption', '-d', suri(bucket_uri)], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage buckets update' ' --clear-default-encryption-key {}'.format( shim_util._get_gcloud_binary_path('fake_dir'), suri(bucket_uri)), info_lines) @mock.patch( 'gslib.cloud_api_delegator.CloudApiDelegator.GetProjectServiceAccount') @mock.patch('gslib.kms_api.KmsApi.GetKeyIamPolicy') @mock.patch('gslib.kms_api.KmsApi.SetKeyIamPolicy') def test_shim_translates_update_encryption_key( self, mock_get_key_iam_policy, mock_set_key_iam_policy, mock_get_project_service_account): bucket_uri = self.CreateBucket() del mock_set_key_iam_policy mock_get_project_service_account.return_value.email_address = 'dummy@google.com' mock_get_key_iam_policy.return_value.bindings = [] with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'kms', ['encryption', '-w', '-k', _DUMMY_KEYNAME, suri(bucket_uri)], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage buckets update' ' --default-encryption-key {} {}'.format( shim_util._get_gcloud_binary_path('fake_dir'), _DUMMY_KEYNAME, suri(bucket_uri)), info_lines) def test_shim_translates_displays_encryption_key(self): bucket_uri = self.CreateBucket() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'kms', ['encryption', suri(bucket_uri)], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage buckets describe ' '--format=value[separator=\": \"](name, encryption' '.defaultKmsKeyName.yesno(no="No default encryption key."))' ' --raw {}'.format(shim_util._get_gcloud_binary_path('fake_dir'), suri(bucket_uri)), info_lines) @mock.patch( 'gslib.cloud_api_delegator.CloudApiDelegator.GetProjectServiceAccount') def test_shim_translates_serviceaccount_command( self, mock_get_project_service_account): mock_get_project_service_account.return_value.email_address = 'dummy@google.com' with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand('kms', ['serviceaccount', '-p', 'foo'], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage service-agent' ' --project foo'.format( shim_util._get_gcloud_binary_path('fake_dir')), info_lines) tests/test_bucketpolicyonly.py 0000644 00000007436 15025013616 0012727 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for bucketpolicyonly command.""" from __future__ import absolute_import import re from gslib.cs_api_map import ApiSelector import gslib.tests.testcase as testcase from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import unittest from gslib.utils.retry_util import Retry class TestBucketPolicyOnly(testcase.GsUtilIntegrationTestCase): """Integration tests for bucketpolicyonly command.""" _set_bpo_cmd = ['bucketpolicyonly', 'set'] _get_bpo_cmd = ['bucketpolicyonly', 'get'] def _AssertEnabled(self, bucket_uri, value): stdout = self.RunGsUtil(self._get_bpo_cmd + [suri(bucket_uri)], return_stdout=True) bucket_policy_only_re = re.compile(r'^\s*Enabled:\s+(?P<enabled_val>.+)$', re.MULTILINE) bucket_policy_only_match = re.search(bucket_policy_only_re, stdout) bucket_policy_only_val = bucket_policy_only_match.group('enabled_val') self.assertEqual(str(value), bucket_policy_only_val) def test_off_on_default_buckets(self): if self.test_api == ApiSelector.XML: return unittest.skip('XML API has no concept of Bucket Policy Only') bucket_uri = self.CreateBucket() self._AssertEnabled(bucket_uri, False) def test_turning_off_on_enabled_buckets(self): if self.test_api == ApiSelector.XML: return unittest.skip('XML API has no concept of Bucket Policy Only') bucket_uri = self.CreateBucket(bucket_policy_only=True, prefer_json_api=True) self._AssertEnabled(bucket_uri, True) self.RunGsUtil(self._set_bpo_cmd + ['off', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, False) def test_turning_on(self): if self.test_api == ApiSelector.XML: return unittest.skip('XML API has no concept of Bucket Policy Only') bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_bpo_cmd + ['on', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, True) def test_turning_on_and_off(self): if self.test_api == ApiSelector.XML: return unittest.skip('XML API has no concept of Bucket Policy Only') bucket_uri = self.CreateBucket() self.RunGsUtil(self._set_bpo_cmd + ['on', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, True) self.RunGsUtil(self._set_bpo_cmd + ['off', suri(bucket_uri)]) self._AssertEnabled(bucket_uri, False) def testTooFewArgumentsFails(self): """Ensures bucketpolicyonly commands fail with too few arguments.""" # No arguments for set, but valid subcommand. stderr = self.RunGsUtil(self._set_bpo_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # No arguments for get, but valid subcommand. stderr = self.RunGsUtil(self._get_bpo_cmd, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Neither arguments nor subcommand. stderr = self.RunGsUtil(['bucketpolicyonly'], return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) tests/__init__.py 0000644 00000001450 15025013616 0010016 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains test cases for gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals tests/test_parallelism_framework.py 0000644 00000102136 15025013616 0013703 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Unit tests for gsutil parallelism framework.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import functools import os import signal import six import threading import textwrap import time from unittest import mock import boto from boto.storage_uri import BucketStorageUri from boto.storage_uri import StorageUri from gslib import cs_api_map from gslib import command from gslib.command import Command from gslib.command import CreateOrGetGsutilLogger from gslib.command import DummyArgChecker from gslib.tests.mock_cloud_api import MockCloudApi from gslib.tests.mock_logging_handler import MockLoggingHandler import gslib.tests.testcase as testcase from gslib.tests.testcase.base import RequiresIsolation from gslib.tests.util import unittest from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit from gslib.utils.parallelism_framework_util import multiprocessing_context from gslib.utils.system_util import IS_OSX from gslib.utils.system_util import IS_WINDOWS # Amount of time for an individual test to run before timing out. We need a # reasonably high value since if many tests are running in parallel, an # individual test may take a while to complete. _TEST_TIMEOUT_SECONDS = 120 PARALLEL_PROCESSING_MESSAGE = ('\n' + textwrap.fill( '==> NOTE: You are performing a sequence of gsutil operations that ' 'may run significantly faster if you instead use gsutil -m fake ...\n' 'Please see the -m section under "gsutil help options" for further ' 'information about when gsutil -m can be advantageous.') + '\n') def Timeout(func): """Decorator used to provide a timeout for functions.""" @functools.wraps(func) def Wrapper(*args, **kwargs): if not IS_WINDOWS: signal.signal(signal.SIGALRM, _HandleAlarm) signal.alarm(_TEST_TIMEOUT_SECONDS) try: func(*args, **kwargs) finally: if not IS_WINDOWS: signal.alarm(0) # Cancel the alarm. return Wrapper # pylint: disable=unused-argument def _HandleAlarm(signal_num, cur_stack_frame): raise Exception('Test timed out.') class CustomException(Exception): def __init__(self, exception_str): super(CustomException, self).__init__(exception_str) def _ReturnOneValue(cls, args, thread_state=None): return 1 def _ReturnProcAndThreadId(cls, args, thread_state=None): return os.getpid(), threading.current_thread().ident def _SleepThenReturnProcAndThreadId(cls, args, thread_state=None): # This can fail if the total time to spawn new processes and threads takes # longer than 5 seconds, but if that occurs, then we have a performance # problem that needs to be addressed. time.sleep(5) return _ReturnProcAndThreadId(cls, args, thread_state=thread_state) def _FailureFunc(cls, args, thread_state=None): raise CustomException('Failing on purpose.') def _FailingExceptionHandler(cls, e): cls.failure_count += 1 raise CustomException('Exception handler failing on purpose.') def _ExceptionHandler(cls, e): cls.logger.exception(e) cls.failure_count += 1 def _IncrementByLength(cls, args, thread_state=None): cls.arg_length_sum += len(args) def _AdjustProcessCountIfWindows(process_count): if IS_WINDOWS: return 1 else: return process_count def _ReApplyWithReplicatedArguments(cls, args, thread_state=None): """Calls Apply with arguments repeated seven times. The first two elements of args should be the process and thread counts, respectively, to be used for the recursive calls. Args: cls: The Command class to call Apply on. args: Arguments to pass to Apply. thread_state: Unused, required by function signature. Returns: Number of values returned by the two calls to Apply. """ new_args = [args] * 7 process_count = _AdjustProcessCountIfWindows(args[0]) thread_count = args[1] return_values = cls.Apply(_PerformNRecursiveCalls, new_args, _ExceptionHandler, arg_checker=DummyArgChecker, process_count=process_count, thread_count=thread_count, should_return_results=True) ret = sum(return_values) return_values = cls.Apply(_ReturnOneValue, new_args, _ExceptionHandler, arg_checker=DummyArgChecker, process_count=process_count, thread_count=thread_count, should_return_results=True) return len(return_values) + ret def _PerformNRecursiveCalls(cls, args, thread_state=None): """Calls Apply to perform N recursive calls. The first two elements of args should be the process and thread counts, respectively, to be used for the recursive calls, while N is the third element (the number of recursive calls to make). Args: cls: The Command class to call Apply on. args: Arguments to pass to Apply. thread_state: Unused, required by function signature. Returns: Number of values returned by the call to Apply. """ process_count = _AdjustProcessCountIfWindows(args[0]) thread_count = args[1] return_values = cls.Apply(_ReturnOneValue, [()] * args[2], _ExceptionHandler, arg_checker=DummyArgChecker, process_count=process_count, thread_count=thread_count, should_return_results=True) return len(return_values) def _SkipEvenNumbersArgChecker(cls, arg): return arg % 2 != 0 class FailingIterator(six.Iterator): def __init__(self, size, failure_indices): self.size = size self.failure_indices = failure_indices self.current_index = 0 def __iter__(self): return self def __next__(self): if self.current_index == self.size: raise StopIteration('') elif self.current_index in self.failure_indices: self.current_index += 1 raise CustomException('Iterator failing on purpose at index %d.' % self.current_index) else: self.current_index += 1 return self.current_index - 1 class FakeCommand(Command): """Fake command class for overriding command instance state.""" command_spec = Command.CreateCommandSpec( 'fake', command_name_aliases=[], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='fake', help_name_aliases=[], help_type='command_help', help_one_line_summary='Something to take up space.', help_text='Something else to take up space.', subcommand_help_text={}, ) def __init__(self, do_parallel): self.bucket_storage_uri_class = BucketStorageUri support_map = {'gs': ['JSON'], 's3': ['XML']} default_map = {'gs': 'JSON', 's3': 'XML'} self.gsutil_api_map = cs_api_map.GsutilApiMapFactory.GetApiMap( cs_api_map.GsutilApiClassMapFactory, support_map, default_map) self.logger = CreateOrGetGsutilLogger('FakeCommand') self.parallel_operations = do_parallel self.failure_count = 0 self.gsutil_api = MockCloudApi() self.multiprocessing_is_available = ( CheckMultiprocessingAvailableAndInit().is_available) self.debug = 0 self.non_metadata_headers = {} self.perf_trace_token = None self.trace_token = None self.user_project = None class FakeCommandWithoutMultiprocessingModule(FakeCommand): def __init__(self, do_parallel): super(FakeCommandWithoutMultiprocessingModule, self).__init__(do_parallel) self.multiprocessing_is_available = False # TODO: Figure out a good way to test that ctrl+C really stops execution, # and also that ctrl+C works when there are still tasks enqueued. class TestParallelismFramework(testcase.GsUtilUnitTestCase): """gsutil parallelism framework test suite.""" command_class = FakeCommand def _RunApply(self, func, args_iterator, process_count, thread_count, command_inst=None, shared_attrs=None, fail_on_error=False, thr_exc_handler=None, arg_checker=DummyArgChecker): command_inst = command_inst or self.command_class(True) exception_handler = thr_exc_handler or _ExceptionHandler return command_inst.Apply(func, args_iterator, exception_handler, thread_count=thread_count, process_count=process_count, arg_checker=arg_checker, should_return_results=True, shared_attrs=shared_attrs, fail_on_error=fail_on_error) @RequiresIsolation def testBasicApplySingleProcessSingleThread(self): self._TestBasicApply(1, 1) @RequiresIsolation def testBasicApplySingleProcessMultiThread(self): self._TestBasicApply(1, 3) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testBasicApplyMultiProcessSingleThread(self): self._TestBasicApply(3, 1) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testBasicApplyMultiProcessMultiThread(self): self._TestBasicApply(3, 3) @Timeout def _TestBasicApply(self, process_count, thread_count): args = [()] * (17 * process_count * thread_count + 1) results = self._RunApply(_ReturnOneValue, args, process_count, thread_count) self.assertEqual(len(args), len(results)) @RequiresIsolation def testNoTasksSingleProcessSingleThread(self): self._TestApplyWithNoTasks(1, 1) @RequiresIsolation def testNoTasksSingleProcessMultiThread(self): self._TestApplyWithNoTasks(1, 3) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testNoTasksMultiProcessSingleThread(self): self._TestApplyWithNoTasks(3, 1) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testNoTasksMultiProcessMultiThread(self): self._TestApplyWithNoTasks(3, 3) @Timeout def _TestApplyWithNoTasks(self, process_count, thread_count): """Tests that calling Apply with no tasks releases locks/semaphores.""" empty_args = [()] for _ in range(process_count * thread_count + 1): self._RunApply(_ReturnOneValue, empty_args, process_count, thread_count) # Ensure that work can still be performed. self._TestBasicApply(process_count, thread_count) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testApplySaturatesMultiProcessSingleThread(self): self._TestApplySaturatesAvailableProcessesAndThreads(3, 1) @RequiresIsolation def testApplySaturatesSingleProcessMultiThread(self): self._TestApplySaturatesAvailableProcessesAndThreads(1, 3) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testApplySaturatesMultiProcessMultiThread(self): self._TestApplySaturatesAvailableProcessesAndThreads(3, 3) @RequiresIsolation def _TestApplySaturatesAvailableProcessesAndThreads(self, process_count, thread_count): """Tests that created processes and threads evenly share tasks.""" calls_per_thread = 2 args = [()] * (process_count * thread_count * calls_per_thread) expected_calls_per_thread = calls_per_thread if not self.command_class(True).multiprocessing_is_available: # When multiprocessing is unavailable, only a single process is used. # Calls should be evenly distributed across threads. expected_calls_per_thread = calls_per_thread * process_count results = self._RunApply(_SleepThenReturnProcAndThreadId, args, process_count, thread_count) usage_dict = {} # (process_id, thread_id): number of tasks performed for (process_id, thread_id) in results: usage_dict[(process_id, thread_id)] = (usage_dict.get( (process_id, thread_id), 0) + 1) for (id_tuple, num_tasks_completed) in six.iteritems(usage_dict): self.assertEqual( num_tasks_completed, expected_calls_per_thread, 'Process %s thread %s completed %s tasks. Expected: %s' % (id_tuple[0], id_tuple[1], num_tasks_completed, expected_calls_per_thread)) @RequiresIsolation def testIteratorFailureSingleProcessSingleThread(self): self._TestIteratorFailure(1, 1) @RequiresIsolation def testIteratorFailureSingleProcessMultiThread(self): self._TestIteratorFailure(1, 3) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testIteratorFailureMultiProcessSingleThread(self): self._TestIteratorFailure(3, 1) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testIteratorFailureMultiProcessMultiThread(self): self._TestIteratorFailure(3, 3) @Timeout def _TestIteratorFailure(self, process_count, thread_count): """Tests apply with a failing iterator.""" # Tests for fail_on_error == False. args = FailingIterator(10, [0]) results = self._RunApply(_ReturnOneValue, args, process_count, thread_count) self.assertEqual(9, len(results)) args = FailingIterator(10, [5]) results = self._RunApply(_ReturnOneValue, args, process_count, thread_count) self.assertEqual(9, len(results)) args = FailingIterator(10, [9]) results = self._RunApply(_ReturnOneValue, args, process_count, thread_count) self.assertEqual(9, len(results)) if process_count * thread_count > 1: # In this case, we should ignore the fail_on_error flag. args = FailingIterator(10, [9]) results = self._RunApply(_ReturnOneValue, args, process_count, thread_count, fail_on_error=True) self.assertEqual(9, len(results)) args = FailingIterator(10, range(10)) results = self._RunApply(_ReturnOneValue, args, process_count, thread_count) self.assertEqual(0, len(results)) args = FailingIterator(0, []) results = self._RunApply(_ReturnOneValue, args, process_count, thread_count) self.assertEqual(0, len(results)) @RequiresIsolation def testTestSharedAttrsWorkSingleProcessSingleThread(self): self._TestSharedAttrsWork(1, 1) @RequiresIsolation def testTestSharedAttrsWorkSingleProcessMultiThread(self): self._TestSharedAttrsWork(1, 3) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testTestSharedAttrsWorkMultiProcessSingleThread(self): self._TestSharedAttrsWork(3, 1) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testTestSharedAttrsWorkMultiProcessMultiThread(self): self._TestSharedAttrsWork(3, 3) @Timeout def _TestSharedAttrsWork(self, process_count, thread_count): """Tests that Apply successfully uses shared_attrs.""" command_inst = self.command_class(True) command_inst.arg_length_sum = 19 args = ['foo', ['bar', 'baz'], [], ['x', 'y'], [], 'abcd'] self._RunApply(_IncrementByLength, args, process_count, thread_count, command_inst=command_inst, shared_attrs=['arg_length_sum']) expected_sum = 19 for arg in args: expected_sum += len(arg) self.assertEqual(expected_sum, command_inst.arg_length_sum) # Test that shared variables work when the iterator fails at the beginning, # middle, and end. for (failing_iterator, expected_failure_count) in ((FailingIterator(5, [0]), 1), (FailingIterator(10, [1, 3, 5]), 3), (FailingIterator(5, [4]), 1)): command_inst = self.command_class(True) args = failing_iterator self._RunApply(_ReturnOneValue, args, process_count, thread_count, command_inst=command_inst, shared_attrs=['failure_count']) self.assertEqual( expected_failure_count, command_inst.failure_count, msg='Failure count did not match. Expected: %s, actual: %s ' 'for failing iterator of size %s, failing indices %s' % (expected_failure_count, command_inst.failure_count, failing_iterator.size, failing_iterator.failure_indices)) @RequiresIsolation def testThreadsSurviveExceptionsInFuncSingleProcessSingleThread(self): self._TestThreadsSurviveExceptionsInFunc(1, 1) @RequiresIsolation def testThreadsSurviveExceptionsInFuncSingleProcessMultiThread(self): self._TestThreadsSurviveExceptionsInFunc(1, 3) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testThreadsSurviveExceptionsInFuncMultiProcessSingleThread(self): self._TestThreadsSurviveExceptionsInFunc(3, 1) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testThreadsSurviveExceptionsInFuncMultiProcessMultiThread(self): self._TestThreadsSurviveExceptionsInFunc(3, 3) @Timeout def _TestThreadsSurviveExceptionsInFunc(self, process_count, thread_count): command_inst = self.command_class(True) args = ([()] * 5) self._RunApply(_FailureFunc, args, process_count, thread_count, command_inst=command_inst, shared_attrs=['failure_count'], thr_exc_handler=_FailingExceptionHandler) self.assertEqual(len(args), command_inst.failure_count) @RequiresIsolation def testThreadsSurviveExceptionsInHandlerSingleProcessSingleThread(self): self._TestThreadsSurviveExceptionsInHandler(1, 1) @RequiresIsolation def testThreadsSurviveExceptionsInHandlerSingleProcessMultiThread(self): self._TestThreadsSurviveExceptionsInHandler(1, 3) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testThreadsSurviveExceptionsInHandlerMultiProcessSingleThread(self): self._TestThreadsSurviveExceptionsInHandler(3, 1) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testThreadsSurviveExceptionsInHandlerMultiProcessMultiThread(self): self._TestThreadsSurviveExceptionsInHandler(3, 3) @Timeout def _TestThreadsSurviveExceptionsInHandler(self, process_count, thread_count): command_inst = self.command_class(True) args = ([()] * 5) self._RunApply(_FailureFunc, args, process_count, thread_count, command_inst=command_inst, shared_attrs=['failure_count'], thr_exc_handler=_FailingExceptionHandler) self.assertEqual(len(args), command_inst.failure_count) @RequiresIsolation @Timeout def testFailOnErrorFlag(self): """Tests that fail_on_error produces the correct exception on failure.""" def _ExpectCustomException(test_func): try: test_func() self.fail( 'Setting fail_on_error should raise any exception encountered.') except CustomException as e: pass except Exception as e: # pylint: disable=broad-except self.fail('Got unexpected error: ' + str(e)) def _RunFailureFunc(): command_inst = self.command_class(True) args = ([()] * 5) self._RunApply(_FailureFunc, args, 1, 1, command_inst=command_inst, shared_attrs=['failure_count'], fail_on_error=True) _ExpectCustomException(_RunFailureFunc) def _RunFailingIteratorFirstPosition(): args = FailingIterator(10, [0]) results = self._RunApply(_ReturnOneValue, args, 1, 1, fail_on_error=True) self.assertEqual(0, len(results)) _ExpectCustomException(_RunFailingIteratorFirstPosition) def _RunFailingIteratorPositionMiddlePosition(): args = FailingIterator(10, [5]) results = self._RunApply(_ReturnOneValue, args, 1, 1, fail_on_error=True) self.assertEqual(5, len(results)) _ExpectCustomException(_RunFailingIteratorPositionMiddlePosition) def _RunFailingIteratorLastPosition(): args = FailingIterator(10, [9]) results = self._RunApply(_ReturnOneValue, args, 1, 1, fail_on_error=True) self.assertEqual(9, len(results)) _ExpectCustomException(_RunFailingIteratorLastPosition) def _RunFailingIteratorMultiplePositions(): args = FailingIterator(10, [1, 3, 5]) results = self._RunApply(_ReturnOneValue, args, 1, 1, fail_on_error=True) self.assertEqual(1, len(results)) _ExpectCustomException(_RunFailingIteratorMultiplePositions) @RequiresIsolation def testRecursiveDepthThreeDifferentFunctionsSingleProcessSingleThread(self): self._TestRecursiveDepthThreeDifferentFunctions(1, 1) @RequiresIsolation def testRecursiveDepthThreeDifferentFunctionsSingleProcessMultiThread(self): self._TestRecursiveDepthThreeDifferentFunctions(1, 3) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testRecursiveDepthThreeDifferentFunctionsMultiProcessSingleThread(self): self._TestRecursiveDepthThreeDifferentFunctions(3, 1) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testRecursiveDepthThreeDifferentFunctionsMultiProcessMultiThread(self): self._TestRecursiveDepthThreeDifferentFunctions(3, 3) @RequiresIsolation @unittest.skipUnless(IS_OSX, 'This warning should only be printed on MacOS') def testMacOSLogsMultiprocessingWarning(self): logger = CreateOrGetGsutilLogger('FakeCommand') mock_log_handler = MockLoggingHandler() logger.addHandler(mock_log_handler) self._TestRecursiveDepthThreeDifferentFunctions(3, 3) macos_message = 'If you experience problems with multiprocessing on MacOS' contains_message = [ message.startswith(macos_message) for message in mock_log_handler.messages['info'] ] self.assertTrue(any(contains_message)) logger.removeHandler(mock_log_handler) @RequiresIsolation @unittest.skipIf(IS_OSX, 'This warning should be printed on MacOS') @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testNonMacOSDoesNotLogMultiprocessingWarning(self): logger = CreateOrGetGsutilLogger('FakeCommand') mock_log_handler = MockLoggingHandler() logger.addHandler(mock_log_handler) self._TestRecursiveDepthThreeDifferentFunctions(3, 3) macos_message = 'If you experience problems with multiprocessing on MacOS' contains_message = [ message.startswith(macos_message) for message in mock_log_handler.messages['info'] ] self.assertFalse(any(contains_message)) logger.removeHandler(mock_log_handler) @RequiresIsolation def testMultithreadingDoesNotLogMacOSWarning(self): logger = CreateOrGetGsutilLogger('FakeCommand') mock_log_handler = MockLoggingHandler() logger.addHandler(mock_log_handler) self._TestRecursiveDepthThreeDifferentFunctions(1, 3) macos_message = 'If you experience problems with multiprocessing on MacOS' contains_message = [ message.startswith(macos_message) for message in mock_log_handler.messages['info'] ] self.assertFalse(any(contains_message)) logger.removeHandler(mock_log_handler) @Timeout def _TestRecursiveDepthThreeDifferentFunctions(self, process_count, thread_count): """Tests recursive application of Apply. Calls Apply(A), where A calls Apply(B) followed by Apply(C) and B calls Apply(C). Args: process_count: Number of processes to use. thread_count: Number of threads to use. """ base_args = [3, 1, 4, 1, 5] args = [[process_count, thread_count, count] for count in base_args] results = self._RunApply(_ReApplyWithReplicatedArguments, args, process_count, thread_count) self.assertEqual(7 * (sum(base_args) + len(base_args)), sum(results)) @RequiresIsolation def testExceptionInProducerRaisesAndTerminatesSingleProcessSingleThread(self): self._TestExceptionInProducerRaisesAndTerminates(1, 1) @RequiresIsolation def testExceptionInProducerRaisesAndTerminatesSingleProcessMultiThread(self): self._TestExceptionInProducerRaisesAndTerminates(1, 3) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testExceptionInProducerRaisesAndTerminatesMultiProcessSingleThread(self): self._TestExceptionInProducerRaisesAndTerminates(3, 1) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testExceptionInProducerRaisesAndTerminatesMultiProcessMultiThread(self): self._TestExceptionInProducerRaisesAndTerminates(3, 3) @Timeout def _TestExceptionInProducerRaisesAndTerminates(self, process_count, thread_count): args = self # The ProducerThread will try and fail to iterate over this. try: self._RunApply(_ReturnOneValue, args, process_count, thread_count) self.fail('Did not raise expected exception.') except TypeError: pass @RequiresIsolation def testSkippedArgumentsSingleThreadSingleProcess(self): self._TestSkippedArguments(1, 1) @RequiresIsolation def testSkippedArgumentsMultiThreadSingleProcess(self): self._TestSkippedArguments(1, 3) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testSkippedArgumentsSingleThreadMultiProcess(self): self._TestSkippedArguments(3, 1) @RequiresIsolation @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testSkippedArgumentsMultiThreadMultiProcess(self): self._TestSkippedArguments(3, 3) @Timeout def _TestSkippedArguments(self, process_count, thread_count): # Skip a proper subset of the arguments. n = 2 * process_count * thread_count args = range(1, n + 1) results = self._RunApply(_ReturnOneValue, args, process_count, thread_count, arg_checker=_SkipEvenNumbersArgChecker) self.assertEqual(n / 2, len(results)) # We know n is even. self.assertEqual(n / 2, sum(results)) # Skip all arguments. args = [2 * x for x in args] results = self._RunApply(_ReturnOneValue, args, process_count, thread_count, arg_checker=_SkipEvenNumbersArgChecker) self.assertEqual(0, len(results)) @mock.patch.object(command, 'OFFER_GSUTIL_M_SUGGESTION_THRESHOLD', 2) @mock.patch.object(command, 'GetTermLines', return_value=100) def testSequentialApplyRecommendsParallelismAfterThreshold( self, mock_get_term_lines): mock_get_term_lines.return_value = 100 logger = CreateOrGetGsutilLogger('FakeCommand') mock_log_handler = MockLoggingHandler() logger.addHandler(mock_log_handler) self._RunApply(_ReturnOneValue, range(2), process_count=1, thread_count=1) contains_message = [ message == PARALLEL_PROCESSING_MESSAGE for message in mock_log_handler.messages['info'] ] self.assertTrue(any(contains_message)) logger.removeHandler(mock_log_handler) @mock.patch.object(command, 'OFFER_GSUTIL_M_SUGGESTION_THRESHOLD', 100) @mock.patch.object(command, 'OFFER_GSUTIL_M_SUGGESTION_FREQUENCY', 10) @mock.patch.object(command, 'GetTermLines', return_value=100) def testSequentialApplyRecommendsParallelismAtSuggestionFrequency( self, mock_get_term_lines): logger = CreateOrGetGsutilLogger('FakeCommand') mock_log_handler = MockLoggingHandler() logger.addHandler(mock_log_handler) self._RunApply(_ReturnOneValue, range(30), process_count=1, thread_count=1) contains_message = [ message == PARALLEL_PROCESSING_MESSAGE for message in mock_log_handler.messages['info'] ] self.assertEqual(sum(contains_message), 3) logger.removeHandler(mock_log_handler) @mock.patch.object(command, 'OFFER_GSUTIL_M_SUGGESTION_THRESHOLD', 100) @mock.patch.object(command, 'OFFER_GSUTIL_M_SUGGESTION_FREQUENCY', 10) @mock.patch.object(command, 'GetTermLines', return_value=2) def testSequentialApplyRecommendsParallelismAtEndIfLastSuggestionIsOutOfView( self, mock_get_term_lines): logger = CreateOrGetGsutilLogger('FakeCommand') mock_log_handler = MockLoggingHandler() logger.addHandler(mock_log_handler) self._RunApply(_ReturnOneValue, range(22), process_count=1, thread_count=1) contains_message = [ message == PARALLEL_PROCESSING_MESSAGE for message in mock_log_handler.messages['info'] ] self.assertEqual(sum(contains_message), 3) logger.removeHandler(mock_log_handler) @mock.patch.object(command, 'OFFER_GSUTIL_M_SUGGESTION_THRESHOLD', 100) @mock.patch.object(command, 'OFFER_GSUTIL_M_SUGGESTION_FREQUENCY', 10) @mock.patch.object(command, 'GetTermLines', return_value=3) def testSequentialApplyDoesNotRecommendParallelismAtEndIfLastSuggestionInView( self, mock_get_term_lines): logger = CreateOrGetGsutilLogger('FakeCommand') mock_log_handler = MockLoggingHandler() logger.addHandler(mock_log_handler) self._RunApply(_ReturnOneValue, range(22), process_count=1, thread_count=1) contains_message = [ message == PARALLEL_PROCESSING_MESSAGE for message in mock_log_handler.messages['info'] ] self.assertEqual(sum(contains_message), 2) logger.removeHandler(mock_log_handler) def testResetConnectionPoolDeletesConnectionState(self): StorageUri.connection = mock.Mock(spec=boto.s3.connection.S3Connection) StorageUri.provider_pool = { 's3': mock.Mock(spec=boto.s3.connection.S3Connection) } self.command_class(True)._ResetConnectionPool() self.assertIsNone(StorageUri.connection) self.assertFalse(StorageUri.provider_pool) # _ResetConnectionPool is only called in child processes, so we need a queue # to track calls. call_queue = multiprocessing_context.Queue() class TestParallelismFrameworkWithMultiprocessing(testcase.GsUtilUnitTestCase): """Tests that only run with multiprocessing enabled.""" @RequiresIsolation @mock.patch.object(FakeCommand, '_ResetConnectionPool', side_effect=functools.partial(call_queue.put, None)) @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def testResetConnectionPoolCalledOncePerProcess(self, mock_reset_connection_pool): expected_call_count = 2 FakeCommand(True).Apply(_ReturnOneValue, [1, 2, 3], _ExceptionHandler, process_count=expected_call_count, thread_count=3, arg_checker=DummyArgChecker) for _ in range(expected_call_count): self.assertIsNone(call_queue.get(timeout=1.0)) class TestParallelismFrameworkWithoutMultiprocessing(TestParallelismFramework): """Tests parallelism framework works with multiprocessing module unavailable. Notably, this test has no way to override previous calls to gslib.util.CheckMultiprocessingAvailableAndInit to prevent the initialization of all of the global variables in command.py, so this still behaves slightly differently than the behavior one would see on a machine where the multiprocessing functionality is actually not available (in particular, it will not catch the case where a global variable that is not available for the sequential path is referenced before initialization). """ command_class = FakeCommandWithoutMultiprocessingModule tests/test_temporary_file_util.py 0000644 00000003055 15025013616 0013377 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for temporary_file_util.py.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import storage_url from gslib.tests import testcase from gslib.utils import temporary_file_util class TestTemporaryUtil(testcase.GsUtilUnitTestCase): """Test temporary file utils.""" def testGetsTemporaryFileName(self): self.assertEqual( temporary_file_util.GetTempFileName( storage_url.StorageUrlFromString('file.txt')), 'file.txt_.gstmp') def testGetsTemporaryZipFileName(self): self.assertEqual( temporary_file_util.GetTempZipFileName( storage_url.StorageUrlFromString('file.txt')), 'file.txt_.gztmp') def testGetsStetTemporaryFileName(self): self.assertEqual( temporary_file_util.GetStetTempFileName( storage_url.StorageUrlFromString('file.txt')), 'file.txt_.stet_tmp') tests/test_hashing_helper.py 0000644 00000025437 15025013616 0012311 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for hashing helper functions and classes.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import hashlib import os import pkgutil from unittest import mock from gslib.exception import CommandException from gslib.storage_url import StorageUrlFromString import gslib.tests.testcase as testcase from gslib.utils.constants import TRANSFER_BUFFER_SIZE from gslib.utils.hashing_helper import CalculateMd5FromContents from gslib.utils.hashing_helper import GetMd5 from gslib.utils.hashing_helper import HashingFileUploadWrapper _TEST_FILE = 'test.txt' class TestGetMd5(testcase.GsUtilUnitTestCase): """Unit tests for the GetMd5 function.""" @mock.patch.object(hashlib, 'md5') def testGetsMd5HashOnNonRedHatSystem(self, mock_md5): # Can't actually compare output to calling hashlib.md5 because that could # trigger an error on a Red Hat system. mock_md5.return_value = 'hash' self.assertEqual(GetMd5(b''), 'hash') mock_md5.assert_called_once_with(b'') @mock.patch.object(hashlib, 'md5') def testGetsMd5HashOnRedHatSystem(self, mock_md5): # Can't actually compare output to calling hashlib.md5 because that could # trigger an error on a non-Red Hat system. # Return one ValueError to simulate a FIPS-mode distribution. mock_md5.side_effect = [ValueError, 'hash'] self.assertEqual(GetMd5(b''), 'hash') self.assertEqual( mock_md5.mock_calls, [mock.call(b''), mock.call(b'', usedforsecurity=False)]) class TestHashingFileUploadWrapper(testcase.GsUtilUnitTestCase): """Unit tests for the HashingFileUploadWrapper class.""" _temp_test_file = None _dummy_url = StorageUrlFromString('gs://bucket/object') def _GetTestFile(self): contents = pkgutil.get_data('gslib', 'tests/test_data/%s' % _TEST_FILE) if not self._temp_test_file: self._temp_test_file = self.CreateTempFile(file_name=_TEST_FILE, contents=contents) return self._temp_test_file def testReadToEOF(self): digesters = {'md5': GetMd5()} tmp_file = self.CreateTempFile(contents=b'a' * TRANSFER_BUFFER_SIZE * 4) with open(tmp_file, 'rb') as stream: wrapper = HashingFileUploadWrapper(stream, digesters, {'md5': GetMd5}, self._dummy_url, self.logger) wrapper.read() with open(tmp_file, 'rb') as stream: actual = CalculateMd5FromContents(stream) self.assertEqual(actual, digesters['md5'].hexdigest()) def _testSeekBack(self, initial_position, seek_back_amount): """Tests reading then seeking backwards. This function simulates an upload that is resumed after a connection break. It reads one transfer buffer at a time until it reaches initial_position, then seeks backwards (as if the server did not receive some of the bytes) and reads to the end of the file, ensuring the hash matches the original file upon completion. Args: initial_position: Initial number of bytes to read before seek. seek_back_amount: Number of bytes to seek backward. Raises: AssertionError on wrong amount of data remaining or hash mismatch. """ tmp_file = self._GetTestFile() tmp_file_len = os.path.getsize(tmp_file) self.assertGreaterEqual( initial_position, seek_back_amount, 'seek_back_amount must be less than initial position %s ' '(but was actually: %s)' % (initial_position, seek_back_amount)) self.assertLess( initial_position, tmp_file_len, 'initial_position must be less than test file size %s ' '(but was actually: %s)' % (tmp_file_len, initial_position)) digesters = {'md5': GetMd5()} with open(tmp_file, 'rb') as stream: wrapper = HashingFileUploadWrapper(stream, digesters, {'md5': GetMd5}, self._dummy_url, self.logger) position = 0 while position < initial_position - TRANSFER_BUFFER_SIZE: data = wrapper.read(TRANSFER_BUFFER_SIZE) position += len(data) wrapper.read(initial_position - position) wrapper.seek(initial_position - seek_back_amount) self.assertEqual(wrapper.tell(), initial_position - seek_back_amount) data = wrapper.read() self.assertEqual(len(data), tmp_file_len - (initial_position - seek_back_amount)) with open(tmp_file, 'rb') as stream: actual = CalculateMd5FromContents(stream) self.assertEqual(actual, digesters['md5'].hexdigest()) def testSeekToBeginning(self): for num_bytes in (TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE + 1, TRANSFER_BUFFER_SIZE * 2 - 1, TRANSFER_BUFFER_SIZE * 2, TRANSFER_BUFFER_SIZE * 2 + 1, TRANSFER_BUFFER_SIZE * 3 - 1, TRANSFER_BUFFER_SIZE * 3, TRANSFER_BUFFER_SIZE * 3 + 1): self._testSeekBack(num_bytes, num_bytes) def testSeekBackAroundOneBuffer(self): for initial_position in (TRANSFER_BUFFER_SIZE + 1, TRANSFER_BUFFER_SIZE * 2 - 1, TRANSFER_BUFFER_SIZE * 2, TRANSFER_BUFFER_SIZE * 2 + 1, TRANSFER_BUFFER_SIZE * 3 - 1, TRANSFER_BUFFER_SIZE * 3, TRANSFER_BUFFER_SIZE * 3 + 1): for seek_back_amount in (TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE + 1): self._testSeekBack(initial_position, seek_back_amount) def testSeekBackMoreThanOneBuffer(self): for initial_position in (TRANSFER_BUFFER_SIZE * 2 + 1, TRANSFER_BUFFER_SIZE * 3 - 1, TRANSFER_BUFFER_SIZE * 3, TRANSFER_BUFFER_SIZE * 3 + 1): for seek_back_amount in (TRANSFER_BUFFER_SIZE * 2 - 1, TRANSFER_BUFFER_SIZE * 2, TRANSFER_BUFFER_SIZE * 2 + 1): self._testSeekBack(initial_position, seek_back_amount) def _testSeekForward(self, initial_seek): """Tests seeking to an initial position and then reading. This function simulates an upload that is resumed after a process break. It seeks from zero to the initial position (as if the server already had those bytes). Then it reads to the end of the file, ensuring the hash matches the original file upon completion. Args: initial_seek: Number of bytes to initially seek. Raises: AssertionError on wrong amount of data remaining or hash mismatch. """ tmp_file = self._GetTestFile() tmp_file_len = os.path.getsize(tmp_file) self.assertLess( initial_seek, tmp_file_len, 'initial_seek must be less than test file size %s ' '(but was actually: %s)' % (tmp_file_len, initial_seek)) digesters = {'md5': GetMd5()} with open(tmp_file, 'rb') as stream: wrapper = HashingFileUploadWrapper(stream, digesters, {'md5': GetMd5}, self._dummy_url, self.logger) wrapper.seek(initial_seek) self.assertEqual(wrapper.tell(), initial_seek) data = wrapper.read() self.assertEqual(len(data), tmp_file_len - initial_seek) with open(tmp_file, 'rb') as stream: actual = CalculateMd5FromContents(stream) self.assertEqual(actual, digesters['md5'].hexdigest()) def testSeekForward(self): for initial_seek in (0, TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE + 1, TRANSFER_BUFFER_SIZE * 2 - 1, TRANSFER_BUFFER_SIZE * 2, TRANSFER_BUFFER_SIZE * 2 + 1): self._testSeekForward(initial_seek) def _testSeekAway(self, initial_read): """Tests reading to an initial position and then seeking to EOF and back. This function simulates an size check on the input file by seeking to the end of the file and then back to the current position. Then it reads to the end of the file, ensuring the hash matches the original file upon completion. Args: initial_read: Number of bytes to initially read. Raises: AssertionError on wrong amount of data remaining or hash mismatch. """ tmp_file = self._GetTestFile() tmp_file_len = os.path.getsize(tmp_file) self.assertLess( initial_read, tmp_file_len, 'initial_read must be less than test file size %s ' '(but was actually: %s)' % (tmp_file_len, initial_read)) digesters = {'md5': GetMd5()} with open(tmp_file, 'rb') as stream: wrapper = HashingFileUploadWrapper(stream, digesters, {'md5': GetMd5}, self._dummy_url, self.logger) wrapper.read(initial_read) self.assertEqual(wrapper.tell(), initial_read) wrapper.seek(0, os.SEEK_END) self.assertEqual(wrapper.tell(), tmp_file_len) wrapper.seek(initial_read, os.SEEK_SET) data = wrapper.read() self.assertEqual(len(data), tmp_file_len - initial_read) with open(tmp_file, 'rb') as stream: actual = CalculateMd5FromContents(stream) self.assertEqual(actual, digesters['md5'].hexdigest()) def testValidSeekAway(self): for initial_read in (0, TRANSFER_BUFFER_SIZE - 1, TRANSFER_BUFFER_SIZE, TRANSFER_BUFFER_SIZE + 1, TRANSFER_BUFFER_SIZE * 2 - 1, TRANSFER_BUFFER_SIZE * 2, TRANSFER_BUFFER_SIZE * 2 + 1): self._testSeekAway(initial_read) def testInvalidSeekAway(self): """Tests seeking to EOF and then reading without first doing a SEEK_SET.""" tmp_file = self._GetTestFile() digesters = {'md5': GetMd5()} with open(tmp_file, 'rb') as stream: wrapper = HashingFileUploadWrapper(stream, digesters, {'md5': GetMd5}, self._dummy_url, self.logger) wrapper.read(TRANSFER_BUFFER_SIZE) wrapper.seek(0, os.SEEK_END) try: wrapper.read() self.fail('Expected CommandException for invalid seek.') except CommandException as e: self.assertIn( 'Read called on hashing file pointer in an unknown position', str(e)) tests/test_rsync.py 0000644 00000452520 15025013616 0010464 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for rsync command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import re from unittest import mock import six from gslib import command from gslib.commands import rsync from gslib.project_id import PopulateProjectId from gslib.storage_url import StorageUrlFromString import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForGS from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import AuthorizeProjectToUseTestingKmsKey from gslib.tests.util import TEST_ENCRYPTION_KEY_S3 from gslib.tests.util import TEST_ENCRYPTION_KEY_S3_MD5 from gslib.tests.util import BuildErrorRegex from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import ORPHANED_FILE from gslib.tests.util import POSIX_GID_ERROR from gslib.tests.util import POSIX_INSUFFICIENT_ACCESS_ERROR from gslib.tests.util import POSIX_MODE_ERROR from gslib.tests.util import POSIX_UID_ERROR from gslib.tests.util import SequentialAndParallelTransfer from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import SetEnvironmentForTest from gslib.tests.util import TailSet from gslib.tests.util import unittest from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.hashing_helper import SLOW_CRCMOD_RSYNC_WARNING from gslib.utils.posix_util import ConvertDatetimeToPOSIX from gslib.utils.posix_util import GID_ATTR from gslib.utils.posix_util import MODE_ATTR from gslib.utils.posix_util import MTIME_ATTR from gslib.utils.posix_util import NA_TIME from gslib.utils.posix_util import UID_ATTR from gslib.utils.retry_util import Retry from gslib.utils.system_util import IS_OSX from gslib.utils.system_util import IS_WINDOWS from gslib.utils import shim_util # These POSIX-specific variables aren't defined for Windows. # pylint: disable=g-import-not-at-top if not IS_WINDOWS: from gslib.tests import util from gslib.tests.util import DEFAULT_MODE from gslib.tests.util import INVALID_GID from gslib.tests.util import INVALID_UID from gslib.tests.util import USER_ID # pylint: enable=g-import-not-at-top if six.PY3: long = int MACOS_WARNING = ( 'If you experience problems with multiprocessing on MacOS, they might be ' 'related to https://bugs.python.org/issue33725. You can disable ' 'multiprocessing by editing your .boto config or by adding the following ' 'flag to your command: `-o "GSUtil:parallel_process_count=1"`. Note that ' 'multithreading is still available even if you disable multiprocessing.\n\n' ) if IS_OSX: NO_CHANGES = ('Building synchronization state...\n' + MACOS_WARNING + 'Starting synchronization...\n') else: NO_CHANGES = ( 'Building synchronization state...\nStarting synchronization...\n') if not UsingCrcmodExtension(): NO_CHANGES = SLOW_CRCMOD_RSYNC_WARNING + '\n' + NO_CHANGES class TestRsyncUnit(testcase.GsUtilUnitTestCase): """Unit tests for methods in the commands.rsync module.""" def testUrlEncodeAndDecode(self): # Test special URL chars as well as unicode: decoded_url = 'gs://bkt/space fslash/plus+tilde~unicodeeè' encoded_url = ( 'gs%3A%2F%2Fbkt%2Fspace+fslash%2Fplus%2Btilde~unicodee%C3%A8') # Encode accepts unicode, returns language-appropriate string type. self.assertEqual(rsync._EncodeUrl(six.ensure_text(decoded_url)), six.ensure_str(encoded_url)) # Decode accepts language-appropriate string type, returns unicode. self.assertEqual(rsync._DecodeUrl(six.ensure_str(encoded_url)), six.ensure_text(decoded_url)) @mock.patch( 'gslib.utils.copy_helper.TriggerReauthForDestinationProviderIfNecessary') @mock.patch('gslib.command.Command._GetProcessAndThreadCount') @mock.patch('gslib.command.Command.Apply', new=mock.Mock(spec=command.Command.Apply)) def testRsyncTriggersReauth(self, mock_get_process_and_thread_count, mock_trigger_reauth): path = self.CreateTempDir() bucket_uri = self.CreateBucket() mock_get_process_and_thread_count.return_value = 2, 3 self.RunCommand('rsync', [path, suri(bucket_uri)]) mock_trigger_reauth.assert_called_once_with( StorageUrlFromString(suri(bucket_uri)), mock.ANY, # Gsutil API. worker_count=6, ) mock_get_process_and_thread_count.assert_called_once_with( process_count=None, thread_count=None, parallel_operations_override=command.Command.ParallelOverrideReason. SPEED, print_macos_warning=False, ) class TestRsyncUnitWithShim(testcase.ShimUnitTestBase): def testShimTranslatesFlags(self): bucket_uri = self.CreateBucket() fpath = self.CreateTempDir() with SetBotoConfigForTest([('GSUtil', 'use_gcloud_storage', 'True'), ('GSUtil', 'hidden_shim_mode', 'dry_run')]): with SetEnvironmentForTest({ 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL': 'True', 'CLOUDSDK_ROOT_DIR': 'fake_dir', }): mock_log_handler = self.RunCommand( 'rsync', ['-a', 'public-read', '-P', suri(bucket_uri), fpath], return_log_handler=True) info_lines = '\n'.join(mock_log_handler.messages['info']) self.assertIn( 'Gcloud Storage Command: {} storage rsync' ' --no-ignore-symlinks --predefined-acl publicRead' ' --preserve-posix {} {}'.format( shim_util._get_gcloud_binary_path('fake_dir'), suri(bucket_uri), fpath), info_lines) warn_lines = '\n'.join(mock_log_handler.messages['warning']) self.assertIn('By default, gsutil copies file symlinks', warn_lines) self.assertIn('For preserving POSIX with rsync downloads', warn_lines) # TODO: Add inspection to the retry wrappers in this test suite where the state # at the end of a retry block is depended upon by subsequent tests (since # listing content can vary depending on which backend server is reached until # eventual consistency is reached). # TODO: Remove retry wrappers and AssertNObjectsInBucket calls if GCS ever # supports strong listing consistency. class TestRsync(testcase.GsUtilIntegrationTestCase): """Integration tests for rsync command.""" def _GetMetadataAttribute(self, bucket_name, object_name, attr_name): """Retrieves and returns an attribute from an objects metadata. Args: bucket_name: The name of the bucket the object is in. object_name: The name of the object itself. attr_name: The name of the custom metadata attribute. Returns: The value at the specified attribute name in the metadata. If not present, returns None. """ gsutil_api = (self.json_api if self.default_provider == 'gs' else self.xml_api) metadata = gsutil_api.GetObjectMetadata(bucket_name, object_name, provider=self.default_provider, fields=[attr_name]) return getattr(metadata, attr_name, None) def _VerifyNoChanges(self, stderr): if self._use_gcloud_storage: self.assertNotIn('Completed', stderr) else: self.assertEqual(NO_CHANGES, stderr) def _VerifyObjectMtime(self, bucket_name, object_name, expected_mtime, expected_present=True): """Retrieves the object's mtime. Args: bucket_name: The name of the bucket the object is in. object_name: The name of the object itself. expected_mtime: The expected retrieved mtime. expected_present: True if the mtime must be present in the object metadata, False if it must not be present. Returns: None """ self.VerifyObjectCustomAttribute(bucket_name, object_name, MTIME_ATTR, expected_mtime, expected_present=expected_present) def test_invalid_args(self): """Tests various invalid argument cases.""" bucket_uri = self.CreateBucket() obj1 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj1', contents=b'obj1') tmpdir = self.CreateTempDir() # rsync object to bucket. self.RunGsUtil(['rsync', suri(obj1), suri(bucket_uri)], expected_status=1) # rsync bucket to object. self.RunGsUtil(['rsync', suri(bucket_uri), suri(obj1)], expected_status=1) # rsync bucket to non-existent bucket. self.RunGsUtil( ['rsync', suri(bucket_uri), 'gs://' + self.nonexistent_bucket_name], expected_status=1) # rsync object to dir. self.RunGsUtil(['rsync', suri(obj1), tmpdir], expected_status=1) if not self._use_gcloud_storage: # rsync dir to object. # Gsutil asks you to add a slash after the object name. # Gcloud automatically adds the slash and creates the prefix or directory. self.RunGsUtil(['rsync', tmpdir, suri(obj1)], expected_status=1) # rsync dir to non-existent bucket. self.RunGsUtil( ['rsync', tmpdir, suri(obj1), 'gs://' + self.nonexistent_bucket_name], expected_status=1) # Note: The tests below exercise the cases # {src_dir, src_bucket} X {dst_dir, dst_bucket}. We use gsutil rsync -d for # all the cases but then have just one test without -d (test_bucket_to_bucket) # as representative of handling without the -d option. This provides # reasonable test coverage because the -d handling it src/dest URI-type # independent, and keeps the test case combinations more manageable. def test_invalid_src_mtime(self): """Tests that an exception is thrown if mtime cannot be cast as a long.""" # Create 1 bucket with 1 file present with mtime set as a string of # non-numeric characters, and as a number. bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents=b'obj1', mtime='xyz') self.CreateObject(bucket_uri=bucket1_uri, object_name='obj2', contents=b'obj2', mtime=123) # This creates an object that has an mtime sometime on 41091-11-25 UTC. It # is used to verify that a warning is thrown for objects set at least a day # in the future. If this test is not updated before that date, this test # will fail because of the hardcoded timestamp. self.CreateObject(bucket_uri=bucket1_uri, object_name='obj3', contents=b'obj3', mtime=long(1234567891011)) # Create objects with a negative mtime. self.CreateObject(bucket_uri=bucket1_uri, object_name='obj4', contents=b'obj4', mtime=-100) self.CreateObject(bucket_uri=bucket1_uri, object_name='obj5', contents=b'obj5', mtime=-1) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stderr = self.RunGsUtil( ['rsync', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) if self._use_gcloud_storage: self.assertRegex( stderr, r'obj1#\d+ metadata did not contain a numeric value for' r' goog-reserved-file-mtime') self.assertNotRegex( stderr, r'obj2#\d+ metadata did not contain a numeric value for' r' goog-reserved-file-mtime') self.assertRegex( stderr, r'obj3#\d+ metadata that is more than one day in the future from' r' the system time') self.assertRegex(stderr, r'Found negative time value in gs://.*/obj4') self.assertRegex(stderr, r'Found negative time value in gs://.*/obj5') else: self.assertIn('obj1 has an invalid mtime in its metadata', stderr) self.assertNotIn('obj2 has an invalid mtime in its metadata', stderr) self.assertIn( 'obj3 has an mtime more than 1 day from current system ' 'time', stderr) self.assertIn('obj4 has a negative mtime in its metadata', stderr) self.assertIn('obj5 has a negative mtime in its metadata', stderr) _Check1() @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_bucket_to_bucket_preserve_posix(self): """Tests that rsync -P works with bucket to bucket.""" # Note that unlike bucket to dir tests POSIX attributes cannot be verified # beyond basic validations because the cloud buckets have no notion of UID # or GID especially considering that these values can come from systems # with vastly different mappings. In addition, the mode will not be # verified. src_bucket = self.CreateBucket() dst_bucket = self.CreateBucket() primary_gid = os.getgid() non_primary_gid = util.GetNonPrimaryGid() # Create source objects. self.CreateObject(bucket_uri=src_bucket, object_name='obj1', contents=b'obj1', mode='444') self.CreateObject(bucket_uri=src_bucket, object_name='obj2', contents=b'obj2', gid=primary_gid) self.CreateObject(bucket_uri=src_bucket, object_name='obj3', contents=b'obj3', gid=non_primary_gid) self.CreateObject(bucket_uri=src_bucket, object_name='obj4', contents=b'obj3', uid=INVALID_UID(), gid=INVALID_GID(), mode='222') self.CreateObject(bucket_uri=src_bucket, object_name='obj5', contents=b'obj5', uid=USER_ID, gid=primary_gid, mode=str(DEFAULT_MODE)) # Create destination objects. # obj5 at the source and destination have the same content so we will only # patch the destination metadata instead of copying the entire object. self.CreateObject(bucket_uri=dst_bucket, object_name='obj5', contents=b'obj5') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Test bucket to bucket rsync with -P flag and verify attributes.""" stderr = self.RunGsUtil( ['rsync', '-P', suri(src_bucket), suri(dst_bucket)], return_stderr=True) listing1 = TailSet(suri(src_bucket), self.FlatListBucket(src_bucket)) listing2 = TailSet(suri(dst_bucket), self.FlatListBucket(dst_bucket)) # First bucket should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/obj2', '/obj3', '/obj4', '/obj5'])) # dst_bucket should have new content from src_bucket. self.assertEqual(listing2, set(['/obj1', '/obj2', '/obj3', '/obj4', '/obj5'])) if self._use_gcloud_storage: self.assertIn('Patching', stderr) else: self.assertIn('Copying POSIX attributes from src to dst for', stderr) _Check1() self.VerifyObjectCustomAttribute(dst_bucket.bucket_name, 'obj1', MODE_ATTR, '444') self.VerifyObjectCustomAttribute(dst_bucket.bucket_name, 'obj2', GID_ATTR, str(primary_gid)) self.VerifyObjectCustomAttribute(dst_bucket.bucket_name, 'obj3', GID_ATTR, str(non_primary_gid)) # Verify all of the attributes for obj4. Even though these are all 'invalid' # values, the file was copied to the destination because bucket to bucket # with preserve POSIX enabled will blindly copy the object metadata. self.VerifyObjectCustomAttribute(dst_bucket.bucket_name, 'obj4', GID_ATTR, str(INVALID_GID())) self.VerifyObjectCustomAttribute(dst_bucket.bucket_name, 'obj4', UID_ATTR, str(INVALID_UID())) self.VerifyObjectCustomAttribute(dst_bucket.bucket_name, 'obj4', MODE_ATTR, '222') # Verify obj5 attributes were copied. self.VerifyObjectCustomAttribute(dst_bucket.bucket_name, 'obj5', UID_ATTR, str(USER_ID)) self.VerifyObjectCustomAttribute(dst_bucket.bucket_name, 'obj5', GID_ATTR, str(primary_gid)) self.VerifyObjectCustomAttribute(dst_bucket.bucket_name, 'obj5', MODE_ATTR, str(DEFAULT_MODE)) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): """Check that we are not patching destination metadata a second time.""" stderr = self.RunGsUtil( ['rsync', '-P', suri(src_bucket), suri(dst_bucket)], return_stderr=True) if self._use_gcloud_storage: self.assertNotIn('Patching', stderr) else: self.assertNotIn('Copying POSIX attributes from src to dst for', stderr) _Check2() def test_bucket_to_bucket_same_objects_src_mtime(self): """Tests bucket to bucket with mtime. Each has the same items but only the source has mtime stored in its metadata. Ensure that destination now also has the mtime of the files in its metadata. """ # Create 2 buckets where the source and destination have 2 objects each with # the same name and content, where mtime is only set on src_bucket. src_bucket = self.CreateBucket() dst_bucket = self.CreateBucket() self.CreateObject(bucket_uri=src_bucket, object_name='obj1', contents=b'obj1', mtime=0) self.CreateObject(bucket_uri=src_bucket, object_name='subdir/obj2', contents=b'subdir/obj2', mtime=1) self.CreateObject(bucket_uri=dst_bucket, object_name='obj1', contents=b'obj1') self.CreateObject(bucket_uri=dst_bucket, object_name='subdir/obj2', contents=b'subdir/obj2') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-r', suri(src_bucket), suri(dst_bucket)]) listing1 = TailSet(suri(src_bucket), self.FlatListBucket(src_bucket)) # First bucket should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/subdir/obj2'])) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stderr = self.RunGsUtil( ['rsync', suri(src_bucket), suri(dst_bucket)], return_stderr=True) # Check that re-running the same rsync command causes no more changes. self._VerifyNoChanges(stderr) _Check2() if self._use_gcloud_storage: # Gcloud skips patching mtimes for cloud-to-cloud to save API calls b/c # rsync uses hash comparisons for cloud-to-cloud anyways. self._VerifyObjectMtime(dst_bucket.bucket_name, 'obj1', NA_TIME, expected_present=False) self._VerifyObjectMtime(dst_bucket.bucket_name, 'subdir/obj2', NA_TIME, expected_present=False) else: # Verify objects' mtime in dst_bucket self._VerifyObjectMtime(dst_bucket.bucket_name, 'obj1', '0') self._VerifyObjectMtime(dst_bucket.bucket_name, 'subdir/obj2', '1') def test_bucket_to_bucket_src_mtime(self): """Tests bucket to bucket where source has mtime in files.""" # Create 2 buckets where the source has 2 objects one at root level and the # other in a subdirectory. The other bucket will be empty. src_bucket = self.CreateBucket() dst_bucket = self.CreateBucket() obj1 = self.CreateObject(bucket_uri=src_bucket, object_name='obj1', contents=b'obj1', mtime=0) obj2 = self.CreateObject(bucket_uri=src_bucket, object_name='subdir/obj2', contents=b'subdir/obj2', mtime=1) # Verify objects' mtime in the buckets self._VerifyObjectMtime(obj1.bucket_name, obj1.object_name, '0') self._VerifyObjectMtime(obj2.bucket_name, obj2.object_name, '1') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-r', suri(src_bucket), suri(dst_bucket)]) listing1 = TailSet(suri(src_bucket), self.FlatListBucket(src_bucket)) listing2 = TailSet(suri(dst_bucket), self.FlatListBucket(dst_bucket)) # First bucket should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/subdir/obj2'])) # Second bucket should have new objects added from source bucket. self.assertEqual(listing2, set(['/obj1', '/subdir/obj2'])) _Check1() # Get and verify the metadata for the 2 objects at the destination. self._VerifyObjectMtime(dst_bucket.bucket_name, 'obj1', '0') self._VerifyObjectMtime(dst_bucket.bucket_name, 'subdir/obj2', '1') def test_bucket_to_bucket_dst_mtime(self): """Tests bucket to bucket where destination has mtime in objects.""" # Create 2 buckets where the source has files without mtime and the # destination has it present in its object's metadata. For obj1 and obj6 # this tests the behavior where the file names are the same but the content # is different. obj1 has no mtime at source, but obj6 has mtime at src that # matches dst. src_bucket = self.CreateBucket() dst_bucket = self.CreateBucket() self.CreateObject(bucket_uri=src_bucket, object_name='obj1', contents=b'OBJ1') self.CreateObject(bucket_uri=src_bucket, object_name='subdir/obj2', contents=b'subdir/obj2') self.CreateObject(bucket_uri=src_bucket, object_name='.obj3', contents=b'.obj3') self.CreateObject(bucket_uri=src_bucket, object_name='subdir/obj4', contents=b'subdir/obj4') self.CreateObject(bucket_uri=src_bucket, object_name='obj6', contents=b'OBJ6', mtime=100) self.CreateObject(bucket_uri=dst_bucket, object_name='obj1', contents=b'obj1', mtime=10) self.CreateObject(bucket_uri=dst_bucket, object_name='subdir/obj2', contents=b'subdir/obj2', mtime=10) self.CreateObject(bucket_uri=dst_bucket, object_name='.obj3', contents=b'.OBJ3', mtime=long(1000000000000)) self.CreateObject(bucket_uri=dst_bucket, object_name='subdir/obj5', contents=b'subdir/obj5', mtime=10) self.CreateObject(bucket_uri=dst_bucket, object_name='obj6', contents=b'obj6', mtime=100) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-r', '-d', suri(src_bucket), suri(dst_bucket)], return_stderr=True), listing1 = TailSet(suri(src_bucket), self.FlatListBucket(src_bucket)) listing2 = TailSet(suri(dst_bucket), self.FlatListBucket(dst_bucket)) # First bucket should have un-altered content. self.assertEqual( listing1, set(['/obj1', '/subdir/obj2', '/.obj3', '/subdir/obj4', '/obj6'])) # Second bucket should have new objects added from source bucket. self.assertEqual( listing2, set(['/obj1', '/subdir/obj2', '/.obj3', '/subdir/obj4', '/obj6'])) _Check1() # Get and verify the metadata for the objects at the destination. self._VerifyObjectMtime(dst_bucket.bucket_name, 'subdir/obj2', '10') if self._use_gcloud_storage: # Gcloud uses creation time if mtime missing. Since source and destination # objects have different hashes, it will trigger a copy. source_o1_time_created = self.GetObjectMetadataWithFields( src_bucket.bucket_name, 'obj1', ['timeCreated']).timeCreated source_o1_posix_time_created = str( ConvertDatetimeToPOSIX(source_o1_time_created)) self._VerifyObjectMtime(dst_bucket.bucket_name, 'obj1', source_o1_posix_time_created) source_o4_time_created = self.GetObjectMetadataWithFields( src_bucket.bucket_name, 'subdir/obj4', ['timeCreated']).timeCreated source_o4_posix_time_created = str( ConvertDatetimeToPOSIX(source_o4_time_created)) self._VerifyObjectMtime(dst_bucket.bucket_name, 'subdir/obj4', source_o4_posix_time_created) else: self._VerifyObjectMtime(dst_bucket.bucket_name, 'obj1', NA_TIME, expected_present=False) self._VerifyObjectMtime(dst_bucket.bucket_name, 'subdir/obj4', NA_TIME, expected_present=False) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check3(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', suri(src_bucket), suri(dst_bucket)], return_stderr=True) self._VerifyNoChanges(stderr) _Check3() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check4(): # Check that obj1 changed because mtime was not available; hashes were # used to compare the objects. self.assertEqual( 'OBJ1', self.RunGsUtil(['cat', suri(dst_bucket, 'obj1')], return_stdout=True)) # Ensure that .obj3 was updated even though its modification time comes # after the creation time of .obj3 at the source. self.assertEqual( '.obj3', self.RunGsUtil(['cat', suri(dst_bucket, '.obj3')], return_stdout=True)) # Check that obj6 was updated even though the mtimes match. In this case # bucket to bucket sync will compare hashes. self.assertEqual( 'OBJ6', self.RunGsUtil(['cat', suri(dst_bucket, 'obj6')], return_stdout=True)) _Check4() # Now rerun the rsync with the -c option. # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check5(): """Tests rsync -c works as expected.""" self.RunGsUtil( ['rsync', '-r', '-d', '-c', suri(src_bucket), suri(dst_bucket)]) listing1 = TailSet(suri(src_bucket), self.FlatListBucket(src_bucket)) listing2 = TailSet(suri(dst_bucket), self.FlatListBucket(dst_bucket)) # First bucket should have un-altered content. self.assertEqual( listing1, set(['/obj1', '/subdir/obj2', '/.obj3', '/subdir/obj4', '/obj6'])) # Second bucket should have new objects added from source bucket. self.assertEqual( listing2, set(['/obj1', '/subdir/obj2', '/.obj3', '/subdir/obj4', '/obj6'])) # Assert that the contents of obj6 have now changed because the -c flag # was used to force checksums. self.assertEqual( 'OBJ6', self.RunGsUtil(['cat', suri(dst_bucket, 'obj6')], return_stdout=True)) # Verify the mtime for obj6 is correct. self._VerifyObjectMtime(dst_bucket.bucket_name, 'obj6', '100') _Check5() def test_bucket_to_bucket(self): """Tests that flat and recursive rsync between 2 buckets works correctly.""" # Create 2 buckets with 1 overlapping object, 1 extra object at root level # in each, and 1 extra object 1 level down in each, where one of the objects # starts with "." to test that we don't skip those objects. Make the # overlapping objects named the same but with different content, to test # that we detect and properly copy in that case. bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents=b'obj1') self.CreateObject(bucket_uri=bucket1_uri, object_name='.obj2', contents=b'.obj2', mtime=10) self.CreateObject(bucket_uri=bucket1_uri, object_name='subdir/obj3', contents=b'subdir/obj3') self.CreateObject(bucket_uri=bucket1_uri, object_name='obj6', contents=b'obj6_', mtime=100) # .obj2 will be replaced and have mtime of 10 self.CreateObject(bucket_uri=bucket2_uri, object_name='.obj2', contents=b'.OBJ2') self.CreateObject(bucket_uri=bucket2_uri, object_name='obj4', contents=b'obj4') self.CreateObject(bucket_uri=bucket2_uri, object_name='subdir/obj5', contents=b'subdir/obj5') self.CreateObject(bucket_uri=bucket2_uri, object_name='obj6', contents=b'obj6', mtime=100) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', suri(bucket1_uri), suri(bucket2_uri)]) listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri)) listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri)) # First bucket should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir/obj3', '/obj6'])) # Second bucket should have new objects added from source bucket (without # removing extraneeous object found in dest bucket), and without the # subdir objects synchronized. self.assertEqual( listing2, set(['/obj1', '/.obj2', '/obj4', '/subdir/obj5', '/obj6'])) # Assert that the src/dest objects that had same length but different # content were correctly synchronized (bucket to bucket rsync uses # checksums). self.assertEqual( '.obj2', self.RunGsUtil(['cat', suri(bucket1_uri, '.obj2')], return_stdout=True)) self.assertEqual( '.obj2', self.RunGsUtil(['cat', suri(bucket2_uri, '.obj2')], return_stdout=True)) self.assertEqual( 'obj6_', self.RunGsUtil(['cat', suri(bucket2_uri, 'obj6')], return_stdout=True)) # Verify that .obj2 had its mtime updated at the destination. self._VerifyObjectMtime(bucket2_uri.bucket_name, '.obj2', '10') _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() # Now add, overwrite, and remove some objects in each bucket and test # rsync -r. self.CreateObject(bucket_uri=bucket1_uri, object_name='obj6', contents=b'obj6') self.CreateObject(bucket_uri=bucket2_uri, object_name='obj7', contents=b'obj7') self.RunGsUtil(['rm', suri(bucket1_uri, 'obj1')]) self.RunGsUtil(['rm', suri(bucket2_uri, '.obj2')]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check3(): self.RunGsUtil(['rsync', '-r', suri(bucket1_uri), suri(bucket2_uri)]) listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri)) listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri)) # First bucket should have un-altered content. self.assertEqual(listing1, set(['/.obj2', '/obj6', '/subdir/obj3'])) # Second bucket should have objects tha were newly added to first bucket # (wihout removing extraneous dest bucket objects), and without the # subdir objects synchronized. self.assertEqual( listing2, set([ '/obj1', '/.obj2', '/obj4', '/obj6', '/obj7', '/subdir/obj3', '/subdir/obj5' ])) _Check3() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check4(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-r', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check4() def test_bucket_to_bucket_minus_d(self): """Tests that flat and recursive rsync between 2 buckets works correctly.""" # Create 2 buckets with 1 overlapping object, 1 extra object at root level # in each, and 1 extra object 1 level down in each, where one of the objects # starts with "." to test that we don't skip those objects. Make the # overlapping objects named the same but with different content, to test # that we detect and properly copy in that case. bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents=b'obj1') self.CreateObject(bucket_uri=bucket1_uri, object_name='.obj2', contents=b'.obj2') self.CreateObject(bucket_uri=bucket1_uri, object_name='subdir/obj3', contents=b'subdir/obj3') self.CreateObject(bucket_uri=bucket2_uri, object_name='.obj2', contents=b'.OBJ2') self.CreateObject(bucket_uri=bucket2_uri, object_name='obj4', contents=b'obj4') self.CreateObject(bucket_uri=bucket2_uri, object_name='subdir/obj5', contents=b'subdir/obj5') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)]) listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri)) listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri)) # First bucket should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir/obj3'])) # Second bucket should have content like first bucket but without the # subdir objects synchronized. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/subdir/obj5'])) # Assert that the src/dest objects that had same length but different # content were correctly synchronized (bucket to bucket rsync uses # checksums). self.assertEqual( '.obj2', self.RunGsUtil(['cat', suri(bucket1_uri, '.obj2')], return_stdout=True)) self.assertEqual( '.obj2', self.RunGsUtil(['cat', suri(bucket2_uri, '.obj2')], return_stdout=True)) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() # Now add and remove some objects in each bucket and test rsync -r. self.CreateObject(bucket_uri=bucket1_uri, object_name='obj6', contents=b'obj6') self.CreateObject(bucket_uri=bucket2_uri, object_name='obj7', contents=b'obj7') self.RunGsUtil(['rm', suri(bucket1_uri, 'obj1')]) self.RunGsUtil(['rm', suri(bucket2_uri, '.obj2')]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check3(): self.RunGsUtil( ['rsync', '-d', '-r', suri(bucket1_uri), suri(bucket2_uri)]) listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri)) listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri)) # First bucket should have un-altered content. self.assertEqual(listing1, set(['/.obj2', '/obj6', '/subdir/obj3'])) # Second bucket should have content like first bucket but without the # subdir objects synchronized. self.assertEqual(listing2, set(['/.obj2', '/obj6', '/subdir/obj3'])) _Check3() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check4(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', '-r', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check4() # Test sequential upload as well as parallel composite upload case. @SequentialAndParallelTransfer @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_dir_to_bucket_mtime(self): """Tests dir to bucket with mtime. Each has the same items, the source has mtime for all objects, whereas dst only has mtime for obj5 and obj6 to test for different a later mtime at src and the same mtime from src to dst, respectively. Ensure that destination now also has the mtime of the files in its metadata. """ # Create directory and bucket, where the directory has different # combinations of mtime and sub-directories. tmpdir = self.CreateTempDir() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1', mtime=10) self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.obj2', mtime=10) self.CreateTempFile(tmpdir=subdir, file_name='obj3', contents=b'subdir/obj3', mtime=10) self.CreateTempFile(tmpdir=subdir, file_name='obj5', contents=b'subdir/obj5', mtime=15) self.CreateTempFile(tmpdir=tmpdir, file_name='obj6', contents=b'obj6', mtime=100) self.CreateTempFile(tmpdir=tmpdir, file_name='obj7', contents=b'obj7_', mtime=100) bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='obj1', contents=b'OBJ1') self.CreateObject(bucket_uri=bucket_uri, object_name='.obj2', contents=b'.obj2') self._SetObjectCustomMetadataAttribute(self.default_provider, bucket_uri.bucket_name, '.obj2', 'test', 'test') self.CreateObject(bucket_uri=bucket_uri, object_name='obj4', contents=b'obj4') self.CreateObject(bucket_uri=bucket_uri, object_name='subdir/obj5', contents=b'subdir/obj5', mtime=10) self.CreateObject(bucket_uri=bucket_uri, object_name='obj6', contents=b'OBJ6', mtime=100) self.CreateObject(bucket_uri=bucket_uri, object_name='obj7', contents=b'obj7', mtime=100) cumulative_stderr = set() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" stderr = self.RunGsUtil(['rsync', '-r', '-d', tmpdir, suri(bucket_uri)], return_stderr=True) cumulative_stderr.update([s for s in stderr.splitlines() if s]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) # Dir should have un-altered content. self.assertEqual( listing1, set([ '/obj1', '/.obj2', '/subdir/obj3', '/subdir/obj5', '/obj6', '/obj7' ])) # Bucket should have content like dir. self.assertEqual( listing2, set([ '/obj1', '/.obj2', '/subdir/obj3', '/subdir/obj5', '/obj6', '/obj7' ])) # Check that obj6 didn't change even though the contents did. This is # because the object/file have the same mtime. self.assertEqual( 'OBJ6', self.RunGsUtil(['cat', suri(bucket_uri, 'obj6')], return_stdout=True)) # Check that obj7 changed because the size was different. self.assertEqual( 'obj7_', self.RunGsUtil(['cat', suri(bucket_uri, 'obj7')], return_stdout=True)) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil(['rsync', '-r', '-d', tmpdir, suri(bucket_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() # Verify objects' mtime in the bucket. self._VerifyObjectMtime(bucket_uri.bucket_name, 'obj1', '10') self._VerifyObjectMtime(bucket_uri.bucket_name, '.obj2', '10') self._VerifyObjectMtime(bucket_uri.bucket_name, 'subdir/obj3', '10') self._VerifyObjectMtime(bucket_uri.bucket_name, 'subdir/obj5', '15') self._VerifyObjectMtime(bucket_uri.bucket_name, 'obj6', '100') # Rsync using S3 without object owner permission will copy over old objects. copied_over_object_notice = ( 'Copying whole file/object for %s instead of patching because you ' 'don\'t have owner permission on the object.' % suri(bucket_uri, '.obj2')) if (copied_over_object_notice not in cumulative_stderr and not self._use_gcloud_storage): # Gsutil: Make sure test attribute wasn't removed when mtime was updated. # Gcloud: Does not fall back on copies to perform patches. self.VerifyObjectCustomAttribute(bucket_uri.bucket_name, '.obj2', 'test', 'test') # Now rerun the rsync with the -c option. # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check4(): """Tests rsync -c works as expected.""" self.RunGsUtil(['rsync', '-r', '-d', '-c', tmpdir, suri(bucket_uri)]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) # Dir should have un-altered content. self.assertEqual( listing1, set([ '/obj1', '/.obj2', '/subdir/obj3', '/subdir/obj5', '/obj6', '/obj7' ])) # Bucket should have content like dir with the subdirectories synced. self.assertEqual( listing2, set([ '/obj1', '/.obj2', '/subdir/obj3', '/subdir/obj5', '/obj6', '/obj7' ])) # Assert that the contents of obj6 have now changed because the -c flag # was used to force checksums. self.assertEqual( 'obj6', self.RunGsUtil(['cat', suri(bucket_uri, 'obj6')], return_stdout=True)) self._VerifyObjectMtime(bucket_uri.bucket_name, 'obj6', '100') _Check4() # Test sequential upload as well as parallel composite upload case. @SequentialAndParallelTransfer @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_dir_to_bucket_seek_ahead(self): """Tests that rsync seek-ahead iterator works correctly.""" # Unfortunately, we have to retry the entire operation in the case of # eventual consistency because the estimated values will differ. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Test estimating an rsync upload operation.""" tmpdir = self.CreateTempDir() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.obj2') self.CreateTempFile(tmpdir=subdir, file_name='obj3', contents=b'subdir/obj3') bucket_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket_uri, object_name='.obj2', contents=b'.OBJ2') self.CreateObject(bucket_uri=bucket_uri, object_name='obj4', contents=b'obj4') self.CreateObject(bucket_uri=bucket_uri, object_name='subdir/obj5', contents=b'subdir/obj5') # Need to make sure the bucket listing is caught-up, otherwise the # first rsync may not see .obj2 and overwrite it. self.AssertNObjectsInBucket(bucket_uri, 3) with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '1'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil( ['-m', 'rsync', '-d', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) # Objects: 4 (2 removed, 2 added) # Bytes: 15 (added objects are 4 bytes and 11 bytes, respectively). self.assertIn( 'Estimated work for this command: objects: 5, total size: 20', stderr) self.AssertNObjectsInBucket(bucket_uri, 3) # Re-running should produce no estimate, because there is no work to do. stderr = self.RunGsUtil( ['-m', 'rsync', '-d', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) self.assertNotIn('Estimated work', stderr) _Check1() tmpdir = self.CreateTempDir(test_files=1) bucket_uri = self.CreateBucket() # Running with task estimation turned off (and work to perform) should not # produce an estimate. with SetBotoConfigForTest([('GSUtil', 'task_estimation_threshold', '0'), ('GSUtil', 'task_estimation_force', 'True')]): stderr = self.RunGsUtil( ['-m', 'rsync', '-d', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) self.assertNotIn('Estimated work', stderr) # Test sequential upload as well as parallel composite upload case. @SequentialAndParallelTransfer @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_dir_to_bucket_minus_d(self): """Tests that flat and recursive rsync dir to bucket works correctly.""" # Create dir and bucket with 1 overlapping object, 1 extra object at root # level in each, and 1 extra object 1 level down in each, where one of the # objects starts with "." to test that we don't skip those objects. Make the # overlapping objects named the same but with different content, to test # that we detect and properly copy in that case. tmpdir = self.CreateTempDir() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) bucket_uri = self.CreateBucket() self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1') o2_path = self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.obj2') self.CreateTempFile(tmpdir=subdir, file_name='obj3', contents=b'subdir/obj3') self.CreateObject(bucket_uri=bucket_uri, object_name='.obj2', contents=b'.OBJ2') self.CreateObject(bucket_uri=bucket_uri, object_name='obj4', contents=b'obj4') self.CreateObject(bucket_uri=bucket_uri, object_name='subdir/obj5', contents=b'subdir/obj5') # Need to make sure the bucket listing is caught-up, otherwise the # first rsync may not see .obj2 and overwrite it. self.AssertNObjectsInBucket(bucket_uri, 3) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-d', tmpdir, suri(bucket_uri)]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) # Dir should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir/obj3'])) # Bucket should have content like dir but without the subdir objects # synchronized. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/subdir/obj5'])) # Assert that the src/dest objects that had same length but different # content were synchronized (dir to bucket rsync uses checksums as a # backup to mtime unless you specify -c to make hashes the priority). with open(os.path.join(tmpdir, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) cloud_obj2_content = self.RunGsUtil( ['cat', suri(bucket_uri, '.obj2')], return_stdout=True) if self._use_gcloud_storage: # Gcloud uses ctime if mtime is missing, so if the creation of cloud # .obj2 is close enough to the local creation, the times may match. # This means no fallback to checksums and then copy occurs. local_obj2_mtime = int(os.path.getmtime(o2_path)) cloud_obj2_ctime = ConvertDatetimeToPOSIX( self._GetMetadataAttribute(bucket_uri.bucket_name, '.obj2', 'timeCreated')) self.assertTrue(cloud_obj2_content == '.obj2' or local_obj2_mtime == cloud_obj2_ctime) else: self.assertEqual('.obj2', cloud_obj2_content) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', tmpdir, suri(bucket_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() # Now rerun the rsync with the -c option. # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check3(): """Tests rsync -c works as expected.""" self.RunGsUtil(['rsync', '-d', '-c', tmpdir, suri(bucket_uri)]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) # Dir should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir/obj3'])) # Bucket should have content like dir but without the subdir objects # synchronized. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/subdir/obj5'])) # Assert that the src/dest objects that had same length but different # content were synchronized (dir to bucket rsync with -c uses checksums). with open(os.path.join(tmpdir, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) self.assertEqual( '.obj2', self.RunGsUtil(['cat', suri(bucket_uri, '.obj2')], return_stdout=True)) _Check3() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check4(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil(['rsync', '-d', '-c', tmpdir, suri(bucket_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check4() # Now add and remove some objects in dir and bucket and test rsync -r. self.CreateTempFile(tmpdir=tmpdir, file_name='obj6', contents=b'obj6') self.CreateObject(bucket_uri=bucket_uri, object_name='obj7', contents=b'obj7') os.unlink(os.path.join(tmpdir, 'obj1')) self.RunGsUtil(['rm', suri(bucket_uri, '.obj2')]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check5(): self.RunGsUtil(['rsync', '-d', '-r', tmpdir, suri(bucket_uri)]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) # Dir should have un-altered content. self.assertEqual(listing1, set(['/.obj2', '/obj6', '/subdir/obj3'])) # Bucket should have content like dir but without the subdir objects # synchronized. self.assertEqual(listing2, set(['/.obj2', '/obj6', '/subdir/obj3'])) _Check5() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check6(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil(['rsync', '-d', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check6() @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_dir_to_dir_mtime(self): """Tests that flat and recursive rsync dir to dir works correctly.""" # Create 2 dirs with 1 overlapping file, 1 extra file at root # level in each, and 1 extra file 1 level down in each, where one of the # objects starts with "." to test that we don't skip those objects. Make the # overlapping files named the same but with different content, to test # that we detect and properly copy in that case. tmpdir1 = self.CreateTempDir() tmpdir2 = self.CreateTempDir() subdir1 = os.path.join(tmpdir1, 'subdir1') subdir2 = os.path.join(tmpdir2, 'subdir2') os.mkdir(subdir1) os.mkdir(subdir2) self.CreateTempFile(tmpdir=tmpdir1, file_name='obj1', contents=b'obj1', mtime=10) self.CreateTempFile(tmpdir=tmpdir1, file_name='.obj2', contents=b'.obj2', mtime=10) self.CreateTempFile(tmpdir=subdir1, file_name='obj3', contents=b'subdir1/obj3', mtime=10) self.CreateTempFile(tmpdir=tmpdir1, file_name='obj6', contents=b'obj6', mtime=100) self.CreateTempFile(tmpdir=tmpdir1, file_name='obj7', contents=b'obj7_', mtime=100) self.CreateTempFile(tmpdir=tmpdir2, file_name='.obj2', contents=b'.OBJ2', mtime=1000) self.CreateTempFile(tmpdir=tmpdir2, file_name='obj4', contents=b'obj4', mtime=10) self.CreateTempFile(tmpdir=subdir2, file_name='obj5', contents=b'subdir2/obj5', mtime=10) self.CreateTempFile(tmpdir=tmpdir2, file_name='obj6', contents=b'OBJ6', mtime=100) self.CreateTempFile(tmpdir=tmpdir2, file_name='obj7', contents=b'obj7', mtime=100) self.RunGsUtil(['rsync', '-r', '-d', tmpdir1, tmpdir2]) listing1 = TailSet(tmpdir1, self.FlatListDir(tmpdir1)) listing2 = TailSet(tmpdir2, self.FlatListDir(tmpdir2)) # dir1 should have un-altered content. self.assertEqual( listing1, set(['/obj1', '/.obj2', '/subdir1/obj3', '/obj6', '/obj7'])) # dir2 should now have content like dir1. self.assertEqual( listing2, set(['/obj1', '/.obj2', '/subdir1/obj3', '/obj6', '/obj7'])) # Assert that the src/dest objects that had same length but different # checksums were synchronized properly according to mtime. with open(os.path.join(tmpdir2, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) with open(os.path.join(tmpdir2, 'obj6')) as f: self.assertEqual('OBJ6', '\n'.join(f.readlines())) with open(os.path.join(tmpdir2, 'obj7')) as f: self.assertEqual('obj7_', '\n'.join(f.readlines())) def _Check1(): # Check that re-running the same rsync command causes no more changes. self.assertEqual( NO_CHANGES, self.RunGsUtil(['rsync', '-d', tmpdir1, tmpdir2], return_stderr=True)) _Check1() # Now rerun the rsync with the -c option. self.RunGsUtil(['rsync', '-r', '-d', '-c', tmpdir1, tmpdir2]) listing1 = TailSet(tmpdir1, self.FlatListDir(tmpdir1)) listing2 = TailSet(tmpdir2, self.FlatListDir(tmpdir2)) # dir1 should have un-altered content. self.assertEqual( listing1, set(['/obj1', '/.obj2', '/subdir1/obj3', '/obj6', '/obj7'])) # dir2 should now have content like dir1. self.assertEqual( listing2, set(['/obj1', '/.obj2', '/subdir1/obj3', '/obj6', '/obj7'])) # Assert that the src/dst objects that had same length, mtime, but different # content were synchronized (dir to dir rsync with -c uses checksums). with open(os.path.join(tmpdir1, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) with open(os.path.join(tmpdir1, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) with open(os.path.join(tmpdir2, 'obj6')) as f: self.assertEqual('obj6', '\n'.join(f.readlines())) def _Check2(): # Check that re-running the same rsync command causes no more changes. self.assertEqual( NO_CHANGES, self.RunGsUtil(['rsync', '-d', '-c', tmpdir1, tmpdir2], return_stderr=True)) _Check2() # Now add and remove some objects in both dirs and test rsync -r. os.unlink(os.path.join(tmpdir1, 'obj7')) os.unlink(os.path.join(tmpdir2, 'obj7')) self.CreateTempFile(tmpdir=tmpdir1, file_name='obj6', contents=b'obj6', mtime=10) self.CreateTempFile(tmpdir=tmpdir2, file_name='obj7', contents=b'obj7', mtime=100) os.unlink(os.path.join(tmpdir1, 'obj1')) os.unlink(os.path.join(tmpdir2, '.obj2')) self.RunGsUtil(['rsync', '-d', '-r', tmpdir1, tmpdir2]) listing1 = TailSet(tmpdir1, self.FlatListDir(tmpdir1)) listing2 = TailSet(tmpdir2, self.FlatListDir(tmpdir2)) # dir1 should have un-altered content. self.assertEqual(listing1, set(['/.obj2', '/obj6', '/subdir1/obj3'])) # dir2 should have content like dir1. self.assertEqual(listing2, set(['/.obj2', '/obj6', '/subdir1/obj3'])) def _Check3(): # Check that re-running the same rsync command causes no more changes. self.assertEqual( NO_CHANGES, self.RunGsUtil(['rsync', '-d', '-r', tmpdir1, tmpdir2], return_stderr=True)) _Check3() @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_dir_to_dir_minus_d(self): """Tests that flat and recursive rsync dir to dir works correctly.""" # Create 2 dirs with 1 overlapping file, 1 extra file at root # level in each, and 1 extra file 1 level down in each, where one of the # objects starts with "." to test that we don't skip those objects. Make the # overlapping files named the same but with different content, to test # that we detect and properly copy in that case. tmpdir1 = self.CreateTempDir() tmpdir2 = self.CreateTempDir() subdir1 = os.path.join(tmpdir1, 'subdir1') subdir2 = os.path.join(tmpdir2, 'subdir2') os.mkdir(subdir1) os.mkdir(subdir2) self.CreateTempFile(tmpdir=tmpdir1, file_name='obj1', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir1, file_name='.obj2', contents=b'.obj2') self.CreateTempFile(tmpdir=subdir1, file_name='obj3', contents=b'subdir1/obj3') self.CreateTempFile(tmpdir=tmpdir2, file_name='.obj2', contents=b'.OBJ2') self.CreateTempFile(tmpdir=tmpdir2, file_name='obj4', contents=b'obj4') self.CreateTempFile(tmpdir=subdir2, file_name='obj5', contents=b'subdir2/obj5') self.RunGsUtil(['rsync', '-d', tmpdir1, tmpdir2]) listing1 = TailSet(tmpdir1, self.FlatListDir(tmpdir1)) listing2 = TailSet(tmpdir2, self.FlatListDir(tmpdir2)) # dir1 should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir1/obj3'])) # dir2 should have content like dir1 but without the subdir1 objects # synchronized. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/subdir2/obj5'])) # Assert that the src/dest objects that had same length but different # checksums were not synchronized (dir to dir rsync doesn't use checksums # unless you specify -c). with open(os.path.join(tmpdir1, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) with open(os.path.join(tmpdir2, '.obj2')) as f: self.assertEqual('.OBJ2', '\n'.join(f.readlines())) # Don't use @Retry since this is a dir-to-dir test, thus we don't need to # worry about eventual consistency of bucket listings. This also allows us # to make sure that we don't miss any unintended behavior when a first # attempt behaves incorrectly and a subsequent retry behaves correctly. def _Check1(): # Check that re-running the same rsync command causes no more changes. self.assertEqual( NO_CHANGES, self.RunGsUtil(['rsync', '-d', tmpdir1, tmpdir2], return_stderr=True)) _Check1() # Now rerun the rsync with the -c option. self.RunGsUtil(['rsync', '-d', '-c', tmpdir1, tmpdir2]) listing1 = TailSet(tmpdir1, self.FlatListDir(tmpdir1)) listing2 = TailSet(tmpdir2, self.FlatListDir(tmpdir2)) # dir1 should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir1/obj3'])) # dir2 should have content like dir but without the subdir objects # synchronized. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/subdir2/obj5'])) # Assert that the src/dest objects that had same length but different # content were synchronized (dir to dir rsync with -c uses checksums). with open(os.path.join(tmpdir1, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) with open(os.path.join(tmpdir1, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) # Don't use @Retry since this is a dir-to-dir test, thus we don't need to # worry about eventual consistency of bucket listings. def _Check2(): # Check that re-running the same rsync command causes no more changes. self.assertEqual( NO_CHANGES, self.RunGsUtil(['rsync', '-d', '-c', tmpdir1, tmpdir2], return_stderr=True)) _Check2() # Now add and remove some objects in both dirs and test rsync -r. self.CreateTempFile(tmpdir=tmpdir1, file_name='obj6', contents=b'obj6') self.CreateTempFile(tmpdir=tmpdir2, file_name='obj7', contents=b'obj7') os.unlink(os.path.join(tmpdir1, 'obj1')) os.unlink(os.path.join(tmpdir2, '.obj2')) self.RunGsUtil(['rsync', '-d', '-r', tmpdir1, tmpdir2]) listing1 = TailSet(tmpdir1, self.FlatListDir(tmpdir1)) listing2 = TailSet(tmpdir2, self.FlatListDir(tmpdir2)) # dir1 should have un-altered content. self.assertEqual(listing1, set(['/.obj2', '/obj6', '/subdir1/obj3'])) # dir2 should have content like dir but without the subdir objects # synchronized. self.assertEqual(listing2, set(['/.obj2', '/obj6', '/subdir1/obj3'])) # Don't use @Retry since this is a dir-to-dir test, thus we don't need to # worry about eventual consistency of bucket listings. def _Check3(): # Check that re-running the same rsync command causes no more changes. self.assertEqual( NO_CHANGES, self.RunGsUtil(['rsync', '-d', '-r', tmpdir1, tmpdir2], return_stderr=True)) _Check3() # Create 2 dirs and add a file to the first. Then create another file and # add it to both dirs, making sure its filename evaluates to greater than # the previous filename. Make sure both files are present in the second # dir after issuing an rsync -d command. tmpdir1 = self.CreateTempDir() tmpdir2 = self.CreateTempDir() self.CreateTempFile(tmpdir=tmpdir1, file_name='obj1', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir1, file_name='obj2', contents=b'obj2') self.CreateTempFile(tmpdir=tmpdir2, file_name='obj2', contents=b'obj2') self.RunGsUtil(['rsync', '-d', tmpdir1, tmpdir2]) listing1 = TailSet(tmpdir1, self.FlatListDir(tmpdir1)) listing2 = TailSet(tmpdir2, self.FlatListDir(tmpdir2)) # First dir should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/obj2'])) # Second dir should have same content as first. self.assertEqual(listing2, set(['/obj1', '/obj2'])) # Don't use @Retry since this is a dir-to-dir test, thus we don't need to # worry about eventual consistency of bucket listings. def _Check4(): # Check that re-running the same rsync command causes no more changes. self.assertEqual( NO_CHANGES, self.RunGsUtil(['rsync', '-d', tmpdir1, tmpdir2], return_stderr=True)) _Check4() def test_dir_to_dir_minus_d_more_files_than_bufsize(self): """Tests concurrently building listing from multiple tmp file ranges.""" # Create 2 dirs, where each dir has 1000 objects and differing names. tmpdir1 = self.CreateTempDir() tmpdir2 = self.CreateTempDir() for i in range(0, 1000): self.CreateTempFile(tmpdir=tmpdir1, file_name='d1-%s' % i, contents=b'x', mtime=(i + 1)) self.CreateTempFile(tmpdir=tmpdir2, file_name='d2-%s' % i, contents=b'y', mtime=i) # We open a new temp file each time we reach rsync_buffer_lines of # listing output. On Windows, this will result in a 'too many open file # handles' error, so choose a larger value so as not to open so many files. rsync_buffer_config = [('GSUtil', 'rsync_buffer_lines', '50' if IS_WINDOWS else '2')] # Run gsutil with config option to make buffer size << # files. with SetBotoConfigForTest(rsync_buffer_config): self.RunGsUtil(['rsync', '-d', tmpdir1, tmpdir2]) listing1 = TailSet(tmpdir1, self.FlatListDir(tmpdir1)) listing2 = TailSet(tmpdir2, self.FlatListDir(tmpdir2)) self.assertEqual(listing1, listing2) for i in range(0, 1000): self.assertEqual( i + 1, long(os.path.getmtime((os.path.join(tmpdir2, 'd1-%s' % i))))) with open(os.path.join(tmpdir2, 'd1-%s' % i)) as f: self.assertEqual('x', '\n'.join(f.readlines())) # Don't use @Retry since this is a dir-to-dir test, thus we don't need to # worry about eventual consistency of bucket listings. def _Check(): # Check that re-running the same rsync command causes no more changes. self.assertEqual( NO_CHANGES, self.RunGsUtil(['rsync', '-d', tmpdir1, tmpdir2], return_stderr=True)) _Check() @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_bucket_to_dir_compressed_encoding(self): temp_file = self.CreateTempFile(contents=b'foo', file_name='bar') bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() self.RunGsUtil(['cp', '-Z', temp_file, suri(bucket_uri)]) stderr = self.RunGsUtil(['rsync', suri(bucket_uri), tmpdir], return_stderr=True) # rsync should decompress the destination file. with open(os.path.join(tmpdir, 'bar'), 'rb') as fp: self.assertEqual(b'foo', fp.read()) self.assertIn('bar has a compressed content-encoding', stderr) @SequentialAndParallelTransfer @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_bucket_to_dir_mtime(self): """Tests bucket to dir with mtime at the source.""" # Create bucket and dir with overlapping content and other combinations of # mtime. bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) self.CreateObject(bucket_uri=bucket_uri, object_name='obj1', contents=b'obj1', mtime=5) self.CreateObject(bucket_uri=bucket_uri, object_name='.obj2', contents=b'.obj2', mtime=5) self.CreateObject(bucket_uri=bucket_uri, object_name='subdir/obj3', contents=b'subdir/obj3') self.CreateObject(bucket_uri=bucket_uri, object_name='obj4', contents=b'OBJ4') self.CreateObject(bucket_uri=bucket_uri, object_name='obj6', contents=b'obj6', mtime=50) self.CreateObject(bucket_uri=bucket_uri, object_name='obj7', contents=b'obj7', mtime=5) self.CreateObject(bucket_uri=bucket_uri, object_name='obj8', contents=b'obj8', mtime=100) self.CreateObject(bucket_uri=bucket_uri, object_name='obj9', contents=b'obj9', mtime=25) self.CreateObject(bucket_uri=bucket_uri, object_name='obj10', contents=b'obj10') time_created = ConvertDatetimeToPOSIX( self._GetMetadataAttribute(bucket_uri.bucket_name, 'obj10', 'timeCreated')) self.CreateObject(bucket_uri=bucket_uri, object_name='obj11', contents=b'obj11_', mtime=75) self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.OBJ2', mtime=10) self.CreateTempFile(tmpdir=tmpdir, file_name='obj4', contents=b'obj4', mtime=100) self.CreateTempFile(tmpdir=subdir, file_name='obj5', contents=b'subdir/obj5', mtime=10) self.CreateTempFile(tmpdir=tmpdir, file_name='obj6', contents=b'obj6', mtime=50) self.CreateTempFile(tmpdir=tmpdir, file_name='obj7', contents=b'OBJ7', mtime=50) self.CreateTempFile(tmpdir=tmpdir, file_name='obj8', contents=b'obj8', mtime=10) self.CreateTempFile(tmpdir=tmpdir, file_name='obj9', contents=b'OBJ9', mtime=25) self.CreateTempFile(tmpdir=tmpdir, file_name='obj10', contents=b'OBJ10', mtime=time_created) self.CreateTempFile(tmpdir=tmpdir, file_name='obj11', contents=b'obj11', mtime=75) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-d', suri(bucket_uri), tmpdir]) listing1 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) listing2 = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Bucket should have un-altered content. self.assertEqual( listing1, set([ '/obj1', '/.obj2', '/subdir/obj3', '/obj4', '/obj6', '/obj7', '/obj8', '/obj9', '/obj10', '/obj11' ])) # Dir should have content like bucket except without sub-directories # synced. self.assertEqual( listing2, set([ '/obj1', '/.obj2', '/obj4', '/subdir/obj5', '/obj6', '/obj7', '/obj8', '/obj9', '/obj10', '/obj11' ])) # Assert that the dst objects that had an earlier mtime were not # synchronized because the source didn't have an mtime. with open(os.path.join(tmpdir, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) # Assert that obj4 was synchronized to dst because mtime cannot be used, # and the hashes are different. with open(os.path.join(tmpdir, 'obj4')) as f: self.assertEqual('OBJ4', '\n'.join(f.readlines())) # Verify obj9 and obj10 content didn't change because mtimes from src and # dst were equal. obj9 had mtimes that matched while the obj10 mtime was # equal to the creation time of the corresponding object. with open(os.path.join(tmpdir, 'obj9')) as f: self.assertEqual('OBJ9', '\n'.join(f.readlines())) # Also verifies if obj10 used time created to determine if a copy was # necessary. with open(os.path.join(tmpdir, 'obj10')) as f: self.assertEqual('OBJ10', '\n'.join(f.readlines())) with open(os.path.join(tmpdir, 'obj11')) as f: self.assertEqual('obj11_', '\n'.join(f.readlines())) _Check1() def _Check2(): """Verify mtime was set for objects at destination.""" self.assertEqual(long(os.path.getmtime(os.path.join(tmpdir, 'obj1'))), 5) self.assertEqual(long(os.path.getmtime(os.path.join(tmpdir, '.obj2'))), 5) self.assertEqual(long(os.path.getmtime(os.path.join(tmpdir, 'obj6'))), 50) self.assertEqual(long(os.path.getmtime(os.path.join(tmpdir, 'obj8'))), 100) self.assertEqual(long(os.path.getmtime(os.path.join(tmpdir, 'obj9'))), 25) _Check2() # Now rerun the rsync with the -c option. # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check3(): """Tests rsync -c works as expected.""" self.RunGsUtil(['rsync', '-r', '-d', '-c', suri(bucket_uri), tmpdir]) listing1 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) listing2 = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Bucket should have un-altered content. self.assertEqual( listing1, set([ '/obj1', '/.obj2', '/subdir/obj3', '/obj4', '/obj6', '/obj7', '/obj8', '/obj9', '/obj10', '/obj11' ])) # Dir should have content like bucket this time with subdirectories # synced. self.assertEqual( listing2, set([ '/obj1', '/.obj2', '/subdir/obj3', '/obj4', '/obj6', '/obj7', '/obj8', '/obj9', '/obj10', '/obj11' ])) # Assert that the contents of obj7 have now changed because the -c flag # was used to force checksums. self.assertEqual( 'obj7', self.RunGsUtil(['cat', suri(bucket_uri, 'obj7')], return_stdout=True)) self._VerifyObjectMtime(bucket_uri.bucket_name, 'obj7', '5') # Check the mtime of obj7 in the destination to see that it changed. self.assertEqual(long(os.path.getmtime(os.path.join(tmpdir, 'obj7'))), 5) # Verify obj9 and obj10 content has changed because hashes were used in # comparisons. with open(os.path.join(tmpdir, 'obj9')) as f: self.assertEqual('obj9', '\n'.join(f.readlines())) with open(os.path.join(tmpdir, 'obj10')) as f: self.assertEqual('obj10', '\n'.join(f.readlines())) _Check3() @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') def test_bucket_to_dir_preserve_posix_errors(self): """Tests that rsync -P works properly with files that would be orphaned.""" bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() primary_gid = os.stat(tmpdir).st_gid non_primary_gid = util.GetNonPrimaryGid() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) # obj1 - Invalid Mode obj1 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj1', contents=b'obj1', mode='222', uid=os.getuid()) # obj2 - Invalid GID obj2 = self.CreateObject(bucket_uri=bucket_uri, object_name='.obj2', contents=b'.obj2', gid=INVALID_GID(), mode='540') self.CreateObject(bucket_uri=bucket_uri, object_name='subdir/obj3', contents=b'subdir/obj3') # obj6 - Invalid GID obj6 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj6', contents=b'obj6', gid=INVALID_GID(), mode='440') # obj7 - Invalid Mode obj7 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj7', contents=b'obj7', gid=non_primary_gid, mode='333') # obj8 - Invalid UID obj8 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj8', contents=b'obj8', uid=INVALID_UID()) # obj9 - Invalid UID w/ mode 777 obj9 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj9', contents=b'obj9', uid=INVALID_UID(), mode='777') # obj10 - Invalid UID & GID obj10 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj10', contents=b'obj10', gid=INVALID_GID(), uid=INVALID_UID()) # obj11 - Invalid UID & GID with mode 554 obj11 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj11', contents=b'obj11', gid=INVALID_GID(), uid=INVALID_UID(), mode='544') # obj12 - Invalid UID & GID obj12 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj12', contents=b'obj12', uid=INVALID_UID(), gid=USER_ID) # obj13 - Invalid UID, good GID, mode 644 obj13 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj13', contents=b'obj13', uid=INVALID_UID(), gid=primary_gid, mode='644') # obj14 - Good UID, Invalid GID, no mode specified obj14 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj14', contents=b'obj14', uid=USER_ID, gid=INVALID_GID()) # obj15 - Good UID, Invalid GID, mode 655 obj15 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj15', contents=b'obj15', uid=USER_ID, gid=INVALID_GID(), mode='655') # obj16 - Invalid Permissions - mode 244 obj16 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj16', contents=b'obj16', uid=USER_ID, mode='244') # obj17 - Invalid Mode obj17 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj17', contents=b'obj17', uid=USER_ID, gid=primary_gid, mode='222') # obj18 - Invalid GID, mode 333 obj18 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj18', contents=b'obj18', uid=USER_ID, gid=non_primary_gid, mode='333') # obj19 - No UID/GID with mod 222 obj19 = self.CreateObject(bucket_uri=bucket_uri, object_name='obj19', contents=b'obj19', mode='222') self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.OBJ2') self.CreateTempFile(tmpdir=tmpdir, file_name='obj4', contents=b'obj4') self.CreateTempFile(tmpdir=subdir, file_name='obj5', contents=b'subdir/obj5') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests that an exception is thrown because files will be orphaned.""" stderr = self.RunGsUtil( ['rsync', '-P', '-r', suri(bucket_uri), tmpdir], expected_status=1, return_stderr=True) if self._use_gcloud_storage: gcloud_preserve_posix_warning = ( r'For preserving POSIX with rsync downloads, gsutil aborts if a single' r' download will result in invalid destination POSIX. However, this' ) self.assertRegex(stderr, gcloud_preserve_posix_warning) read_regex = r"{}#\d+\. User \d+ owns file, but owner does not have read" gid_regex = r"{}#\d+ metadata doesn't exist on current system\. GID" uid_regex = r"{}#\d+ metadata doesn't exist on current system\. UID" self.assertRegex(stderr, read_regex.format('obj1')) self.assertRegex(stderr, gid_regex.format('obj2')) self.assertRegex(stderr, gid_regex.format('obj6')) self.assertRegex(stderr, read_regex.format('obj7')) self.assertRegex(stderr, uid_regex.format('obj8')) self.assertRegex(stderr, uid_regex.format('obj9')) self.assertRegex(stderr, uid_regex.format('obj10')) self.assertRegex(stderr, uid_regex.format('obj11')) self.assertRegex(stderr, uid_regex.format('obj12')) self.assertRegex(stderr, uid_regex.format('obj13')) self.assertRegex(stderr, gid_regex.format('obj14')) self.assertRegex(stderr, gid_regex.format('obj15')) self.assertRegex(stderr, read_regex.format('obj16')) self.assertRegex(stderr, read_regex.format('obj17')) self.assertRegex(stderr, read_regex.format('obj18')) self.assertRegex(stderr, read_regex.format('obj19')) else: self.assertIn(ORPHANED_FILE, stderr) self.assertRegex(stderr, BuildErrorRegex(obj1, POSIX_MODE_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj2, POSIX_GID_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj6, POSIX_GID_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj7, POSIX_MODE_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj8, POSIX_UID_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj9, POSIX_UID_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj10, POSIX_UID_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj11, POSIX_UID_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj12, POSIX_UID_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj13, POSIX_UID_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj14, POSIX_GID_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj15, POSIX_GID_ERROR)) self.assertRegex( stderr, BuildErrorRegex(obj16, POSIX_INSUFFICIENT_ACCESS_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj17, POSIX_MODE_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj18, POSIX_MODE_ERROR)) self.assertRegex(stderr, BuildErrorRegex(obj19, POSIX_MODE_ERROR)) listing1 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) listing2 = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Bucket should have un-altered content. self.assertEqual( listing1, set([ '/obj1', '/.obj2', '/subdir/obj3', '/obj6', '/obj7', '/obj8', '/obj9', '/obj10', '/obj11', '/obj12', '/obj13', '/obj14', '/obj15', '/obj16', '/obj17', '/obj18', '/obj19' ])) if self._use_gcloud_storage: # Gcloud still performs valid copies whereas gsutil aborts if one # will result in invalid POSIX. self.assertEqual( listing2, set(['/.obj2', '/obj4', '/subdir/obj3', '/subdir/obj5'])) else: # Dir should have un-altered content. self.assertEqual(listing2, set(['/.obj2', '/obj4', '/subdir/obj5'])) _Check1() # Set a valid mode for another object. self._SetObjectCustomMetadataAttribute(self.default_provider, bucket_uri.bucket_name, '.obj2', MODE_ATTR, '640') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): """Tests that a file with a valid mode in metadata, nothing changed.""" # Test that even though a file now has a valid mode, no files were copied. stderr = self.RunGsUtil( ['rsync', '-P', '-r', suri(bucket_uri), tmpdir], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn("doesn't exist on current system. GID:", stderr) else: self.assertIn(ORPHANED_FILE, stderr) listing1 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) listing2 = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Bucket should have un-altered content. self.assertEqual( listing1, set([ '/obj1', '/.obj2', '/subdir/obj3', '/obj6', '/obj7', '/obj8', '/obj9', '/obj10', '/obj11', '/obj12', '/obj13', '/obj14', '/obj15', '/obj16', '/obj17', '/obj18', '/obj19' ])) if self._use_gcloud_storage: # See note from Check1. self.assertEqual( listing2, set(['/.obj2', '/obj4', '/subdir/obj3', '/subdir/obj5'])) else: self.assertEqual(listing2, set(['/.obj2', '/obj4', '/subdir/obj5'])) _Check2() @SequentialAndParallelTransfer @unittest.skipIf(IS_WINDOWS, 'POSIX attributes not available on Windows.') @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_bucket_to_dir_preserve_posix_no_errors(self): """Tests that rsync -P works properly with default file attributes.""" bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() primary_gid = os.stat(tmpdir).st_gid non_primary_gid = util.GetNonPrimaryGid() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) self.CreateObject(bucket_uri=bucket_uri, object_name='obj1', contents=b'obj1', mode='444') self.CreateObject(bucket_uri=bucket_uri, object_name='.obj2', contents=b'.obj2', gid=primary_gid) self.CreateObject(bucket_uri=bucket_uri, object_name='subdir/obj3', contents=b'subdir/obj3', gid=non_primary_gid) self.CreateObject(bucket_uri=bucket_uri, object_name='obj6', contents=b'obj6', gid=primary_gid, mode='555') self.CreateObject(bucket_uri=bucket_uri, object_name='obj7', contents=b'obj7', gid=non_primary_gid, mode='444') self.CreateObject(bucket_uri=bucket_uri, object_name='obj8', contents=b'obj8', uid=USER_ID) self.CreateObject(bucket_uri=bucket_uri, object_name='obj9', contents=b'obj9', uid=USER_ID, mode='422') self.CreateObject(bucket_uri=bucket_uri, object_name='obj10', contents=b'obj10', uid=USER_ID, gid=primary_gid) self.CreateObject(bucket_uri=bucket_uri, object_name='obj11', contents=b'obj11', uid=USER_ID, gid=non_primary_gid) self.CreateObject(bucket_uri=bucket_uri, object_name='obj12', contents=b'obj12', uid=USER_ID, gid=primary_gid, mode='400') self.CreateObject(bucket_uri=bucket_uri, object_name='obj13', contents=b'obj13', uid=USER_ID, gid=non_primary_gid, mode='533') self.CreateObject(bucket_uri=bucket_uri, object_name='obj14', contents=b'obj14', uid=USER_ID, mode='444') self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.OBJ2') self.CreateTempFile(tmpdir=tmpdir, file_name='obj4', contents=b'obj4') self.CreateTempFile(tmpdir=subdir, file_name='obj5', contents=b'subdir/obj5') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Verifies that all attributes were copied correctly when -P is used.""" self.RunGsUtil(['rsync', '-P', '-r', suri(bucket_uri), tmpdir]) listing1 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) listing2 = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Bucket should have un-altered content. self.assertEqual( listing1, set([ '/obj1', '/.obj2', '/subdir/obj3', '/obj6', '/obj7', '/obj8', '/obj9', '/obj10', '/obj11', '/obj12', '/obj13', '/obj14' ])) # Dir should have any new content from bucket. self.assertEqual( listing2, set([ '/obj1', '/.obj2', '/subdir/obj3', '/obj4', '/subdir/obj5', '/obj6', '/obj7', '/obj8', '/obj9', '/obj10', '/obj11', '/obj12', '/obj13', '/obj14' ])) _Check1() self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj1'), uid=os.getuid(), mode=0o444) self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, '.obj2'), gid=primary_gid, uid=os.getuid(), mode=DEFAULT_MODE) self.VerifyLocalPOSIXPermissions(os.path.join(subdir, 'obj3'), gid=non_primary_gid, mode=DEFAULT_MODE) self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj6'), gid=primary_gid, mode=0o555) self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj7'), gid=non_primary_gid, mode=0o444) self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj8'), gid=primary_gid, mode=DEFAULT_MODE) self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj9'), uid=USER_ID, mode=0o422) self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj10'), uid=USER_ID, gid=primary_gid, mode=DEFAULT_MODE) self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj11'), uid=USER_ID, gid=non_primary_gid, mode=DEFAULT_MODE) self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj12'), uid=USER_ID, gid=primary_gid, mode=0o400) self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj13'), uid=USER_ID, gid=non_primary_gid, mode=0o533) self.VerifyLocalPOSIXPermissions(os.path.join(tmpdir, 'obj14'), uid=USER_ID, mode=0o444) @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_bucket_to_dir_minus_d(self): """Tests that flat and recursive rsync bucket to dir works correctly.""" # Create bucket and dir with 1 overlapping object, 1 extra object at root # level in each, and 1 extra object 1 level down in each, where one of the # objects starts with "." to test that we don't skip those objects. Make the # overlapping objects named the same but with different content, to test # that we detect and properly copy in that case. bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) self.CreateObject(bucket_uri=bucket_uri, object_name='obj1', contents=b'obj1') # Set the mtime for obj2 because obj2 in the cloud and obj2 on the local # file system have the potential to be created during the same second. self.CreateObject(bucket_uri=bucket_uri, object_name='.obj2', contents=b'.obj2', mtime=0) self.CreateObject(bucket_uri=bucket_uri, object_name='subdir/obj3', contents=b'subdir/obj3') self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.OBJ2') self.CreateTempFile(tmpdir=tmpdir, file_name='obj4', contents=b'obj4') self.CreateTempFile(tmpdir=subdir, file_name='obj5', contents=b'subdir/obj5') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-d', suri(bucket_uri), tmpdir]) listing1 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) listing2 = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Bucket should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir/obj3'])) # Dir should have content like bucket but without the subdir objects # synchronized. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/subdir/obj5'])) # Assert that the src/dest objects that had same length but different # content were synchronized (bucket to dir rsync uses checksums as a # backup to mtime unless you specify -c). self.assertEqual( '.obj2', self.RunGsUtil(['cat', suri(bucket_uri, '.obj2')], return_stdout=True)) with open(os.path.join(tmpdir, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', suri(bucket_uri), tmpdir], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() # Now rerun the rsync with the -c option. # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check3(): """Tests rsync -c works as expected.""" self.RunGsUtil(['rsync', '-d', '-c', suri(bucket_uri), tmpdir]) listing1 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) listing2 = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Bucket should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir/obj3'])) # Dir should have content like bucket but without the subdir objects # synchronized. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/subdir/obj5'])) # Assert that the src/dest objects that had same length but different # content were synchronized (bucket to dir rsync with -c uses checksums). self.assertEqual( '.obj2', self.RunGsUtil(['cat', suri(bucket_uri, '.obj2')], return_stdout=True)) with open(os.path.join(tmpdir, '.obj2')) as f: self.assertEqual('.obj2', '\n'.join(f.readlines())) _Check3() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check4(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', '-c', suri(bucket_uri), tmpdir], return_stderr=True) self._VerifyNoChanges(stderr) _Check4() # Now add and remove some objects in dir and bucket and test rsync -r. self.CreateObject(bucket_uri=bucket_uri, object_name='obj6', contents=b'obj6') self.CreateTempFile(tmpdir=tmpdir, file_name='obj7', contents=b'obj7') self.RunGsUtil(['rm', suri(bucket_uri, 'obj1')]) os.unlink(os.path.join(tmpdir, '.obj2')) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check5(): self.RunGsUtil(['rsync', '-d', '-r', suri(bucket_uri), tmpdir]) listing1 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) listing2 = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Bucket should have un-altered content. self.assertEqual(listing1, set(['/.obj2', '/obj6', '/subdir/obj3'])) # Dir should have content like bucket but without the subdir objects # synchronized. self.assertEqual(listing2, set(['/.obj2', '/obj6', '/subdir/obj3'])) _Check5() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check6(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', '-r', suri(bucket_uri), tmpdir], return_stderr=True) self._VerifyNoChanges(stderr) _Check6() @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') def test_bucket_to_dir_minus_d_with_fname_case_change(self): """Tests that name case changes work correctly. Example: Windows filenames are case-preserving in what you wrote, but case- insensitive when compared. If you synchronize from FS to cloud and then change case-naming in local files, you could end up with this situation: Cloud copy is called .../TiVo/... FS copy is called .../Tivo/... Then, if you rsync from cloud to FS, if rsync doesn't recognize that on Windows these names are identical, each rsync run will cause both a copy and a delete to be executed. """ # Create bucket and dir with same objects, but dir copy has different name # case. bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() self.CreateObject(bucket_uri=bucket_uri, object_name='obj1', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir, file_name='Obj1', contents=b'obj1') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" output = self.RunGsUtil( ['rsync', '-d', '-r', suri(bucket_uri), tmpdir], return_stderr=True) # Nothing should be copied or removed under Windows. if IS_WINDOWS: self.assertEqual(NO_CHANGES, output) else: self.assertNotEqual(NO_CHANGES, output) _Check1() @unittest.skipUnless(UsingCrcmodExtension(), 'Test requires fast crcmod.') @SkipForS3('The boto lib used for S3 does not handle objects' ' starting with slashes if we use V4 signature.') def test_bucket_to_dir_minus_d_with_leftover_dir_placeholder(self): """Tests that we correctly handle leftover dir placeholders. See comments in gslib.commands.rsync._FieldedListingIterator for details. """ bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() self.CreateObject(bucket_uri=bucket_uri, object_name='obj1', contents=b'obj1') # Create a placeholder like what can be left over by web GUI tools. key_uri = self.StorageUriCloneReplaceName(bucket_uri, '/') self.StorageUriSetContentsFromString(key_uri, '') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil( ['rsync', '-d', '-r', suri(bucket_uri), tmpdir], return_stderr=True) listing1 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) listing2 = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Bucket should have un-altered content. self.assertEqual(listing1, set(['/obj1', '//'])) # Bucket should not have the placeholder object. self.assertEqual(listing2, set(['/obj1'])) _Check1() @unittest.skipIf(IS_WINDOWS, 'os.symlink() is not available on Windows.') def test_rsync_minus_r_minus_e(self): """Tests that rsync -e -r ignores symlinks when recursing.""" bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) os.mkdir(os.path.join(tmpdir, 'missing')) # Create a blank directory that is a broken symlink to ensure that we # don't fail recursive enumeration with a bad symlink. os.symlink(os.path.join(tmpdir, 'missing'), os.path.join(subdir, 'missing')) os.rmdir(os.path.join(tmpdir, 'missing')) self.RunGsUtil(['rsync', '-r', '-e', tmpdir, suri(bucket_uri)]) @unittest.skipIf(IS_WINDOWS, 'os.symlink() is not available on Windows.') def test_rsync_minus_d_minus_e(self): """Tests that rsync -e ignores symlinks.""" tmpdir = self.CreateTempDir() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) bucket_uri = self.CreateBucket() fpath1 = self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.obj2') self.CreateTempFile(tmpdir=subdir, file_name='obj3', contents=b'subdir/obj3') good_symlink_path = os.path.join(tmpdir, 'symlink1') os.symlink(fpath1, good_symlink_path) # Make a symlink that points to a non-existent path to test that -e also # handles that case. bad_symlink_path = os.path.join(tmpdir, 'symlink2') os.symlink(os.path.join('/', 'non-existent'), bad_symlink_path) self.CreateObject(bucket_uri=bucket_uri, object_name='.obj2', contents=b'.OBJ2') self.CreateObject(bucket_uri=bucket_uri, object_name='obj4', contents=b'obj4') self.CreateObject(bucket_uri=bucket_uri, object_name='subdir/obj5', contents=b'subdir/obj5') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Ensure listings match the commented expectations.""" self.RunGsUtil(['rsync', '-d', '-e', tmpdir, suri(bucket_uri)]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) # Dir should have un-altered content. self.assertEqual( listing1, set(['/obj1', '/.obj2', '/subdir/obj3', '/symlink1', '/symlink2'])) # Bucket should have content like dir but without the symlink, and # without subdir objects synchronized. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/subdir/obj5'])) _Check1() # Now remove invalid symlink and run without -e, and see that symlink gets # copied (as file to which it points). Use @Retry as hedge against bucket # listing eventual consistency. os.unlink(bad_symlink_path) @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-d', tmpdir, suri(bucket_uri)]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) # Dir should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir/obj3', '/symlink1'])) # Bucket should have content like dir but without the symlink, and # without subdir objects synchronized. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/subdir/obj5', '/symlink1'])) self.assertEqual( 'obj1', self.RunGsUtil(['cat', suri(bucket_uri, 'symlink1')], return_stdout=True)) _Check2() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check3(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', tmpdir, suri(bucket_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check3() @SkipForS3('S3 does not support composite objects') def test_bucket_to_bucket_minus_d_with_composites(self): """Tests that rsync works with composite objects (which don't have MD5s).""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents=b'obj1') self.CreateObject(bucket_uri=bucket1_uri, object_name='.obj2', contents=b'.obj2') self.RunGsUtil([ 'compose', suri(bucket1_uri, 'obj1'), suri(bucket1_uri, '.obj2'), suri(bucket1_uri, 'obj3') ]) self.CreateObject(bucket_uri=bucket2_uri, object_name='.obj2', contents=b'.OBJ2') self.CreateObject(bucket_uri=bucket2_uri, object_name='obj4', contents=b'obj4') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): self.RunGsUtil(['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)]) listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri)) listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri)) # First bucket should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/obj3'])) # Second bucket should have content like first bucket but without the # subdir objects synchronized. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/obj3'])) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() def test_bucket_to_bucket_minus_d_empty_dest(self): """Tests working with empty dest bucket (iter runs out before src iter).""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents=b'obj1') self.CreateObject(bucket_uri=bucket1_uri, object_name='.obj2', contents=b'.obj2') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): self.RunGsUtil(['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)]) listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri)) listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri)) self.assertEqual(listing1, set(['/obj1', '/.obj2'])) self.assertEqual(listing2, set(['/obj1', '/.obj2'])) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() def test_bucket_to_bucket_minus_d_empty_src(self): """Tests working with empty src bucket (iter runs out before dst iter).""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket2_uri, object_name='obj1', contents=b'obj1') self.CreateObject(bucket_uri=bucket2_uri, object_name='.obj2', contents=b'.obj2') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): self.RunGsUtil(['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)]) stderr = self.RunGsUtil(['ls', suri(bucket1_uri, '**')], expected_status=1, return_stderr=True) self.assertIn('One or more URLs matched no objects', stderr) stderr = self.RunGsUtil(['ls', suri(bucket2_uri, '**')], expected_status=1, return_stderr=True) self.assertIn('One or more URLs matched no objects', stderr) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() def test_rsync_minus_d_minus_p(self): """Tests that rsync -p preserves ACLs.""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents=b'obj1') # Set public-read (non-default) ACL so we can verify that rsync -p works. self.RunGsUtil(['acl', 'set', 'public-read', suri(bucket1_uri, 'obj1')]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync -p works as expected.""" self.RunGsUtil( ['rsync', '-d', '-p', suri(bucket1_uri), suri(bucket2_uri)]) listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri)) listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri)) self.assertEqual(listing1, set(['/obj1'])) self.assertEqual(listing2, set(['/obj1'])) acl1_json = self.RunGsUtil( ['acl', 'get', suri(bucket1_uri, 'obj1')], return_stdout=True) acl2_json = self.RunGsUtil( ['acl', 'get', suri(bucket2_uri, 'obj1')], return_stdout=True) self.assertEqual(acl1_json, acl2_json) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', '-p', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() def test_rsync_canned_acl(self): """Tests that rsync -a applies ACLs.""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents=b'obj1') self.RunGsUtil(['acl', 'get', suri(bucket1_uri, 'obj1')]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): """Tests rsync -a works as expected.""" self.RunGsUtil([ 'rsync', '-d', '-a', 'public-read', suri(bucket1_uri), suri(bucket2_uri) ]) listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri)) listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri)) self.assertEqual(listing1, set(['/obj1'])) self.assertEqual(listing2, set(['/obj1'])) # Set public-read on the original key after the rsync so we can compare # the ACLs. self.RunGsUtil(['acl', 'set', 'public-read', suri(bucket1_uri, 'obj1')]) acl1_json = self.RunGsUtil( ['acl', 'get', suri(bucket1_uri, 'obj1')], return_stdout=True) acl2_json = self.RunGsUtil( ['acl', 'get', suri(bucket2_uri, 'obj1')], return_stdout=True) self.assertEqual(acl1_json, acl2_json) _Check() def test_rsync_to_nonexistent_bucket_subdir(self): """Tests that rsync to non-existent bucket subdir works.""" # Create dir with some objects and empty bucket. tmpdir = self.CreateTempDir() subdir = os.path.join(tmpdir, 'subdir') os.mkdir(subdir) bucket_url = self.CreateBucket() self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.obj2') self.CreateTempFile(tmpdir=subdir, file_name='obj3', contents=b'subdir/obj3') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-r', tmpdir, suri(bucket_url, 'subdir')]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet( suri(bucket_url, 'subdir'), self.FlatListBucket( self.StorageUriCloneReplaceName(bucket_url, 'subdir'))) # Dir should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/subdir/obj3'])) # Bucket subdir should have content like dir. self.assertEqual(listing2, set(['/obj1', '/.obj2', '/subdir/obj3'])) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-r', tmpdir, suri(bucket_url, 'subdir')], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() def test_rsync_to_nonexistent_bucket_subdir_prefix_of_existing_obj(self): """Tests that rsync with destination url as a prefix of existing obj works. Test to make sure that a dir/subdir gets created if it does not exist even when the new dir is a prefix of an existing dir. e.g if gs://some_bucket/foobar exists, and we run the command rsync some_dir gs://some_bucket/foo this should create a subdir foo """ # Create dir with some objects and empty bucket. tmpdir = self.CreateTempDir() bucket_url = self.CreateBucket() self.CreateObject(bucket_uri=bucket_url, object_name='foobar', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.obj2') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-r', tmpdir, suri(bucket_url, 'foo')]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet( suri(bucket_url, 'foo'), self.FlatListBucket(self.StorageUriCloneReplaceName( bucket_url, 'foo'))) # Dir should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2'])) # Bucket subdir should have content like dir. self.assertEqual(listing2, set(['/obj1', '/.obj2'])) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil(['rsync', '-r', tmpdir, suri(bucket_url, 'foo')], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() def test_rsync_from_nonexistent_bucket(self): """Tests that rsync from a non-existent bucket subdir fails gracefully.""" tmpdir = self.CreateTempDir() self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.obj2') bucket_url_str = '%s://%s' % (self.default_provider, self.nonexistent_bucket_name) # TODO(b/135780661): Remove retry after bug resolved @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stderr = self.RunGsUtil(['rsync', '-d', bucket_url_str, tmpdir], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('not found: 404', stderr) else: self.assertIn('Caught non-retryable exception', stderr) listing = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Dir should have un-altered content. self.assertEqual(listing, set(['/obj1', '/.obj2'])) _Check() def test_rsync_to_nonexistent_bucket(self): """Tests that rsync from a non-existent bucket subdir fails gracefully.""" tmpdir = self.CreateTempDir() self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.obj2') bucket_url_str = '%s://%s' % (self.default_provider, self.nonexistent_bucket_name) # TODO(b/135780661): Remove retry after bug resolved @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stderr = self.RunGsUtil(['rsync', '-d', bucket_url_str, tmpdir], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn('not found: 404', stderr) else: self.assertIn('Caught non-retryable exception', stderr) listing = TailSet(tmpdir, self.FlatListDir(tmpdir)) # Dir should have un-altered content. self.assertEqual(listing, set(['/obj1', '/.obj2'])) _Check() def test_bucket_to_bucket_minus_d_with_overwrite_and_punc_chars(self): """Tests that punc chars in filenames don't confuse sort order.""" bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() # Create 2 objects in each bucket, with one overwritten with a name that's # less than the next name in destination bucket when encoded, but not when # compared without encoding. self.CreateObject(bucket_uri=bucket1_uri, object_name='e/obj1', contents=b'obj1') self.CreateObject(bucket_uri=bucket1_uri, object_name='e-1/.obj2', contents=b'.obj2') self.CreateObject(bucket_uri=bucket2_uri, object_name='e/obj1', contents=b'OBJ1') self.CreateObject(bucket_uri=bucket2_uri, object_name='e-1/.obj2', contents=b'.obj2') # Need to make sure the bucket listings are caught-up, otherwise the # rsync may not see all objects and fail to synchronize correctly. self.AssertNObjectsInBucket(bucket1_uri, 2) self.AssertNObjectsInBucket(bucket2_uri, 2) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-rd', suri(bucket1_uri), suri(bucket2_uri)]) listing1 = TailSet(suri(bucket1_uri), self.FlatListBucket(bucket1_uri)) listing2 = TailSet(suri(bucket2_uri), self.FlatListBucket(bucket2_uri)) # First bucket should have un-altered content. self.assertEqual(listing1, set(['/e/obj1', '/e-1/.obj2'])) self.assertEqual(listing2, set(['/e/obj1', '/e-1/.obj2'])) # Assert correct contents. self.assertEqual( 'obj1', self.RunGsUtil(['cat', suri(bucket2_uri, 'e/obj1')], return_stdout=True)) self.assertEqual( '.obj2', self.RunGsUtil(['cat', suri(bucket2_uri, 'e-1/.obj2')], return_stdout=True)) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', suri(bucket1_uri), suri(bucket2_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() def _test_dir_to_bucket_regex_paramaterized(self, flag): """Tests that rsync regex exclusions work correctly.""" # Create dir and bucket with 1 overlapping and 2 extra objects in each. tmpdir = self.CreateTempDir() bucket_uri = self.CreateBucket() self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1') self.CreateTempFile(tmpdir=tmpdir, file_name='.obj2', contents=b'.obj2') self.CreateTempFile(tmpdir=tmpdir, file_name='obj3', contents=b'obj3') self.CreateObject(bucket_uri=bucket_uri, object_name='.obj2', contents=b'.obj2') self.CreateObject(bucket_uri=bucket_uri, object_name='obj4', contents=b'obj4') self.CreateObject(bucket_uri=bucket_uri, object_name='obj5', contents=b'obj5') # Need to make sure the bucket listing is caught-up, otherwise the # first rsync may not see .obj2 and overwrite it. self.AssertNObjectsInBucket(bucket_uri, 3) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-d', flag, 'obj[34]', tmpdir, suri(bucket_uri)]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) # Dir should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/.obj2', '/obj3'])) # Bucket should have content like dir but ignoring obj3 from dir and not # deleting obj4 from bucket (per exclude regex). self.assertEqual(listing2, set(['/obj1', '/.obj2', '/obj4'])) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): # Check that re-running the same rsync command causes no more changes. stderr = self.RunGsUtil( ['rsync', '-d', flag, 'obj[34]', tmpdir, suri(bucket_uri)], return_stderr=True) self._VerifyNoChanges(stderr) _Check2() def test_dir_to_bucket_minus_x(self): """Tests that rsync regex exclusions work correctly for -x.""" self._test_dir_to_bucket_regex_paramaterized('-x') def test_dir_to_bucket_minus_y(self): """Tests that rsync regex exclusions work correctly for -y.""" self._test_dir_to_bucket_regex_paramaterized('-y') def _test_dir_to_bucket_regex_negative_lookahead(self, flag, includes): """Tests if negative lookahead includes files/objects.""" bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir(test_files=[ 'a', 'b', 'c', ('data1', 'a.txt'), ('data1', 'ok'), ('data2', 'b.txt'), ('data3', 'data4', 'c.txt') ]) self.RunGsUtil( ['rsync', '-r', flag, r'^(?!.*\.txt$).*', tmpdir, suri(bucket_uri)], return_stderr=True) listing = TailSet(tmpdir, self.FlatListDir(tmpdir)) self.assertEqual( listing, set([ '/a', '/b', '/c', '/data1/a.txt', '/data1/ok', '/data2/b.txt', '/data3/data4/c.txt' ])) if includes: actual = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) self.assertEqual( actual, set(['/data1/a.txt', '/data2/b.txt', '/data3/data4/c.txt'])) else: stderr = self.RunGsUtil(['ls', suri(bucket_uri, '**')], expected_status=1, return_stderr=True) self.assertIn('One or more URLs matched no objects', stderr) def test_dir_to_bucket_negative_lookahead_works_in_minus_x(self): """Test that rsync -x negative lookahead includes objects/files.""" self._test_dir_to_bucket_regex_negative_lookahead('-x', includes=True) def test_dir_to_bucket_negative_lookahead_breaks_in_minux_y(self): """Test that rsync -y nevative lookahead does not includes objects/files.""" self._test_dir_to_bucket_regex_negative_lookahead('-y', includes=False) def _test_dir_to_bucket_relative_regex_paramaterized(self, flag, skip_dirs): """Test that rsync regex options work with a relative regex per the docs.""" tmpdir = self.CreateTempDir(test_files=[ 'a', 'b', 'c', ('data1', 'a.txt'), ('data1', 'ok'), ('data2', 'b.txt'), ('data3', 'data4', 'c.txt') ]) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _check_exclude_regex(exclude_regex, expected): """Tests rsync skips the excluded pattern.""" bucket_uri = self.CreateBucket() stderr = '' # Add a trailing slash to the source directory to ensure its removed. local = tmpdir + ('\\' if IS_WINDOWS else '/') stderr += self.RunGsUtil( ['rsync', '-r', flag, exclude_regex, local, suri(bucket_uri)], return_stderr=True) listing = TailSet(tmpdir, self.FlatListDir(tmpdir)) self.assertEqual( listing, set([ '/a', '/b', '/c', '/data1/a.txt', '/data1/ok', '/data2/b.txt', '/data3/data4/c.txt' ])) actual = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) self.assertEqual(actual, expected) return stderr def _Check1(): """Ensure the example exclude pattern from the docs works as expected.""" _check_exclude_regex('data.[/\\\\].*\\.txt$', set(['/a', '/b', '/c', '/data1/ok'])) _Check1() def _Check2(): """Tests that a regex with a pipe works as expected.""" _check_exclude_regex('^data|[bc]$', set(['/a'])) _Check2() def _Check3(): """Tests that directories are skipped from iteration as expected.""" stderr = _check_exclude_regex( 'data3', set(['/a', '/b', '/c', '/data1/ok', '/data1/a.txt', '/data2/b.txt'])) self.assertIn( 'Skipping excluded directory {}...'.format( os.path.join(tmpdir, 'data3')), stderr) self.assertNotIn( 'Skipping excluded directory {}...'.format( os.path.join(tmpdir, 'data3', 'data4')), stderr) if skip_dirs: _Check3() def test_dir_to_bucket_relative_minus_x(self): """Test that rsync -x option works with a relative regex per the docs.""" self._test_dir_to_bucket_relative_regex_paramaterized('-x', skip_dirs=False) def test_dir_to_bucket_relative_minus_y(self): """Test that rsync -y option works with a relative regex per the docs.""" self._test_dir_to_bucket_relative_regex_paramaterized('-y', skip_dirs=True) @unittest.skipIf(IS_WINDOWS, "os.chmod() won't make file unreadable on Windows.") def test_dir_to_bucket_minus_C(self): """Tests that rsync -C option works correctly.""" # Create dir with 3 objects, the middle of which is unreadable. tmpdir = self.CreateTempDir() bucket_uri = self.CreateBucket() self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1') path = self.CreateTempFile(tmpdir=tmpdir, file_name='obj2', contents=b'obj2') os.chmod(path, 0) self.CreateTempFile(tmpdir=tmpdir, file_name='obj3', contents=b'obj3') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): """Tests rsync works as expected.""" stderr = self.RunGsUtil( ['rsync', '-C', tmpdir, suri(bucket_uri)], expected_status=1, return_stderr=True) if self._use_gcloud_storage: self.assertIn("Permission denied: '{}'".format(path), stderr) else: self.assertIn('1 files/objects could not be copied/removed.', stderr) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) # Dir should have un-altered content. self.assertEqual(listing1, set(['/obj1', '/obj2', '/obj3'])) # Bucket should have obj1 and obj3 even though obj2 was unreadable. self.assertEqual(listing2, set(['/obj1', '/obj3'])) _Check() @unittest.skipIf(IS_WINDOWS, 'Windows Unicode support is problematic in Python 2.x.') def test_dir_to_bucket_with_unicode_chars(self): """Tests that rsync -r works correctly with unicode filenames.""" tmpdir = self.CreateTempDir() bucket_uri = self.CreateBucket() # macOS displays the "Ì" (I\xcc\x80) unicode character the same # as "Ì" (\xc3\x8c) in both the first and second filenames. Despite being # different characters, the second filename will not be created since the OS # detects them as the same name. file_list = [ 'morales_sueños.jpg', # 'morales_suenI\xcc\x80\xc6\x92os.jpg' 'morales_sueños.jpg', # 'morales_suen\xc3\x8c\xc6\x92os.jpg' 'fooꝾoo', # 'foo\xea\x9d\xbeoo' ] for filename in file_list: self.CreateTempFile(tmpdir=tmpdir, file_name=filename) expected_list_results = (frozenset( ['/morales_sueños.jpg', '/fooꝾoo']) if IS_OSX else frozenset( ['/morales_sueños.jpg', '/morales_sueños.jpg', '/fooꝾoo'])) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): """Tests rsync works as expected.""" self.RunGsUtil(['rsync', '-r', tmpdir, suri(bucket_uri)]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) self.assertEqual(set(listing1), expected_list_results) self.assertEqual(set(listing2), expected_list_results) _Check() def test_dir_to_bucket_minus_u(self): """Tests that rsync -u works correctly.""" tmpdir = self.CreateTempDir() dst_bucket = self.CreateBucket() ORIG_MTIME = 10 self.CreateObject(bucket_uri=dst_bucket, object_name='obj1', contents=b'obj1-1', mtime=ORIG_MTIME) self.CreateObject(bucket_uri=dst_bucket, object_name='obj2', contents=b'obj2-1', mtime=ORIG_MTIME) self.CreateObject(bucket_uri=dst_bucket, object_name='obj3', contents=b'obj3-1', mtime=ORIG_MTIME) self.CreateObject(bucket_uri=dst_bucket, object_name='obj4', contents=b'obj4-1', mtime=ORIG_MTIME) self.CreateObject(bucket_uri=dst_bucket, object_name='obj5', contents=b'obj5-1', mtime=ORIG_MTIME) self.CreateObject(bucket_uri=dst_bucket, object_name='obj6', contents=b'obj6-1', mtime=ORIG_MTIME) # Source files with older mtimes should NOT be copied: # Same size, different contents. self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1-2', mtime=ORIG_MTIME - 1) # Same size, same contents. self.CreateTempFile(tmpdir=tmpdir, file_name='obj2', contents=b'obj2-1', mtime=ORIG_MTIME - 1) # Different size and contents. self.CreateTempFile(tmpdir=tmpdir, file_name='obj3', contents=b'obj3-newer', mtime=ORIG_MTIME - 1) # Source files with equal mtimes should fall back to other checks: # Same size and mtime without specifying `-c` => should NOT be copied even # if the contents differ. self.CreateTempFile(tmpdir=tmpdir, file_name='obj4', contents=b'obj4-2', mtime=ORIG_MTIME) # Different file size => SHOULD be copied. self.CreateTempFile(tmpdir=tmpdir, file_name='obj5', contents=b'obj5-bigger', mtime=ORIG_MTIME) # Source files with newer mtimes should be copied: # Same size and contents => SHOULD still be copied. self.CreateTempFile(tmpdir=tmpdir, file_name='obj6', contents=b'obj6-1', mtime=ORIG_MTIME + 1) @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): self.RunGsUtil(['rsync', '-u', tmpdir, suri(dst_bucket)]) # Objects 1-4 should NOT have been overwritten: self.assertEqual( 'obj1-1', self.RunGsUtil(['cat', suri(dst_bucket, 'obj1')], return_stdout=True)) self.assertEqual( 'obj2-1', self.RunGsUtil(['cat', suri(dst_bucket, 'obj2')], return_stdout=True)) self.assertEqual( 'obj3-1', self.RunGsUtil(['cat', suri(dst_bucket, 'obj3')], return_stdout=True)) self.assertEqual( 'obj4-1', self.RunGsUtil(['cat', suri(dst_bucket, 'obj4')], return_stdout=True)) # Objects 5 and 6 SHOULD have been overwritten: self.assertEqual( 'obj5-bigger', self.RunGsUtil(['cat', suri(dst_bucket, 'obj5')], return_stdout=True)) # Contents were equal, so we verify via mtime that the source was copied. self._VerifyObjectMtime(dst_bucket.bucket_name, 'obj6', str(ORIG_MTIME + 1)) _Check() def test_dir_to_bucket_minus_i(self): """Tests that rsync -i works correctly.""" tmpdir = self.CreateTempDir() dst_bucket = self.CreateBucket() ORIG_MTIME = 10 self.CreateObject(bucket_uri=dst_bucket, object_name='obj1', contents=b'obj1-1') self.CreateObject(bucket_uri=dst_bucket, object_name='obj2', contents=b'obj2-1') self.CreateObject(bucket_uri=dst_bucket, object_name='obj3', contents=b'obj3-1', mtime=ORIG_MTIME) # Source files with same name should NOT be copied: # Same size, different contents. self.CreateTempFile(tmpdir=tmpdir, file_name='obj1', contents=b'obj1-2') # Same size, same contents. self.CreateTempFile(tmpdir=tmpdir, file_name='obj2', contents=b'obj2-1', mtime=ORIG_MTIME) # Different size and contents. self.CreateTempFile(tmpdir=tmpdir, file_name='obj3', contents=b'obj3-newer', mtime=ORIG_MTIME - 1) @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): self.RunGsUtil(['rsync', '-i', tmpdir, suri(dst_bucket)]) # Objects 1-3 should NOT have been overwritten: self.assertEqual( 'obj1-1', self.RunGsUtil(['cat', suri(dst_bucket, 'obj1')], return_stdout=True)) self.assertEqual( 'obj2-1', self.RunGsUtil(['cat', suri(dst_bucket, 'obj2')], return_stdout=True)) self.assertEqual( 'obj3-1', self.RunGsUtil(['cat', suri(dst_bucket, 'obj3')], return_stdout=True)) _Check() def test_rsync_files_with_whitespace(self): """Test to ensure filenames with whitespace can be rsynced""" filename = 'foo bar baz.txt' local_uris = [] bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() contents = 'File from rsync test: test_rsync_files_with_whitespace' local_file = self.CreateTempFile(tmpdir, contents, filename) expected_list_results = frozenset(['/foo bar baz.txt']) # Tests rsync works as expected. self.RunGsUtil(['rsync', '-r', tmpdir, suri(bucket_uri)]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) self.assertEqual(set(listing1), expected_list_results) self.assertEqual(set(listing2), expected_list_results) def test_rsync_files_with_special_characters(self): """Test to ensure filenames with special characters can be rsynced""" filename = 'Æ.txt' local_uris = [] bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() contents = 'File from rsync test: test_rsync_files_with_special_characters' local_file = self.CreateTempFile(tmpdir, contents, filename) expected_list_results = frozenset(['/Æ.txt']) # Tests rsync works as expected. self.RunGsUtil(['rsync', '-r', tmpdir, suri(bucket_uri)]) listing1 = TailSet(tmpdir, self.FlatListDir(tmpdir)) listing2 = TailSet(suri(bucket_uri), self.FlatListBucket(bucket_uri)) self.assertEqual(set(listing1), expected_list_results) self.assertEqual(set(listing2), expected_list_results) @SkipForS3('No compressed transport encoding support for S3.') @SkipForXML('No compressed transport encoding support for the XML API.') @SequentialAndParallelTransfer def test_gzip_transport_encoded_all_upload(self): """Test gzip encoded files upload correctly.""" # Setup the bucket and local data. file_names = ('test', 'test.txt', 'test.xml') local_uris = [] bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() contents = b'x' * 10000 # Create local files. for file_name in file_names: local_uris.append(self.CreateTempFile(tmpdir, contents, file_name)) # Upload the data. stderr = self.RunGsUtil( ['-D', 'rsync', '-J', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) self.AssertNObjectsInBucket(bucket_uri, len(local_uris)) # Ensure the correct files were marked for compression. for local_uri in local_uris: self.assertIn( 'Using compressed transport encoding for file://%s.' % (local_uri), stderr) if not self._use_gcloud_storage: # Ensure the progress logger sees a gzip encoding. self.assertIn('send: Using gzip transport encoding for the request.', stderr) @SkipForS3('No compressed transport encoding support for S3.') @SkipForXML('No compressed transport encoding support for the XML API.') @SequentialAndParallelTransfer def test_gzip_transport_encoded_filtered_upload(self): """Test gzip encoded files upload correctly.""" # Setup the bucket and local data. file_names_valid = ('test.txt', 'photo.txt') file_names_invalid = ('file', 'test.png', 'test.xml') local_uris_valid = [] local_uris_invalid = [] bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() contents = b'x' * 10000 # Create local files. for file_name in file_names_valid: local_uris_valid.append(self.CreateTempFile(tmpdir, contents, file_name)) for file_name in file_names_invalid: local_uris_invalid.append(self.CreateTempFile(tmpdir, contents, file_name)) # Upload the data. stderr = self.RunGsUtil( ['-D', 'rsync', '-j', 'txt', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) self.AssertNObjectsInBucket(bucket_uri, len(file_names_valid) + len(file_names_invalid)) # Ensure the correct files were marked for compression. for local_uri in local_uris_valid: self.assertIn( 'Using compressed transport encoding for file://%s.' % (local_uri), stderr) for local_uri in local_uris_invalid: self.assertNotIn( 'Using compressed transport encoding for file://%s.' % (local_uri), stderr) if not self._use_gcloud_storage: # Ensure the progress logger sees a gzip encoding. self.assertIn('send: Using gzip transport encoding for the request.', stderr) @SkipForS3('No compressed transport encoding support for S3.') @SkipForXML('No compressed transport encoding support for the XML API.') @SequentialAndParallelTransfer def test_gzip_transport_encoded_all_upload_parallel(self): """Test gzip encoded files upload correctly.""" # Setup the bucket and local data. file_names = ('test', 'test.txt', 'test.xml') local_uris = [] bucket_uri = self.CreateBucket() tmpdir = self.CreateTempDir() contents = b'x' * 10000 for file_name in file_names: local_uris.append(self.CreateTempFile(tmpdir, contents, file_name)) # Upload the data. stderr = self.RunGsUtil( ['-D', '-m', 'rsync', '-J', '-r', tmpdir, suri(bucket_uri)], return_stderr=True) self.AssertNObjectsInBucket(bucket_uri, len(local_uris)) # Ensure the correct files were marked for compression. for local_uri in local_uris: self.assertIn( 'Using compressed transport encoding for file://%s.' % (local_uri), stderr) if not self._use_gcloud_storage: # Ensure the progress logger sees a gzip encoding. self.assertIn('send: Using gzip transport encoding for the request.', stderr) @SkipForS3('Test uses gs-specific KMS encryption') def test_kms_key_applied_to_dest_objects(self): bucket_uri = self.CreateBucket() cloud_container_suri = suri(bucket_uri) + '/foo' obj_name = 'bar' obj_contents = b'bar' tmp_dir = self.CreateTempDir() self.CreateTempFile(tmpdir=tmp_dir, file_name=obj_name, contents=obj_contents) key_fqn = AuthorizeProjectToUseTestingKmsKey() # Rsync the object from our tmpdir to a GCS bucket, specifying a KMS key. with SetBotoConfigForTest([('GSUtil', 'encryption_key', key_fqn)]): self.RunGsUtil(['rsync', tmp_dir, cloud_container_suri]) # Make sure the new object is encrypted with the specified KMS key. with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): stdout = self.RunGsUtil( ['ls', '-L', '%s/%s' % (cloud_container_suri, obj_name)], return_stdout=True) self.assertRegex(stdout, r'KMS key:\s+%s' % key_fqn) @SkipForGS('Tests that gs-specific encryption settings are skipped for s3.') def test_kms_key_specified_will_not_prevent_non_kms_copy_to_s3(self): tmp_dir = self.CreateTempDir() self.CreateTempFile(tmpdir=tmp_dir, contents=b'foo') bucket_uri = self.CreateBucket() dummy_key = ('projects/myproject/locations/global/keyRings/mykeyring/' 'cryptoKeys/mykey') # Would throw an exception if the command failed because of invalid # formatting (i.e. specifying KMS key in a request to S3's API). with SetBotoConfigForTest([('GSUtil', 'prefer_api', 'json')]): self.RunGsUtil(['rsync', tmp_dir, suri(bucket_uri)]) def test_bucket_to_bucket_includes_arbitrary_headers(self): bucket1_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() self.CreateObject(bucket_uri=bucket1_uri, object_name='obj1', contents=b'obj1') stderr = self.RunGsUtil([ '-DD', '-h', 'arbitrary:header', 'rsync', suri(bucket1_uri), suri(bucket2_uri) ], return_stderr=True) if self._use_gcloud_storage: request_count = len(re.findall(r'= request start =', stderr)) target_header_count = len(re.findall(r"b'arbitrary': b'header'", stderr)) self.assertEqual(request_count, target_header_count) else: headers_for_all_requests = re.findall(r"Headers: \{([\s\S]*?)\}", stderr) self.assertTrue(headers_for_all_requests) for headers in headers_for_all_requests: self.assertIn("'arbitrary': 'header'", headers) @SkipForGS('Tests that S3 SSE-C is handled.') def test_s3_sse_is_handled_with_arbitrary_headers(self): tmp_dir = self.CreateTempDir() tmp_file = self.CreateTempFile(tmpdir=tmp_dir, contents=b'foo') bucket_uri1 = self.CreateBucket() bucket_uri2 = self.CreateBucket() header_flags = [ '-h', '"x-amz-server-side-encryption-customer-algorithm:AES256"', '-h', '"x-amz-server-side-encryption-customer-key:{}"'.format( TEST_ENCRYPTION_KEY_S3), '-h', '"x-amz-server-side-encryption-customer-key-md5:{}"'.format( TEST_ENCRYPTION_KEY_S3_MD5), ] with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]): self.RunGsUtil(header_flags + ['cp', tmp_file, suri(bucket_uri1, 'test')]) self.RunGsUtil(header_flags + ['rsync', suri(bucket_uri1), suri(bucket_uri2)]) contents = self.RunGsUtil(header_flags + ['cat', suri(bucket_uri2, 'test')], return_stdout=True) self.assertEqual(contents, 'foo') tests/test_data/test.json 0000644 00000003575 15025013616 0011541 0 ustar 00 { "private_key_id": "test", "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC/gxdAzO6ef2ZA\nXJFHBraJjRlTp6GzDZ+2h2ch2u2JhKk4PsGq5d8CpMrGYBDVjkeGevbKNbVk+f3r\nbVdVuzL+BLxw8SyNLR7CLewzrH6B/H0WKRxl0oqT/balxaje+PmbTy+lxB2r/MUG\niCf8u7+wgCpJJVSErjT9h3PKL3I0cYDWW2iLTe9xdJl1PxIzBRrwFwClijoB5Cs8\nV5+cdFMglqibI73epe8CoANQulyWzbdzU3T2EoxpNc/dx/XTY1WiZsFgGzZMfu1d\njI+vN1gRWksZ7oxYNwbMf5OMUNgDUTSJUPcJilGB9BwOPrcSTEXbrMsI/ZWRmtzB\nBtsfHHhVAgMBAAECggEAJ4aQyAkDvqagnev89+HEzsLWVVOlpeTL78omWPBsUhOV\nMUySgAkNa+hDCYpOLyU8Ud8TDEzlcT+/TlW0zrbiIVcs5UqBXWnQjVcZBtk1CvLn\nhilZ+RHpE/4rMi+HWO2d5pYDSX0eep6is2RqjFjbiTwKfio8NV4XBbRtC2+bbIMc\n8sM5pzWqwopvJyrssHwbi5TrjCiwnKK05HcoU7KKSrgWiqJKFbZb/BzzEJEWowV/\nKjjrV3fgXkdZ7iHtX+rpTyGqa5IP/dYlto91d7eSiz34rzveUY/W+BSiliUXwJfY\nlIn4Egt5afKinOlm4zk0scoct7tySAm/ZLbZipmsiQKBgQDsSgVq7oIxSaGzfbvh\nIpJPOtxH1tl8kJCtEZAIsZueMn7+ICKkRFLecTdNDm1XaiGrcTJ5WGK/2lWDCWP+\noHhDDRLzY9G7NO1gSWl0BDwjKFMksix1PZSmliGAA7Tqv44GvwDtQi0E0XRYvoRm\nlWmx5It/LKbmG8gXfqFoq4pS8wKBgQDPfNnJews5D3YzuXhK7dXupvMyOC6plSXj\njZzqBsQAWOgPkn7L2kmlIdanFs4J1A4774DpIugDjjzsKa0Rsu6IJ2vs7QXr5V9x\nNJfFQ9mWl+/fLlMdGHlB5MOMMb4SeuwABMkBQvaTow9xQTEji2GJ4Kr++/FaQ0n7\nAH1T1rkJlwKBgQDlv7dy2MQ4flq+9tN2vbG7XgUPyQUhTEOQNFJzUZZ5hKncGIz2\nYgSdi0H1J8EDTQu98kvclyc1WmrpBXl3/LnSoGS7yWQFfhoNFHR3vSJoIO5u8Nuv\nh5gBPlsGaRRBaM4FGzUgfMbfN3XEUELWqBLkSM+mcVk6xuqQha0V5C2UJwKBgDNE\nNQesaFR79ofHPuq8o0xpYdO5wAXUvVzhQ7ErZMh1LzmRx9SUpfewSbpVTylshOvY\nPLDeGBew0GrFzWZ/IKLxoqAS3JCxxwhQp2NtZVjpR7KMXqrkvilC3fJI5B7q0l8a\nz5OJOSW3d3xTUFHpAXuw0G9aTwMvdxsWvuC1QFkXAoGACFHOaD1cynEd93VxW0P+\n6s2QDuaEH+p0ZlJb5AiH6e94mMLxrbqObnSAyTCJ62Q1R/hf/+S0x4BQ26q38BBD\nJfJ+40KXSb0CT4OcPbtnT5LBGWYQ7C2G12GBcYFCqZqLtmo0o1+5RqKzwDf5yCN5\nN117uNtUush0uYBGAh9u7ig=\n-----END PRIVATE KEY-----", "client_email": "test@developer.gserviceaccount.com", "client_id": "test.apps.googleusercontent.com", "type": "service_account" } tests/test_data/test.mp3 0000644 00000022334 15025013616 0011261 0 ustar 00 ID3 TSSE Lavf53.5.0 ��� H LAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU���� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUULAME3.98.2UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU��� H UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU tests/test_data/test.txt 0000644 00002000000 15025013616 0011365 0 ustar 00 SK5uT5Q4DSVadLQ6QkSg9lO23RpCwySn2MreWc2KNJhuJ7M5U5yq/dR+OuF0MRR1Eq0aUjp71lES 4YDuC99T7MMOF7ikPv1rvPdpLWdxhHhLu9tRCfft1rJWbKUL0JglZAvC9TbU+lauQipddQzyIg18 RRuc+MnebnOzQ8xcfx4lI21TqW/S9y4u8T615KYHUoZDGmKNokiBM1MZclXP/026UjEzJCBezwuP RHDk29QF63x9BNC0qOWI+Fg2X2pMPoz+Enshf61WW25Q34EEtb4QFgy/Q/J9G+eKb5293WBIHDCS N3T3OJrDh39/iXPE0/UGV6MH1ptDlEN/mU2TzLMnmoS8qDCb+IfVoE1JKj9JWu9+fGMF4RplXrTG Uzb+unhnE2tdKekjr+HBXnLIxADrSRoDcjVcxsE5fgdzmN75ZYhbXNY2udSwIydB+pDf6ia1X8bH X2HDUqDm3KtJQ4CZp2moNo12DcsBQHK2vpT+HB4DKlUp2wwAYhKAXgWUC6zV7ZLgIHIjmYSRndwi gha627tGt1xov/g66EDRe8CJU/pDKJsUxJeQP594hPi0rh7JzHdA7+lRHZmgWnEnateH3hmldwIS cRFR/jhnVUmR3OIJOJOdrKAUz5H5ldFDb9H/g9g0jQC2GYbQiJC6Qb5QS45I4IeDpbcxqmsO2We0 yVnQuvl9HSbr6s3ONdx3IiWLWjaAcOSDY8OutfD9NEDoS0z5iLbTTsGyUP1pKIDKUiIyJ6OLy2Y0 W2aCJkAnvzylQph+5KtAEZrusNLRtDf/XInXGe2H5PpH5vfmwdzu3tRmt/Zx0j+36+9/1K7I2fcl Wxh7INAdTd2dhYpMc8XNltvVs7srYST/vU3NZ9yx8ZaQdJk82osfjMgZCVellYjePim/skkf03h3 G7Wi0ktikquzuPQDIpIrhah7BZFlUpCkw0+HArjSHz3NhBIM+vccuov6VEiHwL7FL77A3GbDXDV/ Nii91Jgd9gSnIqHunkj3i0vEdQEV9qfA5GAVJizSh7CnQ0v/pNqBKPhagcKqdldCghhsjWcVC4fR 9mWjysAx3mzMOKgvmSbUTyGoUbUiZ+LWh0tMLXMVPd88CbuEouLdEnBBWFPzcrKm+o3Ww8ORhiAq Hfv7r6xlTxGMPO7w7+x/rwPJp00ZIk2g48VWkkDZkbC5WKyHwpfcwRVuFM+f0jFtposz2HJDAWyZ 5ZISHyLZHSkChUZEW+8z1JmYmLWbgtQinQCdHg4bmcs94vyz9T7A/JGQYkvL+Dq5oTvNw/BabWmJ DkaRU1aankMVwk3PZa56NnDbJjgJOMqTBxAwtN83Dr0L733W7YzMpYuV1niKqW0YrDCvzQcgeiIf NnaeYMNR4+KtDW7kNWVEgZiyw/0hYOpt38H22DQoVYA6wQ3KIsUas/zffHS9Rs9BrOY1xfFMCqMT tRvWZ74IakYKWiJnSM2rV+OgKwUr/H+yL2IARIBflhgEtUsXV+IncQxP2KCBPgLqfH0yPK069ZBW Djc6ZX2fAghYrXdpEJaIJfP7wXigj8fvYlH928mW6th8wWA9bnyU0doBOW0aCbXw+J7/3N1pz9Td Ai1lhyWDqJueEHmZK/ntEF8ysyfX0/QulLeVc5f3BTMZ1hQcr5E1/6hO2MaCs/Ek+uf/KYnBrug2 t8T1smRgdE2RYBh/qRZy5xkwHv5evvZ2KOITZRi498p3BvG1fWAGrgU4u2ikH7au93XuQA0BR7gz FAoWTiPvpOX7Liq+CqD36h6pylhyIkT/p4OSnv/bB8SW5+T+delhpZdrEvStd9dTPIaTQQtFGYJN oTCCmnp78pBSf4AE4nKfX9121orZtdE3pnpqJjI3CoqkdmP9FxBEm/U46N0lOe0q5MkZVjyaIQxJ vVXYOTPcoj0C61GpJ15uBDvvh0acyI/ExMGirHTFGPswTH2RuF/3bbnKGTOkpK8s4FZx8sEyXyxs jyGYAup8y0ZzMrfRtwpxBNhEJaIcPRzcLbsdftfqJSu4AmAfLXQIk7RNRyrdQYuYgbizsLP6fd8j MHT5sKp/U/R4tGSp0mN6GAoB02m7ZRu089nEPmBxPdElOMpqy72y5+fcuqzoNyQW2ra3ypd9ZTCY 6XS2ZDNfUbpA5dTU6WI0jFUzAPyORbgdoFmhzv8JFuqDjIAykoZI6sBImwdmmRmr7Q/FoRotbBDM iiEzuVeB7ouyFAd9sbs1B2mIZRvZX5S2mRjSZkvLbdGbiapD2UFz3q8N6IUFCiDG1yMtka32GTNe ysEI8Iw8DRqzz2PcGpw8K7YbVOwbWTAHY6A4poyrKsC8F5ET4xXz8XHrd7hAkAM5P+GiP0FlpIv7 oasUS+QHi03ZuzGULe1XyXt6pV6KQGVfjX9lUcUPPA1a1ZXq5Kxp4XcbdkEjLGSEsBnuuo/SfwGc eCnNsyFBZH+/0BWFdEXSLB1hq5hs50zNV8qVkPUPBkThxr1MHyrj2PMnfa8MX8sBRVDklrhIQ9OQ nXn2PxPf64pFStI/J5rbJ7hWQXSua7652LDN6pQM/Z3Ig5XTirmoRsTSIsA9LLExoQXNNVMxrdDw yjaeRip5xnlXQfw2CTBOacM7/91cLl+5cTOtjCJ7D4d5BTz+AQE1cnxkX1yCP98JQ40li74p4YJW PU61y2KRr1aJR9KoLEGgl44YfLD7uLcAuEjIoPFgfdM4wEQneVVfZWgvklM5cIOXSubNFdzYp0TL fNqRAx2kSOnq3PdIT2EWmDyS2OEDCW4HrJHua2UFOzMFLwqRoTAOSqrKXcOBgrEj6u7HEfiUGMku f1wyXk3n+McObrt1b11ih0YC+XRG9DfRFUX7pyM+XvRFCpJaij6QXHZZeLMxXsMYQYrwNnr42lC5 mLkQNiO+9mBGCzqO/8ivPMPVtpkYsdR04k2+Y/Xfarvu4+w0hjORFDCPEytxSfOdFZBwmadrAfJy Eijc8GEGmJc1P4XqP2kYDJHhlpH0I34tut+QbxmHR84r43dhXXMWZIFi6CLUdwswd2CYnJSdQNfT RJtH+2LdyT5dGoHQCUmQ4KV8AHRLF/0M3ro1B+vfHvkx7x5wH98wu+h1DloeeQ3Sq0rQcWnH4IrV FD9xkiKsP5+PePe6ahoEtkt07vHQsxpGhHGwcBdbXmdiKUm/ujS5WH79RIjMYnn+x5RQrp5YRbd8 1vlwTTRiWYgSrodpsM46LStf00r4wZg7Ec7uVWt40yFszQmhpIXXsmXeS7tdJkl7hRk8s4jIDFRh XpwUoAebwV7hNnmXxF5Rk9eOzZkxulGcIIoX7SyCDYnsGYsb9W+2+0AD/9oszQB4ulcemcPSA7fd 78qkakcmAAwX5cAJ8h3N7kOZnvSrVKI5E740YU77lFoQHLT+DAXNH+1fgj+KGimPKtwVZxlxNJOS 9xsl5rozrJYCdZvc29sLYX7mQ0KfUgOLVWQOt31tbZfB/XuVYacSE7I1emr79YyGLlr3zy/UVoI1 XhQvBxFuHRJNIbOxGTP3SxKTmeWVnFeArSwVgd9ah+yRblZycLDJOLhCPxNTUMyqygJWFpS4uKgN rF1I9tR5d5MhF45lj563iE+l4tpQRZBn2eMBnKy3OMuUdSdm9PVtogiTzulw4SXhIHIGyf/ILLmT EV2RnIpJMCJebAicw99Yfy1aoWTK1iQ7F9HM+JDDWwHW/F4VU+laDBZc005VQijDR7e5hmR0zi2W ABSmlLnrSmzFCuJXLZC3XqKYVg1YaJkRY6pGnrngw4++5lUpuI116EZpXXWui9A+x0D0iJxK0tLY uam/gqIDEImbqofEMdK9flBbeDxfbov94qZAiexW/xExfH38khMqEjoF7Qm0qTBCEq18eaYjOQ+Q wXrLvIDLb9bwgLGQazfQR7q8p97F3A36tM8zJc5+G7a/NutUMQhTI7UU68Pr5VrjcTh0NDXEyFZg FqIPWlaX5GCvUATfOfk/vc91ulYM0YBit0HAEl1fRKL5qBAqXLwFqKyaaz/Xq92Q23hhu00gGzgo PsI59dGPezZX81/bbxHAv9lWv4x2phIvZsIW5QkS2IKa6YY+Q/FsNen9WJgGDxIG65fMDBkRfenJ rxy7wgn2YCsu7CxRSYgEUh1LdPb6W/pLNSPm9vVwttLcPTrYg6QtDzfNXP0Jv231s8B5pyDECXES Gh/aRdqLLpgZRBJPOx0DX6p4ZVlwNi4M7ClpbMvWAO3iwAp/HHj1m2of4+wfznqm3m/ITxlHLp7s wB27UpZuxyapuBQqIYpsm2ioxqgPlZJ5/rP6a8o4TkzTyMKUUY3h6JYd8e9UWPAGjbjjeW/YWbjE vyqn6U5kLZYV20v2aOXX0376bpCrtfLEccfcZV5fTX1v9ftOgs+ged/jDuam+KDiKSRIpFNbR0Uo 4B5k7HCoNzLY+ibsuRhjBTcO1brX3U5TjPnVC2xzqXZOm5MV20LFCEMYyjBf9zN9TRyCobTt4Hfm cI+w0xb7v6Mu+YeCexyiseh38TYnWK0HanDOg9/pAI67jIq9dsQCU1XpdQ/qn6Xoa0kYI9HQROei 8aNtPBaYtKXHNkfKimH17ys/0AVy5UJM94UKKbudgicY4/6iX7kz6242ajDyF6A/Aazj/hRTDwx7 6O/UHmEVftMDz+pWNuRk0lBjA0EU4jy626sHxAWmHvsqsdFtoJ6tUqdABn0C72AVxgItDE+YOo1U MAGzNugdfoA/vYvFWr5iQPRNIKtzsVK3e+NTbOWhFVDbdp3ua1gTToFG8hgY0n0VpjOlVdz/EbTf E9u3rwIXt7/o7qYJdTTe8TSj3nXNBCeL/JqHtZKElRMXgW5wjWZzsKyNMqubMtV9CAGzQs1HOHhJ T5bx2l3yRpMSILqzh+PPwNZRiF2H9M8eTwbtAhkWWgg7rGO+iU76DXchXy8vZ3g5HwCvZcmNLwO+ TFXiM1NIh16UW/wVv0/7VCvfuDfAYlP5vcLQnvWVJSJ86WzMwfOtZQN238WIOJ0efxa/qeQ0ZSQN +DFwgnhZj1QiS0ykRsJ6z948bxz2Sx1DjomqJ2gzrpYEiQi2avq4GmovAHp+TvB4AuxamyWNgb5l dgc34CItp/JZm9RuQ2XqPN6H9pR2tPsDjnPfhDd7Yhx9hmmH6w7cuUxagbtXLhgtWPamaaFNX3mh eJEBqinh9CrPHTPLRuPpNXCfnCCpeT4udXKCREzyt0xJ5Uvn6vMqBstVQcQbsFUiIDAwapNDp84c ZPFjHmBj9wwpjAE/xQ72nftwncRYPzF5iT426HqYA0PHUZOSPYz1eIN8R0vYZ6KDs9M7/1aFusf3 Os7g7U8r+f3DP6aIQyW2/QCJNkkcJjg3AD7C/o6cGdnd1+15cIv3wo/mxCMVSjTjNSXS2Uf2LyBh Akan3vHScUPACsOkhH1WOD6AW6SBxClZrqQCGfSwaJ7HfIzajfqi0M9L3Tz8Vp818gD/1J6oObyL z2WhflRd9L7nJr6Dz9Q2oNaP9PUMm305RVDqBvng+VkK8HnFYTUoafCcqf5tMeDbU+Y2shiITiPX EUgb8PdFvXI5DMgRp+DrshFMIHaC6PlCD2QZS+yEkUcPrac52iFTCceTX2vSN/ysl3E4HReACi8d Jt5PTicKlNItO5sW7FVXKXchhj2qrO+7rqMXUxg5yX+NlWdjVahcybX/thLFxCzpK63S1aKszzNi lfvxuEtrABGPjm0iQ1KTOVi9gVISUmp73JR/FEiY4cQFPNVKRW16Byc1ZzAK0GKZQWbl9u7ujO/H k+mY0mwrGdxnODGIXL2mF2Xo0eBnYrgPku0sUYrw/NBejrRfyR84ekGfwLLSG20s/2XFV+O832TK pCyDJ4737ZJM/LW+7cm4CPyJIVxSQeut9DBgVFCwuuOW87NdzyLWstzK5LeZTXig5RhJzz9tNKhU mGB2V2lOpcEexJSeG+DfnaWDYEHap6TstMLfDe4CERNz9VKy7SCA2lO3l4BmewZyL+kz8cOWb4Qi L/5cllB13kTG78la6hJhNytLKH0+w1p9qBoDd/xvYdWVMw0sDtqjs2AwHoQVaikcG5CQrv+ODAZJ FG1oAlxZzMH6LmfaQkhaQ0AM45RgYiBvp4fOrrg9iiRX6CHqrzm/CJ4aIWSbPNVEBs19Mkj1ATqd NcNLQEd+gts2vxXC6w4DwD2XKyhGKmWRyd3CHM6OSnf9/CA8I1Jb+VX4IMfDYguGtT1hGTyxMxqd wyNhZmPDAoCIlFumYXTJrf7kSBk8c4dlYngXWjvltU+LONu0/LpKHWQCTi/hTAkKKn6zmWpkuMB3 +MrbUS3lIK46OD5DONhd3YzoMg2G4hUxgtvOFwO0ptrUNTTsFTixkKm3RLvWvDQZQq/Myv8gtngP cq6Mj0M7xCKMVDFOteoeDuAfMOXr5Olp4rK07yC3PcaLWBc2A+KDwb5XKnJVLKseZBQ+1qoT5Vf8 tUzC7HvxvuUtYj9im7vD41ISdawvLyHRgJCTG3xh4160V7F+bt8nCn8HRmhVVGXS4M3eTTPKU2aT nRigCfY1hr6TqeOEmInHmSuat3+ANGoifZbaTM5vZ4jBjPtLXAB+XmV84fDXcksGagTR0/+v1OHM lEgit4rbwNsG/QTYSd96kGKgw1JhVgWXCeAbySfmGOUHpdIwWDtN1yPtPPPK127C1Xzja8CIAVrw 530nzWPRJ83He11q1PmjCD8dHBvP8A6vyMNU82ozQU7iD0FjnyWL28qZp/8jZTuLnzO75tI0hUJr FBmvOhmHGs0XBX1FlBWfVVgyHG74lnRCLtflidetx3LkiYBUx73Si+9eJuqNGN+NLPAygKevqZh1 Z2s1fgmmZv0Uw+YeFsWmibQ7ID3/TKKTOYzCEPpfUIQaaZ76VWYLgNrinBW4OApvSEkgYFkulEY6 ++gfzNTHLogTSwqkPHZh4aEQJmCc9n3L5pQmbwazFIOyhnxWhFyzuR7cZyJCGQUHBv3JZg7kvTau X0Ykoi43htUiGdlsCGnnKeDio64J86c8YcnRcOfF5azp51p+oj5CUnhgollkNEyA2NnEGvyMxqjL 0u3UIj27s0G7O1vnEbl5dUEA9K28ujdzD7dgyC7E/sRrujCSidy7dQkzO+3t15lWBkL1tBhV2vpO +YBzSXUpX1Q4P/yk7LRQ4WEvvEZ/QRquqzOztFCOZd6MAgCc+ZI5PnvolOSCkCIPbGuM3FOElLrR 5sf4Lf5XwuQu+9KKUffu8qQDIq5z3ztftqBZznH6sOcSnpuJZnJ4eun9ypl2c1wgc5kgImZIMZBW 64711xe/GaV28alilarvb6+UIe/DraHi06vGu5ND1UyXfKwFzhw7Jt5h4AbTWMuULOTnrhz+WO9D OKE9DRxdXCMW9bWvwoL87oI+yszekDYFQ+rpOzbKkcFn75G9QEzaLiPidVpEJPw8AlgYn0XiT7KH S6LXRLdd9XRMma/1XqROXGNG2ydtngusTe89GjdAlvaGYdpuKlsCZckfyASDWzctIrzbuVIlFmvw ZvkOoQLsuzwrRGEEel4Ihp+ifWkGHKscjNVmIuiO+f1WtLN18xARWOSz5EiTjoHVun1xHdrRtSir QGVqWf0Ga/rJrMt0R/nlB4IfVDyiGgJCsoze9NJALh1b7CEjeTuX1ozieDzKHPfN6uj9Pzck0wiT ukbEjFMgbfMJfy+WN6biBJJFusI8Tjwxk1whMJQnHQ8MAMt5SRwz9L22/8vrTT2k5+ctU801OuAG ojGKcjH3ot61AQKX5v48Jw75svbWN8/Sur1hL2oAORWdYuQo10p2CHQhwWpknzOkE5/H1vK33tR4 vQowZ4UZfjyy95GIJsHtIYSV8NDltPUhndPp0BEkYbJzv3XonVUutTs2AFd72lGFkimkI9GKwOIk MPvYByrOX+zMc6kmeawdxKYzwCcGLQDJhTGKxUjdfAlpjz3b5qSsWBijcTXDucrnRdPaPgNaEWzA zqDgAkjGxx2iJT7VvRotzDz8xqi9T8RWuchIHmOFzeqTfqL2dG1wpkSMGjO2AfG5CEFhpeHJG1d3 iK04sUWyqx8ecHl4jz1z21N0QNGV9/lG1dB8QUaSq2pCBFF3uKCxi7l8U5KDuonVZXcXXewLv3tv cuvfrqqOT1qKJCDK170O1hTQpMio47vWeass4RBD4Rgl4EZCErQqtSjgeeCdoVKfwLwkU3hOZmPy 2J1FNnqPWp5cC50jAuL7Dvx6IKTIX6+r/pnQ9CEMzKKoOgXu7ib9JVeCdH539466Vmkefr04WOJk knza+zmLwHj/tew6s0IHoJ8n2cF/b41oRyuauvb3ZAx7DnuD21ybFHwwSaeqVDHZh8YvV6q5s1vm u1VK5NxCXr/recAc4kur/ywmsk49ftrYVYcHbgL/oEMP6HqPCPYIrgxIrrVk75NHtuQBMwsE6z8S 7bNktD1Q8mihae6ZO3BJ8azlRsq4Kz1VRDUPfSygpioMX9/N4l6tbXy7tX80doqpA85hUCXO3kIh CjGoFw5qMS9rvibGuT+yJF5Fyq7mJYgL2sDFWnd4CSkOtDXoK+s3o0ghoPMHDTX0cLoZAEJ4Vf7S GXJye3tnZCFHrO7GM4Z/rmUzdvuZ2dvmKGETnrx9Mz+Yr3O1tNZpHqd351SS6JAKW/mxEAl8QwQO ep1A2uU1fkHXWSuar47b/owrGHgV50ctkZyDELee5rPqg27zQCJo75NDDyXNt4zbz98wnzQtBbbe 6aszPDguN76UMN1rTVQE+9EQGmT22arYbt2LWDCvtjBAI91Pv6ie4fv37QV/bQ8pSomSAZbNMSa+ vwfk45tknlkQ42eI9y661FqcbJGzcB9igIxi8ubVT82ckCbRR6w+bZ2HuBxqqx+UlgrePtIJ6u/9 VCyPB4ldl9RbJHColgMmjZuPIQ9L+ClA65H8cHo3bRSvscqB1ZRB5xuOKI7pNpTOS8cOfkQO98le EMUrEVv8e0S4F1AZ8SYkFH+BISxAx/HEGBUx2iAfr54a5++HIUYb1ocaxs9aRIrF1P0qgIK75Ech 8UgO/fl9iqDvZrZjSpxGbwPHwSzN4ZX9wRCZCiLD6v9zqugCyvhjFKpWlficoUY4V5PDWckRww0f THMtPyomQlyKtxBASf0keoxRiYCj976tSHhoKLNe3Es1V6w2JVBHaIAThYxHQWShSV7KQZ2b1ARE B5xR1OrVV7qOHjEH/0LniqJ/Y/vCHuneCg+h2OwpOd9wy/ygrcOg1W6f6ShZa2v8599icaYPDfkP AmI4qLstqzd7cwg3VFbD9jSLB698KAYlSn85DRH0gOWG8rY2YCR7v9Y8ctHbToE2VByh24sUGmH7 NZEUgU81lWjqcEUMB46560uSd/vsCWptSdH8Cg1xpBeL6HFgAB5loCsERF2p/7/Ut21QiGaugVVD oCLgUSaVQpYmxcNTCfmIDugvyfiZwDAUCAmhV7V/pBmWmzMNP6CRXysgpTyaBuDkmF3Rl1UgdFWo y5O10st4FevfBw8i4iwj81d25A3+oCW6ceGON0dJlSfGouk0/VlJ7O8nUsHJmCZJD4Y+EJL3p2K4 vCInfAsb6Jm6yk/rUb+TtBvecmG9KjNVlr/RSgpAAhjEsLo+6gNhHYq2oKzGghvSLbnBN3tw1Ixp KayC/0ev+nZxjGw3w7NdQzwJTJjXc67g9iICl+h0zhkZPiyrvALnv3WwjH0AZoRDxogPe+qgMjRm ibu9vWtGP7K4hojx59BcreQtsAJsK5YACceTjMOTYAy/RxtKmrWcykihiRbCL0NhVn88OKpInGdh wl0QDUcOYXYywSsCqGlKQsGluHb2jp3NTR6MzOE9vJ5AueMeEOUn50X+qiUWp/oAQddINZPkTqVf KtgA30+CH11ZIuh2L2WKEQB68sIgUM16T7tYFZlQlRl84vx0jYIgc7rbeu/rgPdnChwzZRapwGYv Jor7atxjREQuFJt3FaKlH4CeLt/Qmh/5CwXytQgdLQdM0T191aqjlOfOHtClAgnIwlYhi35ADDcT xmBjV+hhFN4hckEsC4FZEAUl81RfyrTh75jSd+W2NprlW0tKd/ZYf+9vv8usH9QVch6krwxdI0NF uYlKCDAzjr2mxUWjMxJ0UngC38nstIEJ/Caq5NdEltMVZfWRZawBpGb/LecmOsHHd1hEXNuPghd9 IxIAiVUkH5z4UyjG4/oyw6xX0ABW/s7Igs/lfCVk9KStLh4A/iyNrK70kYMM8JQyhZxBi3rbDpZr Z/yW93k/zVpT7J1zYBzQGH5Gp1a08rRVDep8EUrnQ8FSYaWgMG/QK5cx4QLibGOP7wBKFGQaS7n8 yzrAeuM05vO+A48EPLb1Tc0B2QXvxdYGe4FlJRB8N3bsXmaBy4L+562omPuG33K4kJ6plBv1fHs2 a3gB5Z/JvzBdENM6koM6jffvWcRtGWKcSMunmXAbEMCi6RgHtcBd6EQ4+6rVUM+Jac8RVNn68Qth xpessd2d8WL/6C4KOJgW7tFRNXUaaVIzL0N/HAkvZl93/gQ919opJQXvZOmVn6X3vc+MUGk9lXM/ bxFqmiarcugF6Gdwt0uHKobSK6q9nDmamgqKm6q0hBWOkXLsaGVWd+EG/7LDt7wduhq4SA1RK1H9 HAyF4Vm5TxGUEEk0ce8jcFUf4BVAJIZM6rIcljYGJH+Lk7Gyhm+pz3CwSXUF3B2sWPYBYk8us37F Bmq2yKhI6Uny+mB+bwJeCJr5Hckm5V5E/H7ZejnWREgwZyKcDU932SihzMGjU9NLBIkFAGaKoH9L QAPr8gkOlRzRaStw9+56P0fn3mbjoy67KkWC8/beacgtFFLxVz0YhylVZOFPmsoUD+55C4t/EO9T ELKRW3eH7Hf66vEHxe4V4/j0au1cdDab5PVBDM+XF7p6BWtbd0lqzpILZ+F1Dmr2MPHe5L9BIl7J tRpalXcXUBpt51zI9xhTHnBqwWS6tC/fOEPMj/d/n+KI9urLpD17xGT/4PNCfKdz8gXFwjouomO9 C9Tqj0fUq16LdViHKQbNy3eQa8sMiON4FuYJfY4k42qT5FBqGQfe0TK+4RDRmRcLQ6hJxlHLxBAL oIkpkxZX9s34J9rE2dLdLWNGuz3ju2aeWpxCMTeTsSfC2nCbpDaN1jNlXm3Z8zlf13cpST0S4y4D w/28HYOXX9BP8R1pJaRuPHl/Xvdko1/lgYkGAcimg/iGFLIqda24MKcJTh0F+Q/28U61BSC5CGlk r71n5IWVHbgIa49MxjZfj8WVUQbvdizVQA9cXkpCAaB/IoDJae64gMGkEqCinyDsdvYAkpIy/5LX qVVFR6HJHQgxJV0SkQ//Afmw7AfpDLLDeJf+mYCXTRt3v9Q6kGu6lXRxOY2VUUqeLNUbtvaIfcRP uJg0IH/gYcEbWWlL4wrXl0kyTD1QVmTu8TVQgRnIT2fsQDRk/SJKbG/NggmUYwHYfKhYuw+rhX1+ T5Dvz/z+moA2Ie5ifoNGsUTvRhyiTvzEClwSMQD3jXS8M6ur3DGGqrgFRqIvW1MKICiiePMoQiab qKm/65HV1Aa+SOvWi//RUWB1vMSaVdnYc2plSEeh/Gqa4itjrgibzfPnfU5tUfApK+rdQ8zkUi0H DSTP4VrNI0pVa0vGPvPidqdIP6vE/Zv7caBuJCUbkde40h3j2LZfv5q4wArM+TnZjaVCCpw0eQpz fM67HaBcSZ1K/E6suQiLypEuOVk145c9M0DCokk4kivHWNfAcZ07tDfyOJ5IDInF9yNbfw3bLgXW xcfB+/1Q7q6ya9+wI3lygdC3ST3wo87Q6Xj/s+2qor5JZ5lXg0yQ9uQVT2Su+EqtMUocnePYqfnD 4FrU5IB7kz8rH1b/PXJstdz4MDNqUXsUk2+s7MrUzYVSq2FRwGtWYbsmplLrtDtQcVvuBtOZ3tRJ TLKCqokPKMjhcHKjiPu7c/gZyFMohjxGLLcFFTEh8LBiA6i6Ww5wm3loPoJ0jZwUEgWbrknjJ6CL lup1sdR9sQC7ojgWEr2zn5e9CTB1RQckiAh7vBARce+cCK3Su8aEU44L2RejroKfO+oBcM0MIo0h 02IcQQTBD5+sR87mPi45X9PuDYD41SaXQEScMQHTfGgMy3S8nxxndzm8WAUIqvdGvnJW+3C+dzSD KCQedi9XmykLrC0mEF3NCD98dFA454slzzWYkqDbN8MMVkJpVxq32eZ0CyJZxa7v0jQ1B+aiaVQ2 bMq5eFrcW6v+BlrRWCVwhrZFrNZM30EzCWF3GTz6wQNXsShCMAva58KvLhtvHxIBXBJoxaHk2no+ BQamS2XvbiXBjOa/lK0ZsvCzvMnanqf8Q58uj9gjitQe7kEcD/WyL8kuEX+IFV+wXbTpMDUljgho gOLliIq20GNFmf3t0CtbKxB7eK3AVxx4oppC72RfWLymcXMbvywctg3Ynf7EsZko0AbDh3934hoK xiAjUdNxID+OhwbDpiOyfi6HcBdO8UJ0NcseRWT634NfXT3fMopkuptc88EXelmrZw6QI94PsIr/ rx1ZAeYtBO4g7SiMhK7gVylhJ3AutKie9Rl19ALvoQy3Z9eODmmRPXG84COyspt8gSSwm4Z4ogb4 VAs+0NxKjppE5gmtmy410krNQjiHQuFGJjNOv96vD5/pfqxcANR+ognHhSqt3gBsgQ3PB0xK0oBP rMvR0YtU3LmVXvXM/z3S/0/o1yejanah1/nsg95kcdyIUVVtTMGo/jltW+p1BgI8NhzDa9wD9K0v HtNvl4pU+5TGkowE/6lMpR+gDYAfAbS5xoXhSldFEWVK05ZCzpXsYR5CfNm1+9u+AooXISglAWbn c5M5ywzNCfOd0c9lRCDGupSecwzZlXDZWTf0HCWLXTtXRADAL+q6xS5IlEUk8EyTYb+4kAdPAuvM Qml8xYzat+49LVSy79UtoIVpB6Go4Z2VsIImgQDt6s3wVqakYCA4W16NflLcPbxHoidUWbYJA/m5 uFL+zU4zOYvQnn9fJU69r3teqeCbxdPQiUfSC9DzeTnxG0JpILyDnwsD1BaZQCAH5F4YLCX10SgG bx2Umd+s8xTNiZ0V44uVToBoUoQDnjF1QaXx4sMVLM+b0oq36pTsJB3ZE3NdSxHgXiRkTmTZzHc6 Yfjx5EOP8v2IwrL/CnUA+rK67zuSFHQcssVFEqkbd6ijPzWxQ2VSnnwfl+SUHBKTtfed3z2Lp3ni cH1zhnGpDpM8TtHoLBVFkKai6EFmFBMqzUc6/AOLv7RhdMRo0WMe2ee2foYULbWIkvvrqloizNdE 3l+sEgaOTTrOeTTu+mL81eXowdg8x5W+fsUj7gqvkZnrOmu2dCbhiYij1eVSRp+cDAowEo2bYCNQ VcyVdmqcm00py3D2m6CPLr6CXgxES+3Aaq7v60o9aHV0duqlW82A43LtTa9qxLFdzHb/mMxL0WrW 3de+mvhbo8KbxG8n9Yl8rKmqUXUIuUc6u2l+ABGDej/7xc4Al/0REPSJX6cStRoUlU6c9wbNFUNn 2ZQvn37Q0z8rRXxaP2DTs3XFOUeEAhh7yV1kvRRsxs5zQ17geQEaSmZTjhQ8Sc/mw02128WcKYCO mNjiudqai5fXO7bxhKlwkiuW25VcDpgkCfbsPFeEfrQg8sT4DXpTU16y4fG1ZAJa7eLDNPIHqF0Z Lwh/k6Xy4JksEM/sUqT42MkoXIb0OkrSSe0M0TRqNcym+HtpKolSsN5VCBuzeqPY/Dy4kJCmM0mO 9j4zeVYk9nTA6SYUmALBsviRNy60wBd3hgyr402cbroYQHIdLQ+k/lkzbAf1pCgy46gyXwzCeWYL 0M1RQ6GEj3B7jc6Xn3Xu/f/VOC/WVpf5Lmin03oAR/rzreT2Zzm8opPJG/xxuWLKBXjfMe7VHlHL sRNm4xEgC7SzDA6BHq8XC9jYAZakbdtu/oCl9vk2SmZ9/zmZJIISFMqRx8QcSF9qCdrCMyPAqBB8 KYvtY5ZSyVCz4cv3tLUfzj1z0ogmEQxjcsnm5T2XC+76KD2pMHsc2b9BbybLGZeylHxi833I6NN4 kX+a0E/SLDBDOEigBPXr9krQfHb3edTWmY57LDYNH97cbxlsOXlovLXn7MbcgZaQqjKmUeBfyIvC MVIJQkntCI7BC7hqJ1MJiKYuPfoFAMnGk2EePkeJweK+lb07jPbl2DX0P7ymtVKgG0rrlSkmn1FG 08FRZu6GLmWOChHB6tJybotxUb3/8h/1L0oN/N0RUPyS2mTnlg75bEqX2EN4Q+o0m/wpseSNOzi8 q/Qo0UVAybdrNH8DdZxaSP9xg/vGiCbpgLNmUCOO8vAw605SaiQCyQ8Ycw0NLpBhfVfbPUWNG+ei tAr+XlDxlAlf2rDUzw6dJetptms4/o2EDlc3azL/cH98RHBO3KuupJiQtLVwgQhnZ6PSjzVXwT1z pl8wScu3PDp1h0h0itnDXHUlXuGpeQGZ4Yldy9I6Tumn0MsSR8gkuNW6gq+r+66GaUT+N7o68BQU ShYP+ReBM+Qx3ypWH8dL8zP5ExJruoUG7f3n0Jg76TjcA8u3phM65ADDfh7xy6Nkxt4aV+bcKCLG YwQgpAIsqNJ9pi/0xLBRGqaKQeSfx0MByzEox666tBkBiCTSHxciwIyQmNRnWrXBrwV8viWBWTs2 OfkNimusTzQz4o1FT9Tqz607HDe8kvS2rL6Q3xBpQay531vskD8c+qL36rGeXxjmXfAnmXBUgp6o ukhw6lafqjeWMKf3EK3fQWh5lijJt896Tlxgi3DXC2k3bDJ9oD3vqv9zhJ6xPa2mM8Gyt+VAdDAU +AFBpK0mJUwiRF+p5yoSfFNnhEUzKeDWIPeyorqEc3xKtcI/P3x0m+p5gYylBD3FRW6lxCWoYEO+ kmoaFeq6I0Vd10O/XNekhRoTzBZ2nfp+pBs96q67FrkjYqqga7WxBqzdAfhBZM4Axdv1j3KRHbyr MYbcZYAY8I065aM7Le5WspO6TQSFMOL8wHHBVehv2mcimaudN7XtKn8PwLk6mOO0DlhI6Zbk6ykG sA9UtzvQXb9BhgvMw+e8KTiaZCudDbOzDL8NJrSkH7Ny7whyxy3C+ToUoQoGb6OkrDGjbvYAvpD1 0DJ+J+FWlv3aZHVELV0Ky5lXb8EShJsUzBP8/q0ToCDTNWXZynewXvtH2rv+Y273bgm68iCsu5Yn q7sFKnwulyy13EIMsozwnQm4qiBJht5g7S1zI8c3CKifIsfITC0IPWDdgT6ifnnjTI0/BwAzqQqz WcbLiM72j3Ca/fgQLgIpbdb6KCpFP5vCseEfv0c8YF3oLlYm4mjRb6/OENZ4pA6SJNhNVDR5ldbE EMndEzEKbHrph0QUmCZjfjBq/ocYHmL4AYL8KM0unlCMbE4YP8ejp9zcG3uQEXBDGkhQWlbFROP2 rDkuOI6AOCSgNC8dgyj0O7ZROP4X0ySFjYNRjXQLYOo0fJYq7Xbb61tS0ECZazYe52Fmiaa10Pan UlZP7SUj2a9plsiWvkIlbkq2NUEzvR58yk/QplNrg11fYdg0i4IacwIlrmohW24VNBJrcQXEk1U6 6fZNSOAleqvOJqe8xFUcF0qzZfSllOGNHxvZSa0LVxagrjmXmNT4VPMDCNYOS+nYumPtAApjnChA /RyjiBEnBRZitjpDWOSgGPUhRp0zCEAImKtBALvhM1Xy4ep87dGfYWB7vs8uzqc9+vjFPk3syzpq HBB8DuVq+3hU/Tb3YfsslG+kBpUh7Udz14esiBRAnDVIUgxVPoxjJhgcgtWUscOqRl7urfJOCb/e E9kRlFjrExIWIG9TKns8L++wv7zkeICy86YfQG1/E3Av/8qq/OaUdOYGHaWFPBkBeQgYRBd+WVSK oUXH4D+UVJOB2mGq2BYyCkPyrCtTPPvmdBlIZJ0kNaiIjADg0PmnsCG6q4PLMtBa2eP+Qb6jEFs7 BL9+Lq3Uf6PMAV6zZsmTz7Wql2s9ap8N8AGu1Gr4Kfc2gF3lPbJq0rcsuQp/cXxilW01Fr5Xy9YB q+yl51jJ7YPce8bQ3NJkNcyNIlb/QdoraeK3iy1xCI+XKjbLTJCqiGbwzvHBvjm8CxufSBctMNCw IUHHkvdGQOs4Ztys3M8OzUdLv2oafXJIPETRkmyy4TzNrw3hXZFS6QUBAFZyf4aU+4TvHjtffRSm yPcP0MYsYARDeNA6OXTNZxaERbQpZ/qOsulHuGf5yca39kowtL4VyQuRLlk01XQ1mq4jSNsdz8gG THtSpcW3zI/QXS386Q3N1Zw/tB25W3XG4vgaOlSk8as7jfa+EWh41ZCRxKeuxZZTuIqMXpBPd3l3 406YkqgjsZXsohEAfengr+PtQ/3i4keobqu/1wIks+7Dk6BoTtU6M03lFrk6xbAPh0i2K+H2xUnC 5a+zV1gTQJD2/FviAhPn/cLSQ5NiMe8CW6PCb5RTfw6CXmUbv8u8k15UwqXF0z+aKEfJeq6LGt3C VBaPyFiY9WDiErxEOLRWf8eCW/X8zEF+b6rvioJFf5q/0ko9Xgh8oDdcEWP5sYwvPxrilXY8o3M1 mWrvCXIUsrdNBFOVY1TC1CUzFY91ycgQmD1DX1MQo8F98lJjdOkkM7wih4kQdFQAB7xZcnt4jEk2 2+ScXnYfZ9lKFG59n9GJ5YaWsPWikJyQtu3/1CrTQV7caUlhSuaxKU9hqQJ0WL+o8v3e1sHi1N/A HZSjcItueLbqDZXr+j/wgaYdeU899Vp+cZ0UjjPYXnIJ8kNWWHgpsQ4NfvY17bzZa3eahi5Cys8/ 6cekpjwMfH0CClTlK84BAAKOqvDIoy7f5aKLUYe+idQX0VLOVYeAuDH3ZCLoeNcxO2NU1INtdUgy H0EJya2XZBy+z4XwUWOvzibL/nyGy4kQL6r9y/iRaYGqLrIkHUo/HVrpkU6PbdpUqIxSn1DTrlmf YgBc1LcnbZJgMLWRQqZbSqmTe8jQWtQTJhYZvS+b7fVK9jyF2O+37FRevxeMytKLpQXceZn/rkcg BRE5WSpNwqF/GP4b/UHBB5dRl1ehxs8dj9BJMTPSjmt0nS8sT3jB/EFQajqmeBD4x0e9BsGIN8qu U6Um+POFrTTSpLc+fTq716727xvh9+W+bF7UqwmYmhwc3O76UjxO+NCdLSVzPidQzVMdZY/5b+5W LzTlgtEMakyM176kxut1rjOzIhfTDp5ynjjjoqQwpfYidlbfw+XNXod+I0DzU/+T9YdBRLBX+aCO FETxd5Buo7gtbJBxrVsBTgEPeFAjxEoG0b8nE7D9T8Vybzs4sxEGav4xkSBK7Kf33aPh/02XKv+q y3VE9m5rlH4NsXjP2KJy0FZd3DeCpEzVu8HTFba8UtFbybTQuO+4bsFvNzu8QcBvehgtOBO3VNpU +RHIgV5vva8NM5lhEn8OdKf2tvg6b9CHIzij4BHU5u2zFYTFGNJuDwE8bimAYaRJQ0FzYs596jfZ dxZJqRO+HQQMbxMVE4M6eAahl5ziN4mhARMbSSe/GbzTe6ThhHl5WrM1tUkos+c1FT8saXI/XksQ mDpSblbZzpFcO0EKZJoMinu0d0CRDl5iJhwTScD28q1qutwkMtskAb+U+7sG0PmlSNeZoQ1aimhL SjeMiSaQMF8dbyriraRbTFqB+zKCzYSTuk2iWkJT/tEaV68zmLSpUDw8zYIC/0WC+jfXm1q8kSuu QZhJzWyS7f3bcMGr0rpGfPvPuhMHjugrcOnxW8FptfymOZV2xebiF0xMb9bqY1isgOTb9gKeRCHi JUgjNjFecG3egjuiYIjngnozbhizpjiaRUB3ORXO6qk0ST0xuuVtHI6ZE1xkk3UEnI3VngvsVowV jk4d1VaD7DsohRDFmgBhHvwkSYL0ILdi2LpmQ3HKsLQONr9JxtDr+ljg73NRF1hHMNtX+9Sf+7B7 K+aXm7JBBzgcDyJZKKHOWwsn0j4/vSZINZV+X9bsHQ5f048jftSoltu7kAFf6+6/GWOI9uoJjoL4 XhfenNpJ8prZt93a2/65icBHLSbfv34Cphkf76aRBwwYxKKF3wYBzOmpvsOx0HJl/M+KoCi53yAC oh6/5C9Umfb/jjun2uRLsSOuRpj+eccLTk3blGQG+fcOJuHMG+De/j/OFns7vbsZtB2XPNiO/Pdk LCLPCUIFjsMGZ5AWlxml7jGaDLdWkz1xQ3tvzWofS2Vn8k1CupKSXct9eXoKASG5vYpHRTEq8Ipx TnBDjDoX817xTsasgXmvlQRuE/dlvoLiiyi6S+nQ0KGX3eQjRTst8Q/Y0qAhweCgcfCbxZL21BT+ Jajc4VezBqCrXOlpXPgI2pQES/zbseUi5/ibO0LhGF13JZKm1MABgP4+4AFrFHbijcNKSIHrwpXl ShT4/TdlDubTj8hHneEdM7yqhG9+aAd5eZxn6mtR0ZIq4tokFYABm4Ud8voWQiF+tyxZK4BRWaD+ K3r9eTiUqvMFKTj0DfRQA+QQH4sRDRT1AwTsbI1SL2HVv3SialKOgCAw+q6XzVmMM27RaLRNkCd3 NrRxUY+DhgAQgrIHXiBvdP3SmP+m+DUi+vRfE0lgGntA29hUyEZsjhq7XKlKZdAI84dRsIK0+nOH FfjocmXeKwz60OkHeltQnFdWP/KjKzsIsbfTelWL/YaM8foXG7APXKH5AziArh36DD0NbQ/ES0Fx BPXwKv/+8SCFDYB28u2x6wNOW71OYgNW3FV008XKO5CA9sI0ZtMHbtJk/YPajEEaVOim/qwnjN60 UC46yOkz/AClM3EgVuGttXly9z2E3MkzPDH9AqedxEeOb4hc6SDtqCtUJ63+nzAMR8ZDpM05XQMf HJQOdKTo4i4rFlUL6OOCLBCUh5HmkE1Nyj6tf/OfHAeETdxIrwkUzRZEwZvjgYU5IBYfVMaJapAE IpYH9c21Mk6LzCcpoI6Z+9+vWwJAS5MAYI43XBUaHj92BFHTHcQwjg7RzP0RH3RAVhjlVKH1WDTg ob8HDIy1NqdKqcle7dLr4t4lk7u3r0ny5gA5JxnyiRXoJUAVzndZkXQhRlv0j9gvT6Y5/MP7eaun yfZ50LPtbt8lcRGhXbCy4IqeInvtw+HwK31cGQfTXpTavb1OrIFufLsRI+9N4XFi3jRNKLvKr5qM utw8X0rDBIkYL/ygnS0WfhDZCZj817TaymYV1gjcJzOMp13JEERyTp0fKezpsNyREo/c1cnkvg4x AibfeqLS3Niq8e18TTRLFU2wYe0xaQ9Qet6GrMeifX4OBR/6tjdkzN7GOAaSB+6v40uSQBCLlHW6 MnP3+tg7XemTDPfDPyqb7jscWRMzzHuKOgzLk/dI/+DyL0U8ZT0rjPSSnqUCWn+BJA4d0m9YdCmf cuLzZQwG/T8ZHS1IQgMSEr/JbEaW+ozrKG1XNsi2QjmrZJuKYOksMOdfP5jje7MNcl+LLgo3svD+ aYNdk8o55wROskE/onGErobdgawLf4U7f6KOeV1UV6CK/7rGI8HXCAOjm8IP4KVtP60iEiAQVEh8 iZTQyuG2Zfb5yKeJiXX2KDkuK0CSgAJwiV+z2ZAdkHo1m4UphN6V2SfHQ+1DeACTt5tdjTh/6QdO FK4ssI5FpflQ9o3zUP/njvzH2eC29j0OQfKfJ7oNdfoyu8PNRQwfxkeKCGPgc1uVqcnthqlD558A Y8bQhDheTghXNm0zTRqcCOYj9j9n5M479/9mSbZZIh3L88bjqwlik999Xzm4trgoKRUoQOL0zV6Y CRY0GPtzQM0iuHck45RnrU1b+LsoOm2Apa4B12/fIH//dfvb/Ia2tUYM9E28arPY4NplkE4xGIJ9 xLjOPKuEZEKU25vSiJ4tYF6uIweiVxqd2gVYDEAs2YL5y9i05svEgb16W/AyeD28OREicS5QMS86 dKlWiKjHfDvUyg0s07ZFOtuW2aeUoKnM8ZiDanjdm6iVIwKYDuEbObpXe3q+TH2B/UNLu89n75H8 pNJv73fci/nDsW3/hdkjMydLa2zIJTNIzxZSjHPzamB0Cy3QQB9xG/QhegzjdUgyQrqNFGG1kfhn YEXNOLGs79tZHytbkn/1ONpMcn0ZAjd6HH9+/lcw0X/okRWZbLcyoUXPhfvGiZyHKGjOJ0AOuclR 0cPQlym7bgfhUzkXkLBXUXgbs43NveLxtR7Jn7EKeDscxo1xYZrAu57gIcam5Ff+atYuVcyiyZxd BReal5SVHafkrUIeguHXLvolRo0kcDB8mk15jaywv9I9rDITpfscfvqAdf7PJsMG+c5gPiP6uApr p+8Uh5+3x267j8dGRapyGekMvSyqm6dGPDXpYe061rE2/STw70EP0eCJ4K3w8QLeBVDSy0IdA53W iuc/LnwNk/UGO48xA0ilR3Q6POG068+7jvuP4yPox+5SxPJMcwGVbO+WPAYdm/Ma/BPa0btczM27 JmtteN52Eh/kQ/lDcACWWDlAqFRrbsAfYHRKGI39r8W+lR/UK3Vwr484SAhnSoBZsOu+ClcGxzCo ZyY4vquQIdhf0+5C89MGhsQApdny1BekPFTur6SllOVhbRftTLGH5zX18nxu8yoBzhmRq1RzRn4X mlU054oexBhKDXo+8SEPy/1qYTLUitnbBaeHPmWozzNo9MNDLc3B4SncLXIShZ1LDK9yLMuHpq1G BO63omuRId81gVtwWfle2RuMxOfhplM/Xo0iOuvMuUjdH35XW/y9RPOC92BKU4TBQt1zi/71jPYu IDrvIz25KP/e3q86HlrB0NKQ3MKu+MbHnf+be0HuJc/G9YsVL7Zzv9DxhcGLlOxhd4/L1TqEfnQn zr4s6GWYgR1qCzdXo8gpXldeChxSdDvMtglwLspU6VdwKZrYPw5dlVrRW3DQgqG4Dn8pItaadYcF TxxJmXxgFIyOSf77Is9GnSi8t09nssGHCYKO6Ejwz5tCTfIeA2GMhwn4y/pAmQlvNjlP01QIHofc nfD0jjma+GaM6suiufnIj1kecA8+ZPppDqm96FZ2Jx/QA/CZnCGjN484AGXQf2TWXVfPKL7QposM XRCZNEGNwiF4GZCr+cZkIFapwWkhz24GPyLcD8+Fp/89Hn6HCbr/VDDUG7xSsGGlDw1YqnW7WEbJ l+baBYx48epW+TZw0fncTkol4o9G6RlniJ1U22V9Njbej0e1gAaZeBbZSk6TISKQqh4y4QZkFV7S lHaZXE6y4hcZU2nORhBR7a+3ly/dcKgQO30yx+mZZah/JufHEMBsBIRvd+XHt5nfc5JzeWYbr1b9 y4Xr2rWueY39xRLAv0ZWXEYvn2mUIyBDGAcV9uDwhV6IjsTewybAhPpqlYG+dRrjeXtlp4Uz0vTr LVEot5fcRdaIDMnt8RrZTNmh+j0VrwmUPHauyRF7d2ndCbp9uLahfCbWBgfeZTfaHAsc8Z0MgyhL MzUOqGjHr7kmT+AGf+up6/BwMliHu85BFg7d4VzHkcv93Xf9KsoaD2Rc/1SIj10BvNxQSkUYehOA R/BJka7mlRbLhlHAFQ4bagEakyn37sDBGqguCXJTd3wN+FXyHipR67cKaS+pnOfIiWKuSlzUG8WG qrIEsCZkNGul5AgWG6TGYucvdb0Ty8K7RSPfCypn87HNVZNopmv8zhmZOp7ZBTuwvqJNHrVNiuLH yYYsRs4cAhgBedIrYknhWYHa6lBMX046gOD5w+S5g2gY/dJ9seIc301lZ4KX84AYq661k1q8VWHi 5hvY3+ZX7B2ORILeCM0JiUe5nlMQftw94AkJ5mvBT0zZ7c818uWel2rvhfYDVdAVXpT5WjU2BQcc n/lSZDH+LsY1yQyg4jAahAN1nDtsIdtH1wqyq9lLLK/BENL3gQX3DqUKnXeDU2zgSLaeBtsMTHr2 sqeub56UD9A71A4oPu7EDLPOvvGC3WTcDx2mAJJ6/t90gguB+KgbOTQiuSna8Ku7wFbwIRZ5TvvL YA/9Rfe/Ul3s0CAwhZ0QFr3pOb0ehXWJRdbZ+19lTu9L8ewspFYACaLUxIlmfHd5qypd32nBJm/W zue7081Cr4ePQoAdaGT6qSJsjPavCSxZPD+//By+vuGg3tDDgDAHW05U0vsm+z5Yji4XZvTQlLPN nZuXk3x1PAaMI6RVwo92IPKEoquxnNgTjlZN5Ry6ait91alg0MqYhtqLTZwg3P4kV/CR96fKvN0p LFcKyRWSrELEGRyEJidQHR/cdPerhnv8njo03AlLrKvjNnHv+fR2JzQY8ZRAArsDjAktxpZtTeCn 1zscO6K2XKtRjLjuAtd45FGw7yxdBZMeZ/Ot4ZIfXV9qlxAT1dQ5jxVeSU+LYBBg0EJ7XxaHlY84 ClYFjs1tbHvYYqIzdxieSpTIrMFVQVtsOmhtuPDJ9sAyr07fEfdLD/WsXuW5WawoxiqXyG+wk814 7sTr2nxgsCDtuXIp2QKhFRd8Eq06gGFfCRLW+E2VgsYkaJv4Y8pY74YKTlIDSKg10yPTMWiBAaWR pNbmabfwdtRJryj+dv7NYRQpLI6eYvGD5D0TNNpdpILH5Oet0R2W3PdluHN4mjeahcxRSYI4wziU powjJbum2aUYb/HlDgs1xFWv9CLXCpk6vT6+8u3UOSdmjY8+qMxJCU0TdLoV+ZN7iGeeEiIIM9fT VOq5LvY88IDAN/cY2erPSzO35Kkh0rd2G6pjry3UkXFzP9SajWk1dzl/hnRPXdPP9OiByvFKWEAJ mHtehxg4/N9gg9pQIZIwHGMCwdKicXM7AMqHgoPSl2CAZcsa/2e5qaQBn6xRq6+zAKQkYdTSHc7P JhGmYOaD4SRycMTWLpaAc5I5uMkng4gXN6WsWu02kgmD94gJfv0B49nSKm/HKaYnHRmrqla4mwdz jr42g1+Mh4gjkGwWGHGEsUQyzQnr1QHhGjxQ7vawNS3g1cuSwMumzakeqhY5WvNtaF2wBpTbKZV5 yDKXC/kZeszlzQQV37PXCkHMnB3YTqqfQ100dsaN3FSZZCMXDxAVtJvBDy70sEIHFUm5hwYXVaEt kJOOg6Xr75OYekwBwnsMAojdCznPqyXT7c1HDrksTnKJT/dEA4582/EqabYtmsXDYELoaCzD0lPl 35YYsrr4OQK6bUkVBHRsD2xSMOjrw94n+DI4K8W0g7GX3FzuFmN0KT8kOkXi8CNDkyTaVvwaARkM CNP2wsmt8we6hE0DO2/ogniuT5EYn8oQ31bisUQsAvtP7QzA7GcqbYN0J6WCWvolz3oJW4iz1uFZ wmCruH3hpmbqv+1qcBtIzhxKbzXAk1zFxjABEOXzCtjSxjzYVXw/OVaU4ypamtFHGX97K0YI/AFW kez7kEX/YHHE8gCjfPqWbWSIvstr9NkuJpmjnKwKHFJGDyK8NVsWpa/CRPp87Iq32DQHs73I6KEj f48ErcSwzCqGCAHYflYCafeFsSulpyH6WhdK8bPFmC4wTxyy1AcZL9G8Kz61XB4DyByLGNoGh764 Rld2u1eLsgFZGUpFh5mY5s70bzuLEevY7rVafjzEqdnnXjblPoV2C8nA0OQGuBncfW9ZilaWdz7j A+3VbPx280O86SoVTVs5ksnR8X61hzeisU4JXOkXiGFZSUghWVzp0xct0ZIDVr+0oBzrZe8a8sfJ r6bUBwfIG+ZhPDK87a0nWDY9Tgg0xbeWrlo5L9Zyk5GRBu+UoLYbdDrahL5hQ6KPvio3hEVxOMjF sVURFpZDRVkyxk0UPc3kRpcy28mRQ7h98qQyflTibccOLDkykl717Tx9Ke+7GhenO22HJKSWZI3c nScrCK0w3HjP3aj5NlZX+jDPZr5AVdUdNpyWiNvy+dcBHlLVL1XsGd023zix8EBJauEwuLOd2LRC TfTWi66Z5HFcPyTA7ga4dS6aI4LTmKe/3IsT1kUr6E2sswaWO570QDWFXC+dHauQa64qR7zxpuii RhOr0Zm1ySJkVcH4rEICq+YXyMofE6/31RxtZB0I76iXEcOU4KDltB3aB50TLTsawm71TKppcMXR Y/hY+u40BMu6nojdwOJCExBgVMT6rjOnAU7P+v5JFO6yelEgv0qeQIW7xJuFXU+7B0FPWIbxwt8/ XoNv7RPkKHd76UVHuQcktQUVncbxEkt9c9bHKrou6xDu9CWz7QmwlyBHQa1RBU3Nzxne8HWXDIjj GI3Dw0Ez2GHk02XLI0R97L6fvoEutulUeWtzsFALFtyUO0T1tJjmWRF4rv/Vt86luQ4Jr5a9IOZE wcL+XMGVRtshdhyCi2vcDrfp11zNeHNUUBEGN2sMVrLSSAW5Dx43XIUkNH3hhF3oDM+hBcVpP1Wj WzhToHgC6brK1iUwMDynBh5UsLyncAg4VXM8gYtFWqn4RaYAfN5cVv65Cyu2glhpznBCEFkkcLVz pirzAvuSmQ7yzqM8SWtsna0v1ncIFUsN2wxAO/WzvldbTJmrSNUw709oIz/l5e81byBW6iw4y6HP qJAZJ5tZd9TqWuEmjLoNsCV0/3SzZSQH0SDzWCKWv/XnZ6q547R4f6MeGZPGFR/pX98Fsx2bjxQC +ZsJRRmJ3L1KUae6GxY9j2fUIrtR6qp9LaIX6LeUXN0B5Sbzyoww+ermu3ZT2xIx2Bhio5qmE9FT iRFpk+ZFFUb/n5HYzUBngmXyL82B+vd+hJ8t8xCaFCtmnzheQsa2EjzAiItKk6ldEzn5PW6WDra5 I5wvJnGFKzYK0ySbRc+o8BSsAnpyAuRjc69jxKeTYagQaJTo2RKza1t7x5dL7CStC3Xo2yJDR/Ii dk/MHAlCffchZqXXA86h2gSzwrnwWq9SmeSYLyz+tTm0S5JJasx3+iboGlJttWjIRuUoNG1uB68V qjTpxZLPjePG/s6Jh86xSBwseg/qxvl4WfRapv+gdPKw8KAyujjbmdnyCQd4ufYW8RYdrWOPb62j 9db6GV+eNrfzB+lb9bWkqlkVQ7wXbjMhi10JtGoa24asQGi4AviFoeTShqzBCXmy5Xhxk1Gntj+G 7iYp9mBq/jwCoHbTtQbPHdc9rPwe6jPPt/zdpnWHucNOEN00w6ZRO4xQEdixTj7wQPWi6AjTCjtX wnWRtGYuVBqWvE2gd3CVTUj0UymDQq6qbvV41LHzTvh6xisWy3jWjd7V9dEHQ4l1EobKBO78Kfs+ 2T1xXb+a7Bzs/muEnMcdw94qfnV2fiO157GSqr2F5Vj3WE32cVFg/WSbTjunz0X96i2OOvrbb2rk YZ+kY/6Wt8vkZ9BM3MSj2qYdFqyKuWlBYZRfIPdE52ZCGX+TNDvWVur8SRpV74Mp24iBkPzpLjrn PL2RdXzkrHG8lPStbcKdvt17EDPP8IZgRxVIOBCaRAWWKqdRArKoufhF9+FobeLeUeLXjMokZvo4 VbJxZy29DfdbH8T3pt5J4VT2DhM7cC1NOTgIGx5rgLhYW+JxYyyuNy7BAdsl5scYONFvYsBEWk4e 2ecBbBTmyST6AhH9NiyTKlLqaqWsja8rMXT5ZfSpdZpemI6jxEx5HsJmuXRD4mZk5Z9LoUg9rj3l 1Pbyzx+mBWFc4aLnfO3xANmr4Is1O5T2OEW9TKxnOOIFk0BO8yGlpidbYWRF65UxFlARLyXyjuk7 G603fUYOwEIrBbBqsc0eMQyaVxEtOndPighMWZ9ZxTY41zP0lcuNCau5b9erBVqsKVB8+1P8wK5B mozKAWH25M/Eg9jfVtPHHUBegtqHrChvZJG9SBzYUh5KQqPQAvhBKf5xzy/7JATpjmx8yVxjbrRC qf/GKkdGEz0ehiBWmC7KzN7kS2gVOGVBflYgvI2iYmrjVZ2IXpyhcXJBOugKo1Dm7J33/uXNt3ri 6XMIkN1BBp58daCz5ecNRKXdpiDGv9GGJz/5XiKvkZau+Olk7tzBcKSzYVcis2wXdPxL1VtpJ7lJ 0dzw4D8ipNsxyo4oyfESTK7TGWMBx93eYs/XVUp4SPKvTfdXPyuC16fVhvECP5bRN6u5FThjGKah JQlSzbVZMLYp138te12DR0QaifI+tL3+ZeQYlyyVy7A2emSxdK9CcjPDLomSC0l0B4tQFDlPurlo c6SfZNsI0DEK0anIiqUOjjUciI5dPqcR1MMSSFjQeXrPpRGvJ5tYdtt9rFKkhltxz7gc+UcF+spT QEznn7DDP0ZPKpBEa1rq4gPEikpQGjmbcXT2vlQHNwt2NP06xiLShx0mWrt8Nrjry9eKefQYWAYj K3hKOu8GGFupsRfu0IJ/P36ZtcKaA68Z73QdS7iRZ11UnkeaMA33hCN2G3tdkJV+Pr2qr1/y9PMj Z985DuYs1+ExSYUUUELUx1XErDVwJMytU1yNwkcBN9iSNUckiCWCnuyeV99a9gAQOJWN+2JuJ7uj drPafYvLQ8oMgTZfULBAsU/hVa3kIMQRJIk0Q/LEU64Bst9LDE3/u8ZOlxL2q6+YX+cQ/DVTorD9 sTGps8d2Hr/OSc3MA8EmfZAZQkA6aoaG5hwhuZ5xTG22N+0DjLFYzk/3uhucfhJfQvEf9GDimWYD 5Yykxd8lHYpXQ1BIsW5Ayry1O2DEaZUC9FfiGnfBew5xcHxwTCJpQmfYKguEiOvoV3Gyxzg851zb b6JsdzFvhqAe02vCuPl8Yz/Rf9c/MmBQKtK81eC7SDI2slqtNmpk7ckXOZDbzUIjHza/FmUmzY3w Z6klPKVceEnXwaTlJqx6/RDXBvUpaFMQD6pmL02/bFve2rR9lgiPZZLFXqYfMGGxpoZpFxm4cwHp JN0Jy/TRxDdYltv0fdI1kUaN0+hFZOtJaGMnTX6fiNeSMH1HVHgMzD/acOqaXJrr+2YrrT7hu5t2 OByihQ0DfYhE8705E3v8FKUL+35K5Hh5X+/G65JHuSdATeECGoA+LtH/42AuRmT3Fo6NvvsFAtI5 wjTTSpHEtA/PRWe/Vx7w91hUjbCib4kJokZjfxzk8aCsNiZCBP8sPuP1WTzITOJy9UXR/eWQMUnw CX/PJo4g6gmvW0mdIdMrCxuaHj0/rQn3rWFzkFAY7Wxyv/JzWuT3kcG/UZLUd2n++VZrLKsWQVX1 sUC8sEjsfYC8ZXeabR0SjVyhl4QBu/L4Ax3mblTts+Cat7WnFAHHAxz3SWCp1IKEVdhXnP1OURpJ slMJ02+O+wB0iiIlW+SSRKAG4ahtQlaj9c//9nIKJnhS/AGTUZMVVLs7nHFs/+5VJhD1caC1zcFL i61WLEXl+bYuuR6amURhvGpbZy42x6Bm6vZkpJVxYXQ9VG1iwZivHaPv8JyYtN9b4aMDrbJp4fmu h2zAHzdGGw7vAjaq20ZypvGuV8WMEVIEMIxxw/VrGyL+Ix5JGmrF662/ZNcQQlGV7DD3QaFO2WdB /2Qv/GJZ2Ksx/orO2pON+xoMqLBFPCU02RuCSkq00hAzvuMD6Df8iPBqhkQI6HnfX/c/yq7e1Ohv aUwcGmTJ6EJv8zbAryOu/4/jLFJ8YEIbsMRUKVQORr63rdzmeMdZG0PXzy2A8q35fnaByB8xE972 3u0tWYWHncUBHctO8ldQyKSdZLZWsrFLtA5asIzanicSXIabnd63UZRDYPSOutDdBG7RcuOkR9a0 g6rUxq9dqE6AUvujg4FbQpWnmBO8vS8AWEB36ty1zhg2L37amsBYWu0jg9xbPV3PUPDjy2Ko7KL6 Tx9woPgtMBQiIzwWkSpr3yir1l2Jwpa/601mu4reJisSDEPGn8SXPEUd4yVfepCM5T9xBUURl8Fr /cvb/yT/XEhPfdI5Hbod1YF+ljm+EobR1brY95PSnz1XURcNOfqGE3yN8xWuGpNMGaazI9GscHHC wrCGIFbGM4l1UW3PoX6E1cyVP8yGQXjpt55zhrHGCiSKGpAtCN1bGdZQZTQVqK7yBRaN3HUDTb+B DqdDDgh0fpp2VYUq55ZVppwQEHnRXO9GALnUUppe2zV34RDb5/wuOpIxoiTH0+8iO0TJofrioXUo mHRdLVYgmA85lAPmhqgW+TSKPzlci7ahqdN06/13P6lSTUhwH3p1a6M10i60JWZhGRuzcuvsymtN my3gabRmf/VQGmfEBmLsOQcJ9kt7ofpbjz8wkYKuZ+FwuJSkZCr4pgNd26AbvxSoLXHt9J+qWXvz c7W5Z9GfM+ZQIi3WNgRRqG63G9rJVUYhHvdGmuclzICsAYBm0yVJ0/8UNnk19xlqSfMriemCZcV8 0ce2UWCmgxI/GI6/OWZtqSmME2HZiideFXmvJUDrl05pDZgqd9obbvWjHcEAwxY0/eNMqXPDdVwp CPZD1yzG3cXdjPnYRW+p0+gw2UfO6s9hgowtSOgk5ElsE/gu067xjXoZqEMFUfwKcY6zOF1Y8Ff7 PA02AFZJuR0D2c63v3vUPEGQQe8o2i2yLDt98I/in2XPdn6flbmJj6+nutm/CpMLZxtyrxniONJc IVMxB3wLhRQEguVH3tUDolR5/iDJLwRRCPAUngs7MTK7VBr/oRdX6R8x0q3ryI9DaoFgUbQybMET LCD82n4sPYGvJd69eS/a0L2BY5Q/FaZx4EM162auKsyTJvj8ttM0O0/LHn6ZcG5gnqVf02/kjThv pK/+C610UQEV/Cs64be+2jynBxxGyK/TBOWplMEwbpnprVZB9bLVmky1uAvu28AgRYLr+iKgJZDq 2xSvBkZ/afBU5xkKTEdohLAnir+nVtBG7xSEfJWoYhfQlv/FrHZXo6CKQs5iYBRJ8JnfJ+QHXMES LrCR0v/B/gZJAZ0kBYrfxleN1RZS7EL/IxeIErFokdTyGE70zaSSZyvFT5uuaYgTTgmt2ioby9jP pV/AcoX7ovkc6+4rrCXJz0RfivFh2+3XW+aX8TwskFpkSsSAC6GUttPd9KxXhAhOws8bYSm1iK3S oM5RzP4Jct76GN7QCb/1gp63mOH/J7lyP3ZuJzHLnoOz/GUDtssuXE0SSUqY3m+3Rxs7DXPUDahG pzPOHOlSJyeIeN3GmBhUhG95r8kwVeAOzhIsNmB5jMDMJXSSlbxOFD6ukxIxX2PUMgY3zJCHYo9u MGmKhrvlqM63F0Wk09r+j43WFRDRWyVuayftrcXDnpteFNBaBK38mhob7uprik2GY1wPnkBmlY/s PlYGduqhpTxLsOcuEQ7uHgQ7MWqF1oyT7Vnbh19AOkcZaFTfpK5QqkyFK9tc0spvVs9EF33Dw4ey ycAh7V1IALmEGIeUhYQ6Z7EdnZz6J0sqUPILgXGh9b2wzUjzriCtf4GsNCsYsJ5yoXXQCBWCLwwN BXyOdNK7zEY4RxCYVgRRfsoM2cYRn/AK34lnhQjsKMrEbeSpIxJCU9vHSuaejqCxVfQmmt16hX5J VXjO8of562Qzzct2k9GqHwmhw+ZMq9bgmHW9EiH6hrMJmD3OIci3Occ3uI6KJkd8lbvqjtXrnfbd oQ6AwRm5ZO/MkSpp7RXLxI7SWulRlRlDC4Nm8SrkM/jTi6HBGfdE1/zI2s5lefzNj9V7chhAF1cc XamOalQJ4QpH4fQiN4P1CS2lPoIKhdeQorl/EcPfcL2HLyyiOXwCeuUypf8v28ymtnbN2O9VL3YG OfGPwVTRO4G2RsVEi2mmhTmtJ8XX6ACV3rHcwXOvKFYp2K45HlzraRO3MH76vKz5RKIdOm7chEny fvul/Nhrnjq5ydfS+SxFSwspOKFGoJMgGzuLHgQg4lwDCqPbqx2xtg5wFLl4XNoFfaM75+mRQFbw CNLOdel56BGbdCXfxo/m5JQmffsXkSHW9+c/LKsyvvonrZFIZcoouVL9fm4GfzOuy+NGTr4TDH8i qj0NlfKqpcYeRGNeT/xT0ES7EsYH6WbppJ7zWkBW/1JRIX+bSrzMclSC7dUYELZwU4rU2T8wIjjN LixAqo7r7xIdqq2LdIGTqponk77Scd6myjjILqjqe7uYTVCZS3ewvZGI43HAvagR1IE9Hy+r3kiP OX3e3eAe3Otx+VJKNdeGhUf5yhDGQmIhaXun0OTgJMVwZ+p8fCfUHo6pAgVU+uALyaPV5r/EUQgK Bdm2nHU7pR8urioSvNtdRTc/sEozcTd1ZupuphnU4BlCsFvithKCxYQZ6m5sBGZgWH6gP1GSwtLm VrdlW6UQ+6k6yRIL4VKF/y+XZKEVDuneVECyRphglcsEWrQdzAhSZGiCGEZtS1Hyjzqgw0XGBIde CQ9klgKbwMyogir/FUz9yDaTwx4Hx9ZF9euvrrag245Ei55ny25hBvqP9H9txR9T8pFljPJJ6RPX YdMbQ8LyHquj4t/uRZcLetiqGWLYnfXOruY0ZR2jOntKS90+dZ0os1cLBMdhu4SfbfeDdLfkmqlj vS8l9Qu3QNUzS7rGgPV1J7DpWuOY3qUZl5zv3oSpkguXIIheaRB7gibGzSPVOW/uOOoXE7UKzppk xoa3amK5exil8YXk9Znx5Nwwi+2Dk5eNbiMuD8+HyIV4JKKwKJmECt2gEh0ex8pXdcHVTJhMlrpa w2n6lvKn2LfcJ/lWh6faK02GaOOXexYaP6893ysaXjzjPCfQwF1rVS/qa1eYooVfKgUWB4teRC9I pNycxwqohaRsfE1TMxgC6zhBH94PNq1oA2pS2bZwjZqq2LuU9i83VWWJb7WC2apDfKCrY4in32dR 2qDhzlMpgsorGvl3f2ag1CQTsuDiUqjS2KHN+OdojgDbcH6i6RnPPwzxr2CsdMVbvOPaVSrpGpL2 //y5WhnUTebr4QRuAkW6RVothT79W0hj8HsIdePBufC30sKRU4ZfKPlCiC4kH2GfadljtPfQtN43 Usy0h4f5gn7gK1b2fDSWVtvnA4+b7K3NX4IOgIaJRBBEQ+fvq+Xk5+EfgCYiWODabkk6aWYQH1sf QV6u7ZFwIj0wPpcsZ+Sg8QR51GsoLiTAAZ71WbtTdMKk5hWz3w128tD1hLv3BfEn7881d54rGSRU 6v2mWgsmXVA3f//LLwI2k+2lPZjsbVsEqo9skVr4iJWlragGPNJfj/WwDl5IcRI7yKks0YRR+tl0 1+PrNYrBxntIRJFm7lrLSA5hz3wtzfZ8TCFL8kMmAF2Jnmq+M8+PPGPk5NnhLZ0I7I0j/6yIv/YK gQlBFKnZqnHgQKGYkUDkE3jBok7OVhXNo9RLXEYUsHv3kH9I7CKAdOBVV2YeILK0QAJUIBL1xZZw ugqzqtLRkq7MyyPGtafakqiv5h6k3oqd5dUbElqrbpiv6ZU9Mo2MfC40B56EKC0JACSrBkqeTA/E uDYGzxu+hxEe3dt7ID35qarVgXUysMbMeY0hLkR95xUToBMWYLlowLX1ctIUSg1PV3AwiClzRPio wW2FaThR4FMX5cwfOnzfcrR7djR3VCzNqCb/yTr6WZHQgEdkVe0UWYUJcp3J44fgGneXjvSAD0mL lu7D6zv2c6x7cEJJPleThtz+N1is4WQ8LPvWWtdTO2HAP9IwocE31O6nwhjJiAOCVUzvllc/+HWl 6cJRfGbdBvz12xDOodYZSR9KByH1Ob4rybj61+MhdPtUZvqloptgpLD3KjBEfeQCWaedtj/ZqwPH 3yTJYJEEka+UrmHImryb0T4uwziikRfq+1jSc80Tu7SNxa9GT/n8a1aJtq3+JOJsSeWNdmbLM/PJ 7AwOELPnmBa2I1Ky8FbWCg+KIUpnYurESo6dNnC3TZL3GgMFAzhPD612SKVcDHgUyYKseKQuhrua 62NqUte9+7ZShV+HDsaAkl3CZz5ek46HqmJQajFOXUjb7ymtOok4XUI3mvA14/t7mVRG0KL2Qf4y 1zilHxT5qsTWqSfyRF5EMfYP6lwsoQVz8Bs+IR0Sx1xri6I5FFpEQaxMRcCUUx2KWl4IQYSntevf itcn/6UlmC2jBLqvb9+Gl0nrqTQDsr3soNhJFyY2YULwV2qltf7XA9wMgG4qnMlzj+K4raJ3IOmO CsafHu8MUeMweTtvi7qe1WsF6wxadLQv63oFtuqCwxXFe9Ul7xKEj3hAtdQ7xwqM85IY5lE3SoHB bvjpRml7ztX6SmJ5wvOjXZw+qvpBFgbRYfZHonWL16A+zxBrQ7OzqfIw8kgmTNFObsSNBM22KP1z FQ4PAvYXrzyC7EV6TmckbZ9uO4D06gZTgK5SJxkuoIVKL2dbuvOGI5p8DOm+wm4ZaGU3EpfdcvWi hZz3l9BrRyDxWaQpBSebMhXqHcYhocXij95wHlP3aO1yZ/9DAH4acfzppnHuW7O+AX1HOhrBcPKZ maQxLeM44HllQUhMqun/y3D9oaNOGPrC598lDPFzmndMK2k2Lkm6sknd+uODqJRdNbJ2FY7pd2XN nWncCRKVPPsvF0vnsk55MJLZMZYrAnjyF2JoV0ZNQDdrB+XzMC4f7X7v0+tmLC0MWOUvJztqLLqb iqQZ2IkMKUjEGCA/G6uTNria6yZAjGR7bEmwgr7E0sAAToQ4cBrquYhwIRvSRVl8LHIkUr4coK/O 3mTFqb9DjmTuRTiMK4VLPAzeBivwwXwRO6RVqo/53lLb3HJ8hMJjC0v0K7MqnUc0DJ251WSuf6ZD bPr9JEG1UHbgmUDKbQu6wtNJdizrSf+JO9SaJiRKP9/NW+RTq5t5BiBHQtoERrF0xNcwDSDkpSno eKRfB2kbXzx1VknF8B0hc8w4dTU2IGKGqgGAmd1ufYjjtjr31YCr0kam+cNH3sjy70foS/oknNKe B+dzoaS6t4vfJlFJgSZxB33N+fH6PMYCsyrcaqxEcfO2iZtbXOVNY8H4AY+BVATbsXewhgtm3xhN 4Ht4kXxRt976/Fe5a11s+OPXt56knXY6jcxotRjdwU9DGcr8T7XJ+8WPVZVByfimAcFABgs99YwU SWIUAfooOR5rZL3OIVlohqUWoTllMnFpSaoxS27gPC3AHzApDostKxYul7IilFXm3PMw7SkisAKk PIlaXLoW6Lb/QEPOdCSIMP8/AtxVmHv8EzXGNnz4xdCGeDIOMBsZWsYwmW4HVWoj2MNz0W3mCUOP wRp8H8I27FiPgUYMp5/JYV/01IiIxbfLUT5K7WKREVKnEdCmJ/91BLuidnPJqc6ols5ID2Vvns5e wq7VgIyHKCHMHXg7hPip8FgPeOpwYu35/9O8cV/ECIy28Usw/A1iEm/h1decTYirwAT1PPgC6Noh OfW/VbZORpWyirOTqBBVsprfUB6RiEN0fhhD19bZA3MMRaaQjUhp5CNJpcii1Q/SNElx7z7saWuW swIUGCKduZMAHw3jIuu3zEvSUY4dE6FNnTRmMnm0YV31mTTxRUyLRdm153Ers+FZpRD2J/YEc1hN ntzlpSzDmcHX7dPPh8okWWrxoHZRYng6HUnLguY7zzrIukNblQ1s2fBbXJCI9sYIdBIH1UQV60te KNRxLI07Ehe88nBuZvQOK8TxuYSQsMElesO7L40Um3l4OrTFq+HjySsubRLOrIOXcESgSvTYQXNt NsKs/MBfNtwrZsljW92oncpTU7EDYVh47DyL7l9pxtFOXqrigyyH1mwS5wOQ2dJIAD5koHgyAzPD 385wJ2yBp6S77m5D4lEtPkAFIhDULlRpGqpM+IUWc4UI7IwI2d56mim+ZJrWqLORFu+pwalF7PRL 1Y2/zi9GjZHxsT7BjJMV/s7OqS4QteZatfGWWzqJWfHUBxDhJh1MeLLOFxIqlr4nwgVc1qfYEw2V 1pikqcLqJM0Dx1ZsdQH6FEddesUh4SDc91Q5rVWn349KpfOfTi4shbKoaYkQcHDe3t2P2hYf4ESg Ib+pt0znb5lWc4aYPkcq9UbxQAHeMJ9XAxWTtKIszXeV45RErk/mx/Khx0sW6KngywXWAFQobnOt 9vkVsBhJng5V47OByoMwLnuC0B8LOO4VKTw8gBRJqkD+9qUEXXwHHfX/W7yVVe064NqjpAvb07qt B8Fnq5pmlwfeTCNM35H4+1F9RwvkoijbJYWv7QCkz+AjpjbCj/Nndz/u3iRrQ3xiLr+cC6g5rmpk opv/QnQORXgDLfOgCvYeVzdEk3tx3D8Gha6IWoIsTh/8/ZM3Xv8jK3m5TQyKvQUHNbfo6U9STG/f QF6o2M6Ma2DpX3zMKlAiYCOzHDsds4dE5mZPp0SlHYxiWB/j6TAToOmLNfwnxkUVsIRSd5CyyMaU F9Fh3hO1I6O9tUem9vDXaaWcHUknjSx4kR2ttkP4FQZq5DfiWTs6F78uUtSB8lG3bG1nWjiVHwLG cFA+DIdmteISn3RUi3Mz6MMVbvHHChjtNOsJUq1mPRgaiV4xYyuwavuiDzj9k5HpRolgb0BSP2OP zEut5XYrBFM7dYAxG9B6V5cim46pHpqAF1CXfZQC5O/68owsQ+YvVlLsqfw6UMAknvuekAbT46Gc 82Xdy3iKVR135DX2ZeTEjkf2VO7tOAumjm7cAfU/SiuvJwGfPAiWKqhuB8F/TzgCQX3kzJPYSsqR kmU8QDTTTuy0SLJYx53rw5qYms4+TfALLrsEBNkc1bgA7EwRemoxk2vUXwe1MUCg0XcWvSFrOVYC 05lXqyl+NXeqG08mq83oRiEvNd0NS1cuTbokGVHfk0I+3MFOMMrf8PUMX2xucVKtxU2f/L9nFGdE mKc+xCHqouVw0VTmo2FuJVr5IztyKCG4YO9FUU8CChubfMD1SzyX/Qo8/s1VujbYhVX2ml/ynCqn EXNGLJdoMD4zjh+tlDKe/HCnlPg4p9Cn7ocGg7V3meyDmiQ+zUxZxzA+nMEVpdigb6RKiHgHILSd xV6LSN8jUmavGzF06jbTIlcqXiO4ULqo3ELQnjs1DtFOdbbivRgWswOUogzt3OC3g0GSEJnUiv+F y7VuEIffl+x45/km/2k1ktW8nDf0d0eageIEUa8xg5BtLCV1nB+30+MbM39gju+7TV1DaEpLHXAz wIDG9B8ugYe28hi3rKpYKYKJywS1KSqLN20BoEWyjSgkleFbK5ZRRroqgsf0/Y/MUh6mi9qp8t2z VDMoabSzxeVG5Ssa9CUUQ75uTACvEjR576UPm3o0b15UovvNROMI+U7mBVSHEXC5VEEFOevcAF0d VfJuKPo0nyCM9yTZ433OLnsPAWefgK27d7ctEdfp97mclXZaQyqPguZdgeOc8TYCm0I10zgpLRNF fxQcUEePJ09hy2xgHkl9xgmEVal+76ceGZgz+IyVlbOe+7km2T2wGEUMqvxl3SVCRfXpFB/fOowc jbvstEgbKWQwEED64DTvA7cgE2GDAbR0W1dQf1I1NcS6cifJIDd2Fdd1l8m+MKvwFbZLPz7sWdwq RW5fH6gPorWpYcsnO4/fbWNu24VPSffsuUwAzBuJ3zWBZy3mBLlxLLGgJEOmISUWft2NuRMdlNuS 0yrRj2KF51+Ye6QXpBvMasiKS5+4guk7RvNV90CGRtePvBi/VQSXjMqlu6iL1mK6Yz66bKg0vcQM w3NiQOCS2rsqyCW8xLxmVaYhnWzkOhDKbi5PAM536X0m2WhBdGRlrE/nH9CIj6cLd//yZyzvbawb djO+YEdFedANy4A5oF6YF+h9Zq3KY7WJpPIwCVMr58TVO4tWmE836wH/5AciS3RCh6iTdUr89TCd c9OCAHXk1u2wB91qugpbPvDDvm602XPj9V0Kg6eiv8pfzDRSn/JT/iFJCd4iMwJJaxAQ6H+DwcVd hYIJXOb0kd3IuGZ09HsiHIxKX163yVCg6l/x3oGfCOMwWFeuvYS7XcpC8cguYuNXBqbkVwjUSAC9 zWmRg1q8PhlCfBqHbsIzfot92l4y4w/byPuF88ajz7h7N1d943X+WFK+3rqDb9qXY5BULT/RKI0a b2Gs9T0H6JSSowHOP6wf+WeQlE4gkMoOmkL8vC1/FCrcYOr0p696b3xnJCiMOy5CH14AuTZpCxIL F+SdwM6/V8wss21JIlYxXV76K8JO3F7YDi/xzIMUS3qJkXecahTZ+oEwWZzODlsvgYvcK60iG/ZS zuTBhKKMlw9tPTEE07up4hiPABn7lzIcD6ZJ5xVBsmpsZqRaIm6yiS1gNbIS5T6SrQnbLoLc1xTg 9EVLhTNtaJY0n2zBU39VXxc/2VPF5FSsRqWAR/Zj5C0sFMES6NrFR3UpxXRRjNl4mvm5ot+PiKYC 4lwL98mDuu1rftHC7ZacVbmHfgUesUGXlU72Y+yYhyji47IS/iFKbTl5YJN1wyCEEYI75xln8gn7 9KETgAwPiAje8F9WOYHVTR+kCMIJMNTOD084ksLrm9IhxCxVXVrVoPZ1AQDPjlv5eQunMHjFQ3XZ QOkfc4zErusv5t3nFCQxjE0LOnv231u734Mh1mUXe0g7tVwDKDV+7DO4i5KzTUJ/Z7WnD0dcGszI 8bsX3MM6GGmE+nn2v6Jdho2SBbAif+IHFjxA4aHHsgcQyNu/UiQCg7vEBGP0oq6VeIKKZrYbXsTV YEDGzmALE1zmD8m1pg9bm8cNKwiS4teKqX8Gj2itoCrNYyL8YTFYD/avHFoYILicqU6Yk4Le8Azn ihvJM3Dhpd/aernHYwp5gdtzp+IX9I6L3RopI0ull6I8H1dCNUa+Ch0H0D5o21FMk1L/awDNO87y APQooOewSpfMyKS7gXi1Y6vu+sStofv3Z9viqyIQF40H2Ivm1xmTjtOKZovS+XfLWSz/SaQps6KQ AZs6h8B1utnnlqb9W865ywxDsi+UcjBa4ldWBkL8h6WT23DLqrq6KMv3tb7ed/C2V5dDb6hR7iAM IS+vW66/0FjfmcpIa7YvMRnfaE7cD5nW9Wu9pVHJxdJmR7DBgHzUGGfdTwUD19o+Fu+uVMhPHl5l x8pGkYG3OCLFl/sQA7fkDU2OrI6vkstJzx3ukeCHOnsmgXCjJsY6mI3Ok5wUiwzfn3StyaunFBCE fOatQb38TZhOqaRezOKjXkxsXidYilrCCQT0ZTZ4uKv+9Y53O+f+l0+fVppxFuyikZf7EmEcaYcx OdM4Yw6uZvb6BQWtemF4glna6FrdEkKZQdiqyrXB0OW2P7Fup20Le3VdIojkFsdlK7ao3STsLqsO umKwmYry2Qas4IRoHzt9SQyMVASQyFl823kLoaqOQl5fM/YUr5adeHPctjR7++1yihl8l03Pov88 fLF7Nb0+qoV8JS0R3xLylyU5VWbusl1CROnhJrMdQUO+iOnRZ2y88+oB3lOn2McZivJaQROAgeTp NiUrzldA+6SvTmKXFgbDive0ELXWLOiY90vY2taDkd9xeZlCdxlTNYuklNqWp9QCBRpaxqODVVKd 1xsdWGRDrEec0Wpd8eQ8oPLxlvYVsJ22sr9hW88CDE072JXOSHkkjrBpo8PGRUri1Sr8TecMnfzQ fLtLRH08PfGx+tysHS5tUWr4EzaUOV+mJT9RIReWZAWd6DdUc4XFLb++EcnM/1Fi4V/P1Ia8s2zO kiPt3ov08KTKbTzdfqB25A1IXsJaQRekzYOJIAQdzKsHFS59wTTlmMFn8496ywLomCGE9bx2CbZG gLeQsHgPasqm9AUbvLhT1T8+vy0+YkJv062Oji7aXq+cmH8S7wnKGJFI8hxy4y7HaFf36XBrD4Xq ePndnV9ayi51DBX/A8exDfutUzbfiBwtzF3PybFLE/DBCMUOkdiqEnpvCmJBnwkx3o8lIUhQ+TMv kJohFyzBLBGSB0K44HYXE0T/YFhis+imsnVJ1FyloixG+9N5TOrZtII2h/650XA1NFfT8usM4b1m gkXqvr84ul38KmsenMMfSXRqhCzMnogexAmRVc+i6CwOgmi7GQ1tlNVE01/CD7PxwKoGgMQrEZT+ p5tgnHKZfTh6EZeDjASNuDyVb6uB3TQP22QgN1O8h0vuD4IiGfYS6fwz79FxyC+w/9OLcDOLlcZ5 YIJ7TiPKcTYtK1TyWcXd9VlpZWOS18arxk/20FJ/HwOg/Q9cBKHtkWp0NjbQ1WGz6Exd/6M+L+WR tsUaVGVUs9/7R3hoeONXs2xbfTsjkKMWVt7p4XFWEUN0LnMtmgGkIuYnlkRk0zEJJACMDiQq+IaH ATyp7be/+NyU1uLEqt7We9GbHEiVdSvH45JuqHBTi5jawjSN+P1FSc6P7IdERsllmRBCdHkrJqs7 XuytVQ1QC0fcH/HcWYcPIpr+DUJP8OGs5GkCaHze2CYaWKq73EKZkuzPNdtZ9rXr0HF3FhEux5tJ bbWEfCV2rgCD1wbSlGi0NvlyA9yOoheT6a5erXJ31ZXBtkGKeBuF/LcCIb9+KprNMm0S6rNygWtU jrTO9mBpZhug+SDv3eqXNMF1NJkF+i7KpaFqwP4H6be13p1qzkVEojWW4/3BdJLp+2dSulYdR1CA T+AzgxvI2gdee199x2/fIZQg02niD1d0oecXTY5K/HQgLAnfDUkLn/77uzh6t5Sv/m8wUR3HFDJJ jv6PmnnE7rStUGf8WtfM1/JUrJdDGWGlyIDiFWaLokYDw982Nfa2DJ32+r9Iaj3xzc3iIKZv7oSf RcVva9rb5hFyeroS/JA/ImqaX9NsEPTUHtJ/xqcI0fG+WKnYMgFneUKMs6wkewyRkUYtdlgWjTrX Lcg4iiC+ZG9ncRFIYd306KGjTxujZpvqFSq9WQ0G0w0lRhiOHFpGj7iuVcXO6UnvszTHtXcHRpGe NMihhBOcY4xF/jK4Yis4Py8NTa/m43foR7zEeVNbn3xYDKqZ+nGKnwPGM3M60dj6qwVD1FgICuHO NvgYS2XUOvK7pAbCGGeQa8moizuF7GJ5OFZv/acUd+iW8f/V6Nda9HbpILe2+QF594ZEztej8iQF fOBYWR08h4Snxg9AS2DYl/JIRC2D8HGl/z4X6m8tHT0dbjDKntGXn9CvLbwsaT1EREGcjX3yKhvP ffPyUvupkYFhaXT83ufAloyF8clwUzF/MZZQesjCd9CxPhtruci9g/jQ30OMlnWwJVyPXxAh74az koDZv5b8sCnCKCjqzDORKamGelxoHPlm2A1IfKKOfQEfCStrwF1N+8KV8Lh9kZ829MhONSqIrN0I Zs9yHxzLVCstvuU4Ianm6CHT8TZ1NI94PEExcQ2tlh2SyI31XJhU6G0yaLFuYFa8yEkXI9eCGRBR 1mdpeoi1BVFjh9i1440JI7S4a5RH1bf/JxAJCK7YF92hN4MEPm3bTCB/6mCrIFam0LgJz4p4qUxV TpbZWvdg4pZpPZ+ObjDolf5jnSE1VrLy98eOK1sUDocUSQ8zNiAOP0/+dI4c1rI1n5JfAF69aZQL ztjNFYxw8SOm3RFcXM6xX/55cGt3V4UeJR785J54FyVsUJ4YrcZdFNQxrspfqRIeljphyjlG1Quf KKe0g2wmv75G88W9xeNeyTxotAlqP3oDLXCOSZLaqSlnTkkEl5h/4nBkK3cXFTJ9J/CnaZcDKgiX /CQbyClsMB59lZNEd+3FRDUOpMzGYkz5dpjWAIk/b/uXoNlt41C6QffX4OH7PfBoRvpkKHaXLf+o UhyCQHPnZJBu1+ZbwZyaLnDev67AmBcVD0jVs8Yv+wHCkF2RLXUhoA4JBlJJDThDGWl5NXWE5Cn9 6FmhwBcAkwCKokz/y9AXBZ9dWFUJW8GLQaJmutSEMByFvmn+WaFnFIlSutKOugQcIst/UKSFYaWw Mip/Z1l1e6JnGgYMZgO5tvcBdW5eVto/njqLU7Km0ehpq6MhWJRlZ24RH5/XLo5U5U0JgKwbIpIe hGFd7j6Jd/WO0yqjW6Bk+hHE5EhnKa9YysCM/LuF1d7ZQRP0U5O7h4vI//JMJdavOLicxzRCOTkL mJFpzGsTrNogK9Own82iNdNEagWl3k1MLhpfQQsbvVWjfEYpTSJ/UPUp2dZwWaf5NC4CWNqOtKHi /asw2b6aUktAh6uZUMM5TLj8Uc0czNzYIOaoTB7EAl6SQfsUsLLknVGDuLnKQ6ayFx1DtBwBHzhq U+5IxvhZQ4swViZERctMnOKDnPRYY3NHDRGR9gMuP3mZgeTTNklynhwG/PYAMBKlyHE1d/jf58bL nGm6SHXjX6QrhkYjwieqn2rBdqvtFAzSCi5RU1wg0y2+kGtak1KFoTpDM75yW9wHHD6oenWDX2BZ PQRDhZZX2PscuIgXu85BUuain0SQpBIBNtLAOcc0Q0+aa/Z6157p15jXW6+1e3YUhJkbmCW/TDq/ MnXex6xuqS9wgDM8P9gtoz5OWpRWOay6UQoT+l0aLSvCOeg48gejkmy5JLELxIutrlKN6rcFc7do UbH/Wemi8BdzwBpf92F1AqJiXJ5kCNdErXrsXba78cM/yGX8+u0XJ6MS4aMybRCeFk1H0CJe9FUp r12gHZYuwJy8ixvKAIymerY34ubw0F/4y2dOWYOrnvCRKaPy2TJaXQ7mJrCiNKJVHy0RUxUNfqQ3 oWTvp4IZvP3dYucZl92WTd8xKABhwkEifQuAqo4E+otN8p6SZajcBoT2ukEw4Y5Vsj3cALr3IiaK FpMmirnRMZ84M2PVHYcVHXnyV9qWhKbmiuT8n5jGzfemGkxqnB/4GrHbw8vOQbEaHtP/66QJOTzd vpZwjX4PHinFaJ6cJICrgZ593byop18SzWAJ1sFof/CkUJ5DDOLZDIOfmPbh5mJ98qnTMcUfXLdJ 5U172n+uWmvSiq3pvTnsFrxM8xTw5irYmDkeJRln/EC7T2FWYRqz+gUCyBiW/g7+G+axo2HifU8v F/wMBqEyvEtDys3+pLtQIKXKi+XDWm+IoT6PYrYom5fJTZrk+6685uGnvAMxKpuTVlTBUKLaVvp3 jPi0rbQuh71ZA27FIbepoWzto+rKuP6HMQh0ETmbZGelopioVXOij/sR5+gfzyjO6ZnAvvkxt4Pf 24ZU3ytF3PTwOTFN+zEaPWIZB6UR9xyplFpWOx686zaHhT6PUIT+jNQ47pp3FcOIvPiM2ZkO0u9i e1X/1AY0sDfOuP8PgLWb3YURVpLtAvDm5H62t1p6k0jGttaZ5y2KIrVStJttsSlDufq5tQTrSLdJ UifkC5lf/xpA6pfi0ULtRuPpgvJIOapzN+o6AYYeJX4ZARqs+MCnvTRuBu8bYrGkkyzuyGY+/C+A Mq6fwWJXf21mMLCaLu8YSd0T/yYBe4Y8yDpuSCoOm9SRPnxms2pQLZCbw1hQpFVw0cR5uAcMdrDo MDPUFcbWlBqDa6b7nEBilUWxgZSbsxY8VPSN8Au6qqUxmKLWjqQtJde3EZBXYLIuKWhXtvpq6XTt FWDg34R1q6R/xzBs0IuB4YMrjHTkBjw4tRaEJ9HNJGt9mdFHINrJGuPoX0CysBQXUOz53mLFbN+k 3/vGZPdL6z9Z//lzt8ZPhVzz6lTz7c1/M8uRo6QBdmHEKdR9BNHi7hduX27KodedYw3fIsnlJUez UZ/LWzxbqWz+TN+ZIE0DWCDr9vGElTnRlL7CZ57jRwC0ZCnLD/FteDHIVTbO9a0QjM6OmzLD9z+0 PzSO6FKrLDYY1FQGvl1vS8DDSXZqFPi1h27RTIFpeaoHpVfNNTOMaDboNOMNrXq5AOK0p3bRNE3E zjgvAu2FNPIJQbBir+hA/ruJjeUZggcrgZJraa9iDOmL4Cn1SeQVcB8pk/124wdGIPLMq88Miunf bwDPDnFbKs7oMTpb0j1jiKome4K1DpSuBMAbv+q+LcaY+XoKGxFo9T1oPSv2mnxzx5NzIknhfeVK 2MuIw6TNYilryR1WDZoyvrdRRduUzg+u1x4ukN5pxv8lgPcFHmv8WNReRua6Pmwd63ahOYYlpM+L xhy9WZAOQe+qdOmksU+Q9MoqYLh+8jzA6y+smMJr7uz1jhBGz+DyHYs/tY6EaVlmLMZUgugdjqTb LitPMTkOSX8/Wjfbwsxll1IJC0XvzAM//I81QvQibmQMoOFZ5v0QW2OKL+CqhfVad5v2VfMPjdJd FHyjKEljrW3w9hoiwOnZmrmGD7RsPASDlQBogjNca7LnXbh+Ss1IPPZG7aAo9SOmFCULKoRD6fcr qwNYE61Tj4HmBYf9Ev5jFSKeABQivsBSNIt0Vd3+E9/Bxs0rVu9yHXIFTNdlc8WdL0Y0vzwqoiQo cXD95MGNv6qbgmq+VV4NH0bpRdO5By77WJ8YO+Dy2rzFEKaFbprelhr84FU4hBTD6wRVhI1su/r3 suFR5NxGc/b1qFeXB+kZqfkn8R+1jYiE3DmHpFDGFS/msowYjpvViRo2i4YazB4DVjnSlR3q4tLO gsc+l7ktBPgsVV1XlnDiSS7uolEIdqXnwchrsCaz5c0FZ6/Y+MqXHe85BW2yiE8anFpxt6IW1FUk 65GwYS7V2TeqrosxlNAdBh4PqRVUqnNfmd+HRtW+tBwVf4sshYZ3xHn5IQRSjYFbrRJDMaRwxBEL x+Lc1fEeA4M9dwSpL8b5WLYg4v7AYsjM14pK7B5NcS7HRjmH3QJKXoPoJtT187pvA61i2qW5C1m6 LhJhJuHmgfaa/RcwEBCbpxcPlMYTXYzE7PlxqeyTCwBs+p80DeAr+w/Pofemc4mr4YngL6En5anE iUPEvqHDH2VEFIwGjrfVpKKetV2Xh+lWj1dP0GD+VwOecSRPZLDPU/LRdflGbNChUoY4jGlZkh4S 3W9xlWE4RraE+fcSjE8wnRKWlXgen8QA2/Yz4lBAJyMVV8+VYJWu9KFgZNTzyeQvPKb5CKb5JKwL wT+knrGyz9CEWyankug7kxOP9xTgodntZd+ydqA4bOw/dd7IYRczRkVY6G3Yv+O0o9pqENE9w3qa JY5kyoFQEEhKsRBzC7+q+IP50pHATj7ruRzRuphGenD//N7SR1b2qo+Aw6qfXH+5z1+TdCd39bxg szXCTmzh9TlTTiMJNj0UTxVZtI8K6nzVvf6ONeWO/N3OZwjBVAWGgaqjzAKhayPofLWgdRD75Zg8 i5g6syhw3p7HoOVXng50kJsFSunydCk/rYzIiK23BR9jlJPc0ZulmYR/3iG0DpFRimzKzpK5K8zw eBaKjwVrPZJ2ix0EIvSgiq0+U8jECFP6OFV7VgGdQL7UESKGdZLX13HdEHrjpYR6o7YaQl/F4ksZ EeVloC7Dh4RUZWqhWNcEnEVFdvGNYEzzdFz2Qx1GdXM17Q7IDRHMto41a2ASKw78CxG0ow5jDQG4 lvC8rdRCrVYzRQTg4NButWWeW6rANKwWH8oePDvIhhVWodNUouqN8QicgrjDUGlA7fTvNcyS6o2E eEvM2FmHCiJW+hQLrjtS24ikXd944dB9EOukXydrUvb88nJg0Ovu3Ppc/fKR+7I+1NmukF2Fczek ZH/pJ+5kdjjESMBDvIjFVRcTlXGm7cB8LfOP8iQuNV684N3fJVfqTLDMkixmDlabE4fiUxjDsVw/ 7ZEznXTVTKuTWfHV+/xTucv2qtx0bqazlqHKhFpKnL2z/dArvA+t1rRzek+IklgioD6cId2jb+vL L/Ad+YUsbGcvnOtvA0aRluJ6fRPGslTtJNKaODF+eD5OH+xj5ybpc4PT+UUn25pEM3N4rWa4eHzK x52CKL2Qy6aD8PXEXem8wv0mhJQvS8igIHP4cpwB8ysa0AcTYju7Rxb0qjr5Knp7QhYzRnb0BHar 50xxiFzsChnLUqnYS+/hRRR5PcHVEJ+0+XoWz+gZlNXOyL7HEXYFGYT3f8b78h1HMp4xSlgXUVms LSH25CYvhhQd2jvBSxUp7/P9ApFhRjkNvH7KyamhjGRWkoYp8VmyMKr7Zgl7ey0N+qnY00XcWd0R C/E5YOQ6HrAGQT6pIiJCx8tjPV4cHpXk+VmpEY6MK1KKQ2xdmJ4OhknMt2TTNsBJZhzYJ09yt7Q/ VF7wMFBSdjt4oeeWvBWTNrCnS1v8k5SEAihkoKQCbp1pC3khbatkefoVqadoA5wL5LPdjPjV8/oO fddqMS5NHtGVRYKkGG84YY1/E7CbFZDAXr79vLe0XxinATYA1x6DS3PJ3b+dgDjvLNR2E0rK4Faj VgCGP5IlDx0HTUl+vnt5Ff/saUfG0Ygyllm/w/uroQdKOxSQER64zx+nzL5lrdSqqNWTfgPc2j7g hb5kPLtg8tKtlJsOO/jpC/i95WZnRTMrAJKpq+FCAG4sFbqTUg8yjlVG79kpql6INUw3GdpniCHh nF/p6un3f7dMoAQV6T1hFR3INK6CkRsU1nmoOyYma/EqE4Cy6k0QG1EJRdz9ZRsQ4m0Wn9OeOgM8 FhzsRHYPjXnkVp+XNvn/l4+FImJfI/2KebEGFjOUMexEOZAthVhhY8wQyf08eA9dGF2kxfR4Qd85 2fIwK9Yp7OroXZ2e+aEb1Xqg7P0rOX8p2VKVRLfUtbqAJ1DDSzda/dW4F1j4jLI5h1kW8iaxb7Xj nSRRwNEizGAMmyVH7WrfP5U5AuLzRgXL2uZrjOQv0lDUqa2AjHH2Ic+3Ea3mfQP8CQnfmjS54hLE 6V2hBDlEfXkmK/verA6r0KBo6Q2DQedWm+1BNVv2OSY8ZQYrgcE7t9Fj3HhXYBE8yrhcOIDZXEQB E8LQiaAyRz9P2vLba15PCrCd6JroZgD7+UIpLEX6zzgihcAjn+1I9z2BD4oniktUIwrDCg5esRnj x7sqjzX4GycmGEqvHLZG+GrA/nLCC6g065c6oj+QLvoowxGSVDbbHZ6CXrZkxDYtT2wgZUXg+O9c B7C+CC6jbkUCMkNi2XINy6em7HUo7uNrWFy4s4Kx2WACv6GyrJy5Qbks2tFASh/hmRBKTyUWXLB+ jt6Wbj2k04dDvJWapPfeE0/Q/Lh6IRllICIu1ZWreDKSKh0TRE7cyni3mxuemMV+attZXM4K5BxP 1CZy4rraDnlVuZcnutci5UBj2lpVOWE7N/s+DcPD9j9S9M8AxpGAjyTMs7Jlc7MH+Nc5kE0YaW9/ HhkYLD2MoolPCfE93BFuBifd4cymIXmldwfZRV6qt/TFIwRb3eSSXLZCqmLhWI3UoRuE7Qo9AjPZ dpXP9hk1vUDW8LOkKbQAa2Nwa/wHG+XnnDclFEiR7kMP0NgSp2qnKNKKnN9OuMfCROLQ6ga4Hgv3 gdNLRi716BJansXjtoH3O/R7CWVPa1zTOb3Wz1B1+mgOu3VbsHQl2W329qVqSMd7JkSSlQCun9K7 2fI1yO+piZEHrW7dGKsgAOKhs+cL82K5ZQ31krFLN/IhH91kOcZxYINPKUJf3ENGAYObNiRbfNa9 lHcc5u65DqgEAG+eSM+5kE5Y34YRnf/+EIzCSO7+tivLz6tm7wEPg3pujyrdKiWvggIuKeQHSSSD D9Ai2wkBUts+dt22FeXIMgnKBGeI5MUx/oRMRe+8IRYF9FAp4TKYA192yoxY+Ia7q6kdIl4xj48Q xkrBxfYg2rh265ZpUb/nUn1HXrRl5XzUJ5j0mBnFvps/91uqfrXGYyVzD2LrogC4IBS8780CTmsl yqwVzdh3hwpowfmD/FLEVczGH5pSWgmSfRENQqCkUwx2YIHpTprorO85dRKyX4NH7v6Y/b/CZ37I nSeoBS7fkngFKBpO+pBBhXaExCvyd57MNiGX7H17BOXwIANwCS2VV3BGOdZsbzu1K+AidT3Gj42f Ps0COlBuCFSAoOAN5Dhk5WQYuEHKYWScILUjLnM8IVZ9FLsnKP7L3BsJdk1+KcGBZpaabE5vKTjY pcJA7rirj81WzwBpDJhKPV8TSbCVK0Xa4pQ7o8i5aO88mh2wGjZp7uy4Xiiuwd0Uswi/hI5DZ8j1 q4eVpiFZfgGi+Rv/jetTaDeoKOI4pmYk9YaxcsIKt2BSpDJjxb5VPmxDaTW34QOOVPULzA9t5Ptd BrokME92qK3jTa2yMM91NHeYbKQA+5sx4Jzg2LXwHk2KJnaI+3nP+IYv1y3klGMiY6AoICi0LqeQ gv+ztjMcmmzHZccXT+MgXpbLSo7LWWefum5Zq14EkfRndohkCF/ZNLz1AcMe8WUfMZyChk1p+5ne zI3Fb9WKI3kj/6bk60LQbKZyg0SMfNnoZaf1hTfLoAEQ/a2IA++ulpo5XRzevnxz8rvHUUMqzkwi ORnN1H/AAZezi1CCTKG1QtM5+J7sUzKLP53MwrSKNYaC05+Npn8CD7wRYOa9dEOChQpsyLDufjkw 3qz76rwSCBcs7QhcosDPh+9agbYVBmHMcBs3teWN9tum5E6rK+DrFO2fOjJwKidZI7/b1pjXRkdN aqEuMVncEx93Fo7NXSvtT5tBIhd/p34gxqS8fXwS7aaq6eqDeWAD62knvLpsDGhDWmFG0G5rnjSe AJWoTFgH6N2jNtQvIv1066DcmS5KNmxZrkb9JZT5QrESwGpijSgbSNfmKhlJCP7hntbMpFwyExk3 em1sowNgc96CFSfyR6gluuRVWLTCmTbNbf4HqlNGwsXJGQd3pg03+Jni7kpU02DUi3K7lkGHr8Y3 +aMOgyLV89DdDkE9L/gTqdVYPoKAXNQuwF7tj7p+nbakUOjPnwWEQhmOBHy5uD4b6BInrvk2AtiX JMJU4OOmDWCc9uRo2fLgMNN2O7nQsp4KK3JLEO4h4ZSS7GjQHxWqDG+FbgI9A/YGcklVJ5RoMnHz uqmBWVHcL5Jr5I2/lYAlweaNrMaxHeznY6YTXVsT43dreOTnpuZYNzfku8eDlPhIIm8OXjt9wWk+ QdqzIdDxtimDVnEDJ705q9FoVZJn45LsJW3C840DnsTI1t/Pq8feYWCxII2xKwMYdD/43ZnvjDvS 5LtUextua0r725e1WuI3g3G001eXrvABsCFEXyeSSpvlYot1PyNSax0CFEey3bCUYEL8t/HkJyzt i578V8qiP2dWTgMqH9U80DD67htjhhoV/w1e7qyS9xUasDUpMo5RFqTG0oZB6Rx5r9YMErGwSVdk 6llLi0wBc0yHbFfzPeaPTZF6OC9RWVaX2aLsNqddv3g9h4D3xkaO7GsDs4jrAk0dTyjCvIG1XLrD MULj47rz7ln+RfGBLrUvuB3E9UquhM0u8bA2S/iGanMaxSgFFsw9uBGv4n3kuQHlWnfQHsSPJPXi KnIr+WGmq9gRfsQirCZFea3ExdkBBvgWH+LXSqgyR8JNBndoQC/CiQBzyMSoAPR3Rq2N/sfuP3FZ X+XPqYnQ+lL1fjQqsWUl1ywT03zpjXxY86VWjoTTQnNQqzO7xux6u0Dvb6N8hh1cZbH7mrwSIxij blkVefxrd2cAi8Km9Rmh35YwDDzdSUKDQx21MKItpc+27F9dM/wSCHsKglrDtOPCvNBc3eCEwbAn U9hZC4ecHKOr8K+SfijQlThp7AsjcYVi9GKnmhdvP4zFBMPloQV3VuqGZtFc/vR9yNyfXINeXJjh 6U1FHawa20YagjTbKTvIUpFw256t00mhobnfcHnUPvCbvwPR0TWFaBM1nJA5HMYULZqj47IyKUZW 2BaXA/7Iqg4kKz+JnOL51+5s4wz/M9X0ezUGBBYKNSFFatZkHbes8kbivdukVIOC077ZjqhQ/Guj 6JYpQ5be+vda6NlDQA42cbjyid6dIkStXg4A6yNR/UgpdpsPV23RziYmZMB3gHFc1jc+V0g2/5Ik tAOAUCGEvzqL7vUt1W7M0qwpD0Fw928um65HQ539+y73LH6zZ6r4YUz6soH0jYsna4IVf3jHMaXp Ql8Ie3TWQPxBfoD0kNY4/V5hP22dCXR+ST1obhqtNpkNvzwU+o9zPYOcOT2+cOcLXf1fyzACeRi/ gJ12cyYhgJVV4s0ByobgAcshAT7a45D6JJWyySERtVCa5Q4BAgM5Z28XgLNiXiRwMK8ANhcDFU7t KK/jQN2c4Lhl+AwmgnxaROYSEHmH5XzEoZd+ZbM9c3s5HBWG7oaIjU2gR5bzXEuIjOpYbJO3zy6b 56rrBdcCif72lmzX5LRBn7I0Vcypwr9E1WiSTe3CNMiZmpi8mn30RtXlaDUAA8oFLsjZvSRPbQsG 9ETxmlgXdK6pI7AFXNUepBkX5lTikZsJUyDt4kYGU6/wb9UzLF/R836GtRngih/ZgA5w7DmTCCi0 Nh0m9FZ2IITKfbl46MJrXS9sv6zUNxqDNCzOQsYPjowUaleqiVasoE07xWGXUrYmt0vo/hy5bWPy G+cbHdk0EcdseT31XQ/jXtYZkpgMy48v2bh3yfGAFLtsZnKv2+HqxpuX+tsz5Sj/dbb0c8mOhObR bC2ZwXyBNNLCJWmxt6AC1K20Jov4WaKMqfHbQmn1Y7jDZHxTnkouwydtjFyDpMhaIdfTXAjV+Wn4 nd1dYPTvcOmC4P+s79YYFV8HYM80TffIuuj13DvQhoO05HGxW8NJJ7b7WjRPUytSu68wQf8Ud5AI Qv6fkV3BXuElwhF0UFOZehWWO9YH6Kz5XyLmH0ReTMxXUemYAudyf3lusmwBg3n4I5t/81DK+p+8 R23XHafLKtBxlyO1ZK3Xmsydp3h1LRc0Muhy/q8kvuR1WhurXMq6HwLmjdOpWtHXPdXVeQWS6uf8 CVks+bJHqM0Ou9Mj5PyZi+VPniijzwXTd/DWv98KH3fNCiwTN5dBWwMPsj4eLAn334KATclZA5Xo 5Lbn45+BlEolwGi2cAuKXLt6trbG972donfZOP0Z8NH33uU0Y7AoTmNGznr0JaJ72lHv2/oNzQNG IcIML2oCT0IpGbiof4qQ3iZ726QB4gfhRc2gNSIxF60C+pKUiJpSAkxYdP8+EeHFj8jhuXBGZwGF j7XqTb0QP30+rIccRPWkzQxdYmrGX1q+rMrh+gSOTa/2R1W7IRbQqVPumxUgyoHw78O8L5ssifrD RKt2+4+vrBjHGHpLSpVwfco83ptRjc8xxLPqqNL4MCu9rFnNaGs76U4rckVzgKA9SPS8DkX6gszq YlVvUGm7TvyF9qgJPyJoFEEVzxnzs1HBZfdPg0js6KU0y60S/YBo05rKpwZrdU+0PPYPHpkYzERI +ygEcH74O/G7mWEscIH43J/WSNllH7lpzp67kPtjN0AJPIywJ0g7lCAc1VsXHIe9ooa/wJI1Pfl9 bek7mo+N4YZ17+06dV51d8ksmlqg+SD5dw4oIPKLAIhnQ6kjJzq3H/LjF20SnZAXRRQ3GR24iyh8 B80Fby04XBMqPetBrf31Fz29G0mcJfhVGoS5gPzGyAB7q3ykfvDAfq3bJqxlLDxQJt92TUNct08C hsxuDqeMPn2ZCy5r0tgQRomaaB00LK+P9FXe2ANV5bGkhXx0juFLFWBw8kgWjCAF16PnUT2UNsTs qbf+9AsA+l3yyefciBx78Z4NZwV1IllC16vyA5giXgBEOxoWmjXzmWFtiVmgQw3CTbdMoCJ/qpXl CSdT6UJXpGxpsC/4T12EZsweIs7+KEaePX+lIwBKUTEoEtfGrR2uBDtG8i6QCXUX4oym7sPcd/pO GaMiB/E+mgXQBFcd3RCJBu39kr+c1UYPhdnXB60X7pnr8a2pJ6hGZ1M1CkA7j0/WMO3dvhjs96bX GA6+c/0O8/MqZ9MTgwrVp9zCxkW+38btUuqxWAN/YnUbcTixgP7q3MmWLNX/rmme6RyAFq3uoXDk OkX2ZDKE0WvL8OWVm7W4BEzRJ/u3G4NfQAdL9XvOT2fO/H70z9Urt73/6pQbQF43YfZi9h0GR3Mx i8TfdWVHeL96e6Ke/B9mgRodqXaX2PkvjtKVZ1wy4Tq+flQOuMjpXKU+FnKTJGKHbZHApPWgL1/a b7rb4bH0PgaekMu7ShkebhQ1+vkkA/YmF15en4ZKddymYuBat/QeYcJ0OoPzwZ/zuUdMp+BgqXcZ dQTqXZuDcUCwuqUrpHKgrwJ5DHtV+Nz6E0jU6xmUzMVJc3UiKHzc2nSeqFOQZwTzckx66miGCZ73 UhX3Q6izKrbyu1ss6d3o6uukBCOec2rshwsH8DeA7L9GGVElbKJ30WIVEkCW7Q6XO8AOZE5aHe4S qVIQRTysWmcnUwCS9AOxDY+I1zZErrIJw7QvWihLHH4ZD4kGp8tAtU3kjSVRNCQIOaApYVbFP/fF RJbHI2XoZTYTebW4Cpdt42GIV3+3vNpQJwnzt5wNjcc1fMcJMwsQPS7C846tj05hpyX3WpYtuXco wwUs+lgGdaDQAmQv7J5k0X1kYtzbMibzR8FVCcyQOAzvB+N+odGc7u3K4VlTVpZwFzsEx0fkLssE 1xp1k7lEm0O2UCum9ngEcHlCgFamSP7uIHBZukG5OVIvi3bZCePHMH+OIlWHf/gNILjMo01RKh8m JBkQN+gVHlVFChBYIxdA3Rx5Hsf1sGyuL1rZAK1EUww0pbQGYzfs1gCUn7nQpPuSDMz3Ufnjx1+4 2tmT8QjQHNZ7gJY0VJ0u5Ag/N4T9OWMPZVKqZF1lp3HqOq4FMnHz/g8kWlmDATdzMB6hZ5keUoW1 Tf3V8MZOjRn1e1ewgBOHUyQ/I1qTpLVCKU4VdVL6m0YOcKdy+CZd6jfoAa5X+jn4BS66XRNe1sb3 zQRLbFUdSdmPjn0m/BoJL7wj6h9SrQMugIT5GiH1Udpbj/u0e8kz/vM0QkQgFc7DCR24BA3mdxjf XCf7y/tMoS02XXXOJ5RPM2fr3FzNDdlwcZCw0Ft+Dj/m3RaoU1JZoJZDsYkTGy3v86b40KM+GOdG wcjgTUtyslA95gj/aVFaXMM2syE0p8mfVVFcrPrRdgV+1dh+nYlzbX33QChbRbOe0ro8cNGcFqNR 1KE5AxDhB/HBs6jFWmDFXikI+hh7t5ApYsPSCkRMWpj/eEbLXT6HGYnTPcUe97e5MIZqooUo+PbM 07/B4vnYJAgdCBgDjRa8iOsWFyziicIYSpn0V7o4UDx+BvAXKwoNuiWELl17hPHS7y4mP+Art6aD fx2ysduuk5HKX8fICdHN3ezXTyjG3H7QQwy/9az4I5eOycx8KWWDDuJG9n5BIUHrHdymzdH3qtl8 f4zd9LHXKdozombAqZ7Jn34tAGDqs8X+LbcQe3vmzM4jkHvRjtYfTWRXuO9xJFFht8s+SKRQEZ5y poslc+XL9aBHSkbkQhdGFeaLagiwsgGcBMctlNVO0jJlgdt2YNqo3ptg1hRDEIHw9xaqMMNW24d6 NYqdbUKURMnXZdWSfpyNO9yXcR7KO9NUJHPoV8/uS+j/5uKTMqdk3r+sGXyArWFBnMCxubImZZDW JQAntESYZMUBa00RqUgVx3KBA8OlyHeAvYjvN+kvHMUqrasGI1lrB3TTuWorYjAwFmpqsfU8ivXk bFdHsHizoaBVy4KKSxrPvLGw1ZEZEmlgOB71DdIQNxfnl/2ST81ywmN5RdhgQpowvYtR2HBiHJnG /L/uF+leztyq6hbhhbD+cI9kJEWEGUAA/eN5jxB8bs8VR1pE7lSGJkGejPcxUKIQiZWBgZs51+O6 cKEsJZCGeOm51lmoMm0E1xXypmG5zkXw6EVnl/HvyYg8xcIDvdmAV64LEj1xGN0QieSE/oGay6uj SNKoh5o+g8LQAPk24tLS6CRDwl8nsLl4Wz2Xxz6jMZm0BpgVxmDO/Mi4pXlW4mTrc+9xsUsHlu/a 0ex/eBoJLD1VrKGCfO3M5kukATOVQGZ09+1woTPD3mYOiaFBU7s6qamu9p70qpERCqFjIZdTcrPM yP7xmIUMnkv1+73P+g49+IiDaa3bdJUvhxuYJ92CRVvG15ste8MmLlUQsH7Ft99DNEzZuaHvrUP4 yJ7hblSfVNg7l7vwEHiKVIi0J68HxUIu5GzcNUvC/QagJRdAf8Lw8sJwP0e4fh3B6RkzAkVkdLi1 O+t97CbLQ6T+NAZ6TqO1NU3GkCHzRy7NUk3PmlbHxGAZ8/qA3Fh/p6YcDYPCEWSRZJS1tm6dMCr6 mRjtDbqlQzLflEKXlqAn/DhIqKXoJ+Ytp1claJ6JFvHNOxa6TGrmkK/envzn813yEneLsc0WFNHz Yx+d5vzxhbciay6Hm5NheK2HWe6GmnRR88MRRWEoQ02q4WZVh3JFcJpVy3MO+yzU+OYFRETcQeCu a1NUSEZFBxUsiDOh7lBaVmqlEKO0d2cAQmt4zgUTLpBChBcy6aDbGLlAtlG5gcDG4XiB7qwdvNNl h8VGr2Rt+HP+X/y0mvfLfOsXiDTP4WyCLAx2i9VyXxN/9KDzmB7KCII3Bz78nEAZUOY8sxUh99Oz lk0MoV+tcaSBMfyyUqz7JJERIL1FqCMCwRUN9Piygvu/doYT//nJbiSVHQtdzmmhdkhTO9+IkbmS bD7HZD/ICVZKSfvg1noZax5BBOEzgfNMFvzipKGjT03uey+ek6es9qTp8ZiMtxj3ROOXebop9NPm Gnri4CmzgSK79DDRek9WzqvwZ9QEks3+EJQOjmjmvI83paUzKSXOF4JDk78mbUwzzu6SzDbKg4S/ ku+E07sbQuWke8yZYb9RoFw5F9zQEc8Zbmt42NcdzvnCpAIOtajpKN5pBODd/X6OR+0cCaXU8WbA 4g3tqlNBnyzdulyshWTB+T5WB029T75ouqUVSarnRPj/YWZbBbeU6taIkwmStsoGGqaarWMuKgCt w70pl9hXAbB5PbqHjhQkvJGCo3ulR5dODu2yR8OEoBvLfntM28D03ELaZ7yDrX2H/LKlStBmdguN Vwnm4SBF6DVJ6DgNizFSSxWKC5sk+PcCXXV8CO9+T7E9SjT6Y2Q3JYuJS1YWVavnZ0rp+caXnNqz 9r+5ocLSuKIu55B4W596qrJiodKyfh+M+JgEyyj++IMvhprTHRDvxgA2hklz3UE6Nc0rgh+WYJNq jKwi2s0EXctS+pNGTQ5jdhi17keHUcoisJo2ZuScqMDi43aWKeQHTSdkToreevbKHQJKk+7imCAl QMRkkZtCqk/GcqCQvoNGwOyOf6knfP3FGjVUu1PupRVE5fgQZFcqQ/aN0kekWeeko6myOIkeM55e uSFdZOsxx+lCSMe0TPg4r7h5qcPXZL+nnd0Nw6LH6FtjNVNVU4/Ocy3SokT1DIfK0nF3G9XZI6Jr c+DnBm0M/IeLnWqvH9rmmjRoB4RyMhaKqiXOTIRz+eZWBH64qff6MFfoiWVR886CT3vgr7iiks4O l+D/URwUl9Uw8aeniDO1hZnvhU44UOkSb3w5dhT6RGIr33/DwJG6JmkFl9nJaBtZBF0Oqn0NnTBw R6lLEVbHPt4gtKvqjh7petug9wIAYjhErjy+RARE0nif24cQeT51lJPu6laLU2L5E5Si8YdknfJE MGWY4iBORgMAxUaUtCTsdmKhS8IsDCZ7f6EF/eS83Aux5VrZD+HE18H+r2T25eeaCAFIfNhl96Op aNXWDNwyLUeK+9bzimEVYZLjHBAPneVkyMg1IIGQ9yUzVgWYPWDlV/vI/I3wfU1ntvArnrEGb1PJ yHnVodas8ivzighjs/pfwnZn9d9tfJQdtjpLMWSkEiaQZbrub6gevy5MQLEq8rw9qPhJ/5ECQ/uJ KOkBD4nlTdjrng7uTLkBxlVGIZ4VzpOlR3z9WzO7rUDCyaJrahhdOE2WF1RATK4D/BTyAalr8Rw6 887Jv2roxyO9J7PyUKUhyXEwBHZNe+hRAr5/yu93ag0tbNkZiPm6PEhxgXY6xU/0urJhBnTJRwQP p77fnK86n3XpNUw/oxtHPL1GicB7ZJVz2/2VZiNb1OrCsuCBpunTMXV5fSk5krpN4K1qnkys7PZE yGrXj2+T8yOVQ9BY43D086qhxYDXROach0WyF/bGH+nYYGEMdfuJZ16BcjJbHkrMwQL0gpitGB/n wl7gDqC6JT3IDJ3M1XaZt2tQDsKejHtmgqZBOXW1zNDp30WnaNkiF7edm9gRuZEMVUGfmUcJ1fot bBwn95kqGHtA+MQN5YAoeOAF5ZAhuOrUMSb2wWXygKV8JpC4gFTQvAjPmiZXOBQagzUabm3uHog6 k/1ncqtSauW9JyKzoGI3e3jXL3AiUIaqjxoV3OnNiDjDiMs2VmSP8LdbJ89WsTBsWuEKGeCHLeFq S5J/e9cbXMKNnOGlp3VJBeMIgH35dEOsVsrS5V03p95xJN/KSJUMG93y1HpLsseWp9BLZkBtm2BQ NLv0imBs18PwoWGI70lSCA/8orIMHPfGtrbW046dC8glolEC0peDHZnwjGfwl73W5tY92VKxpkPh BflLkqw1xhX62C75ky3W6KteARiyevDYrDmAWeyn4UywHAoc7dewjjEbONL6JxDev8ESh4NRLxgB nYEBciA3KvNI6Uwha1GmZY4m44na1ZIO9L1fjJsMVIqBM07t2DCt5TZY88XqFPqY9x/z6q3cwUod LfX7enfg3j+E6HUNNif+UYzasa3K07VoStPI/PwCSGoiFxpfpZ77WLoNpn0X6mcXrLbTzGpb2ZSA KyT6tbW2rBPJ5ju1yijJC2rVXuzO4h0FhzawsjWVKhhjAhdL/bHrnGRX80lw19wjBHZ37u62RTuz yNzbJYBy7K0GMfemKpmUmc+ElGX1UjFv8XhU70P0Blxn4wehTo7+sMUu/UGxtKlWN1MmIJHdMKnv X/OCLjkZIjZR9Y2+//gu80+wwVGPks2gLssPxqw4r7DbWxU/ukYm17pnkORkRZt3e5fg9RGAO7BT IxaF3kC4pjslUwn6rVFK4MZP7HyxUBkQvkz1uzS8Vz1zyvsbumVTCMMaxKPtLj1nQG6P+e68RHDb kGJytu0wzVJ7zgrqNT/NlArmLoVm0wN7/q1AYqpUpobqj6/yEUy9W7ZZz/c8LBRcjd2EfbVednfT 6kQrlUU+dkO3kfGkSNN/4YONTE5WG3MnF4FF7R4etbVBweFhIo+1PeCFq55W2pl+zvaJ8ritM7qM 7NIFgGsvg8LrwjB8omo8jR9HFLAMdz5Wlwzx9D+QEGAGqEPg00s3Lz3t2jZyy4PT/Egn3una5qC3 iP70rXXqt44/rPcyrcpuGzQT+Hoxy2lbVujabApq6aZanWNXraUjGktc4h64TKNbZJdKE1hZgNYo RsGPPTlfwXkKdrc9fe4ysMh2Nc1BcxqC9M1LLUkVPT5mqQ1K1ZmMsG2VRbwz2SKjyvOmcSEJFe1v oL94Uhy9OnEZR+O+NzgO5yopgDiRC+u2pFF0YyQxBcKie5gNDgTe3Xj+veSED36YP6igxZrdrBD5 JMS3SQwocz0vV0SacqoMqh9HP4BYKYdJWw/fAnJnll2mzzdAYD4SKdYYBTba6bWFxGsh18HecoYP qIjgjN6ChPZAoMwrtesNCxgd1FDApZxc5pA1MwWOEN4P/SI/qU/2Qkk4IJJI+3ASgoHudPEPuNjg ehOBeagaMn9WnXwLaYHAoJIzcaMlK6nRdjVAR8DmeRPAi6AtBEXJ3Rn3G0qy7w3VZX/qdDdBdLRE GMeoDmBvkrzqlCcrgFHxrY20fDHWVk4wl4J2e3jCEJ22hXrezH705J9Mmfue0U23MF0V/sNd5/vu dTf1VNO+TnZpfjolMCuZPzmfKhhF6wTdI4idTKZ+6Kmj7jQC5SR5Y6iL4212EDguTfouiK6BLbCd bwnLrHXCdZZ6MDWc0NDHdukuqC8RtMokEN/3KXmM5PJqLryuOFQt6wmgJQ+W4a6ELJZPX1513X9r pIS51v5BLFd7B142KRGf2dXSaXXOtcnS9jLYsI+VT4q3tr6lzvBflUDqyH5LOv2XO3P03HZ5WmtC HbGVZ7RyL455lt4kyEjl8eMDZ7M6qtE/ko2WvnVRjHQpdYmq5x62yyuDGJM+3qBGcQdfr2Juibq1 LXtjEoNa3HeJudi2t3Gc+USP30MqVcCYZXCl980E0klqT6h7BRGItHTaJFRd8j8cTbVdUTTzE2LI LY9cxb0H12hbdzkW2cQnRcANhOaKrUUtuiQRL7OZSd9qbMx3pf9C22D8uqeLvjP6WTimV2M17tmD vXom9tWVi1nvtEhDy64p002oA2TtQ6S96AMS18/UyH54EKtM3D6S/KLt0PULgMwYdHdNwXUoQUMu dvnMNh09zp8K3qYs98NLi21OdafVcoC/qUncrb3SfTHb1YbvYWwMvj4rMnatsWm+wQ7j+LpCEBzn RPIGR2+c2O2LGtMo73v5Q1/rjEi14JKICF9hraOHwsjSH/sxu6gpSAyEbZcA+R8Kj+QquuIvtGx+ +O0FkOQdvQjH3r/mXdn7sVZTiA9bZk2FFQTA1c0jG1LThqYrHoa4jV4p2UZI/IaQ7H6bi2Q6adEQ bk4x7lH3tzignoborWZhI1wiDDzbkEzqkDUl/8Y/fWqA2y0TjVZVmUyqJX4ukZVuN2NIQt93kZAZ GXdQ8EJO+UW3yvfXz4nKXyA/YweHjoENQf8NwLzlIdSCA91jXQtYOQ9LX6e6qYbfAQRV6LInIfH7 s04paDU+rQIIz7ZVL6DE5Nk2qIRVY3bZBJRV/iYFiIEeGKtLSer2zoEqI5X8WV0Gmsuo3JyaiRZt 4uTRizXSo5keycpU0WMtd6y3T6lTvHdnGI1D0UPTAxzwOwUt77z6sBeu4eKN2BNgcQ3XQbU7E0ki 6PS/21wOVW9KDla4Xxi7Fmq5KuiYKiMFXqTl4fAFtBoglRYdE46znJREfD9GPYgBOY/QioGVaWlW uviyRWzd3LAGfcjwNovU7ufWAsSeC1UQBktFkOycp4is19MTpaIaae8IUQ7S8/xtvbvFoj25YpHw QQMyqmnQoFgUV+N839Q3wevZZ0skj3JgAaf1avqR64Ei/uMFs59NhT7fzhlmjfcNeRk1zg4Qhl3h ccTzAGT91R8/LFQ8FJOjggMbt218b+DQBQOaAiQIhEOVtkiaoYEPdY8c5VPRDvTrQdN3TWbmVFfa lOuhU7btonGcUSUFSA6VqbOuG5N9u6RV3t89oogPfmc19TSVqRktHGdF56nhe1/wRCDwtJJiqJch 6Wf1RsopF/7vEyJg2ntu64PFEKqkRbpGm++BOi+S2ibeRY2+PkwbSwAy6Uzl7hjtMad7JdAiuDWS HxkgVaHE5kxrOwvbqSrh+3LZGVVRpXrcuGCHrt5TG7rjWfT5gEJSgYYlJBzasH3rePdAaZXVqqR4 plygrUe2yK7PmMKoV5Mt2Riqp8GF8VI5XP4vRRjFcvduymCaadJDhEaESuCBa4Qor4yF/sILTCVs 6utN3pZ7BnaC3YLdvNnBl5afpwuU7dsH8QWPdXiGLLRSe0ZY5Uv91IeA9PtQGmz3Xv0VmkdDOZwY 4aMiH6eOp5iNkwTeO0VnHRFIQfSPJfzbfTHay+L8KSWHk94Yca4g0DRkGvsDb/8FBF0pAHvx4NI2 4oTqZFk5GArL6hQrGKJVoOKHVZJMF26Kivl+jNHWOpw4Nf+v5ux9pg7WzLnCuhKOm6t5ZpZQXjRm 1HoIPT/iyg10GtQxpSJi31MOqO+080VBUWBHqbHJwsV0agbLITjDipbXIfUUbiSbHNK3SU2lDgOu u5L3LOXHjBP3cHXhCNF4w/xvMCO9azFoLSSt0OfLgAWmfDOi13CWb8/bNezU0coZzw0ipAplastK rIPvp3KZGSgKIXpKlmyDfs4HJvGEyMs1HCC0Adt4EI8AuS/pVwSOdUpLuMO44zm7nw+r8W4kqZFk o5fjyd1qhHDTPvnSQRbwcfACdYiIjaw0zuQfsyyxpddqdX3D5T4O8n9qkrTXYdEeI7cyITTCGZ+d YRGwqq8l35grBS/mpkfHVvio6cSPcGcbRZcIyDCK9vWsHp0RqVr1h68GgS/g9PJE7mzmrMsBponw 6+CvSU2XdT9H9aZh1yhWZm4gX9cKLh1ZXZf2lGx9c35btnSYbHor4fQqhVdWT+QDPB6nrCPJ4v9a nlk0V8TNxyEaV4zbLMWNMYO+4sKLQg8KA+Yqv/Fkt5/SZLfMfo31Vg+JDSO/aA5QhLMdKBRbqnSt PXfEIT7pAyJRTMtl5IDNqcY+hDOCI32grow64u1s1eCvUaLLwzXnNn8QfMgt5h2PJPAW0e7NbMZF G4VOI+rcTlUrNypUkBiHaTsp52njNlAjSWYXuWKJJrVy704yzgXSd+sh9xK1YsJV5Uuq2xD/bl57 5S4q5nr+Ggsx1KIuInQgNLCrvEhZxcAVNW0TSSlPgBKoeImBYjzIcS42hG5B+Rho71pRL+O186Dn bzPSPwCpge8sY+5wQzHi4Cnv53SmVLkccKhSpFRkXXWy+DvyrB8xufW0Fmi2/fvtulDRr/l5hXDO +a71TKstP7YcUS3Qsj13p2E+w99JCyQEh/hhxnsl+iNBxRYHlm0eT4oaEiJwzMD9Ms1nbOL4CMnA CxSkJAL/fOFjYC00Kx6Vq+zblhCzH665eYoZDy6+FCWhaw5TXZg/dvLutnxvIVkAruguJFAFEN3r TvHa6sMv0mHDNnRdGq11S7BQSvSYyYpPdhqS5jjkz/7e9I2oBRsVPQL5KD+mLg8n4Aj8y1geLm8k Y2csSqk5lO4CSbnVJX3uvnGQi/BnSwUdMCtzmTkXrWfI2iWShAkTP0Pz9fNlIFlVy6w3jCsXVO9D zJzG7roYtRDtPTJY6H1UXTomxZVHBS5TFThDVvxqFAMb0FQX5GTFpPYg27CX3WiR8dihUcPc2EiF TJrIHTtMq+VYPv3l/WMU+owejvvIdiWtNhqvO0N/fN+yJC8FyG4lZLWGiiiPJWiJweFJWRZIa0QJ 7mxTkvD6aFMa7ddWfxDfWEbiBij+zuDeTrY1Pnt+tqUIJy9qbhUxnB4ZRd9sN7We6frMeKaTGLsg DeBpW53ItEsp6ZVHABnPZSB9JifZIr1dY10ykpjV26S+gVVQ1Z0/ZprMT2PUCP7fSba9HyeTfDug x8ZuUN6MGO/NHRGFIkuxRiJ+Rt4HqSYrCHXQkqlazw1PNYwJ+Up05roRSNQMKZLtXEHBPUHFlDwn oPrrPU0ifbIPqNG6AUmsLU3gi0wYgLT+Nlg/pl5UxEZ3Ym06SpUsRcV6/P+BLbF/yTE3TasRtsHv 74B64WLpBgUy3GlUxaY9q2ucjkxBwdyhZy58g7lTP+WnPWmlSclJuT4QFlhBi+ZFy/aoXzYGbG9d AepbRRLLYm/7+PW6BVw7NDXfHU74K6E5fFcDl4hcbYj8AVozzbsTiUggDEEkJwggwpYdVx9nQNiZ ZWfKACVolQtOrEn2MkAWcymDhFnZfJlM/BqWUsQDrqqdp4L/qXr15sizxw9kpQfFGrM7gbozCnY+ GtDF7yDIfdOzv1c380dt/C954qPMjBGYumgNRRloN52Jh6ZvsCh6h0Di/thbOfLZ+RDntGDQkFRw 7RotjTEe/AwLaq7UkHg74C+nXB72SJ4pOiy2vGmhojOLnFTv1rSMbS+6VtcDk80PdSILZOFcIaCC 37KRZmvqq17FGxgCkgh1W1eKQfImlctbLCIlrj7yG4CjiFAWGUI7qFS2Vmlvsbf0TKs4b8haE/Em fyRBOYvmDaY4+wJgdliBHjK+ep83qIITYUAkhoWiUjTBUPB+5zAiEGQeC6HD/iPNSI37zrY1eRbi S2GpmIC/fwF38TqCgGUVkAGBKpVsWHqRjLIpRQdAwf40VsHP1favm8g2Wqgd2MqM5cozDFAiVDOK YhteD770JQ/OJr60tpZpSmp8VOdS5G14eNONl8ugrwx/wHV47e7EdJvNwLQOfP2Mlt/goJmsrBeA Q5daohtlmkAvscFyNROk/FTKfrUcexCCTsnJ4ij+ozxI7E2vzopfu3wkqfWP+9KU+6lIFYlTjdNE Hv6ULE7PNV6FPC9G3HjBZtug4rxCeZSvmBXHOWc8sGqRiSJ4kEBc2xsmmJhaKp1noCf7D3s7xnLe tIwpB0Y36k/RvYmbvgNRORPxZ4eyJuZVig4slKnsbqWZUITYEth9BiQhqnasDdcm4DvAXd4MBe4l OzdUSo7lXNnw2DiMxiwLae+56LXsTyma16VFLkHTKlYajAL9nL+y9qyNkfkra9YVFo/XjMTCfA3J 42OlvhW/rSKV0B0I+DYqm0B4+SGf51wtp8R9GGSf4zx4LYm8pviexTfyw4i0Xnui0yBXYbjqp39Q RGwjoX1opBfDEPauum0riFqbXblocotqMazvQfk2kdbwWV1rli4W3go42u6NWQmVEHFb25UgbT8K NoYvgNQtd5FE9Pt9S+FU8SFwAeA/ig6fMK97yvYqLXBQWeWUSVp9B+A23wiybeT8/YW71Tkoh+/F xHQBujkSXVHGD8bsGpcUTgcwb/+XLzJd3gy2VM27OrwZfbzndxk8z+T4wlKf3FsDqrKwCNSJBCnd eTY3EJbI1FvJ+wqOv0ZGHE4pMwoyy9VSMwaUyudQI9lsQcqb1y/GQO/wFb/l2j6nxA/VO53Y9hCp JLxZz10cyoHbrWXmVAsGXkVOPiFESl7K7Uaib/iKzj/WIsj2PonanJIRL2PSY16/xgX0HVfc+1LE F3tCcos71rFnaxxA77DE4sOtI1DXnaEPZePd5Nw74rEXDIlK1LKiMq0iKY2J+8qKnCh03pD15car cW0OYpxMrRu+LMs1rDd3HxPL+Xh6ui3lPs0whPybG1u8DdoBVAN0b9WwKxludBu5+WgUhtpX9GtN L+8Fw5LHQTxVKg5jhOKtMcmGTdYevBkBssc5t5m8OrurG62c8XwknwA/4eofRs6+xMi/Kyr8CH5P Xqu1lZ9+afuWCxWslQiLsx49QHjKuMeFBSMeLaLSllHiVrkj0Tx7CorXWsKtbSC4+kLdq9l8DsY2 vtf1wWZfNV8BvdHNPn/ZlSoJeidylNrbPYRDy0BtiJJdcURZtcf15RzfT23cLtlwgojg0iKS1dnI NUXQOVBBKPu9u2ndww3xYbVvYhDUmbXe6RBU4JEPA5MB9EFGb7t3xf/1Vy99b8kLiNZeDxDnKVSL 9+AGChYtFQa3SR53QNCVijXaUCrBwG5i0g6kbjCyoS1DzddEFMC26F47K75Gm/pIHRWyW/KGf8u2 V2zAnlIX/iRPhR7ImIm7/vnGjzbC+/NclF2uOFiFOQvfYcfQFqXA0wn3+RGnIZMzeDWC3vS4HbQh TFnXJMq7KoqYtUyzhqelx5BDR3EqV8GiInCQt4zI72bAE4enOgx85nklmX1xSR3EuBhnKipeFsjH bA1RSRzjuVaFPWbuQW16ELNqctH41c1hO9LJE6pSRsH/erysC1cDmM2M1pTGy7RrgcSE0hN8HoEj xOTgV0c4Tj6y2MiXIIggI8Mar6V2v7wuR30H5Te1r0zmYG0mH8PbzlM3hxnLpPNyBU4fF/GnyQR2 W0zh+rnDj+tebZyhzoyBOZJRLORRTMhBYDPUk3ChCMLtYVy2gNhQTeiCUR7IE2ov+1rV9JqFPIxO WLz2v4HvyzFnrWGY8r9mCqDtRUPMv1pj/mfAPy/9+eT3X7SNj+C5p1H/KD5BQUxFLJ1+k1h0u/F6 B/qGGvrqG0tokJRmNK5PaQLxdqB5FArNzLQwuMiqUh3b/9+0DMj9B5OLMI6ueRqyeOOWe53dI22q TydNaRVObJobnSu+NN8P2G3zxDPQLLlGNOmjv0dLDgrrpFWlEMElHBrJxDIgDETQL61L6cSFO8Fb rxOW2yeu5/lS2Vlue2i0v+X3ls1l3s3TNDDcr8nTANQy6t6M23sUfiLR1jtYTGkuzzlp2/ReCyBY E33dYyH2bta4AXuk6zTZIzaCO8FCRgLXrbi7c/GTKyUJMQ/z+pA6gXG1+dm4WMa0OiQbZ35qTDCi a5uIqzNMQjZnBBhDTV9BQiK0KcLJ6y1BJiByWlkYXSDpl9A68aMr5y9gV6Lyr+Si6lczLJWFevvJ l+5YoPc06LTX2/wVZzlTuye34uA7GqoO5Cfo1pEOPIHMh4wM7tMNTXRbGNb/sMA9R5Fl4Ve0O4uS dATrQOlnHgbianTJrKCNIssreD+ohPvm/3NKZuTUDTPuWhz5emaOd0TFs0DSG2EwOnoweOZb7t3Q Qe/grZIE/0IA4ml6xnEY4j4mzOewEBnz9zo6wKnqDR25Gg+lpCH63mVcxnRCvvyl3/HcerVbVjIk T5saDsuGHJqkfH31soIEM5JMGrsDM2/bawVgLq18/IzH6ke8UPtc/7tmxdc/WodkoW3pJdQyWZT1 TLSEIDVUk3afECRzCw4oIJsnPSJKeWkmXlmfMNfD37GL/+b+aQQGnrn16zC/u1lNhDysv9A64IEB OCHIvsE5KGz9fPTBLr+t12fNt61GU3nwAj/H60zX5skMUN6quneLBMvW3l94vbeyz1s+f7o/svhu 8kBKHyTEQ0vzKJb5k5mL+a+fJQAZQPrpXw9Yht2HGOy4lmz8R1gWHr92YtkSMpOvl9UtgDDVn7V5 PYP/CWQpaVO9zVZ6FMxN2tFjhvd1TgK17S9pJGXKUGxt9ZNP205GWds3vyQ9GmPhLzeuU+me4IcF d0iJhmfzPDi4sN26oAvbQcjJPw7pzyFk9NJAYwRntOwtv2iI4VL2lhrEcnrkjlvHk99cRdrjG2+S 5x3m9VIINKDHV4BITT9RDKOB7HxrBBCGqmejllQIniKHRo0VD6+iICWGOmLOtLEYvcynU18oU35G VnBTHG4c2SGYzsMbEk6/bUzUjFXli4c9PnOgWnkjeFh8WwZi/t09IVEZdD53VOY3+RQNBImGKIXu yzfs21ipiXoy6M89x+47HWHAEIyRH9ec0bR4rMaRe9GknxBKiC+Y/LfQi3VgnJjd9ZLMMOK8wjPV gycdaGu+/XP3koSkpMddQzWcTJV7+fx3HOHRQSrHzGo2FB2XQYTxrtV3FqsFToFRU2XOAyAA8Coh wI/JzBU6KzUoHmtJ7UInnndRxko6GCHJnx7zUw7Lrl5Mf3diZySNj8gqfAzmI0naLKOIXzUhYnHw +qmqfLzwTq3FWqgrQEwrWANAQBI6Rc2ehIhI6gpZqVm1I4RsqZOGUQXLW6ihIlIBI76gePCCpk8C nmZttFUyqQyNOxT/AP+2UZp3wo4rAN/JuglvPTGQRvFGbUtbAYlrU7NMMuo68gF0IbyLk7/fPAH/ ieXZ/fJyVE/3RpQ8iaUDl5hOnnESkORqHooXPUNiMMsbORnS25frUSTXiXIslclajL6L+3gIVf25 II1NwAzkAoCOS0tdhwY91LbNEVKbgfvx2q3EH6k4MKT+2uHhDnZJU6iIfp9jX8Yr5Bv7dbygXvsM 3dbYP4+IzgzdWOOia+5knnt3rmybdBGeEaeqYLhPqQhR7maT0sbb++eJVDTyLaoCoxbz+2VeRBme UBzjA77IB7rNq2Ox/sAwttl8rXuU3o8CDoUqTSCVm/wVpwT3eacEeBVxsh4RLB4ha/OFU7/7BCxn wW8BzCZJHSgK/VRMRm3vMVvyFpk+ldcTr+88YxMPkyzuu1eNpZnIbjZf+d7hZ4tYVcGlwqQotL3F mUPlfQymh1cyDePTf1osgL07dr3ib6qo6sEU8jT8o0h2dLejLOc9k2eewVSKO5YY75mXvtbalpGk 5hJcXP+u8pqGeRAqQ1vR9Zi8dABRRXXz8ElUPw2D/OzAsMUa57GWAi6pnbIYYa+sIk1ZGSwj0tgp B4AWitrfJmzQnrsgdGuB+S6B0Un/VjlhzO7x50xy22J+8h3HewBYq/5+NxnIN4/6DPKI5ptZhJhR TzfaKBE4MuF0Zv3foB9OmW6GAbvlXn/WOHkHHf+Rq3qOhaam6u3BQXU+S9/ymuAzLyAsi3npvurt 5vZvX3Qa3+CbOXQSfuZhqj5P+d+fzR9n4LQRNYiseTOk878vMvqOi8mXeA0zB7c0bxh2AF2tljoy aixPHOd1hZROSfNMQhrvjuzQDINrffwpP0R2w03D6UxYutqwCh+EBN7gXEJ8aExPReSKGhIZCUv8 lfe9u9lFHiHmnbCnjeeCDrM9TbFIRUt1CZ3QAbh709oXM+IkIpyFvCsfJe62IoLql1NsqqBcHlMl MCe03bVxuJNWghEkrhQ6sJ/lJB9wxUuCJ7WcIHgLvMMk6QKtILXQLHGdaozEzwbT3MK9+E/R7dGf uzOgGL8SFQUTM/vQFap7kIBHgAFBgFDHSUZYsS27IC9BSLOuRtHT3a647GqE+jKPo5zK2NG51KvV wVsFt2CqXLXRlKaS+f2t+zUE850zI4ihhvuhVSkV/zloxO3mEui86NluoD8bBuOglwRylbxSjwE7 52JEILopm5wD8KYw5haR12ZjUMpdZ199hcLyDwThs1+gGIvJX5U7Q8Mkeos58Z1kBXOANu0e9asg 32f925Q2i97WPAddvLsDIagdu1IPsAXfIfmYd4iHRVmKoqhijQtmbZv8Rfj33+ABN7AV4dOxYybk SNLFH2c6WvtdP0DYTiLs+4M3lrVYbNi1JhO8K3/MlO+uxdB1YRQW/a3wxnfJhe7CnCEHxe8JX/A4 iuu7REUj5cfy61FobNcAMb2xad2xmSAWFfQRllyuMz8tN50EAFC1W2B5vuE/eB/slvng0lyQmfpO JsH2rXvcKgFXXZealrfkSipy4GDPUo+J/It1iukQSieiFn+Nig6uNbrXUswxYHVFaZ4hFnCoQV57 gX8K3QdwdmqJFk9LxzazivGYMHSTg2l5H6dfr6lYYtGTSGu5zJ7jSCEB+/Emhy0Ff+LwMtsEn0r8 oI7oLNVpjl5RDIs2loaDyeyXOd+fqlKu7WvsKlB5fDV6B/M0ygyF9nSzG0GQUkBROrnkUN2jA652 +o+O4JZwqqf80P6hr2kQHCiUSb6LqKBZrOdSaV77kBHpU+9j0Ho6bGKIiDrBy0jpVEYYXZB8GBQ+ P/RTNmc1MmMUXvkgOG+5MnuxsU+djg7D53jXyoxNxHtxH6rTQro6Uf72idpB7gWtEYCekCmYXxFd ukS0LjeGxZRyHyrrNSA42KMS0YwC2rrrpytaQxVpNwe0MgwYiyYjXC9NuIO/ToegvTdxxV1XrZhe BJW92uMSB06EhzjkYkyGM2tEnItC77r1+jKVLvr9duzR6fm58S0cnBfVeMS1qvade7y1y53KyKFL rPj3XOUF9FDmxaxXGccZuYxE+tfpYT/jv/PkOfiqezpMesaKzI3DS8DPh5dd/nyv+JjI1Dk0w011 PkGNXiEBnSOeydJZ8CiD6pM5CEj+qBAW6jgZXBEP+Rn5P/mv3zCnW9jvd0wTzN+bryf6eqeMXhLd jbGB5rUG9xsxg4WE23yPzdra3DN12rlpJCxlPog6wi9wwzF86KRNL+QJ/u6BP54vobF8iS/E9UzF CTIpIuvltFv1byVVpAl9R3tKfjDWR3mASEdtRC//g7FG7THgI8kcOEqjZ/KWxY9sUHNCWLHZH4qZ DBcGU0bF6gr9uV0qtdSOCk1W98i1SY6xOUZ9oHSv7mxWvem1mn7s9++B3s2hZb4dUMhuvYECWos/ PqKwheLIRaSLNySeDZ/M4rBUfngJwNthViQONOJx4InLDp1ZnOmEA7BGLNn1RVkdVDqTLv1yB4Co He1VpVXRu97rNrHOBnAB29TKJZt5aTvRjig26Tdl8yXrzWKYVMDs3sX14i5gNSGk55OQlRpqvn7U 6igB94HpKO8p1Elgsfvy3dO+9cKvann0Md9GGnGy1JDsU5eliOQGtr2JGZV1qhw0INg6eQrxdCFe 2X7esjFUIChH/qybnTQTK/iJ7lKwoZeVW/uMk5o+0k1o7YLC601QvXxgTtKK6rozuznYfmkhMyFd QhnafniNsJY5FNDsvGO83oJgc86v+ithpaZ9lVU5w7cA3COD+kvKp8aFGn4+VXsUVHZFGqc9i5W3 QsJm7cM/Fi2IiMRabGupSyOxKrsYB7PeYij75mYWIQsCIdT48UoBYUdCo8ujxssKXJWhXKRd3Ipa geoFWfk48+DfpqL5XQC8mNFH3RAwSOtZNqNa4b3qPSFh1eEgxikQv8ScxMAr3/q6hZX4pLpT7Bqe Np6lPZSqMmAz+kks8p/KTCzy8/+ZbO23aIIJBEvy36VIY62z2rqyHpcKR1BaXNEdRvuVqv5pLDbm K4QqH+cJaMS4OJBiO4AOV8iYQmL9jHuvZZZplWJxsOflEb7pRo16gywcV+4ZCfymo+qI7aiRFrBe ToOZNI9109Lja22AnhmJWoBWf8Bx3A08ErMSa1EKNcPv+6k7kvY7ubvKlm6HRkdrCzXSBlve5BKl 7RJfRAYGRIuzcQClIg3mzr96PalILNhz728PgsVk1OhODYF6FWGa++dI1Z6s7IASWplsqw12sRQo urYu8Z3B8BKSYfJR3qusrj627qeVV3DAyd8CbH+flPlaeI5CONR+8taHUSkufJfm/CTfs3Y4BB4v AS9sBQGS9i1ZyiENT0TMSGWOHLSMqVhFNNetyYQ1MFdP4TqULRoqTQez+3mn4ex1aL4u/njT2aTQ qTY5Mu44mgjv0eCHJimwN9aGxppIcSS0Jftst4JubLtWgE+prfWpmXkmrLk+UvJaPW/JTfcwhtPl eVseFGCitknLlOjnw4khfALVXpHP5Go1FPFuxcbUWwfjWsL0PdLAJwEw+cwt6QCaLdGAYKmxL89u UmKCxpsMTlRkgjQHtAIlYeLVWmoDx0pmzBsroLsoGhEGi7vaLcq+Suvm+cD7btuJtB9DD/VDn2Kf 0Aiu1GgiLdMmqEajoD7hn4+ApgrSAyo50UI6sGS84G9/1SDoCM0U7EmYpD/X5/FY7hbKPVhCaYfv WoXg0weaNpVstgoUMpYcIj3qp/htRZ5ucqIVF6qJMqFheac7qRUyLSzBVGQwuhWL+uaa8xenB0yG ZIDbe/BHDouyFjEbDsVkkn8kQ7rKg0lsjFcSfifJrcQsKO6o4JY3M7tkwjeslX2vOUxEa2kH6fRl /ILPTCQKbBDMkk6fy2ZTW9IqqR9dt5/1wmcjMGNNa4+qflw6+cl5mw9+UIbQQgYiyvy5BEz/0zTZ vW/7/1qKOZkiXtyTPd4WgAFBEtR8UJwzT93Y/8vrOLmNEgIwJ0pA5ATZ7ZOy04rK22Fu6OyYnNf6 3V3mOMNwHl50+2LwrfdHfhJRfxslLgyVARH3fDCF6Fwsx1/u1ADGdqQ1LS9NYq41X97kkxuSk9MH HPfHCH86zxOvY3GREqX2yxPTssFokl7qDeNLe8x0H3Y2Izg1QhqEC4l6ToQlc5yxjV5zJIJMEwBv Fwq0ZrqPhHH1dfDuyfzd2ulhy/YnrhJjenRB8lpqQ0T5BVdPEfyU/eMY2lttCeiiSi0mwaDk12SC NrnWuaGwf8htDGQjfA3mon83Z23VD8Zy2jIZ/nbIgK/l1ddRtr7ahHIOxNCPNn5vQ27S4+xHacPM uA8j8+SWcyn5YIaL84efSjUGJMkC2zPAUtowPENzmF+raNbFwBJPQm4Un+7kFoY7HvhuwSLhrKnF WgOvOJExYc9AeRHDfn02lLJ6B04qxpC2r+kCgxHh/zmZxYOLO3OPXRuSxZy5/HSIpdQPkOiwwbCZ ssn/UMabTZjIk+To97HOqYW2qWNQ01UvrRbB484wyBoESzS5/dWdV1xp332x4ZCgf7jQVzzayjhn DWMGUApEE1eX5u+S3b+8T0jUZeGHwZDJ1jZjzg59Sezdb98sShjzd8jnNNUBG3UuDS7DQzVzNL/b mCjT1DZ2UCm9UrIsZz7xN3Jh3zWEHu+9ZMfY1T5xgWMhtX/vS3Igs8mhiVZzQXc/DTaIRkXIvati TKYA+ZggnmLOkUtRs2XG9s3e6mTOVhZt0/ofYFE50Df3qIb9THnMuyRUCfewEcq/8s6Pgizz5Qov sM79cGZiOMIcatvFk+yzl+SEPyYIy7gQWcP1SXwW1UCWnIncPzDTrVt+L9tXcaGFZe4uvzlxISDm GAQ13qUM6Nk73WELFzB8MDN5GF0x5g0jaIKXfgthGdgL+vlVYSasGOZZJ2yfQnNxNtqd6vcSrXh0 zI+R/JHEVWuIzUFhWeQdiYP0b22gGP1Paf0Qsxh9kFY6kbStKuTJsiK3bZIBF8eK6WhfX/SklW2J +RP+Gw9spWzgtBSCv5SGulGtm+0vrCEI8SPAalxPMYcoVtreSPHZfBs9ZNVizEJNzXclYTbw7Q8W O+1DAVY2kyB90qJHX6v78eVWQcRg5ExlXcfwkGr4NoN2kPffk/u3lbH91kXOyRYNCNGOiYTHhtVU OpUl46d1GHnWdJa9A7TXBXGUHcSx4Hrt2N08qc50SeXYPS62GQbU7XAheulFfpFepFV7mGVNTG6L V3O63L/k5+6XKwNZFlRZC0WBediF8XLV/E4LulTTLUB0n52flWR7Offqz7z5XJMGM5QZG0ekARVo cPGHSje7PI+9cBF72HPWMebUoP62qKG25RPX2lIPfJR0v9Ruw7QByJDL2NDND56wLljXWdsJgHP5 K6/LhDKQrs3jlwgjXqHb+1qAtB4Hu6mUbnXIuNMge5CdqPNHy7YVpziG4GXnrQqfMrTD9yEbF5xP b8jMrib+NPOxBQodHOBDr1ytjvXLBwEkJM084bIEYN9lsQg4N3xCl2KmN/TKAjCjQdzsmKjOy4qH 0/tUnl3fwWEweNUtcKGCaMzSLFh/hjOnSvrNlfrMbnEl30n37rw1eS2q6esKrU9UvNDiOFY2Amtl A2EDGRUXk1OFHF6HjQkFN7wZfkKzQHVm996KEzjoqLq9mkRuJMCsc+ZJrvxelyIhh839qi12CmGB n/sOgk+qC2IOzeKqfKvxcCaFtO5AGe8BDaN0CSM3u05rgfClbT+xsNfqoR2/mEj4Ukric0IUz/fx fFf+tzUM3K34TxoFQr84G72H+jLbFFpIbpNg+wBAJ2zpfARXzvibav2HtvwtUs5YbWndWWZbSYte LIFSWHpoizdXBSfSpi+PXmSf1NPyEfm4PRR36E9lgJgS+Nl+0zqTwcRL2Tk5amxDjGYws2EBlBv8 cJsplfHnhIAcbcWsR3xODGqcMqMDpLRXzngvMQ3Ldgr+tWqIOd7379FWPGvuB+hcU+E8cWdsMc0Y JbIFcRoFgwa8JqrPRLAs5l/Fgtxt+j8iRyfE/6kHSPInP2IkxymePCtjSiPPWNK+hqHVwbtXGjAF kYj96cH5k5rS3ivXnicPfqmku7AMRSuqsdbc+cng9oK5LhYC/vkN9mfOHSl6RkHyoWlz7QR9gBLi ++O3vjquJmh68uCbnbH0LLM2uZibkqaFNmksv+2dEJVGTJVGm0QUY7fAQ6+xJ/tm4ff8xo3i4p2M 2tgZmgl6zvOhrw8k0eQPV11Q9NXKNxKAx/Oh/43s4hWTwXmeDj4/LPyyzDphnN8f4KQHpMYBpmbN d4WjwTjXt4IlzpVuhTSM2y+LQg9nI7EG34M9/kKm35uhOImEzMkm8DSTFYWCx7HFjYzdE6OC5RRs OMcDv7LnKn6NbIAGcGqhe8UusJP7tWJ3Btl5V2mtX5QqWvSHD2YQPkPBozxGTYAWfjx98Zj1aarr 849DVScTpMlr4Uhz0QdkCb8a8Z67Ynp5JZ0uqTtNjdLCqSCRzFCJBvWRUkdVBiYAqhdESgkC93/R SiGuPWHjiUpju8iggh1Fm0dWq8UHFRiCZbNej5Q9vS29f1gy1PNGNvCWs7+NfDZaHJDdnWV1R2O6 OLKUGihm/Ve1N1ORM/4zuVMoLIvm9i30aiXE03yZEq2tFvTEf3VwQX1nt3aKJUAxJwZnDW4Odc7/ RQEnWPCtDkoaKX73uMA2gtfdRILFFMURjvTbYo4J1y5IfcD5F/SS/47k/pBScAO/s0Z1GgR6dPtC dT/S8OBMs7T6tYcFz2hyl+5lDyx/iBmNbSqnSpFw2zq7juxx2Lz4ffFHST0jrB36m2KKWvjmQYJJ dcq0peba5Awp0fySdcRQuO8Lwju/T0KHx6pDgFXjVxCmepxfJNoHQyOhK6bE1rZ2GCbFrP65ORJS xT5xMM5Gt3iX89GMjgIsSBsG4tV6sanLE5wWt21yZmtl8yBU8bvP3oYlwVnWF/h6aBsEw9XCeiYc eLMY3yDQkM2Q4sgl17tcBHhSrAodTaiijfvqnZ4k9vMe452dCfKRFI753CpWBXfSmVdvZkMcI68x 9fUcO+W2hNfzgKTUt+Qqr2ZQlmZbrIHkdxPbLiBAMdDbJU3HuL5YTBcnz52S8GCnyu/TZZdKR1TT FXgzTl35BqUzqfVEPhKikeUPGUIfAgXP1FkN6aRDH3rb9NvM0qtqgqZmaruv9yzsCuPT0rqd0qDJ ddkEcvhQFVEIMUSsB7SqHs7FtX0MEwc0GHVm22U4LApsiw5rLuzoUsIpebRldw5hcuRDRFVbsMB+ X+1gv9d2FjEb4vQIwnFP1YEXIJzGFunH3x5O2hCqNnLedDi1JPfSTqa6q7OcOWUBQ9/o0ejmXP5d 8q8BB/dcTHyBX1K3tyhiO61tLvskFO2+zdY3tsZmOAU++vujgdvw3UzCxdc4Cie9wCxOzHjvx83u VJ1uzRTtRopdvhPBXWunACJD6hxzChGMZ6Ufomhzwz+WUWbMqMIqi4FYN6sLGLEW6h8WUyphByOC 4oAVfAU3s3bMmpY/gdaQVoA/UGyexdZO/QMVdAa77M4xvz0hSm6TRhkVmgemaOihQUEj5qPDBsYk w0m7gEyFyunmpEqONTX/QNHSXqpCVcW2W0T+kzyHBWjgw8OrqxF3ZTFbwEs98VCDwniGfb0U5BQ4 fUd0BJOwYb1/7vDLjn+wfKHutoXDsE57jtXiDcdZNp3sPfDrbhir17KiiY/lSYFCiVAczlUdnWog fgQziXXUDd1O9tCQUF6Miea4CXSutx8dxZ94H2a+4a6QIopXU/WgjQPabBmra9I8DeSb6BhUEYQt wbguoBm4SV4r577kwJLoznkb/9m3NNm3DWIT2Ae7nZPZfAvtmw5sm0sOR3x3ospFsrfO2Bk5l/Bt s8+3BtAwf0MICPDv4yfQMEeNvTbVNHqxZh31evCmVkp3tCA2e/g722b5X3PxyAkfJ4E2Fz7cxAY9 KCw+hqr65h75sRkxShr6k4R5R7e0cIjuD6uugRI8e9rSvzaNdJHW/2Nu5MLyBdbGTkCaxzjXTpr7 KFN5bfzkOcbeI58W16NvoCxR+fnQ7fT5a/FZB3yVdPdLZNZ4vrwjzaLKs9tEAEkRwOE0GrPuzg75 qXfRUy/mWuHpRz/G14yZtjR+q4nZCg3i6vneOAzsCVXpPLPQljHfTEoKPhinpBGYJFaHivYNzLPj r+NszcWXFK2XkbyioiQb2v14ZE9kp9R3rjzp1K7vZtYWm57unVX8gnrqAcFRaMxEZUPhY0E5lSDr lED0qfWihBr60IYmrrefQ5A9vIKWbL9aQggXN1t48nRi9lLuERHEG5yMFgmTDCMbVyyzGBtQvJdA a7B/0D7+yUahTzwtcGky+tJqxZuq54tkheN3Yh0vXrWOKgpcQBLkzgPbKd+ulVNAcgyA7CklhzWe kHLaswJHvbxUrahDB3mz33uNxQs0KA2nTlczq2QqgNX0UCRd8TOsW57GsrkIH76s9PXyRG58xh+M xfNlolZ8gtAL3Bu826QfJHeqpMck01klUJf5k41RpQvptcuuKhk3++lnG3N4xrL6Hn2M35FFt4XO m1kq/QzUNKrknvaY84Xo6mz6Pj8VU+maES9+2lE+6n6YJxvm1ITbH2hT1Iwl6xqJp4o2KVU/pjPN GT9pYmZwozTdrsJ1K6jyBvXNXe7dVRJxczXdxUNh2H1qpGChL17pSNE9hjJTvgyuTf6al/+WQ2eT B+dSu6SdR+0MwcrH0oLs+D828YnUARcGa1tERdUVilp/cKMVHVRMoxQ7gzi1j9qFhbVRinEmdrC4 QAl8aoZSO1g5Tcy6upIfpbs2oGXaPSRv8+VMxmSmFUbH0yZbKyehWJw3uso2oQ5BpUIZKz4lOefI 5rX7h0sJ37YgrpagGro8CnD3XRWx0G4cL4HQT4kh7ZnySSRMODpAktUTxdi2Dt1HlRmwHNpsWU2v Tdhie5vdX6qH1JWLN84mouelJDFiXSeYDalmr/GA55I7RFikhK9QSyRZzMpZNh/55dlQek3WjDLJ lbe8050AJMiffjUYlHJjqaoe+Bc/xi6ja46tRvycKqAa8sewdw9vTa4N3nMrxAbszqWrcnLsdBpx HzSf5ZoR5ULinBhxJuywqBn9LVdSx1ZeUPc+TrAtKvzhtOWsWLoLSkDbacplz3zWCweT7q2LfoG0 sU6E5WON1AtjhdVSFHfdmU0NfgufwZS6qgxErF90B3l9S7PVTaQ/mFycF7QNdD3HeU0ZyEVCRdeI BLh3ZP4cR34oZ9cGOsafdmYma6+HTAaaYs+iAUtkPTRvzOu2rHb80e/d625UeRLU27jiDkErObZI v87sB17bsKXl7dQIgjU06B8ZftJWz3zdmQhmA3JrKMxOA7twGQLoEe43u401UQHYzcfmS3cZamF/ KQD8sfdjU5JgyI9N9rvtVpkjlFTipgV21YWwmzwVIgwYzeKUvE9YqrNIkijqC7nqhJkGVW9EkGAg 0GDOziWzt7RPmP5nenGk7eySh8JfimKh1OXhjZ3yNw4XR/4WhaOvxMTt9GJqgSo4LYPJ9iTIBeD5 +55kstU8LnufoHjo/dSfCIgBdg9iQr4B6252U/VqJLWXovac6cHPv4SPSmrE/PQBz89szbKzp9Cq T0QGjZuAB0EmFNtgz/3ynsi7k+jvnoBoc9VtDUOYmYF7UIsMoimokGCe3q3Y9L7G5uSSD+sSl58G 9PBdcGVUYhmxFZq+L285SNPgcsPXrlyrvTcnT07aboO8cUqlpPLT7O3wXBt5sQOJFRwAlQO0jr+A EEPNTDxSC54ZsA6UQId3yvzmH1xk/DWOiTHE+n4ql0z1FJaglFJBmnRshdeiXzPT+IrP8SRYPin9 EZElZ+3CxT+ZI6AQmhgB/LsGZvC22RGK/8todqfbFJp1OE01G2+X/26/fj6Lk0AEr04bSz0f+QOV rSwCpwlQOyruW/ZVPbTukqcqQs8AjOEvzL36cjN5HECAVMgwQIlf3L/A1rSfpNtss7oLYkF8MAe2 J+rARqYr1H0PgN+QzHoavCCEg42z4YYmPUazE9LtjfHatmQZwIahV/uOwWm1R9nt+zna/1NrhTu/ 3nawu0gneBv0YxV58y3+LfmtR3MXm7hIN/ns12Exh+rwKb41F4unxI3SHA6kX2vDtEo7dEdmbxmf V7K9isK2g2lSh3iD1TPDu7Xr0Lp2Wanfn9iguspf1pZE0I1XI+EhYRqy4Mosrlty744S4hFZ1gH4 U/qNdVMMn/RIu/SrfWKTNRKk6/QdyY86mo873FIx5i+T26lJHYsbgYTY9BVWQGazZTnN0V4O6S/u AR39Cx/nTVJ6RvIvW+LfAUMx4nO0nfNz7iQmmVumcBTNpppWGYDJBbuCwzOKl0Us15H164xwfsQV 49UlplulPQ/BKD25qNPYOkt0lJigdkekTEcGz+N9nIVNzwl8A8wE7wYgSwsDtBg1ccHP1FxESPfa EabnWjvXc6sgl2gImGitPIxoCaV5MXrIFG/YZ8FB+AKG3aZchby6HFZZmI28BAV6QfRXMI0qpql9 miPrTetsgebjf21v49DmAVJuUzsBj7ar3Gnfn3DBjE9GB5TImy+lxlhpECpEUuRQ632+9Og640Iv u4ZrdLUO7kBfLF/zQzyrE1PhIiHBpMey9sHHOLlh+OPpzJP0vwzMPsbSt6VCuZ4Twde0q0rgJWYh 2l7Xwza1eGp10xuZjl6txKuVfRjlinGFOPD95z1auqOMPp/A2PU9KtvHSrisDUi0IU6Rp2vvRypg TtJ+dKe+Zjd35Y05rriz6TfZYhSjLQ7b4gaRr0EkMpoqIH+2u3yNYgVf0BGQVUcSfGW+bzDW8lWW 5zHi8KrnWOs9GnT1m2FrevXNKmUWRsmwCi5XTTUNTnbJQd4KY+Axkq+5WKec1LyHHeIMpFfr5gGm WTdwFAsLdeDM431UTIvrCcBdiT3hzb1r9mKym8pC07lWzymq0+7wr3MD2PhEr3wDDSgQQv+2fPH3 QIoY/f/Hmb6LijidOb7Pst7PawANei7rCgf58AVf8F9MB9Y9/iGL8GR0YUG4tB19XLJ6ltBMXtVu T4JQzCiaFfVsHdcpu6LEFa78ZGYFf0YTKb+FjS8zPOImfA7UmNwEGYtaRrwfvgDmGNj2sklmA4K9 nH6Z0d/QxnRYLXU4MSj9DWoQjy4KDQ3rMLERMww/ncuMCEgD03cG+CMdqtz16tOVYrYo5WGvsSEa EYhyfX9WQV1m3L9xrvDBzfBpI+ESBwRjGezyzOHND1JaSmZCxmb019tFTI+zk8qyoHRl0KlgKTmA KFTLTe7bxNNVcwEC61LqneEK9+Jk5ng8TSKGIFJpkq2rzF2bs9W7R3yURDuVkD0bgLAKuSubba/z OM3k8Fze+Ik+z/dYvcqucWiDkjIAoM3J9yysF3ywwUvULtM3wdr5qreXnNfUtCWtwGONqYdA0LJV Z+ZpiCv9YRolA+H4ggR+30t+4BF/nN6q6kTZaKgtQNlpjb8raMYBFSWnMr0SUJLsZ/QGUo0n+N+8 4KGajT6fqKfckd+je/n1oIHuTqNDWuVc1/+xE5Gk3rpHyEiB3MFz7KzPWBq9lFa4WZ5y7d0iMwER E+H36ZjNtS1T7uZgh/s+MKWfnEP1pZHlXb3zswumeL2jaPzvHNAPPvuq6UCBzPtIgCA2RFNt0eej eVU6xsiWxMniiGa6YSN/ob62qVfJSeFS8PZ+/3qgqAXuktqbp+HxfUDzeiiRC86FkLI4JO6WNls1 msnAZ57Flvgj5mXXwdFSM5bxnsn3UVdG1sptuCB8AzzEoXji4sIe0cdVSDZCZUF9JQMFK+VGSSXs 7mHGu47om6p3npZiOJrgtPiNsF4ux56HHI7ToutSzktOH9ZywB8tZ3C6WRbEr+KRAtm1yVGNYIFY UHiNCrKfGWaEUfOp4JJ6BHa5tYlEsI/War9XNG4W+UA8fYqOXFh6LOQq9xY6j4zugGf2nYsQsJdp NMWh/8WsZSzHSl5NhU8q1Vcp6UTHtPZgbcSopUIrQWlMmXBkMSiqfcQphdKaVxBDU9+UQwUEDyM2 Hoq9vMR5mBCyEmcwTn0cSILSo+kk8uXGwPEElZSKPlWlVvnFD/E9FAloB5nktJ/t3uREzEklREfE 2MQR8Ph96Iq+CC/i5vVXOEU/FxlwBhjQnr6PHvobXPHoEYMfW/GNxPi58XM2HrjuYVz1UeypEKXv 4n9zC9eioajzwNo2nrbZDIS1YYiM1VfeXfeLhrWPrizqjXh40qQCxemRnBlJKWrKtyh3ByUNPAAD 0IpT6YNmgIdFlruKDgig6XmniSZ/A8aI1tnmagwvEh3+pNVC7TEwp3LNG3KErayvFz0seeoz3rEw aOmoKdIGD2YoN26nIo5Io0aPyk1L5aJ/Mz82bM9jQPBUwz5qzzVcw5G2uxIW+7zSaRsrAXGC03MT biZIZJqJYsXPowBwOqysKPZ7RFCd9ZsejJowPBUI1bweoQfABUWP3Va1hbLpMDtpC0usGS2RVutF rFuqOJ0msmmKI7jymg3Gl6+Jh0WYLMLYPB5YdC9XpFcBo26M/8u4WI1Lp5VKJp6XDqDCbw/6Ix18 0Hw0jGwFlJuhQThVHER4a8xHezko+NZl1cl1uzkB2d5pNRpy/NRAOac4mWdtD8qkHsfXhX/Hb+w6 SReQvxaFeUSMc/CIp2UzgTmlKxMAXq5dVbJXpH5JTkpfyruBiqDzljIREAQ29zX6fYfLJDG4wDkd PdkIMd5f7AuictCIySHofwroO39gBEDKwg/n13S4AJKD7pSS77/ThDjInROTNfTmAUNUmfhVx4uA tohI6aU2dVguLhn0n69KcaILlVVYy3KRHB8gfaVlQK01X9qIlRlxZIxehz/VEz3XaBL/jmUb70Sf +OmOfKoym4VWDZ5rLDEoayH9lTQYHsxdxvqfR3kP/+lPK0448KIgtpSZYE08MHzoeIoyPOzWay+5 5Q6YFpWP4KISCath6N002iGHVjASSgk7983owF9JYsMQFi/L+v811W+jXFY9CIm1ytRN0/P3WtI7 w1q6NYXR9sLCx+6BnbsUl///ZucstK0e7ekOoVNWFp4oiok0XDlWO90wqC4L95WayZPvgmWCPufN K+g+BC7Vc8pjrnWcvZCbEo2RbzJ+UzsNFZiu/D4/CbF/mEFsRbAAezyQT00TsKNoDHvedeCoGIjM H6Dax6bIrkAvo0o48gYllyKS+hEssbomtcwaJazGOwfuWQLByHZOzHVahpiBPevIHLDbfkCL5zQ1 rN0DTCycOvkOiSXvQ8JGt5W/kSRGqORdd2pcY2ninuCDcLyz/JZaoFuWaApFGytdX/Eim25jRoF9 dJ+BiP4Q1AhgJz3KZsK3JrwuUNHj4If/LsSQDygb50p7oK2GP3F1c6dLeHQfQe8p1qdAjdJJF/fG Wfoprh0Y4Q4vjnGQzkQrIwyVay7L7E+90mNHSOWxJWRWKXPRdleZo92R9m6wgo87TS4Fm0fyu6WO Qmf1kLUp4FLeGpkPSlbpoOKMyGqsUkByyvN+IKp528Sd6TJhb4BIVYUmnFaskRkTEEBaPO/CqL5S iItO1EJqJLGaa6CHQqpZNDSZ5f/2Tx6px5bTLhVjsxIZEgyrd/WilJmMqNYLorBSGJ08XF7xzzKl QlvoYWr1l4lnPIqSHlKVtqekoMfYjDvxASbzkRytWpWsMGUKpylHRE6mGAmzsav4SgV+iKWdyvNs udj609uRxbyv5dzLF2C/0X+YVSX7/IfeKP970CTy39weXCZUsJe2X5LHZJ5PP7RxXMLMCcgyNVIa l+t+4rQtZEI34gPCKL+q0KyP245svVdnaPa8/QLk1GJe4O+F+EcQQRlUImjKODyL+cCQDYvZsgl8 PdVSiFzmfhyK6VQnEG0pg+2twmDu7r+sQKfhsrrgthOVMxXlz3ormYaxu6RNx71vldE47ymkunKV dBtHn1yW9zpSfef9LmH3bu7CVzUig+l1JH8/UrylJPMLSfQDIw9Oaa/5eeIheUWX+vbLtoVWhNET a3vP4lSOaUfZTatYmqY4J7WKNfGdG2bT4attbRIlSGX2+lPXj2A6S7l/mFxOIOx6R/6SgY59G48G NSCz7e8aBAg6VWsjo2zrDcdbf9FRoP298d57AGyZSWGkfXdyup1F9J9MPF6fihzxAGNfXu9OcnUf DMKeDiiUFAWvzTroaKw1jIkVxy7SCo0Beibr642GB/we+ktRvwmIJcOj6jXovjWslprKGYvzwYT8 1H61RLD1aJTsG2n8P7aAG3yC/medLLXYwCUGeZnuLfyP+AUC+K009vZBWU+z3RDWLJ5L367xGK/x wHTH5y3D3mUtVy6ItqalBWPgxUpth2m853uvUvon5aH+3hSwGkK1SoN0jJCFFopz4ipilZUOFlMG 2NrIxJ8VyI712DHPP+z34e3CUGj4zksXx9LFiWLIPPZeGodH0T0JOvLo5J8I+FvRPzQc9Gsrlpii UNiX/zhcoruwO3lz2WpC2fLouWiv60kGIBUhCk5y1SmPsECh4qm8CR7hCLCoJmwntXnCmXu/poWd 2o97LEEsOTxbXB2rvCLIj6B0BHQbp1Mb1nir2b5FUIjd3OabY+Ht0bLB33C9BMT3Ca7Uwd0tpyHw uvpz1Xc5VBus/ebfRHusnZCrADIswmkqKIOyK2Nlt+5+oKfw4w3tFW+/8SFvclGyYr8t6131/oQ+ czZkv21xOtB37m01iIhSVAcCrRUhsrNr2ohaT1gKQodqjbDRV5ro7knqJ5+Ap2z6kIWuDBGxCiCS hoaK6O+BVfFsABaWG1gLZi+WyCP2UVVXqtGx7zfQRzUs7Uah0r/SrLGdp1U0m2oeWz2FNQ4v2I1p XjfvuqXfDro5d1OddaRjrKYZKmygA2L0/0i2WTGIvF9dDsFRCESr6AGuGghRVEKPKqQ3th52XwEB 0NJSsVMcGizaanEbW7tcdMr4N6lME78JfShduafN6rmWfojNRGlLxl7jqo0nad+L4PvorcDx1N2B kg5QH13dhJfKexsuHx021SKueQ9GIP/T+SNvJAPC/OuZpMg9vDLMPtctqLzSDisnR4p23L1HCfqr iu8LQyhVDdsSpbz3AeFovWfeKu+tSQCe6JFHrteJFZetMd9jXaA5RSJjfjFNchHji1NQGYRi/oto uOopVASbw9YrEu+isziWm1gpa29bED4I4N9Hh8pa/bOaGPnj/IX6bt83CNu1XP07gy/2PDrn4SjF GmLi9d4cvjQZy1Zj3/dZP1x12qTfMnGZe7OHB5i4zocxX0ZOCn5p4OPYscoSAx63ZcqflbMN8pQG 5oXJn8oHMbBzmIfLXuo36uNJJ9vKMPfM7p87JQbhp1s9D7MzKSwyUSUHlDpOBpQHZbJIZK/TP6US uK+0mfxBwdB5s5Qq4ELt3i4X//lbj8pjhJXofm4dv5+k9wKFrOBDMpllhthjcHLWUun/Ud8VV4rm act9mfYH8WgjPYeTUqg+czz0rTK6TXpj7ZJfMx9uW2DepHC3nG1+WSEE8xniBaskjxCar0UujD+b lNQiZws1JK2eNGDGkeY6brgavUh41K6E7QqYebcYqKm44kYhtewU57eqyl3t5w/K5kz7hds/OfYL DmKnbPCcl9i73Dyrp4uNr+0ADf25ZQnQWfn9Go6zqFge4hnon1dvGulaRkeztq5Gt6KNOryMUPGW QaFt9KEIgVfRvZfTuSfRYxDL/irbvHEw+vealPIICFyHVnU+SCwFXSAc+/VyrK0irTYcFwRmAT7q FkzrijgIa59j60GxXe/lavi/G8Uf5PjMVEbJXD9MNjroY2/n386Rg+kJGGWWCghpvRI2lal4UYgr SBkuR/rcs0yGeOezDwvWMJ+MIFBReZYGjPjIWmaB81lgXS7Z/xVkT9eytoaLGcTx03me+RYITzDi HW2W+OeVSeeRUbl3L+N+qHrCZtf7JLRuLc53UItcynjK+P3osy7wsckA2mi+xy1Rr66WPgy41x6W wutOmib19OfvsRv1IO6BlmKWqSEVldmEo93G0URfBbEKPSz5J9qg5pd6KuBjtwSN0JJKYNsIFU+A 1mx9lOhKh8vSrqs9ieLYhzpp/9REESOJo7AzSrtfljXftd/KwNGnCrN+spTTkkZFDS36I7Dy3Vwf wEo6BsMSgxnte3aLbwNqI5CYgMVOT2rX64DMM86hb8Og5xeYmuMldt8krMCYNjpX9xTWgymxKtd5 CTtpVCGG4er+EAWOPROr22RBw8Hr8mR7pwLJnXgligmC7ohBBBOliG3QLKxLaAwNznYLJMJ8ycqy WfRrjr8SHAro3fXmUYNlankbYuyH6v8QiqVlxyzP7CRNEisVAbAOcNXjRysQN+yMvqO+0pFhsrYK Hc8mgm2dMAgM/gX3zPJXzIy6ZZA3dt4aITzoZFfmgnD1UdbKimRypfSWWqISrU1HBL4SUM3bhfUp qtLZKWv0caqvxSA6n9pW7wkHK6XUuWCbR7SdxrPsGORjLDJgKtPKqriQNdSYqrl3rG3cXBHh4V0S uNf+Sl4WbmuJLqgpwtSS+kjYE8+56KoDKwDFSMuL70yHclFLOLUEm+RIbWe8KVn0dHfXlf9OmFzs LBIhm3prbF2dtyPoMuyyZyYPEeSsuw0y6t0npqJ0cMyPmJlbZxdfNNdmQTQQZilaTRQdIEneVe6/ pPUUkmw53FqHlKArsHADbPrwEtburIhz02qNEMb7amr8Jm6AT2z16rj/CHkzLbjAmV9UuxATGZ+G 9t6e/8sbvlBua1YPqDQM4pLzBqXl8N49/ozr1wdCTDv2/noGnAoW7OL/BydvmDow4+9xsGHccA4G omad137ek6jL6j8XmBzyquoixaYrGdePqZOKKMc9I/Wjm2E6j+yyioiBLpAQnkcmTs9Qq3ZhXt3/ NWCRjBRkfqXTX2iqWE1bw9KxI+c7GNQkiHkJLcBQjsryYaJMr3/833URugZsGDmG2PMRqh/sEURb q6Z64BLHgb6ffjn5HuhgoWxPNvdZ6SSddRmbrJL52wZts9OpPfgIUfKfr9pTJlwPaABBsJX2Sbkl DUsjD2R2ix52hJLatkjTUrUsgYUzqS9ML6QMhrxBi8ANB4CjNUECe5DPXcGSLv4Fw6diaeBPukc8 Y0elLFcx6Wc/67ExI/yxn6ZRcunTb8JaLkR3tn9Sr+i5objwzhS3dBIP3AQkSpEDXNpaoOwdmUyq 1Mzw2muBJddyc8vr8vHvBNVGRxWK0uh69SyDVKBIzIJnXjuR6Dt+LU9qZaZkCh3IpFInq9lmn0s2 CqSwScrecYNP+kjVL0Qc6VEjsTPBqNRoLE/1tNZdne6upgzN+7YklmcvyA1ImRHwkMmzSvFYhc0q zHGnsJywTEJGZGT54XSKUVWCy6qmziMT03IuiRZCVbR+TJq2DW05ohBoCVGVz4RzO+rPtGgmaOlZ UoT2qhNRZEoJQUz4ysasTpC0zgjXgnk9weU9r3m2PyubKLx7i+aHKQ9KQdZ5gcWHfuQkhvPcYjHA 50fN3MdLtUojYFbCZqx80qhcXzfnyr2m4rcCetvHaTkh5AOm8dKnuim+AS7ZvSeokrXoQscq13uR hrTKFwh1V8zBpeyaXDdYaynCMeRWeBljsy/0YgKC1x+ABbb/Ri/Wf9B6zsX2LgV+WpaL8rlESptL CYXLE027XoZcwYwLT/16AZJ9txj85dv3uLPUlD3uBfFgJ90hGJgW1m3iwfF6PIH/dpsmj9ldFmVB OWhFW8wmNnoP1xLAHhjJ9sDAsvZV3FAvlzz/VQRKpPnk9fj28VI855H/zO39TB7JeBlOV3/66xtf yqe0+tmxJcnQzdTT8q8KfwWb8VzViALQeFwrabu0YIb5lqijwGpSwuxgSlCnEOFiB0tr1nVeLJJX la/QMQTQi2hJ1wX9HCjlYoWSuEmjI5NGCgn6X4djucLRLaGAraIif/yMIMan3NvN4f2ybTMaAfzK FUqrs/H3X0T+bmE+3wst7/LRe7w88G9dNITCxHgbS1hgidPo3MF9rVJ5Hl6qm+7zhzMQ4VYhdfOw I20g8DEpx7Z0QD8BVFuyQHhmZ/9plnrQQCGrpn7vdF2dUpj65Ztmx/KrlA4HG1jKueP6W/nE+b/+ 8ddk9lJMZCT+yiHStSeOrj9z0vKtXtWVtTTm2h/6qJz9yOXVV4RskAmgUo84WH2rvcWuT6PRp1gc keDLJ7RMID+pvEava5vYULQDsGtXa2bMsHKlhPyqxycLVbdbdcpgwrgNpRxn/z83lilMOjQr7KnU eGdPZaOWYA8OPboiTO7t1nL6wf5ennIQf5g7AnIEiNZcdXqv6/gF81+qKlSoX6PIBQJQgb7pQCZR 8Nw5snEmdWCK1oXFIAZxUjF0fHnr1MdlSnxe4d1qxNnhWcaa/bxQaGSlxXrVFtnpqMF3gChZvz+Y UPMgcAbF3ugPppH8rJ3fWl9zjZ1aM0AXCCpsA6xBxfQBnPGpGZtk38orMhDjkdgTE80pvleo5lWh wGoJlwwF+9k9i2aLYXEja+04Tng1r9OZN4AoHrzHF6oXxAMKqvE45FQsUX3GyV9+yb+wke/SzzIp KrXwX+QVJBSY7j534sVIaJYiXe/OfBoFigc6/XVvmitmb+iYQEOlx4sepMY3+aH3F8dZVPPeqxBX TPkzO5JGEokzE2cFuYZXfN/sAVAtQz4y7V75lczND2bLqy8jlGdb88VDvs9t9IXX5a2YWBKayP2w OMeYa0jLmsMEZLNydWFKTGJUjzWwXiRxn6r79RNHz5pfDYn9ucZ5NPAB9ShRSh3xTeOja3EbGAko 9T9d3tFuIo6E4sGJIlGQyNjaavwYA0vieVS/9to4K4R8rsr7SHsQS2+kJMNkZiMRJpN8fG+BN0br foclVqMljLf91J00iPUnbGX9pirzO4v5iyAsjK140AKXSYoFwiu1glLqyxDpzN96iLMCgM3kodiI 53dVSU23kyE5+7HUFB/f8IHoKoFyg7tvx68sJ17F3rrpaKmZVCYgZhhfHNSvjkgDg28GcDcBjUgL nNT5kOFF33F6lxaFHZuhxEeaYWd1AmbrVtPhOktWDBgR4AW8bWGZQkqdOMoocXSLjaIx0/lflfpI EpGhiOkAJfJmzPeKNbxUyUz6IjCuYnCa2oDA61zBgVEuHtoBJDHPZ7Gwq/OEH+fx/lFJYUZVOQlR KMXQc1bAYtbUXft6lvCG+JQ+ned0mvlXQRrVzZSrZ3JPEViGBGSYDqAaRZXjcjErD4NGwm5w5kll TRZlv2h+F/e+9zLB8QcYZUxmWJlfetr6bEQectsBFP1rvkx4oo7ROn0vhdqgxpERw5iR+ZpMqkZd 7wmQ6CAhd5LPLm3VMxLrJm3AqMEJlowK2sZD9jNyjXLJPbpVF/gjIuiT6RJs7qKvYIa7NdHrl0Ys S6XDVWM3yB0T3ZOWjwLLAYTPFdBp8j8auBiJzGKK1DqpILBZe7fX+pv3+1w8TyrXSPCq7dkECFcn 7YL4t3PEnRi1b8kKEIOkTFp6OfC/R10us+T6MGIrOnlEK9qwtV45LjqA3CH/zODagTlb5NUc/S9T n4DHDAEKqrikRYgsp2ij7VFUTEEZX/4rmmrufFBW+HgxzPu20xI1Oea0SC9Lfu8PQqogJ69Et9wz KSaGQfaTh+UsLJeroIVaDtot/Rmv4fN8S6wvhaRzkay6TM7+Ay2IHfU7PtUF0z7IoUlXbFoIQMFZ R3PpdjzPXn3EWu9Rue7lrOXd+1Jb2t+R5RL06golOJyTSWA/8dQpRsLNYElCaFFj3p7wlxRreGp2 G8fUCIyQYorVm1RJrn4WykWtfwFjKNNKSERaUb8N5S36en3wynUrwaDOW+ydut/GYQN8gBmxn6fH 1ZEp0Rdr/AwCHKzeceAHDyoQIFRUWJuP6lciSs0DrEq+QaU6nZVQBBl1zf38H2nZztvZmZS056FM 0q4h4cA5LNDyg/x9gqQPVHJWqf2NwskXw1lvb+t1/+6Te7+KzOGGH88YCmMGCxWHL49X8rM8I3kN /4Gxk9G0VMnrO8yNAFU7MHSY94p/kaRe4CukYuUzbKYOjYvJgdB3m++u4ONjzCnPu6hPSdqcZe9Q sB3dA0Aw4meIvccqAsgmNI8p9PvmsH/Ad+c41Ba5RkbOx3T6TK4uN4GQLkspEAiFLOh+n5CeQSGz 7Pe/wfpZ5WviuZHM6p4rGNCZFQd+rrd0uKTwQUvuELxFFB61GVS8RWQPo+i/U5KIinMKQJCs0qzC izHiJ4kPbHlRwYYYC143gS7QLJ+5a23H8pPR63jlbQa+9n6SPiVdRVrLNqY1L4D+QUFkRrBnOeQS Z73DSLwUjJSEPq4hv+rwi9aOeKv4KlryhCumueF27MHpzqXROiV+hrZLTqy+JdzFsFP6tXuClYKX rpEBcbGYK96WMpg7S8RtB6Dfy4Ul0g8zgfRoCzvhYQFyfLaRJvqiSFEcoGRbhExpiKgPY9nn+zv5 DlKUI3SAgy/7idop9fOEB4BYX950yaI8VUgW0G/j9FrJSTOIv50Y9QMUosTRlWLuSNPbxaUv88Kb njWRggNMG+V1Y4SjU+Nmqle3SnS7H0wtF8QdSl7IAVbh7+BVddzPZKAoqnG+u1FUUicmP0oOBIr9 UQwkOvchgqWyP523/SVZPcJq+5bFQl5phoAY+tWnUSSnpeoGw/69iAXfUff9s11buW+OzTL8Yvlg H2cN2v2tFqbHVFpko85Ya+2r/PqzM1nkQMg+NWqxkLN4mw7nCX15FzO7Rh0gHuDIoPr5W1byygVj lnpVscgDyF8cij4wacF75gYbB9s1MH5vd9zX+gip4hDcIGonNANVkbeNaLlmpenXek40KKWo3evf W3QXcJMc3MDaTnqnbLh6BrbSRu9hMFAiNwNsreW8m+ztupj3AO5PIv/Ze+EA/xpnr0Qr51T8tp4c ULhv9gnhPZudVHxkpsTv0HU4yEI5gnWFre6ydbcbMn5RkBNDh3Z5rqh+eh5lBwa+uiWgIe1hoVU6 rr/IBTuYOPY02Ny2ItgwX9UPuuNTGcOetHEnpetjD5lEH+VLMuM/vocBmx+s6Y12rOYrkpZsZRqE BjW+cSJM/x6S8FVVMRT0qghycbgyEIbEV8SFJj4Ebfm3vri7WnmtaDAjcaQlsKTOmX+K3xJeYTdf dF+JxD9sVb5wqHsnuFSI0nd5M4wX4o06Z++ofCKJpF53J9dpHnD28Zl7u7rQzJ6AlQ+89vQMyRPv E/OBz318gzPZkYPXHePUheUrIT4OfxrshXoqwjYrr+EzWt+LW4KxgVj3tRtBDKFW3Mzt1bEJBLdF 2sTORL1Chk4SQOxFrb1utNbNQXZ0yMJO8C0X8oB4w1m0L6r0rBNcS8oUG3iKVjrcIx/2GMSfE+n7 ge2Q1d/hoApeE87lthNpjA5ZwJ/xtnVdGlXVeqxPjXzPYNG4Qd4/t1MKSp9xVMMBOefeeXUnzt3a xisK4V76XmS40WBPNOVcEly4G+lUL4ZVvJ7yEtCjxgeOkU8tddn450u0TIjRVmWjVlBHK4FQANhZ gIaedHT0SD7AKOMLQ56pQtkfPeFYt3NSYyaWSqu9lw3JHoQHmvJAJtPIu/nSSBiK5u2zJaTjjJqq D9X/+EMzR7d76wB8eSwY+XiCiiYVrfamCVq/T+Ff8weZ2H7gqVH/TuKOa7iPXjOG8jVdQCXYUD3e +FyGeb3l8ujYGwW8pYLQ8wMbxF4MTVnqey6p43ICxWyn+dYIK66cyn+bz+fAiX8d6eQLAzPyMxz9 kUeeHAB3bZ3f+TJ5G6CxZgLY5rGciKW+bh+MLImJTdTYGfHoLpdU4otyzOS0WzLfuCvZdhpomBTd t9QEYDF8+fQNBOtPby6isxiRdPUYV5h0bYEdcWaDZuNSTA7DvrU4nWdRFUF9HXNSldQ3STUOj7cA mAO4hdlItBZUk2NveqPFhHGlZYfJWTQoWeBMrcUwM5jTKg95m83Ipt5DMCHa2YaG3erMl7P/5MhU T33FqLqKY/0g/0r2LHF6QOK3Kb3eP5kjJRVA+wx4/7l/cCDyjFtTMWBrCP9xON+JuX32Cndg3I9w pkUEjeFEKfLQX2DjPhOnVN9KC0AkOUh2kkpELaRJHJLY9RBrgcscbhogqFhVdr6vyUQxbHzcmGYA thNmrOZJ1jvh4vNd8ZPgKq72DVFoZFATU98sd5mwe65r3ww+qjFtl74qTmW3H04DEd4tm40pSxtW MspEzE1FXrQSrJUxUatgSN2koub4ujKZttp9iQXk1gLr+RqMBa5PwYMtCVVimqXzpBVFDJYvrVPb PW9cGV9wkqhkkGZF/VsLsmz41owrNoD2zJ+v9Zwr06+0Lt5O90LxDH+Uu/IeSFeHPYyGFs6yBlCw OSd1tLo/Hcyt+MPNYqZFSUXp591E1JfXgaHfXmV+YapmVzOWkKfBbynYfq9GIQRHg1v1Qs/OTafZ B5j0LcK6QU4R9keHO2mwTGl1JVZ3iYHtzk4ztYGqOp+AmDw/oxVpKDyGKFv28b2uWcZhKR+Hopxl ZyezWNcE3HIm8Bdmwbt+/k3kJ2er6VWLHhCPnLRfTZZRLQkZ4BFpSVr3sGc+poc4JiW3l0dYLBu2 POKwyw1NOdCCd3cf716h7zF1FmLrb/7eflxsBn1ZAtaGcnoSmahaYAnnlmp2VBXYW4y4m4dljETe CDjBVZJQannoee7LVI0ovl/fUu8h0CK0xhaDWatQE/P+Io6N1FW/0Sbo7/DdNW+wHQQNXXqOiQ0G Nv0FNYI8SeRJAkpSig8MnchrZ0T+8b9X//qN+bLfFgczWM7G1ZHAcahG5T2McnmcmXzBhKwlABjR vcORwAoWZIFobf9HKdq4SNzLle7v+3AIa+h0E+Wa9d2l6WHmADXlKoptaVGqGGcIplxVtSwzczYV zazXuJiJk//+JNQ3ev1hhIFOklhaSoseZb7VGdIO2AZ7Y9DMuUiG5eJkuTymd2uY3j/RusOijSTi F1Bjsn7HMX9CxWtENomkeAPPlkbFJ0V6HYRKPB0mTHxpW8XtE0UkEPEVgToK+eN+6/st2iNVoUCs a/8PaK5QFOjhY1kqzZmpEiYZAiFzTiL4PA1ZJOCei1FbHtnVz1nn5zhVUIcX1rTUgK/auRkZXAkN hnX3U2ScWLGbEpoza2eO6/6FS38tYm9SG93EJXDnYegVMoHkGavymLmwKWIX8rPWnN45W3Tz9e1g pJ0McRtNIspN3PgDHzPQx2DJBzbLx4eH+FMPKuiH1IxoqVpVWLlTvTKthz4BUQgkd2b31ntmbQtf nuVmbC6yFfSOQUxRrQ2y94yRysMKZoPwZc4acJgsKKoDypB4JHf6v4tH2/bekgDAF2GdZJISoKzo w2TDmINgxzx6GVxxvGQeiDLpnHNQdI7YWNni0I/9jHh8zhyXGDOIP5HORGcUGuTIqWyi+hX4oV9A jwDpsEinRDvktv7Mo1waE0Ha2if4k3pK+1JHI94HJkv2YN/b/UuBkit2pI0COGh9VV1TBHP+e3RE U97VAJSnyryQ7mQfkwDtIllHhKT2pD7vp2Ud/qK3tIGQR/aRf62urJcV5WtX6qx6SG0yJ4WYqC9N yhUUXbaga0/3F/vEeqBlhz/21yvV85kIi0yu0QntyRWzaYtDqZ8MSU7UyM5KTo0QLIA+jJ9kwQiZ JH/hucbptGLx8gyA1Qk7uCjqlzYsJraexhYlqmDZT/GzeLwKi0BP6hEH+QX5w09Ixfy+x71lT0I/ msvgnQj0DWdPFarLqWmpMqGGHrS9NwJX8o6DD88KxDTFzqRUJis8ykjGCmjPIPHv9Fk7xh3pCrcw JXro8rRA23Z00qSibsYc6lVpAY9eu8I4wJY2vfl7B/WUTZD2q98exmX8qk7ZMbapKufnCbifP2Jr URLyQt4dtIEwd4pvs7w/dj1mrQekcq4KeepQP9G8NOREbUUzcTUcjgOX0jxojoftUp7w9eo88j9k m6eLIbJqW/4Ue+ECY6SH15N1Rm3+1nH+FnfeYnTESNh2/84id/RTzK51MER0MGTcbINoPMaqV03F 71kUAovImoQ8PzWgFVsGMHo1HJ5oMWB3aen/CQssCJ0rh2zbJeMDbfoXljJQ1lLT1vOq6l2V6M/P 2vz627gwdkaQuhDYUuhvpsPnn3tk+bP+fYchHE8uhCroXpQHEv6M6TkAEI0eOgiKrQt+zagdSM/F /9sde3k9rmGExs8lJ+55aHHzNs+dXPzkf2H7TN26hpMbVwDY8FcRHI4kUWaeAMr3Sa7dh0J4TAKM +ufWMk3FLMiito7PGySfWEJ0RefIdGtPTT4T/m0zldZRi3j/gPxlw8gICVgM7iOqkzYrreIwgM63 9UYhQOs0uTEWT8vaQVdxoKedmi01CnP0N1wbSb90rHdmsIok1lun1soMKeMYeYzS0i345tg3gvPA hO1ZvTwHq7QnszRnAltytwcI+1hHw+6J0yMw4yzSiCJXS8KuYMW7rrfvNQhnL28UpPXw0Fyay3Mc +yPKe9O6yRnszWWcV2xPfSU6sqKF3T1nuePCTbmojHjT+y1g3JQNvvBAjr4qtJ/G2Cp/fMFjEqD4 x6ffbjq6Pnlypb1H29OB+f/dPFX4ZqMXrHJq6tYoDUxz/nE+yD4bVsjXcU489g3SxoRDfDWvYpWN yxdhRDnEYtsvQvzmnWSZi94E23dwg5WuvXvgrH9ER1vGew3QXrJ1X1HW0ge3M5Zu0rOUH4S5HOzv LiheDv7BMPIEDsG6Z1UzegwHanR+2puzjbvfsK6XTn4vl7vtjISkfHyW/m3ayPqD6rOKY5qZFkgG TppjC+jkmujflt+ReB0px0PDtZEavwhWl0VuGKSFblqcNVZcmrnUw1A2Ms+y/0aRagU04jrASah8 3xbFOcjMQ7BQIkUxPILhm9kLo/BcDWiFKcupfKnNFv83HvRLbm6jwxHVzOY6Pf6Zqt8V5yBd1bSe jk2UQ4OTOcSBIXkbqS1Ye5jy4kyNsVuW6e4trKPHKZC+ERbM/oM7gxGlfO7YyjaAoGDsXQ5bIHq2 lo3RkTwEXJPLwyvez1mjUemN1t2CrGC6TqUSWpKgak+HpjYxfLsDAAnX+MpZ68omqEPZrEbJp/4e l6In8ONdqecyc+lCmrpdUr+aKZbk5LDWyHjOQMw1mrm9kwmIxSSRgus1RLUSR/YGXzrPrdbVRsEq KNW+ojmNH8bMUUtvHVaQZgobIdwDlc56cphm1SYSkbPMAwlMdiu90r79EKYSixRG+CNbFNjE1VbO sRk7R7omgVVUesYv64mqiFaMWLWD9fga5knvAF6vpYd4wjuE9ewjATrp46mv1pRgS5tujlyXb8zZ SJC7iAxBGBqKECDqxTe/3prHYnmTHwGqtK0BtEkgbJCzQEpUfwmn1Ij3JVwN5+ahDb1jHMUCXh+w 0iNofVZIAOmhrQxyvBmqjWBodsQf1DG/lFX/6YLBM5TWzz35imNS6ylldUcO9ACY5FqKL7G1zJtv SohLycySyRDCFbslBt4DIBvLsDL4KCq/iYzg1vwKWfTpUhcF1QmxnAFr5zJll9yiGRW0NzFS8rqi fQP6uU1J49PrxAex5AkkkjgIekYeaZQdY2Z4VcooEFAuatl8oay2pKYKDFA3tCA5nGqrKPTyfITz /DwgSeeWgg+iRkn6RFYed59DwgZ2ZkCX9Tj3RIaWKQRcM0h6qRTYyQqpjrUv4FbnZ6R9G1AJLuKe tNkuLnU5CisQtkZj+4vPLAUXfbQvsnzqA2xOWajio2ZO9rlMzhZiRh9s3V8r7QTK4lHid+r3w5Gk +HDpBv5ye3yF5/XJlUBweOtmfq5T1yUemdX8QNSUMyoZ1KHWDsOwuBi1695TX9RkynmKTdm+cLC7 K8xWQkLRbz4yufpWfqC3PmlBbGH8QTJDSIcLubSvjDoOcfqafeV5OGCHB9mt2WTTBXPStFXiXk0c IAajAcbdlfDZ97EQzpSVWYZRSBuRHwdrCMAhq5A5XgZ1MkjPxO4AWY3vjSYCrQe6w/ywT8LEzOCb HS+OPeldZRXABKDZZqKwh0awSpsgYMuaZY/mk1lpIqcrR5l3eGfwEsgX9x8hTLzLJLrlEg/KXAGF 2dhVwDpgPLYosr2ljzGkNfeDT5KwzhI4ReLrUItfnsKo6YxyNUK8AGhKhDMCgyhC0sM/RO0jr9WT Pgl1FhTIidvYvE7EX0z6DnU/uvcjJqyh/znPGc2BgDyYcV7FrW7SLFiFwcGsKduAmt5bydCS7zwl 3UCTc4E6hb6LaQygyWO5x35DWFTObDl43txw6zIFMNoJopFDPbdcKuzeagzyIqtoZTj31oAU+QrZ hgzHu+pKxWY1y0nK19EMEnCpeq440aXNGJlu8H62Uv4+QqpQGJ18zSpVCbRd0WA4WBUmbRLB4xgh MawRBQi+Q6HyonEZggntMIbY8SJNc/kMbvKNpjR1nVAs06iuD6Ilr/766jPc/TRobbZGM2jEiHgm BTDHiBCZGyYhdGc0YnzAPfUy6tymtTNHYXIbYsm3bXrgd41/XfAQb6+YMXYB16xM5wfryTGAiD30 8M+FIQBNfzB/mXLhwRWkiDQHwK6INE0kqtyZDP0JpFB+vhIMTCJIwNvkIKNPPy+ZrCo2pfAFWytq pvg1QfD5rr1Wl1JtMXutFAiI7SgD9PVTDd9NFDnT/MmMZk+7KPxqQXgxVAqsaRkGX/EQfptuNjtT eycZx1VsoYOjzpetrNPf9wMIIYHpCTVV9fdeQHMhhKjTvyEomSBfIyj1qKdbx878PnICLlIACzc6 xUpptrxz2lt1znnmOp66udpSLCj30a3xlOb868svAr5B8TEgb/F65qSGubSqDAXgyoKmw2LKYmqJ wASgo8PZ93L90nxh21HQLHiFM/fKutvksGNL1bvZkLHy5uN/vR9TrG9hEPPylO4J80V2+CMuEAbi OOwkeDxdJua0H4P8O36vZ/GIJxPXfPPU6vBf0APA3OvOvRTN5ErjWIr6ofX1px8n+2sl5Q/9h2H/ XqfEK+XytC6ywFJ4qppzTabAEERGB2MFhqWWgu/A6ABt0ZjlgNyBSu/S+8lTXpUdovsvfeCrsLZr 7v1OzB816nmgmp7+PAQMkZMNc2rgjRjwPdP6KbGXokNG/5GXZLoy9bPEpgoLVp41GvDTNz9JHZmR MSQcUwAsKIhkJD44r9kfFl2WuUTpkfjYQLS5K+x68iRuq6aHfsKJZnBHd/VHPGPCh45v2I/4rPwc sZzO9QM7uCxGqGgR/RriYC0iJwxR1FR0h9lK1gFQ+4oiEmDhHc4v62tdPBpfS4SPiqpXT+0XzAZK jLG+cwWYSlUX8S+mM7T5BrMfqR9B5Xop4AL0hx4BWiD3z/2CgCk2hMOghpRg4vgH9loa7FFvW/p5 jiGM3bNh2+YEdiIG6nEI12acxFn8YwWw770Bs5xbuLKjD6v2VUwXOUSNLN5AQyRDpxnjQ36u74bi lkHQ2TcA83i2YlhCEqVUgT5QRVMwLU/JOUhMPs0A3W1KNr9SYr3HYbxaoBMVk2/d17zJn33MVOP2 3zdEDjijg8AwJN2h1d78RI5UKfgGLBNP8xcpESfSncr4uAlSEdEM5V7B8GPfr0k3DCEXoORFIQaf 5J9lP5RYKf+NSaQUBKZ3M3qGt3dHa31u3xVUfztNlXJK6JS7R5C6sKwQN9Q/TWYJYYIPC7fYhvCF DHemoNDvtTegDIi1OOLP53WZGzXmBxF4KBFJVNTXyDm7ga3y32erIiY1lga0ZBRZMrmRk/DcGIbE YrYN2ROCAntzXmxRuol52rAIoX+u15kJFfrHRuY8Q30h9C0RpLVI+roW3qgLkQaOmxXdy5XugewG YgMAMK20kdC1hkpYMlDkKiwBdlTy3rDOA9b9udj0aXTQ3k0FbGvEx6DJ4+5hh8q+qxlDetUloEUw N4LlfCVF9xk0jSxpxc0J8bWBDwlmRn80JQz6pfs3+1QNolLTV6PzS8U7+KJ6HI4t7YjyQqHeO13u Qrkc6BLTEr6uyF/IQwbdKwni7k1A88tf0F99qf09loopN5s3m/rJd/C+2gA/OOd7B95Fvaqy5Pr+ bMZgPmASKRlLh2gDUUqz/Q/+H41Ad7wiSdbOoClou82naL9b7jH4dV5iBsl+S3zI5qD39A//7nHS DcftDbH4BjCcys02zL8VWoUBJ2ZRNy6irP81PcGibEqPG3ZYLWDpHE9Zcu0NCLzGmW5idH1y8tzT 7lASsm8W560yliOhQNsri2h4abQy/lq1wr927F0sLsDPKfgM1/r+DbNQNwutBA1mE1vBp/rglkxn qDIbyguxoED3Nsmuqb0XsU1G7cG0xzzonRsVGZST6ZDHhjqeSJL98RRHzVwWoaQl+xP2nfXo3YhS eQzukfMqORaZT3ME43CS9h9/IqWQmWspruELYqtYP82HVan562zisgAeBoqi+RRtDr1spVa7gXf0 ju87GPQRKs9NjUviU/GzOo64s5gELho721chVikdBp9ybIRsrvkg4GDtESBgp5K5ZFKgJa40qvH/ cmFX6U3+7dY7Ki0Wiu3OEm5f0br0GXUKXbq/fkSrv5SuBr8gDmthWMjv8c0PO2YHJ0MGXFcIOrXU 1hMjU4dfhNDXGoT1EJDjYHcDPD+ObBZadwIIJ152Yi6Nh3iCrKzHPoGc+V/h2SQThYRY6KWoe9GO nGu7mIMZQytUR3zNKLTDmLmQMzU6f/CMFXY0RBVslUrOfkN55CzF1vZlE3TIuiiY17UhJ+sULzzY g1NELaipw7EnmdWtqQDnLxRq3lJMpb6mXPRGSsdpqSGkJZMAIUknhodp9gMysOAtp9Aaruxkk73W ndK3aK7J0fkfoHW8UMU38PxLSj8gbn2pSgaYwQ48NAm+Mq/Z4DTzDUbXDMuZavA96rJRh6eyQvff ZoFf8ClFjcg0+o6rehMr1Nkvr/Zcl9jwn7cTlIHMxVIVvKvVtLEu4lOjNWDejyAZISDGx0ERU2zH XL4l9Sjmb/Stlafsez9jH8Y+C0cU2gGDiG0EBap8lHZmQRGACFcmd8w5XcxrrBrKVrssfOsiSPSv YUKm1yT72gwCRcNBQLhUD6ksc8HIuT205Ls8m+sgZ0KgHkUDNQDbfy+7HIJstbxec/fJbONYpXAK Lh4WYjn3FhAXuGf05Qvjjbwfej3EZdV4lPlDgBeEUuepgqKRTYNimvDboKg4PY6DcqPC6XfHHJ0j vEDJ+I9w4yOmDs2c2VYHfzLAYoTbmMEn0/qsef8VaH2hIAGBst+N59f2tZCRmLM2Krr3sUCJljyB Z9Wb8pBNR1nPLQryW9+jfk1qw5nbuMAMSFl45SNI9hEcFXxcWayKJayIZRYtJIUQLj/Q6CaPo/v3 tltWOF+/BKYq+MlthOaihDzYzbfC0cIzuxlX1eBmqQdZitQrVzx1qqBieDGGFUQ1nUAx0bbTt0Wh ONQdWoBA+jB14DHTfuBcqecH/FQvxS53wNvJ3jSmGuzfas0RMtCqqbmSilumvvZrf3UWVk2lKLCI u/76tSEw9/C8tXwbWAmF96DZROqcDQEtFyTqposDgN3atG9GvJayfRq9WHVupHZ2Iz73AfcOp+Tx HWbwjHFj/p+oEMZ2mqLolxgzDBsKdmRynI+LLtrHg+AfqKkZZbzSalK1GOrQkCjaTYSolTvDOBL9 EJQ+iKB0DPkjtoOFEg0fxO7SgQR5acZgrrHpKqCdSkRJQHkFIoSfNoL7oDORVRAi8SOI74D53xa9 5gIvw249aB4hLAjrttS4BD/1nz7uZF4BBDHy9z05lxeSNbZVAYY2OT1M+IyhrE/KD69lYb4gHfOj Q9kxm3ondydu9BmXoYg7nBHfIYSQx39Ny5NTVSQjNXwKuiUMlwj3e9aQ7dEnJs6elH3bslISIUD/ y/GFySpR1uZ1mvsSpwDkMDxfdDAIBND3MMEvxhC7v+245DJ/P5xK1ztQcKJaEMh/9Adqa4JeB0s1 v7rNmTJp67nJ5ujgvEg7QK/L+4C12GeNm0VFeGjqQu7cThoyIeiYYQhgZ+QcpHO2JjIKqJYvvlxx 0uJVRSjrikavyG4NGQEhF71MBdGlNBf1eMRHVbYpwujJm1DnQ2hm+QhhF0uF+uLSeIO/MmtH/tjd oLfR3Gj5NIJaByeXvQio0G9uN08Cgc+4vWcFMnwQ94FGHriN83w+pC6jNDHD49cfNqfJ4VtR1Qvo sfCuWf2eH47SLhIsf5VxJG56LtYs1XEXCYtE9PMgHbm1L/UvhIrP/LeYVNmyzf7s7w8g21/SAatX clLk7y1fAlUd+s7/YGpK4YiIB01aU9dLxcgnVdOv9CXvzLEoDwZwG1ItT6TQY73+8UjYpqbFIu2X W7PlTEqHdwHpahP3BFhGjHtuxmTP0W+XhxiCZ5s/1+k9sxZOL1913kK3d5YDDBmNvUA+9Gf6WsWM vGexmY0YA1CaBSy2xBVzytKtwKOM0WUsyvimfSSeH7rNrcFsl0kVBTme28YcDnI6w827JV1InxtZ LuTSLbq+eIUPwobEM/xwkp3yEsBPY0VFEoBSxRT/xyycxXs2t09pK+UedsvwqqgGMRPkbnwaCIWY xlzVvY61IWALzKaj3pGddTmh5tsdvxbUXExHC3QOilIlITvzUTK4VByOcehB5yebiPniDOD0mNHl 1uZEr7tH16Ieat1M7Z9QVM/tpInDB+HNTykXaUb1JhMpwwktONrtAqrGtS2aSUI1ze6JWtScNE2f 0H2qwYKq1xNc/o1KTJkf6DVIjUUN4cuzVyHWroQH2MCask/SIfm6s6gaLqgcZm4PKE3P4OAGp8Qk Ka2msp6CMHchGTFMQCf8yl3kHhH3fVC7JKJfeAW1ZqTccjyp6cUmKepb53VbyC234nFUYvlj0N1s FbVSqzsRjXGxB1OiwtUIMNDXs5ZWxML20Rug9KE7zlz3z1GOdyKGVOFVTBjZpz1/WVRdcy3GT8Zq RekTg2Oc0scsOl0knW3Lt02N8pj78QK9oNpIj4vMTO+yrzkyTpTApjhTo01xOenS2GJXaF3AnR6B 6eKloR5G57RZErGLnTK3xrEznRDVRfUSxBlXdeEsqREYh8wzMffmgj1lbnC/xZKHD8AjirICbEeF 2mtlP/Zsn4whYQpnxqhoJvIdXdzGGjW8d2qqL9KolLS0SkUkGiY4teKfy599j+czP8DYdrlSrAcD n/Xc3pzUAAZjCTiZfV9LBfhOQLdSzNgMHhTtprThjfnVUewGcVL+xxBdGsZKxz5WL/J7sgcqSKU7 lMrzoIMEFATQ3ZYGLgU3CJ2xKS8NjUW6f3t5rnRHWwNn3+UtCglpAxKhAz7b+yiizlriIQOx95nP fnJsl4oO+07g7oT9xE4eujOyrpPgUa6t+MoDGNEST/zpOr/7SOPv4UCY8ZwY7CfZ8JdK69OWPW26 3uuaOKbp2qSNqaZLy3G672joVEmUhFLQeaRAeFMW3kFJlvZx5XZeOAEcjsOBdbD1/YX+9o+dk77s OPt4VNwjPKIhgGIHyEVSLOru3bVdnKmkPLpU0AbtNMaeBfuzI5nsFrsP+TIVL+s6cDLltT5MQ44X wYkjpzi/yvCnctCRlAqDhOEhSb3/dVpJoQ2C3Wstc3cA6uhiqpUhtp581ofvCFuT0SbIGfMq24V8 JppzyX5OBERvQexoScJ1ghPM7SEnEhckYHvLhJqjGdo/MMlp6RXAPVTRkV5K1x3NC5Ciwl43Ryg3 xPTJ3j27NDhBRn2V2PXRyzRFd06TWZUltYG8Uleg4ArB8BKgQm/SF258AdMDA/jo7Q4NUMRJcsGF PcrWLk/KfQyMbJCJM56BPiHZXxe8laxGq/e528yYWjbrmuNP8hp6oZbAj/6nQcQxikJvJUVE9Ix2 p+G1ZjB4rlL7wWAoYMaWed1d6VTjM+kmeNCHAdd9HFfVxgfWxHX/mZiioPGkyEFl+gGS4ucH4q9D rxsANF4m9EZzwcmNgqC8Pg7vMqwASMLEj6QfGXxAVsBIcAI3KzALP450M8vEx/CXdxGCMS1VcWbw 6Ss0bkmt2TB7GM9r5M1RP0URXTqS6fS78NOzDe3Sy6dtrYDv/HUde+M/q6o8fxL6KBAlXH3+aYmF 3rZpPLwCbt2acY+LCxPrfRGYtTFb4vUjcUhy92wEBYxKQWVl+VvoC+To3tm6eJ7BhWZI8R1ZooW8 b/d9+y9Q4nfF2rAvLEprqTxxBhH6juH5+1+VeaTc21fWP1FTXxy4bedEvZh21Rbk2kpiTFbDCQHD 1lvXiD+w8ovrrVdHpX5JIdPKkE56k/vvCyTaDEs4f/EtIPvcM5ziJ1mcliVsp/3mbURtFRMLghPF hUzGEcUeUoeEWnpQrL7Mpnu6yjZcfGE/tHROlt7gTCg6PniGo3VQU+oPjjLj0ZZOuF8EYeQTNQHh +DFfTN6/+t+XIf1SATegACqvT5hYw/DpeJr82oGrjbFhF6f0+oJnLjLN5+cnqVHL8/EC3wHyp/wF IcehHorKv/H6i+Undg0DgSxxId6tZ4lrglOc+YA6ViQMloYRUt09QjTSDgUMHkzq6C5gPwtUydzt 3HLmhJl05xs9wOkMEB7nzdZtyjflXmXAHBAL+TNskkDviMxzMC4zd0FZ8A2d1U2hnE94nMXXgAE/ B05r4NqvRBQ0v8achCMSywBqSuqPdg49viY+VyfY0iuSjRPyxqnb3b76IE2ULBGxnNENIYM7GXPa BxU6QsNGJVhwWOFsXZOGiByHcx5vm4mnVkgwEPfTuW847Xx8KAdMq1TXA9NEYumJNJNNOhnw6jtI VzrSIwD46wypMYA3RQeCVySFi5y9aeaaVLMq6m0wIw1rcQy/BqnXR7J2YcIK9NhYa66/BfMR4X8Q Jl29RIUpRLeRmCsEy7R+XmgUWLI0fzEBWbobV4H6HAcmeV91ugwFZNZ5l7aZ7KaxGM1ckR6zU+IZ tt5Z243OlhFKl8+NAQ4cSUFLRfcGGjllNC5oIDN/Fb/yN7pQ/iu7H9ZSt+0tocGtmQvkuX+z5019 aMo7GGr5kkUky4SqmYRUYubtkx7NDDcOe6xRripVkmS51HKAWonw96dVwvZjRawAszALiFbnGnTv 9i3/R6eHXT2J4gXaO47V6cDVcMsCK+lVnyA/71f7RAxc31Uuv1P8N0ufq+5R39P9VmtI3VS/zb5m dP+EN+Add8VgeSbiXCfvEsDYUqjSKYUvydhgQMjxRKrNvplXnYn75WPtKWqc2Nh938OHxJEdclvI OArgSZViXgbCyCFOWFfZBP8NxOm2QEMLfeTR8eBxDM4wi/e23Jm+lZfrmuUHunBOyvRE2mTawJOv bEAqYdF+lRmF8ftQoJpRcKLtFcMKbAvrVXFR8B5HLQZAHXBL9XnpX+SuyNqLCEoCVonElHUEGxPG C8ufB6mFuV5VyLqHvv+WO6/BROWYlGw9GZmNSy0MKK/4hl81wK6IE4nfjesEbQJPthmf+lr9/L1s YzxReyXc5sTB8lIfjEHCvRNo0MViHjr+7LVrykGTZKFflfvP4bKaZUmiByv4RrOsCs/9vrcv3y12 T6LXEYwomJK9hpjZuXTpag30ZtL3M1LiqPXEDhL36IbIMFP8y8FAeS7ONtvZ+7yyMS9UqabszAcZ QQGjio1wp84CppmqZtVjYgqw/kLKLSfd124sTRUkG+An6MjxL6IRyQWdA5C9mZyQKQl3rXxeWTev cmfhhbNxB0UUOgUFJ1nVhgPu+2x8IWenXZ5qb403bhB6Ge7t45vn9Wmr7WSlUVfaOQ7sCcEdfhKa YLO7XmRJFWO1j+Jl/NXhXXEsBPvA7sfS2Rr+4D3BQvTo1XImA9dQiZr2XfxWcXGiYhc8amJS2u6B BK2IHes42DCARRrnKGoYWtiK1mbHTOmO18m9nTwaqwY5Dhp96dSXC9LW15LwHWWVEenXnTjpy4W7 HiTuZ4YRke/yG0chcInm1WkV4mW71zWFkA8vVyVyr7qdlxapZJhma10B9QHuMTX9H7Eo4JObP/kG Ml2NGYyfDE8DJiQx25cdRTJMVUGFTf8Ik+jpQ6LX1NEAfuzNyItXEIRlwZRBYbR1S/FBadt64K0A 5nkBidn+zo9I/XXRTXQ5TSkal5D8acSuDpNU1Q3m9IWJHLvJe6PAxQQsSGvStbByTjOgYe5qXq9h pdo5ISpTOizIZ2X6q2YJRtShrEYnISAmGikXPTbYmPpfaHENIBsJkLzTMPUFEc8uvtvpKIY0RMOG nyG+c5JoQqAfrePtDYb0zesm/FRXRhTeYZTdg6evIEnpky5EEOHh/i3oybBpawgMCcuxH/96aCE4 lPbFGEeC9odhf5ww7VBoCXlG5j5xiyvovOPnS6Az+kGba8BwglEt3nuI7uKs4HevL3Iw9AEFDKwj irOtjRCCnK/RnJhsloPB2GzpzAZ4TjkAiCf2wn9JXkVlCnWrX7YyV8ULix/rpqqj0F+cDpXkjJLP b+F8t+ma14y25sGLPsEu7c8pgLJVlZyWsgsrlS8pxwT08csL3WGKJeJ3Ls/ROtGLDyTWI/iCF8uv 2Nrqf3iAp0JLRNC71l/kGIYV+3rO/APS+qTWWGMnb2XtIETW9pUEJK0h16UaCvaAGA77MvUMxxQl Vx93/5GM6nIq3KLU3tJjsuMaM6wmdJQbJFWzlFwpJEC1I5nlwk/06xU7XfBE0QFEX2yd8iQa0wdc yzIIOwwnV1ixBOFOreYnK5noJ9hfwT5SLJ3pmgXtWoQOy92pfQny1eYGoyL3rhrgqbsooxIyP+Pe Rf7OwsjQBBb1UTEuZjNUgO99RrHCpCGVhas5NzqEW61u2PF24CIAHvh1huaL0bSp3It3IqcysMwz vTUE1+wY+eX6O7bpAycVYvZ+Edmh0jKKBTNeV8l6sjFQFfmHu9Nnb65rN8r8JLFfTlACkGVtOOjg TDPlhTSo3tRNNzJ4y4OyUQQ1/Tg0rJYBNaqAWKn799RI5+Le1/eFA7JsaNzRapDpSlitRyasOslE uhs+i9We2TauXRf0th5ZUNz7wBE4GWCDnWUNxY6trWYwyKWfrsQXLoCDkeb/d1bghxEJRx7JB1x/ Uqnp2InMWuEja9T5GMvQdgwsMUpOY2KrNk0NcTs9ggZ5hol6Jl1D4QdA1ZlU1UIW0lxBjg0NQN07 jIzRMHTgKeozcHoi61e/0FhO61OVD7Cev/RkIFg/CJaGzsDQBslhVGKhOusOwEx2ec2LjPRlRd1/ 8MITh7IFRX6vxxmlIp7P2MZXNhsn5ivfsB79DhOHV//Yj0AIaYedvF1jLAwsy6NFgGG2REp7squs xydpHmdippqDr2J8DhBvZkwq8bpLqcTFExSWOFzGRv2iZWibL0G50b5cGpdtw93XFVqUII1jLDmd BAMy0LIdTgqZIjBHpi0QS/rpiU91a04DqvLZKFPzvplILjfXynUUV3M1X4hb3b9bbH8ZWffJVjzF xB8sOnSecuJBq0LeNHGhJG66+5isgAvvHAyYF+3vHrwAoEEV7+ycTeBXSA7VMFRTrkIrPYve6rsf XI2GBTPCT5oLtPZ6DeBhUmnmW9IScGs3SZw7ATbjjwTygeSy+j7DpNPveU+coQ9kO1ngPlgoS8bh ZnbnBvPG/Sfd90g1M7Pr+KobXnMhfz82uEmIyfWqlF00n6pkSnCrv7wbrsfUG98nj3/+r4y0UQ91 6JjgZ6RViSQAZojwVQsNLutgyLxHj0RkY9RHjD12L7KG8hhkz/5ZawvV0OJdiJ2i89/tMPtwuMo7 xyIbPUz8uE1JGn4c7L7Jcz1zEDhra8CcL7UuZqkZMKviUW9yMXQoIkzGG6frvlGOVaiK1H3YWPpJ 9+IT7rytJgOHhcwvtC+zVdtmIf37J8ipOZv8dtt3RL/RwV3colrtdTEyEAqtfLiBs+UOrol5rt63 EcWZy+edSj0KSK1au4nPeOchqhir5/50khU9AEIqnsES2Vyg7yFJThzcxGS6Xkq1W0faxid27M3f ySJ7jspp230j9+xd+NSBr4WE6o11WTRrg4tzXtNd5pShb51WafRGF/rrOiQX8EUWqHVwbkhfe6JA a+SxMh0nlqkk3PO6grk3GHh3b9nPNU6c6xnyzFX/KtjfRHAQhjogvfFY+nl9eHcDCflVY7ZBvbum 2ovz9QMtPcw62+uN8pcrhoH5HjMJDPP849gTWPwFs406mQ6NiwPLprVcz1lrqnx19hxaIZmqu6kw j0E09vSO5pgY8xzbT8lLvHYWolPOLJYr9bnezf1Wf8exqkXn64mpMLN5+FyYnrMKvbjUtQYMszXN dkkt/SEY2/kfQ8fzwJI9E4Qac5sqDHGEWEJg4Dt9aKSOEkwj/0S4IlJ+vdrDvBTgQcH4G5w7i/v3 X5FUemN4j4kNwGmMUjtkNhXJvwOWF9SN/4gzw7pkmSz5F/RF6IC4JKNm+LSCRQ/t/De9n+pg8df9 4fZ82AMN7GHuRXNIDa6b6+M6TtxVtl3P4W/WJEQKbZuNDC+v/nhqE3x5gVrP7o/FRXbBUqiZnWII YkZxLbugiI8HyBWp3coKtGsqMtQXer2zGeIEzQ22VLe2EdIcZvwW+SbroDCsPu0Jx/2GES7J4XYI iKQ0zKQzq5dqv3ZrLpb6JO5tfydJWLnr4NHTpj/gvuvotXsiAmJoxkScKXidMQraawK2reyQmrWq KPHu1dgjSCkjGBNCvqCqRRUFWrM5Eyr9jhQa53/Y7uA5MYJgR8iq1uEMhNFlQAm9G9Eo9k9g2trw UtmCzXIe9JRYE9KQciWV3CpthFTObZ1uPZICbcvNRngpmE8MdgjHV2Q3UrKxHpgFXFdvk3VxDg/a Koj9yr+4hcRW49TgHTMqSlGZpCuI9k0fuWGNbU94D2zUxUjAABXZsoAbHO2qbO84gGYcKik4PpCh mQ97X48jWva//TgGoB9SbKes5X3hDdsTVWK+5jIej9F7+lbta8dc+KGjL8cpJoB3kBJvU+CeRzrg IybwQ6I0Cwe7wJCleYC3VVotCB632LUJPZIBFb3PItDifdCiGBCjleljrUlaHLVElas4+MjDyKC5 mmXgGWptuDCGk+Lul8BHOFH9pGiGEr362dur9NSlklUdjes9z80IU8ka9VNtxoJYYF5MmMp140Pm UBNxps2JnOXWfr0In13L740dXJxU1K5+BHHwkBzPNS+/y1jUbNrle+9F+tDz4KcPvXTyrv9Q90xg ryNkLpqzIr69f3K6B4f4uLjl24OM86x0f2c7koFyBUUO75QRU50xs1e8RJVgbsDZEBg2DYPwJkCG 3W2Q7/nInxPXaEbj4+GVNgR2PkKqISjGjXTP+kDzBLtvZNkhdQf1ak5Zmp0B7vWheQXu9+VO2Xnx 6c9Twt/r9Qla4NvkM2qiouP3+lq3NV+sYjVz3YW2cOFMyDuhL0aCMVOVRuXpM2Pfa60JYjTpQiPP q1v8089OO8WmfL1Q7Jp/yvryRsK12PJ6nSK4iPyNzfWxd/uZlmi5u1R8AaArakRLm36H+mX1JfVp ZAHs7y8MpZA7+c0BgTZfdjUmfq02idP+QfKwlTZZX31QOm0mXPLGhcg63Yby7CWGs9ja/ibKL6bb XaLhEyjkTSWtngJURS4y6YCwFmVYmdKbxHozV5fQFmD8cOzhWbTeXD5xgntLApmQfYDNt3K43pae zWgmYlZl0bBgLaViIrZBAlOZ8oQf6wvNp/wcn1oD4mPXC3lvb3ny+b8owizO2DOYMzpTvIY68BcI oE45+gcmy3Gq9Ssv0DtQfg/eQ6Kr0+el0RaY400YBj3hx93w506TYpe3LcQOp5a58jMBoia0YbJH zZBj0VFIu/wOHB9/2pHu/fPhFrH8CVwCT83I/nRaQTGV09Wu1SGGkeVsCJc8jRzUsbFcHLFFpQKU u5vvTVaoaj8+QfQgTYA08wDbMe3HUiZm1XnOhfcSE7xGO8P/Tw9R6YSANTJDlHRHcrPO9C/XkD8A oBZAuGZxyzv5mKpIbNw8f251hlA2i89BqYApX1ykLA/gnJiO9zKTvG7sAwda+/sHYWvDQulCSn3d f0TtuZ/D14x5KuU/2QWS440OtgOOb8TnVec76aCv4x6Ud8mlF78TmkDCGbKOaGUNmfKBrTA2QQlh rofCenKQM8vex8zo84zqCvWXrFnwmXMMCA8Qd8Sgv5JlUpz4azk7CtHSGKxHtveswMf5khe5BwS9 trQD3Wa5TAQt5nSDSE/acRY016G29pykXuvl82zGcIk6j8AUkKIy0eFXy3KdBmgGjewE44gKboTT myyZaILXrVG2L2CGyhuFC90XzHU4HtcdKcjFz23sUYYxAhVuBGiT0wPJUJJ+vaudmmqLUFICTTtU Y4e78bWbIkIRoAq/oiSQFJArUgbOh7LFsewdCChcVGARc+9gui0M4EmGStDQ5AUNZ8uq8nu35bda tQ8MWhkPEeklpygl4JCU/qurHK6swXCQU+rxFOJAWrABmErqpof5yG8MWfMSOamxqPAs2/kCIJKc NrLRpg/YAyxSur5uhWPmxEXpmKrXfmDBQB4y0ONDV0aSmOEcORJsBfkomPXpOjc/Z6LaK3Nu/0qi qbtKaicDWKOuDqRbCRkJ8w70eRyVZti0uS4ZgXUxDZ9oyPKkUfPT+G25VrirGH4Q6ghjSgx3eN+8 vN+ln70XX4BRWXTm1EHiznVXjgHP6ll6OfPlc6wWwBhfHrBFSxTO0mUS4xoFnbDbWFccu4mUqTDI xiI0oF096A+wm98CJf6lEo0zStbB+5mhI535LkR5Awucj8cXUzHAcBEc00ihjsLVmJRROoA2fEhI 44IMSzmbZjnb7re3VHK863cA/lLQ+9PYVS0UuxTtqwZxJyZDxFwQHLb2118rr3MbiAgkzyeTCaZx 9zhRSGvz1PlUz50guJlgJsOhmshZPxuD9qAevTuPFzTInirJJE8qJVWARV1RRZ5LK0CSn7t0eMYh EvNZmipJyY7RShGHqY9GN6TNbUEFQV4ugSmU9z6h90c425uWyl/ZFL/ri5CuzWkSgVEmBgapMQFK 0/w34rarkbQXzq6lk+aFq8TRj6vQiF8lbf8M4GSNVC05RklZr0kW1q3sprouBjq7ZGVs+mYwfV99 OEoRVDkjTehM2d1L4W665Sfw37LvBBBpbOPtxaeRUkuqjH8ebaDqJczTxb2O167s0lzCOnqRPdA0 8kjw0gmE5NACwLI6ZLbEZ0AzpoFk7NEcgVm5BCLgACAKuJj66y6jFkQwkzmZ1z7WkiHU1kGve1h9 apEtMNQEccYnXxWhspgvR4X6oGZyYwaZ548W9dnQ54J5kawrwmCsPDEcCTJQyx9Dg9s57aE6Rvp5 H3PcFM5/EKD05eowMSHgTKivD89injfd3wBaHs6elSFIsyTNx5f3Y4VEYFljlwDJdk94Apuunf55 oUIoaQokA/qJ/qdxKGnjNdXtyL3OhdZJMo5OKKWgJMzBGlArasvpVWxlkBqI/cceH2f2L1r9iFYO 5tbO6/fE9N7rClBbTVONTvN6CItnIttEEB1gUN+GGpkblyukvy2GXohYo2AAgYyotO2a0l/mi/Hh bGLoggMjYM9iEZ3AcuO5eSUF84Vn1kBZAKfcxx24tvIujtBZH7vPtfuOCWn4IcvqVSuys6MiHRia CZUFURuQ8JVBGt4obpzNi7tk0qlhu4psaBDaDOZ/Z5S9IKsrRjXOT4ZA/jTHl607A8ZjdbuZW0ft 4fMbERrfnBcPKfhcAr0QT4VWj/hQRrs26wBk2WskaFg16UItMzO9MVgoUF8qyj145q8zY5pcEwh2 pkNbSdK7hQacokoipJ9TpPvDjUpRB0BVH9S85rQBzUNO03lEYbzXyx13KNGh3gNAAw6uCdxvjl9L inOynTQaQ8uX9b9xHBFmmxelhfCBkJyjQHkLRJrwhptDrI75fHNk1Bl9owFgRUxVB9fN+MoefmRM jWewW0mpJlEw+bRSfegdnZulT2CZyJNJnJqV/6xLzE6jC/Vc5/DyV861T6IdR5WnsOZq/2XyE5Wt HfScsH7m9KRapXZnb4EX5VR9VlSLVN71hVg/uzL1zlLWENOWR4kPpXe+KPRDY/aQzMYa/+LzZ8/c W/tZmtIK8DJPhzp4ma9bUhF1ORyk835rhTBMW+crDHA7y6JNEDwFn5/VPPqwolYRj/NC3l2LDvi3 eya93AoOxXYnm4npWDRx28PDdSyLoBfQHJTGS3kZttg+XgbBq3/LDuMWZpK2DFea4vssXhBdVI7R ZDF9pH3yL8zbdzOuCSGnoEpjeMWTHCBHFEHzlh0ZqJf0N+0lSudN6ZBNjpOPjCz72QfS/KkaadVW 04FN1sjO7wMcI+avbiqJ2AiuRe/Mw0ae3dFhYDdATcOya/2Bs4s7g44zhGM2hwEWmOS4+kng0OWo yW6aVT/XQ/f5nMp1rEh4vZbRqU1V3Lw05D3XEHgehMuir0mCEZsqiY15HkA2GGmXd/MgrLmPYvcn Z9L+ge7hHzECQzX/NdCdbEd50HbosU2ezkKcdG91/KImhQtPX739OsihUd9ktZVyekdbC9iu4oXl 3xCBV8ktkdLSzB5RQUNZiN/JUeU4uez2sL3dyHfqwjFlBJmEXZG62EjkWct0VOnIIa3K4/qJLbdy SJ33uMCp4EZmZsdYnVrpg64vMhs+no6Wl9jPYHPw2WpK4ROmthrirFmb38AkDIF4i0LODAX9UKHM cE/aH7UJKrToTzx7KhWZAjpzPJlW0imLBLoTFgjrh6fHyVUyYcmPYVsiTT+tEFIeOfYUG27kmmyh 25Sbz1apF23IysZNOn5+rS9s0BuISQicnuLFwBhuswe8SZoFf6Y5TGH3dJ9aQYpwiWdJziV+Hv36 2OyGmNmSYNIIclRHCMksx2rcRxuNfT6F3Bzn1XwcbBx57B3KTdSUIrmjYJUH+7NOc8T1NcgNCC91 Aa5q2SZWJwGlJN9us/Y8V7zJo18N8ZiR3yCs4WMsUVQo1n7BOEQgrsg/U4mwt++L1oR3G/mvFvSi 5vzCVT9f78YxRF/EVFyWvTjMJqEAwkmZt8jM9gB/PgPDG1JS8uWMW/7dlMOQbu/90dNegenGhBb/ THXqmVR2GVzmgA+JQ4X40w69EFkfoOo00/p9PPZI/hBSkcexoXgEqKRpUtS489bZfaPC9k/mADDo EiZ6UR7orMZyGfj3Cogwj6TTbmOtTjQB3ORImFFiBUIbnChhhzZmDTVq9RlCTFJLmCgmFBOgQdDD gE9LvkfeqiVRBvBuXbBY6P0YTBQOhyriT6C5VUjUe7VbPLSmKdMXtNCyplpFbyZG3AOir8B2QS2Y 8RXFFsT2iP9umml/b1CBIdV6L7o10pMuAF7kbLgjHHvueEvp5oFbViMKn56H2mSjcWvDKPCwTBox xVQSonYxOy4alryFfwl8wuZvWk55fWxPnYcCZ7AKNroHmIHGtYXd8uAcyqafePHHt1Lp7bCnr4a0 OzXXGStH2l2l3CKmPAFX1MI0PCDaO3iUmSjCflSP1uR9C6IxFMOHDSo8BgVQ4JowF33Cpaa0k5cz Squcf6xwXcOo/Ns/CcP/mldbnR1gKXbQk5FNcNrybLOliFbyp4cawokemttS/KwG1etftZfTxniN Yeze5C3fIViruIrkkT8lVv26e3A10tliFoYiq6EtoIZZkVzNhnJloV8uZEYCXGFbbiQcdJsLdZf0 xw93cP5oLkYtFZbONHTQmWGYddEdDK95kJR8W9//aGjcWmgSXV6f5RdBimeMI6VfNSQAQv22ftw5 Y5cFHx7Z6snBy38ep7TaZa2SWxghj1jG59wHSRh1kOqMJBNvaufahIVzaTnAB/45YOc3+ELf7xOx 1HlhU51MOwfQcJpLDFVaP1Be+Zjpldf+hTqBShPpt48YUhS8hpRI84Z3s3848w0vlj5PmDH04EOX hoyyQaNsJixfnvvXOa4WF0Sb21BYxaWODuZlrtwQsfCJs0B5/3OsXjo883lMkI3WJUFm16r12cyC NRb2REf3JyccI8Quk4wYglOr5kz6aLGKRoqSBlTCxRdqG+cCfNqc6VHdwVbGvhsyzSGhnrw5QoEp ZOZy6LQmO8KsBGJGFbkkzpMiHVxp7Evg/S6cCuKHf68n8lWKLQOoTRgzkcn7gUxg09UUyBxetGgc Cv3OWtrKMOXJIQr12S5gng60VfUgA8lYWFLhyPnAT5aGxYqMrBbGJjfNNjuRhHGuTs++sLKhaOJU u9Oe22OERqk7VlTcpi2ojUTyL2tL8F81f9urhXhD5+vtpwX7NS6j+e6guoyP3fuSMKfbCTxeCZMf /b/pw0pMEr42M+1683Sq4VY50TkF+LK15GNTjYong5QIyHpA+kIrX6vdfUe91zPHY1BskbFW/cZZ 6ZM3y28P/Lu7wc5ixO+nz0QXbjsbKUcu8fqXD+f3BFCLPLglcvdCTcvLghh7Pprd20K9d0QVQ3Qw q2hJjN9+qe6mEcEJzVo3O5IAnKN2Ksb5F04ICxReGijcKzsM5pj+4aWxJLfUp8btIqD6ToOQ6DDX J7TlNOV0IgTFh3JyEoPJXh7HWUy2rpQTLHNx9buaxpXgedJy5YQXzCHarYS0UhQX1sc1hbOwrSIb Tob+rQRtBULjedh103pULoCMxxDqlaJDEZTVdiVQhGRWGgULm/UXMB59L3h94Bb2r6ugMkaCrKfp tg7RRTVnZQavv6/grxZYmZkh9B1+8dA9YuVoUOcnLlkQ5dpjaG0rMd4dLkR7xQSEk+n+YGVpf9Xy DdU0QZUdtVncpp5Kj9L7qX04kJMypdbu/uOWk4kf9jCfPWWJ2IdqObZRfo+cNEzKQF6E8EGgAh0C NxRP5Nh+vvc03nv/YRqdCsscKZILEj/OteRIj6MZjHSNz3GMoirMuKoc+ITEmJ8074OLrwvUvRWq 404FJyjqCHhUXnFSn7XxST7qW1Le4Nqp64An1LYv9+cATe61hBVfMr5wextIUDDoEW06vRDFLcG/ k86uIacqSF2fy4mONHUHcUcFev3X3KMqItmlAAhQlf6gT431C5eRQoKFoXX7sKfRNdEIMoWoVEx9 sySZRW/T4FffTD/IDJDgEyykvN2VtVPwiFjPyEf3qC790EEZxQdHPSol0/aREBazeIwQhRUBcEdb BYDTSAodgjfP2/yMae0F1kjbCgd72X9reYzsHsOkJI4fzmnIippheu1tQpUdWnyCeEdJwk1NTR+x aFf4jlgdXjuOcOLooSsrgvYd/x834xtKhEhAFGMUM+B4mRvGVlNCM+cbnwgI7lE3b/GIPzPDkxVQ SBQEhgXySGiGclpMfq9fM4eVvyZhev6XI8A+MoI1a/4utoMS26V26iB1zp5OuIzhDTh23FQWJE1+ a5+oV694za592bY/6wqCPv4ptT7dy9q+bBdQR6Jr6h4KTw83dykNGDF9/+l4MQI5w6WCg/XvPg6H E2XJ8pyN/VU3n0SNEyBYQ7+zwvrUVYBZGE3a3o+Mxzgxfquud6z4HpKCDFfVal8vuGNVWdMmrLV9 bkjxjwxK3JWzNI+5yOM8XQgj2AZMOyelB+yeUK+h89BUQQ16cIVUJcFrQdpvrkR1cSD1AVNlpcfA j6fg/Kkq8nhe+fsQEAXR1nrGHuxOWSbNbO1LSV0Vb5IolkvBhxb3/ii0ITwTJ7+g2JqoN/zbHgZF TjyGTvIMA0ZStf/9UcSfSgldZqKwx06DSnbbMS+11jTg0OI147kr5uMH9EbtNwUh435Y+oYnQMjk aAVBXmGokLmOxDnKHnpL7PyRx6A/SvOgrkp90oUc014E1PiQmU+HpcmN31XuKqwdRWWD35GJ2gB3 XDoVW9ZuSp6pCp9WJ2A7TQdKbiuHTO77M+dV7LwigUf43WOK2ERiP80GRZBZMNRcy+W1KzOlKDIB M2wNYnQeF2zfZvXTLxt7d02kGEnlRq6e0glOoQa1o3PzUU4om0YjpU+tbP2zqXOtg0hqgX0M2FKx l/eW3kGP+YrbW0pqjwqIKtmvFQ3MvGE8WBhvKQ5Dt9YK9fYFIGWlkINFHQrkQ/RKV8MY1coocq2L c1hO5x1OITSITHZ/TFgKe6m93G80mR9xA2iiIzEIIXgMtjOik7NSmc0iap56sIoIqBmUSIyb/S4Q k7Yx4/77m3Sz1o7+WbhO+bXuLxOeqfJfZ7ZnBANxvwhiKnLdtgoLvRQKXh1D4RM4m9ggFPJp01aG le3lxYmgxSd02JzhnAM9OiR99KHxVPCoz5iBkhTNVcz+QeUOA7PjvdRVD2+QyG4t1X1/0bVpweoS nhxwwthZI9qRZeaiaWfz8nGS7T0+lwWjEewRFHYINerh5ZZULT9Yhmch8AbJmHABfh+dGldvRHXL LfSmiqxiGvE82Cg0HUTEX4kgYuEe+Rd/bZCVKzL04pDZ/vbUI7NBvA3dlNQlGhEROnAXtkjcZUde 1Tr/OSDsd7eapXvTUZxudsxUlRbEntfaIbk/KojADm1BlvhLFvFKg7eHsaisw31LX5bd2rko/kEt aOoDkKSuXKxdJpZxK02JPgVGCBedbpqAAwa/4qKnxxqy/YjPkiR++A8xMIwmX+kZ4y/90gooLWZc xTYMK7QvVANJQlmiZ8gr0I6ldD2F7+DZHAEP6yNLJriWWsqZA7Z+CH4qiqHD+C2Ubee4BEi8QSTK HzWn83wyl3WgqBXqyFYvPuH7Vg24RIgrT8qrUP0ecr7Wasm8hZsAbOxkCjDgnzQLmnOFQ45WwBgk V6BnUp3ZzEE1JYgHyiUKjFu5XEA0OoeYO/0SxpRitA+tjAqzYqu4O+pmOCEqDV55K+iLJUb//4Uo Bf9G7+SB5fsUv77MaAP1lwENBDotxamgtZYkaOc3W82cOOKL6zocmA9+14Xdkzzh7Nq3wwqvjWeF Ap/Y14e/Qu4I3hagipmPlzzrj48eHOgsR+oNNCCqqk3KYe4odAUbweYK4Iht3e8JtV4mDatM6JX5 ekHCZ7K3d9Ygmp9vsNJeowMMygmjQOquduoby/XkvbL+EOupS50lphe8Pw8KbiicZp39Ib3MAK5k qfm1GDlMAYoaP8Ib0i/Bvt8DAbhF43OChTMmGuGRarZuT0hYDthi2skN3tUDDXxUjAHatiCsCyOM lKlERvd1+3Hu8hUOTLrTsLKP+N8RKg73OjkuGja/SXvE5xBG7c+ujSjYfyEAfP7reFTQ6ZyKUFAr 9dn0iMXE0GADNH7WNcaqevPWgvLfDTgWwtP4qaAH+XSvzLuN6NzvGywzrocLbp55picTZTe1lBJ7 PNDF9GzeNO+8qroHLfWxxeCT19mc3ABCTDdIeiwxxdGHvGVeVK5Wm9CBk4976hrhWWGdixnHDj77 FrIhmFkAFyRRb3/gDVtngcpOFZwPhjDJPVaJR//Brpg9RVPqzpqvqQYnwj/K+iLdv+Juc/tktlNj 1jONOwieKfF8MFZKvCzdoF6D45KBT427OcLNO3cYD5cjDJBCQs7SKkQTvXjLbs76pGvECAMEkNyN rzHu5eSOKE+MLgpEaa/Kfi/xgkFS4QlsSldh6UYR28eG8WuWd9PlOOP5Hy5gR2JVI9zo8TZZVcCu 8uUP1Mv/RucRQndg2wYj7ybvH1bJhJok6rH/tMHgQWEk9B3XzQ3cmxaBd28onNKYzYm6R+xaVV3l svIYor60gEEME1Elq4LiUUiGCEpNzV64Gdqh8mG0yx1PNumTlaUR9wTl7gGAyOnxq+0YI4xRQnfa 4i3nCzH2nv/7gN8P667OP5ChaxxIdVgB72E9FEUAvhqGxUjEXi/Kz5BxxwbYLJ/WPtm52wEiT9Dv Swt0Co/Ey3YyiFQTzfOXczmau/H41bH8hseIjobpothl8rLlFhvZ879i+VoqAuzWHkdnZQBA4lpq JH5UYnHbNRpp87Vg1W338L3KHvuxv99TAGu5HAyjmVv7KA3G+gokRxTsAz0jv/A4juvwEn9ScKEt J+F/Iyj39ZWrkhKEh/gvRodyU2S99b614l1RBoRp67smmUTxl6fv1s1DSpY9iiDKc8bnQpm6s7/C SwU8KHa7V5FzvAIT+Xm/f5hixoIYI2xAj8VNY7oexdz09jGHaRjXUDH37IE4TOZ9oygqJOg0iAVE BRzb7LwMN+cUyhay6z+v7f7lA7D/ZIhYBUVJlYZ7JjPiRYoouiEcvMtzwXlV72yb249Uvz5iFZwB PT6aCOwezdK7uhYMBxN6Ddhtf5VZfCG4123roS+8VCyoPbT2YvZ3bVT8VbamsJItAEy0XTN+1RhI SdphyQhp/JeU1XRJNCA3+wgOBr5b8Bw+DtULKLzS1eS/tudLf7W1vUOobydHhd1SfsuR4d8DxHuD 8FP3j+MCLkQRSfVu5rZpXPkhN1Uq3B6bM8ga3T94dyzcKT5EH5W57jjnubp72mdywHIsUInkrkxR CfYY7GTCMXbTzgw7IRbxOyLvljPU/cG+Z7qqnwYj/VJRdYuqVf4dSeEyPGDR8NX0Gy5DBkdDMYdr oEBB3WCX08ripGYyAiv8GUF5BeAk9JVM5E/d1VBkTk3jiqm1u/x0PKH1mDdYT4e3zeARA0INXkkV odb0/uhk2jwmmc921GOzVUxlSgcSutuErPAGqMoFDjUSbkkOOnIUrTlPzHKHSFBxVy0S00P0ZE7k ZRO3e1fxd7Lit2MzvVWHR7c7F2RdN3nLICZdwTPQKVo7yVdJDzI8At7mJJHq1Jpkvv8551UZ7YLb PdGEhZFM58xHa7DZO/Jn4gpcyQeAYA0vsQFEtNq1uz83V0RnQId0gvgRf1edUgYqZ2ILseFcMLH3 +Q24EysjBND8RbYIfExSzcsnRNy57vKbYL/FGVAIbwZDyXt/XEBPJtwQLXphl+XHmdaHkmr8gcY7 q8vbn/atcPuRqt0qoEcQalkhbswA5AmuwrbIWFy04AmSl8sYsT1oZR4cmajS7GVT00PIfUF3aVf7 0OU/m90yhZxN1OKFqmyYoIKupz7mlQQys58HSPi2aaI4K6iTlwl8rs5Hq4Ih0MYgS9VHKws4gjcG LJngTGmesBbRlEdlJgBFWV5bT5BevQzCCd2N4giCfKJAfwdrnoVswKaBUHp7BtM3orJq0+pprVHM 1FZw62wjZbLyDfei9Erk51FTxblfNeT9vFq8bQNLrlZijhmqVAYBMVFYJ4tLkT2iiMGci2bF3Iud 4Lur2rFvyyJ7omsN3Qx+sOrpynQTjpwsNCD9iiL5dkGVsmxmV3oUi9W8b6nwgjqjLPLZrV+3rbZG paos3ZTvTIwDvzQcjZDyqiduJ/JnLi5zNU5ormMzccFkSRrLsWT4cGfVdyBtsFQnoRTEsNCeKG86 eNyYT5r0ZtSGJy5zaydcWWSsuPV4zfCJgtg4IgvtrnAEYyFGN2hv/CMkZEcSuzoXv386VxoE2fVq Z3KeKznAfYEK0wnv2/PW9qCkWxEbFoEbWoXDL+sBnveLJes+HZsjUmEJzdheh/BNtc4wHi2WWfaM W9tIGIu2WNPIg21WE8qJM3txzBCGzuf00kX+Z1s/Igt3hiWicf2sicotSiBdCGU8b9l5ilOsYrVT VoWscKHidnLbTNOey//f3lAWaIV36/nPkHlrE7LR28itu7rhnr10d3NCH6cgwKBnN0xY3pHhjK/O W36RDqgd8ApogiJm40BgBx1EnF51nyPXku3/mEZ6mBoS6tr/tLfVwpFWk/Gj3XCyW7Uek35WFAtE OkvoSnN3pBDrnIJldk3oevzQ8mFmu+gWWO1SS18W42s9YyH9E6p8KW8Sr+F51cu+Cq2NW3n6OVv1 Usip/h4EKkzQSM7Y2aduUDVl+OlNVRQY7B1Ed2I5ziyFt+lUpdHFUOOXQ713u9TIiVG1TtyHqKk1 gL7atyenqyLpmeo2AfzlPTSGkPheCZCi3tryHg7/xmcdHCxnmbyLhbaw7URQRCYEAmNr+RtKmh+9 htTrswPISXfknvZwaHAC/HujTwWghydht2VepeCu9Bh274Lbj0/uw8XyMcT2HCTkSDNC6c2AKhco APuilnjMOLcwOblm02UAsnWdB0vzAxxMvMbRow3hTZFfqLXnkB73/ThzvRSGfO47eir/z7rwewH1 sErtcMEfr5ZicqrRui1mrk1Zfh1l6jP1rp9sM7TCuSokFzNyABi9Da5fmHoKunjlw0BHqEig8Ha/ YWriE9VdbhBjkYc+OOwUdY6lxaCAzcdEHEP+qirOfj2FHx1ppqMP70Pm126x0U3QKkacbIsbppmH qV5HV1heBWUzFzfWC0PJ0xnUJE/OU5BBHRbPx/gjwOlBFVAUJDBCm9HnDvWQOZSvcxaycD2fsVxM 2jtF07yI+3FBJm28AnMHsKSTuKABdLzfPYDjrIBon/MtuBcyWDGFTQ6cga5eqnh9HMtECk5RF9p9 snfhZy2Qx4yj0Wz34TaJbAAnozLTuz7rMMm7TQcSim5GIHsdoEzzuBUI176MfXN5naLHW9wc305Y a0LKQMfWcDNhsIio+N3Ho5bfihT0rS45e1uY7HOydFF5QNAulsprCNHdqjnH4HuNBJWcyVfaOG7U 8uuP2URulxeb/fPZJeRbl+Uo0V92O3tDcOSHjdgUocEH4d/hveH37mMtwAoD003/AX8A3eiHnEsZ +EvF+e2xJJuxQnG17vnWzz+LqbQj0UOt9TOuiCMTVE8vjmzWwOv+tg7xjDuf1oDdyB59K2G/uvZ5 OSTOHLZcN4KLI4hPHkkWPHoG78W9MrmPiYK6jsICfyLL6fwXGEncSqoT6AVMaa84ZsxE5OTBPFY8 jJgOwWrx0Y7/vYmNKdXhF8TmBybvmNhmHXIvYuLonmqJtF4ao+eTJwhohs0tQ8CMRnJ9aIexGr9a BNjT3S8NitK+JuAs9iXvVLrv/tAbgFKDpOKdsbZtmpXGgHmCu+oajaFgRVrKoX85X/1nntnEJXmP yWDcZWLurM1X0+GwOi9vNfQNPMMKhMyq6uDKzH1SxNj0HckVWADo27WsfR/fAjaPs9UU0KO9bxr6 kx8WE/iZiTG7iLG/uveNG1HnygaueBMOHwAWOh5jfL2RQghsUu4fhmNFZlO6wF2oLXyOK/TCmxPX Fn4BcZdXqji1z4tuwivotbdwbP8zu85/kY2l8AEDTHQ39NiOCozisfptE19debsVu6xOKQ0EWQ9L RwOuuRTf23pvB6QI34CiM5Xh0+xPY0iufakw5KDHLufzKKDb6iBxEz/2HpJ2IdCd3qkW8wRbb1fO xec4+zU/vDNTYhTHDKwhsOHT24LbxX8YzAHhje1jkjxtqWaOkLePj6o8b3wPlMkeXCMEEMsLAfh5 1RBYyZb+OEzCDbfXOApR2UhIeJpKFBjuqbCpYr01ssS8k8erLFSx6kBRWJxByZvbyrNpBMYHTlfH J7eg3Y1Z0Ff5TFe1vMMSNWxEeJKoPnewUge2vyRts3Arv2nGlytrb5l6t3RIF9g2N6jqbPSu4PMX PsE+Xli+I33UOzBuBI/HM1GV4Bq2t4ORl7mATWGlFevfI1O7PluNxC+Yht2E46T1abbyy9v+1GVR S1ru+ZjwwfdvhMsZc/LreIwyyR57VkN3yvgQsIvcEu5cqcV6AwmcQCrk3wQ9Eb9VHghFMLtOvBeI QBgVT0ISGgmByyUbuFt7avqvcCC8kDnOqRcTRWpI0Aif0N+9dFq7rT7R+NOEtSQxfXbVBxjxWGgb fb9c9xutUeIzMXggff9f3xTw7QaeQ+3DpE9mdeODpb9AyZWZWMls4EuK9tzgRFUlsIqtaKHKLBUA BtLLYTM6HnZxuHIYI8fv9vSaf8/JfazlI/L0TBmJrDJ0z8XdD+9P+6hFU/KzOXR/istsmth82tcc vjT2w/qq7/w+H0zZXSLwYW6RdY0cqFCzwVzxcXSXNXaMJV54bfyAnlMQ4hfJ0on0d9aULp+uEUR8 1MzrcIouYtMJ4FXGSEEOfqXcCiYNLcxbbqCPKAwJH3V4rAsABq2LFASGeIespWji9PP1WXZlrjN6 Ztmg3nIeOOy/ZO3Z660Q3YwienCyh5PPOQr46GTfxnd/s4soZS09/SNSNwLIyIi6Rs+odMxAUe9I qKAzB3u7hoJ37SnOuHC48F/Wpzu7MYKZCfl3zojBGgXcgQ1Bo2ai7kHuhuk3dJGgJssPpvR4h2mG d4TQLTc7Jd2nteOCRb4Tol7O9U7wrwqDQz4a5xarVZo6bvFeFuxM5ay3k++8hfNZb7ygydV7kZca zBIfHb0uEH3KTiEGr32ZGrrajB8kLhH4dtzwXOzMMazPgBG2TvU9w4CUzcVIt2OBq1BhIdFTkyep 564MCjHjzctaXRrNd0CIqKGjwaYDFwv1SM5CW5EPeacPTWL2wn+De//hsHDLiHZpXRRomQsdiezG p+dWJ0ng8A6/QX46x82pCoGR7Pjf6cmhDSW3OVDhuMpSyEwibitTiooS7Q9YAIIYgdNAUePY1+dC ojs2PawKvp/WA7LuFwMghsHejM8j7Khju1/Y58n09ms1xQ+6wYUFW6agEJ4Q+6P0ZY8pmzWI70X6 sebgx+cs20eOM/W0NPykqn5Sm7MElNIMxmkVYS660061Y96Puuyle7Lo8fhwUG+j6EA8ocL+i7iw roFpGF9hi8xLuyyRXbJ2wDu/VgLUJJLr1o+gRJVDof0EQaH7RmcCkRj9daMfvmd0Dm093/Vy1Dcz dBEUHdF+fPV2+bbwyg3iJCZTyTTcj80wIIo+WX+Uv3v4/orr89k0nP15zqUiITsIbVy0E86VrHfM 6ZUWjROY1KX1f+WVFanXemL00TNhZnILmt/7aTet9eQO4Kt78/ER+3NTjvLOvP58yLFB+LAzTDOz EfYrypwp6AnytVDWzd6mDGP8m0YxjaFYym9W8f5P1XOoeb9S1W9YY+3Cf8ASjvw5fnaWQPPUQt+O xrRhp23zTRpCtVJVSjmoz7DlhQHq2JtuBElrcCUt9Vzq4rs1fQhZagQ92AGCf9gcMGXiPDq2Yo+b gvXpJjPMGrkDws6bX3mhtTcejVAlRbSfRWATYRy9v9AJtsaoeS/Cdjwj9/iMJCBAkre1lbs9mKY7 PHBxZNqqXZ5Zr3deS1PD6z2xfxCSJ7PrTk2CBmUjWINTcCvKKn4srPUh/fDaGP+duRbYe8n3P3Lv 3UmogkrgJ1PAGPvC3SUQ5bYN8kh+ofassi6C9cLTPG19m9fe5Ktma4b5dlYAeGDk/xsMYNRPEQAL e8tk2XzG5fO1b6Y+LySw8fjAC+IgEOwCLXtFt06rvzqVT/xPuAaxWyivOS3cuzLPAwe5Sql1N7Is DyjSOfkJRnN+HYnF8OPriAHEFGJq5JSyxmYCl2WlORe7j3sZJT9mMrBZYVr/XYuylDH6WSz2MEaw xfUDRbyY8a4ePEWJbhhMZpINzNrYjEowgPU4ZUjsGpWERz2LstdwCtx7OiGYNmixjiEoWhMogYrh sfh6K683yIQ/mLrIK6tVmLl6OYOOQWPB3muB1/Hjp6TCk6X/tvy5jHhpUwFF/R/h5Xbd5kLzbOB2 gHO+pzKOAovPY7qiXy+W98hv/2YN3VR4otdK8TQ3idQPJ2qKaxnMrjkccAJmxMPnTSjH4X+m+Ko3 wS1Y4eQ1Ec5LWfF17PpFskfJijIrU6gT8Q7IXz/9rjDUI0I9d29dimUJ690jUamN2zmnSoB53d/A bceTosnGjFbj7Rm4K5SAsDMBOgARlscz6Tlxx5LnuFOnoBuisxX4DLPI0eoXoXZKf8zbyAiq/nE3 R8pBHLYhK1MHbPCXnyDrgYWl9im4fkeiZ+z8st/hgtQbiycEyvT0sHBcWRuDGBUmiGnkgOi0lRrq w+rfprZLlXbcWPiHl4DfbRq6h+/lXPFYg9nGStvPBEvg/52/mnec0ukVZ95K6L4qJUJoputM3Hq7 XozB+K1iJ/i+ok+FZcPDi6QOp0+cOeDq614FASCHbkFbDOlmmqeckJcJEMjqc0mJyOPhl38Km369 c+9ZN+Vq6kS2xdrWN12ZskiX1T3kQat5U1ERazWFTUVCuBQvGX8ghVAne6LoUb9Uw2YMCd01SgXY /lIjSxsEIlsVj7tbdP7b6h46JsnVoDUaE/SRwoCMGHTsW+CiWg1SvTjqmxBE3UwdtMhYGdQUr/H7 2is+O2W9gAxQf+jWmwAbXg0ocuLfe3wFXtvrylJHxKbvW8F2HRamcRAtlJjvsiT5DjuVOaGz+4Fe SE4m5ifBMQvcJhPTkSMFYG8BimbWiAL8WsVqGCn6e/8S5vrrnUh1A9Om/+/VvNthMiGNEKEtSS8O VYKKT28gTAUqCXux4TpHoHMiXH+Hw/72aBg/+V7DBpMs0GXjTeZv1gYfLn4P92SsG5XV69XWpRmM ckZRi9NdJ3/aDjZjKIhNhi2z1d4glPz5hn0k5etBpnNzXH8BiiCr/OMzE9JZE7SK8KTuO5C81pFk yOtHHPxL+In4wAYqmtQPod9BaSEO/U75Jqsr0BTOoq9+PccLxRSnYd7a3ptJVEh/j8QoqihgbCec cn363wXyZEYrx21Z+ZVeHmLZ4q5qhOUTTQBi4fWChL1tT6H+GM+jr/x+2q0Vo3O3NUENJOxI8XYP cFlkN2IYT9XOxgmkGicr6dgqL+KsdXXdG/1RvWb8ZdfVabLYehiT/1kz9/Zv30UcPO/nqznjU1Ej mzZi3VAZHSoUQGHkR9Xyak+frKzDUZnm1M64brjAq43NJz0m78z8El3paHGSAD+3JFjQ+LN/nc27 aFqOSSwTcR/R4Ff7J6b75PH0raSlCaewE8WvCXVUIrIpTefdJHZNCoHIkuYvmZIw3oaTXdmuU+db bAQBg5OdRRxIh69nPEAO0DN+mkwAJ/drimtuwbEKFx/jrSpi4teefHcGC0w/3GJa0W0giKn3X2H+ 02mVsQg07pJOVJvIdUIZpfRZ4J3tB0MckDlWMQtYs3ulEtaG1S4tjaQ4oo7gwyW09/aD2kjSmUU3 blFSW/4r3N4fKTPXkgBJPToXjYvVks1wdwlFvsF0UEvsztgnWQc75/mSY27jwLmirZeGGdecHuGR 2VtEd8nER3G9f9dw9nVQdjGkwljgfL00SkdOH5xiG+pz3ZQjZLqsmb0SrtSZS+iUwuEQV36x2CYg VMACcXSC/oGAL5yTuGbAYo/gj+bSb/RqIZzKyUV9VEz1FIex0xdTxuxd4NoIy+JQzFer7i22JdP0 eonNvcBdegNKxsGp9VkLPzBWZrQbVB5XtVr61uPZA70pJh5CqiOAkGBmj27ciql4ll6v6h0U187F WJstRqmxcOWHxJHbqH92GAMfjIKmRAlLmYpHBYy29QcHwA9+ns4RfV16n/nKfaefaMq5pN+qRdxw +d5iap5sxy5V6neZV6N1k8SqbOcuU+LfT2wRLMTfff1DAww+uMKrHbD+//0cRi8T31S8WWz07grd 6f0iUKSJLO9yUhGuBdL73TQwMRsvmWxPWIXwQ5YXSuR78KjV7/kxCbcYAW9jdojc4/VTbnmQOElU BM8ab0H2LlRJMK8cXS6hCQldNWdkFgY1Hw87SGVywvvFeC11VaLHuM8DRV2C/bglb3GeqeJS81wk YPJrNCVW7QyEFm2z868cFcvzSPhI713eMxzaXA6znJ1OfX5P38CoMu0YkDMWSXeuNFNZvs6HWBbt Bs2+16XJ3zY1d+m0vnf41idv0JQw35lkFo3osLwBOaN7c9CeZnxBa3fNFj/d1oVJuZ53oVYrAWr+ W6jsE0ZPjjqWStEXJFv/57DE9UTZoiC7o0JsAa5We36mkzSYwVLxU21R4KN0iRKThf4CefCU/1hb E63uk88NtGuJhQj0lQC1eOOxrY6klePK+ZzKNbnl/xxiU2/nsgbMBndA/kK2aFyIpcz9fGA6fFI3 PVs/bEQo0TkpPPjXsZuLWD4FtTjMF6EIZ77EkcVrIUKomr4qieCMZSsZ2HfEal3O0K2fDnivmk9I 2GWHOqDU7+HXfBfXg9qLw8CKNQeiGD6HkRO32DMCNS1baLZ/tI3nxleLiVisslqwavnoVuLzAAtl eoIJ4VU51LMTUURJ4GL77Ox49ZMc2GqMH6ksak/tcwpe+nnJ8jmrFPexnLr+TFpkY69BxkV2FM7e cJqzJnby2AVpW79DmXBkpYZparsYtxsx/S3uA2pa1+vzeUcjEFUgn3c0RB3uJpzlHtxIyaP9TEQv S+IE3fkdjp/c1KWsgzkx6JLDRX8GXzJUBtUUDUDQ89jfoBLybwVBkPmsf2R/WkiW6U4pcqZpq8XY HTrTwt/oUGrMGsV9U7KgAj/9JKBHx5Ow9PdYxd0k2Z5By4rUoBQvkzv8uMuWxSlWuHHt//7Nc9Zt odU9UJjeIbgttX7y+iii4iChMeICX2VK3XJceiTD9BZnJFsOwB0h5bsa0653KkHI38z60ZE3mR50 y1K9QXo0Kn+QJiKcqzwankHZjk7PoiZzjzQz8Hyq2NepKWerLMs92WGTIL8r3nAufX37+G1W0g7E qomeJvvWjK0bq3IXmEOWbjYvaJDt5DQfFaUjyTAbRGEkuF6xHRiay4K6D/WvnMHK2B3wK1L89EO5 gfXYOmCVkSoNC46jnM0/UHF5h0U6tABdle96WsA9TbH6xyUlnX9M8Smf2fQeKgR/V5379a1TgtKr I+ufiQmdtFOlzvGbRRCpQLr0CAGmDhHZV8THmRlnPwE+TgPlhWeXRYh39SCQCJbU1ZK1k6ZcxcX0 iEuqwKytVlLSa82rqIYRdT9kFIHos7IcyD2KbrbmshRUreOgr6I6p4i/rafPeCj+FY77GmccE07C e5lmLqEjexwpMAs8fnWcehJ7oIsOpmn6LNlgrHzRukuOL7LvBVE1KLCNIGKvlRUdt8aAmkim/UFF H4aEydWxL2e0vlhPnq+16E3N16SYHqhzSZ5MhRQYcHzkfo3yG1tid9P6hNTFFk3XQVs5NGF0ONdR 1J+IqSmu/Cmh3Bl3112idyf9rhVA+nj4hw8pRuaj0Fk0iBo2Jvv5O/VxT0bCoDp8F5hP+u+4AE61 rsvkcSCUdAMJalQ0cxs7zy45vl6sHjvgll5/PZKnjlryOWB6B0rXepXCoRwGzNd9OJ4S8zp2joYt kCdL2QqqNzG0mbCCvd6hn0CTUUQ8UfFBp6jG+r1kukNAw0Xgsnf0/fALGECua9iiOVlS4rrOdGIw gQFs5h2FRmdY/U97qO8vkcvEKFYwdldI/DIeoMs3XSs+Rz2IaiESSGP1o6oDr0tH/A013sOEFf5Q boRweHxilv4Y6NG9sx9wE2b7yhnRYh77cgb2R9QpPf+fwZP1WN+AJeMozUE7EhXZUQGSmwxQNrG7 C8l7+dWiOETUkwGPiD220f0XjiZxHRM1r1VM1wjsdxESX6vIzy2qAJiJXc3cBF1Dw+ecT+QMtBn9 RzDMgOOAKXKYr3SLo7OZ7MSFy8rl7api4Tw5s9zqXJIs93REFwe9mgtrcPUY0SqS+YELr+5N432w SvY4MCyhNIxFC/KnOCbmVuHYRQYitdSqsKoy56EKMCKKL96yU6IuIhWDAbJhH6Wqvz87JzLvh1PW tiPZ7Pi7lRezSbGgFbjQt/CpNtDXYFVKMRR9E5zHEZYCrxC9Hpc+vZT6kOA7mihSF5Z9rBkvee2v nWEp6w4pMmSVsanStCrnS7ihjc12NljHyeJr0Uwt+d0LPVe49FkljImcrimqPhmvqVUtuE87Y18M Q3TTXTzeG16zp787HCD5XrOqRjvJPPYY7BDvJJseHcGe0JGDSxKMvU557rdWEhHIShaHnIrG9/mR Bfv8tZU8gPqoinQe5EOGfGMWsvQdlAbFMoDeb9UID4JatYuAP9QzNAytE3hCw3YKkV0X5+Ytt87c Dfw7M4RFPLK3YD3xwvpigYvzZ9m4ykU1RXGxSgJ6a5+vq3LmlbX0z/A1SG4EIAKhVja5Z82X1MFC QXI7Ct5FW17LUupyS5xigW/UdSIgJoKSRx1hsS2ginibQ1vFkeNjND2BP8PkXkk/Of9xII18690E TLhYXVGlXYXlFisF4dVqqiu79sxxXv7kncmpZjhZXj5WD9EkV2y+8gFt2Cq8Nn1+rzBrLebRrNBU GKhWj4IBXShv+5J9erEPY6LGE7t/dpckfBPG5o5XuvOYkSUv2GDm2YLB7XAxoYGq/0jvBf7M+Am4 9djKrr/VnvZWb4zzQmvh+FaPethuDBsCR5uDoiUYLkdK5nMB+u5H1cDx7BU1lbP1uQKllmCtYuZb tQXfrmmVwAwlx4ZIjggX3+KK04yNhNHhMnUbpbpOj1h3+SFu25eKsl0ppfDyRtGvCN2eYMKhwWRF IFYnMXjTH2j8PDk9mtQt58ZekEviklCVdAVTciG3VQRKhQq6Bm5ynDCESTpVcYfxaGgkOxXkt9TE ZhDi/1VtqyYCh8AN5Ciq5CeAaWzaBgMZPydsI+NURrevFHzO8vcgdYlJQRH5buNbuNWGr9usIX7q TcwRKK8gWEssnrPf9Ot7GLWN2TVZ2nbuLHkHQgw1jiJQyy/B0slmhg2NmpmpUlCQWLcY825P+2vS H6bL9DbI3dho/tg/wtC/lqPEWpKgIFmDNLjK52w7bWV+90uz6IgcpIqLH368VAkrrlF/FMFu4X2D mHGr6yh8b27p0Bib2qcfg6yIsrGJGSp6JKAgZQ9xJ5qAIr0SeqgEaWMvcEdsz7liaR3xxORL2WaH Ccpoc7HnuEzYW+mDLOtBaQnPj16x4oKWdJxZEER7vyhSSFdHZbJrcBtathtWZdkZuIWNE/ri1Yvg 3AMQ4u6FXR42xLFVDigCaj1AOLIirlZA9SMl1KrpUjMpqumCDxGZNpsMEMTuij/xZYb01kHY3HMz fiwdwSoE3EKqUujTv7/r7yL+aIq6fgfxIpT61wAiUEa+ArFPF03Vr0N9IjatgucVcUE83hvlDcRR ZsEWEx+/m3KZ2Mw8HLZUoi+fBW/CnqxHU2zgVfEFfDkYYXFmhgRoBYNZUQGlvUdgheV3f5mSEeox XIdK2Ysdwb0rfMznKcs9ir9cLm60Z7LPBCz8/nxQUYZwdyTaaZ6f56szaKzq8jiFqeJx9YdkD8Q1 Ixf+jUdBA7Ctp2g+QndiZTiRd5vboa1KaHdt3tIlhwRO7lGw4D583ZOo8TdVDLXM4ozntbQIN1lq d1P/XPDUaaibKSjGd88uHG8fyEvfHIUcOkywX7bau/wuDTWskQhFpDUdSrH8cWHfqOdljBKG16FX DxDirzPuyoCnAqes0ITzNEcdcCZBlGPvhwdGu6aRy/EPfVrEp4CGOXRDjLP7cRQfwwRPiohpdATo JWuQtKhdKZ2XPpQDBiH5P1kf5LF2GO2w7Di1Y+RruCrQVlCj+mMfI+0fXnun7K/bpbDkGSUUiH/6 6Li46aivOpg/qT7RH7nLLqTWQ/3l2nUyyMMNrkavmRXHKdP6ltS4dhH7Vjq92CRqUIzzCV8tNqOm nsRlN6pxGk7ODSjdga3QF7MNabiIN5+1KYk0Ckb6loXFEicfCILSFRMYQXKFhPvnsZazq3zC6CAj hh0j3dSiQ2KFdM24zSP58JRVi8r5OBs38PTMiqPaJ1+vNBSgXGc939N+A2AkirHyJTyGR8xVUfJ3 AvfIOPBbZfKUdJJG2LXfDFw11H9noQlqkWpHZkFdp2OrkpEi+Szo6I2D22CWfjd0Fm+ihoP2Z+2y 5QnyPjQTtzsDGhpwfL6kTXYEkou+i9abX8gqYiJ7E1vse/5Pp5HoZ//05DXNW5/HT6lB9TM64Dya rVGfHl6wBEJ7eoEDdg/3hOzxYjs/9wXxQu/8euJqtNNfhuYp99SzsDbjW+Evx71oBcWz3cMDhE1K y0q1cGiPVIZBhD6g380FURB7id0hUt1J8h4QkTkuKFRX/UOVXWJr5UchV2M9r7UB0mhqk4O3JHH9 5TuWVuB2e3Gkifo/l9CuNKqrpKvGth0ncAEMmZbmBDdBoINu+1kMvBCl0p/6dcH0mhIXJ0DX40QW zmrmxlCZvPWOSIXjApm06neVIMUmvhYkthUYS0NVy84VPNZDDr+X9AmsTuJRfWwkPDEIIj8Bw9RV NjtgJq+wCAa1OtpnYCFbUAKzHbz0gAqaPhMkZdj+XYQKEZqwsqWR8TjVqhxQHkeaq+Yrro6II3+e N9ixAJgyJA/qCiTJ+FDKRwE287EhDxm4tjovvISkzlGUyTYTZq5Md1D0kWDVn7EB0C1BZD+CLuPe M5ErWoG11EIFPpqH/M/Xi1ueoeTKh+9HfFtUjBwi9N1jVi0eMgSUTTsAHTMix1yczKOidJWP0hR9 oyypOy5/ExhfJVdNHFOMMfBVlaWkO4R7m8fVP1zxVePRdJCOzu0shD+eXDfvRVctWFcth7yA7MIx K/dn8Zu4i6/QjBEmhrsBHD+R8cFlRe4KsBzlH4z3RyDGDNcRtNTCm8ittjmFvFQYKhEddpIRbnSx Ngbivm77JEzm7ZgvwmiNe60aX9irCzJ6yTrHfXJQefOfQ42rOZUIQ9jhKx9rNhTIIlF2xvl4AnWS 8j5zZzKYEOdcO5FnDI5hm+cqZutM/MpxDCXm8b86gVfsnD2VRU1LDkcC2vPAtAF+hDNbQC//SEmC H4nBubHZjsH2TPJKt8mDU0rhzvhPrEuZNH9JyuSHVLHS+3MaGeIJo5t+Sc/N5TfDlyJPBPnYX7Xe 3X9bNF+pw0rl8u9gtI+hwVHBR8lFTdPzhfW6gUbKirEtc+pXAfT7K/BehSvnmzCWar3g2drmFj0k WvwqzJCSBAOf/ttR5j23JHi03z0vMZcLRc+2LP/IASmlo8q8AYMoHDFWtyY9fB9RYIKZ6JN/NUkR SvGedxO/FokwOpPZxyobmHcp+qv8pLa3pvswmJ74o3NFMiu7d6sFP3GVV8XXKzfMKMRF4kr/yCo2 M5B6A4OugPxDS2rtZiYTaVumNZNe80L5zoCb5QClqSCNjBcQUAScayzLrVZAm9qRs4pMFOlwSh68 drKinFIXY7K2QcP/3vCVaPX0P/czvqHPDzn2j2nuZsBghrEsaSK0VM23162kDFy97KxcvolUftfX Dp1A3DstwFqJ4t+ZWOJRjHHmAQQYTOovY7A5ktvQ4o4cs05Ho0bHnluC9NOhotgPLqCId/JAPnS/ Sd/758EYEGKhS71TCpKztNhSNrlhZuFMhLlrG1upRMjQRqTbjiHnMd5zZrA6aErmfCN/Juhw2zpN W56lCPeNEnN6DKT0ck4vdaHa+54bOmHTNwuVv5GOaeBb5GBRNq+Q+yppvYNNmg8TD+ji8Qjx3ipU ++UBHiCYdmAXNEkRzm5zLluQw3RFOtV6Cd0DAuZEwr1076x6XVjU9HAVArN+g5YKYAcqlHckNJLZ Z1FfJu/luzokh/4YFytymYWDZL4nna2Y8LYvJCI9jqaY0szpjpvCcQzR4NRqzOFD+Nd297xgkbJ3 PZZkmYBWm2G2sLez/GrYRR0a+BrhL478LkvW/e3y2lAIumtgdrPlNi/4x7lASBcK48MvZfl058F5 aA9dtjc79nWaJAJaRKxXoHJ+g3tkAdv2lrjqV0p8KzeW18e7Lh6Eupz6ew0N5Loq6zhVeU+5Xswg AwKImzbJPOdT3AsY3VCDD0rvxb6Ll+JsBsgMuGQlSiWuN0oXuA/FmM9DPqK66jZYC8i5Sfu0stvo WrBFWfU3cijkpQZ3PYgFJOPAD6RrynVAXvrb9x0FwLgXU1d3KMaek+IpObEoUoqw27sRW2ufNid8 +V/phIMpokgQ2J4sj0hgDv0PL/Y9gA1aA0C8tyl5rQcEzHqJWlDTeWSGJiErIM9+vKGeyd+V6A3G qDCFHxAkLs2pAb7nHXmBhwQcjHMKBujXO9KVZWTtHn8Cs4v0SUkfQDY6X1CyV42LfQysRLM2AkMQ 6D4WzMCP2WTqzgHWOWorNfRFjNaEO6kWnMjWaVi3Ii/nWOW9TpJpaldxzcouPHnG0XputDRNOlW8 BurG6CPNDtjnVqwwDktUn3xaVJ9Cu9oO/ELMh6o3uCCuscsvkTvn0XvrssMCR0FHj1HnNODAxOxx UT+s4MULxLWoZsTDEyisn5q5LCzxgGD/YnmqKIu5MTQUllB8m7pDXsZDlGO0vpLdz8USLDTfl3+P Y0S05bJLRjVXcyI8c528uwhO+I6aKoW8AEKrRc+opp8gd59xidBa/Hb7SXthv7bFhgkDJMXJ0o14 nYl9DYU0O4jNmx8j0z94nvOZXIA+UipCXz8IMsKrMAs5t7lrFaN0C1snav/8ML4FFGC2LjD4VhJJ EddPBCALZq4CfxmpGxcT9dRROkj/gcfEJMyBCH4E0fVqMMOromWko6k7ZuKsJJ4BvY8RciRWRJEu 0uJgENaUQU03BNdpfLITxLVT/FyhBnuqMeufEe+f9879jyKtmp8SS72SJv7VBIkM/GgVlCSRpcHC o2Ysq1pOwgac55tD9CGiSeHzfzY88njqdghz0TJj1eoTsfbh1tn6VohfEbdGr5B2WcfFpGwjSamM OWv2MsaGhJp2J6gsdBVO7SYQiW/H5hBjqZ1Gbxkeoy0SKkGIbCacNJkLeNTkZyzvRYM88tm6TfX9 uzcew1+L3etCtnCZvtBwjqdUxmE3bsq0t/QpQXs2HaWYLQom5TgqwrqpGcpn8FhNNAEkaHHvJvLP 1uCUlC0P/ZXzTB27W0x9kb4lhDjNpwlV3otd5Lqch8N5N8yr/89jsmcUzsySwGcuBhq3j28a7IE8 WZi2eDnOsv9DrbH4fK9xQXBo/xhpNUOFYSDlFwhzMahZX7olADmC9NnigFZLdxW1OS3aPNiH0GDt iHIJUblViqpUrIA7OaIwYptI4IhaeWxIp2y6QA0dHasygQIUTUBEilLV0VioXU2NmrFnQKAPkbLX C92gvHDPfSZa279u3otQQ+bwWHIKvgZtdVySDnYOkSorlUpQBxzNNUXh/9icVuRb9Pq+uVxMetyr x5ckgh8KT47CFZNmsLmM9zi8t5P+OhTFcCihrg+5SUZyuNJbnv7WzJsbEm3QEWV/8vTgCD1GRci9 R2qjkgt1UcqA76SNEqJh9wjmfZy7NcYKTNKLiI1mn6jCm/6qKbWgHwS7R8++Hs7e9zvEVMshNI5x zopiDgw3XfWbHwiUeUN4/3JaI0fFop99Z3VTvMC8hH/sXTGp7ehJWt/p5gZQNnbWB/aFfR3tVGf3 y9YucEZl6ndaJPkhdsqIx5ofDa1Mjx+BnGauGG+O5zqRLtlseq1TZx70Sk6v6bTu2Opkg1dbvSSx fdbunVNewMs02HOtcaavtqzNH8hZ+MPQuC0G/Chrw2P0ixyJLIscFRynRlKBkdw6ucM7ZbbO5wVS g93YAXmzMdRoceqVDXHT6k78626i5OV8i73hTERNhNVhHI6v99DSIWMS4zCtIY9knvpz6yYCl41g v6fYXpg9p4KO0Rp+viBuLDS7AzHWfr7DvcpR3hneF/5PHkf1h2w7OSf4dvSy9MJ4abZIQPPN0M36 hucN6uDZT3XWc6ervCkH3dmvWjWCosmJYjRH93ClUXayzKBUjgA4mk8I1Yrqd5Utex8408bpFV/7 W1q4IpDtxUZjuPub0GSeOVJWVsvtE3iW06MUHIO+WgzHMyWfbWys0lr7+c8/avD3k/XbBR2naDhT 9EEmmK+LW0zsrsIYcTd+JcNvBsDUglUXvlN6w6XeH1d5sgB0+9lhtQ+Iv8o6Yc4+Euk23kPLfqQD COLVkSQZ52bMCJPyGRBG7bXkoEFzNTSGYBnVFxyS2pUrY56QAILeKhRIihX6hTS01UhsJmVp8Cns F86ZmMfZm7KdSg1QEOCfAyOL7/D4BahJ+S8V/+Ou3prudojQroWa+0kacCnZr7YP3+pJma8zTLA9 qq7o8eWbTxnAx54vEUqcTg+TxLl6/Mv3HjT+bX96zrPxK1pEshJLq6IuVqeJ6vPQxf2skpDfSfX8 fh7YpVMpLVR+Hx1YSyYWUowvYo5JfGnkoJPrhQf8CaJF9dVfj81z4JadAdqbvvrtLb7kGbShzJAq RgA7qCQBRdFKwQHvPZq19wBFg80yexO8D0dfjb8y+q8MgnANh2uNAx3JduUaB2EIlv5MHKVvdh0y v09JWXmeHoD0LwNpCqQMIEZ2MaxG6nJJ/MBX8ZCpci2xssMyr1HlrqcEZQ1bFT4hkP+XD3ujGpqx Dyudvb6on+OxXqffKXjgeYoUdPFFQf0Vk5zAHfLhkN0TzxzGn+GBsFrRn4qTqpsSie0BXrrqWpFv zvA5qzWWHWCo3hx+Ts2P3Y1iJjgGu3yEv+YmhCeveasPD0jLKo610+b/SY1HrX536iXZeaXum8L9 xy2PVAp7dbs68G4wvTfsuhd5ZHCI89ouYs6v+Abd7nSO0Xolq8tnTZdSVEyhVml7Pe4Yuvr8BHXy /1VJTKl5qKiBi8d/oDzynUWFEToIA7XpkXWt+Mp6dQ4MY2fYtGy1yXtE0+kOcE3UwyKbLB9jU67g +QThsEnXGP7y7q3t9uTOeoBAA9W9jNpuiFtFo94St3XZqZ+DsYTFxOYxXIJoNFWRhT+ba/3qYeEV psrnVPFBqwx0STn8lGim/NKGErq81vVnEvJNcgE60vy8RQHyjYT4y9opWuudtUrZxcm4M+avP+AX laQz53YToK058nHuZ9rNXt5zaBbR5Yvjx3WKP5AMSvb+cJepFCBV30Rw1BOPm6af1pC0/zkm3+40 ziNvIQPmQ9P0/qQbXomIk6XsUhyzWMFPcH+WGboa0QIMX11ZpXtKjLudjG+fzv2b5xtPkwFEX4EI ek2q/ERtVPCJBb/zwX5eLLqxtqa5ro8yPn3uSHr9Be3YLcvZ8wfedF0vccrurChULcKOQ8gyTQja KWfPNRY58hy1yhHipQD7mqoKQvpVLFlx2PyDwnJ564yaVyEO8tCCgyIkyTGjmWK8IcDrU4MdbFhH 32ofB8/ZxAbEApGu+mCqyI68KdeFlXGJknazveX2ekEK34RHibsywsLpjkDqcvmAEl/Z7/1aRsjT 2r7cZQWiPgmwgY/ihWbH01usfjhWevxaogOOC35dyD/7Ow0wkSkCDI/dWN4Fz8dsFhelG5N1Hqc/ tBBLaUYF4FGgV58nG7WCEBcb1tjV7YG36GaLT9mrjm1n0p+rR/1PboGpjSCyt+ssmUGBwnZLdDrz hj122h0bxE6O4P5iPf3EJY3ULxh6GEv3fW3BcXBUx+df+LOdMhI8GpSZX1VlUEZHYKsi+TqmA/Yp Ma6lGBjBS/ruxw009lAdefdEDaxkf0+GBNZMx9SDq1gBpOhM7d7j0AauRKeJKk4zjn+lH/wkoBqq eeL8+vCjE1Z3DHntG1kg/AO8qDOpK0rQgXEVX89x9QKLrnvtCxp0kRkRQusLbrllf754g0gmXrFm yVv3McwjK5SO86tgSmJhBO7M2wg2yK8gukR/3i5brAnzB0EevrHkXK94B6Cofp3PqK9JnrDHodRX Hxg5LdyConmvA8pE7dMCc97Af1szPpn8RWuvRQglhF3KLBReiMct1pgBjna4xHAAkUXw+u8PHBmJ 4dWTeNXmcg1/QY+MdJZIqqJbfUZYrozD7yFCXJC6/Q9IRksACBbPUf/vxTZkYq8fuhSchazBv+Yu VBwm17r1QwzhaGyJM3+JbVAK1d4lPKY5rUjOaomUAdPoI6RkB6TP+oJk6RT7M9/kugDvjVKDV6HR U8mtBzTlkyPd9JDDEMUsKchZlFToSuOoaIxxJnK+rz6f5wAxZTsvYFOgVH2U3x/tsYoNwTb1H1Tt grS9CJ9GexKjizFAAKInjoXiLwLl0WWfGV3APJrQ6OKKwIP+BRSbfeQJpWHFQWZeYcf0OAiFXDiz hQZd9KUkWdPXv80plLIjBN/RVceNfwg5Es3WLTi2jjLtFLiPLV5OE0HD+w1fHWryFivkcepsoSS8 wa7OfkPzzmjN2Z7RtBtiryY51sAYFK2Fl5/0hzjLaA5cyiqxTwK7I6iN3pjEJWa6g4WKp6GsuQoH 7B3VKxQt25DyoFuMlHl/uqWx/gCvj862R8Kk3bVUgdLzR8Mgkzs/B0dCAwovi6cZbOL/OU/yv3Ue RJ2sdJFdSWAWt7adWX4gfavwWpG9hjjI4EsbMpKG+GvbcrW5BfMQbWktD2p4728V673gb2TbLw6Q vNoBos+XPpQglRhcLaYHdtRqVgn33uzwqTtTzpLxWTLM+ehJVx1r8Hl8i7L7CqbNFXpRpLDWtbnp L3fYSgiPx7y3S3GUaV3Cg4QsaOZLMJxH8+BIOSwRU2eRS1ob6DP3QloeyESjAUi2OFX9g4NSurW3 gX88jm/vKSMhDxCNaf0gMUOLR11l97fc1HqpQBUSNkj5LPizXlXsYB5bnJ6R/jk5N20jjy421mfh x/x0eATdLgKhvrMCLjqzujuMfQ7Fu2TEOH+t6UpBGkGu7Ebmlkz5UF9V3pkuittfO4YzYK9Zkc2R XKe/EKTjRe0VEc97GjDlZDsV3tXTADEOhCvAVSPYdxksG8qM09On3tBs9HR31ivGLRCmkux4eOvi qf+w97mXRZyLu3ss4bqpowUu8qPM3JdCeEBxer+uImuP7D7YwOlAWUhof4waotXeAAuoX1PTuv1e lunJrjbLZWWm3pgtAPcdwprHpYuxTKvdz6ElUmyz2q27Z/ppGkie4DyF3H8PmYc0vL/PYMJ+TeCO Bgk3J0/LkIFk5OEPoLJ9u5/eElSSGZa/LhIuQVy/2yF7LVoEJXnkDRKela9juXENJHlicDbjVnXQ HhxNgfjz/kbq8Tf3Ggfe/Md2RY3nGeejN4vvxlqdOLsn3uarOFz3tH4yQDWvg25Unzfj9Yk83qGZ c75xsz99i1299Dil+grLuKK9kJh3m3G50S4O0tBtYikNTtycIcmcVYvMNsiJ5Z+6GPYcoktoIqQD hDw8qVldofa9Q913XlvgCZzYIiFJZKZlhwFvehJAIXrnDb/dMpfBAxV8bO2IsI8pljnsN9x20dHy JgdaI1b2B1T9CrDGJHo+FakF/cZQpgC3bKibmeA4KZnnvypseu6NMtJZpJbkY+oFDn/nsOFRkyX2 +iJNW0bohopH7q5o2LikhHqfVnMl4QqSvCpluYwTar/qZyWQTRFIDZT/4bbnPjBbQCyotrJ6uYhB q7jmCKQAW3UvMY6VCQ6jCxGvhU3rmhLqBZy7fQ0SEan4oIL0WH6cnpvOsaFLuPralkr4a/bfBGxY lmRBmrHkNIoW2XbG6c3YYdyqZNkwr7heEiRv1YLHhNqFBTHJJq56bsdvxRetdsrQcXiLMlMFTbFr 4eOMOM5mQf2M4MkDYxpC2kNl0etErFyFVRrbwRUD0jk7HRjT29P32qwuiGZL9RhGoDYpFe2WPNnC 0KV206VPdSKovbbq8JfySuOhKlrEAOS+VmBcL5SjzxAicGZ1WCmictIpKVqivDD50DNW4N5IxqxF CAE6LszqkP9/AxBAewQeS5qYs5kITWdxrmce07tzt4LMn2EHgdpl28oyX+ZwSMdrg+383GEVowF8 mGEuivX2wEg99Gv1Ow/yn5cE2IEvF3GKL7wgOqIy2iwGEPLLWlxy6KceWAzYjQOrKV1Mn1YoU+Hg UurFoSVJhuD9TjxN/6P3EXXg2yc9TN4rNbmHjuwoCDIy9H2wPJ5O3j35ViBpoA3VT5eY1PVOXYA/ NFBHaJVKsqReNFKKTtHxeYjDKPPLbyZEBVG2jVbcbBaG/uChcjWwq4eav+9QxBrysvFke6rPs0tt jGkVCQKycAZPTHAF2r+qTW/c9I4y4JofpA7NQQhJA1yImvVUfphFNXSesgOqzMcOPZ3e2X5j9+vj iFYPXL6aWLHbcg8gCjVxZGjAXL8LPqTa8Vlf/K2jTxyk1OhPtr0vxJmHcRwe8W+vFIa0aweoRbC7 W79IkbpOO20w3han87J034W6TeGuD+Qfcp81oqL6lyCTKktaCs8LURmnJz266mFr817+PNfxMmxJ rguMPd24R81Cg9Ol7T7gex73/2B619Saa116pr3IZU9s+jta3o8Sq2mBC651be0w1hTUz+jdJocv 3FynV/c7/brDEvYnziErSo7u6HRWFVcqHcdiUVCiEH6JmSciT6yAUZUz2IXl8pGAYMN98pMVz9FI fk/i0bUC30tQUq/ai7dbNwILZliBdmmj8l37wSVDNNVmWI0lSiC+kJ/bX/nn/eEPRzc8xxz7bbXl aqbki2L55XDmjAjRKEhf29BqimNpIwi+cqo/32kV6PXUbk8LFnQz13EHhimmcV1dPFuxBkp5/yi5 nj1WnJnor6Q16SPxHxm8CEimWgypMwcHvX7Te+RyFunL+7010SFXY5FMxr+27XaePhyLBT5LGgfD Em0TNoj7SVgcYbvGhTWBwLgKt92HK+32ENgIYv/mWbfSicQK8LSXyXSqQa6Zppb6ABBThhaOJdOA TRcPHQ3iGIJ8GY6g0PROiO8MK/iNPOT3D7hrXR0PIkIQVvsoxzJCEnhrzWpDbfXqfGfGgEVqU002 gMgGZCh4w1L/brqRukgtw/dxmKHg8pCyoAvgWiMadHX/JEbQ27TikYKD7dhiJ0ZjCtd/JvnhORRC g/FKg4HETjjDaIUqVY4ccvDjMWnQdlvb24ofRL/Bfh0pwNJeLV6B3+Wxc8PtGyApGkxhBN51z66B Lb9GS4LWKnniC+85f35zi2oRpCNt5lQ/Jt/2g9gIlkLb5kv9rucKhsZSh+GWQMelwI4kgx16r6og 8OWpbXzoPDdBtzBKy0I7VgLMmnYmEL7NJn0pdBJ0DkHGBDu57+a4004DheOvq565+BXQBH0nc9fE T9kakjG+uJPbTQBXkKn3R85r/HvZvFuKY1ZTptF7IpUbmCX1M37R0tB/7WvJQvGEFUGNjQnwB7I3 l/GXu+dNXRvOzZGvtxUEiwuk37BwU0YpP2tPruEQAT5kjTgIFuGLmPlsP9n2aGTFKNc2kn2ik31j gL3yb/sI4SX6VbZMw6FaaG0z8Ki9JtcgFVlDlUeDNugEePDJkUipCYCHiRA7tdNDdUjFEDootpH/ b2N81NcjQ8SMyYsZjSn4OnootSBLHrZw/P/1r2py3qI8Zlea4i+U6Ab/PkVqBYuzEOlTcGanizDl 8/Bd6jjrM6Ng4MZ3/4N65cUt44rFZ4Lw91b61UOP6T6zIdIHYEiXZ1cSpXDLvOCztfwjYrdSzOiB A/aUHC6cPzQ6PmzW1gKn7EUcgsWsgVK66qjGPzS3LYFTS36qVJWdj2pNlzHjuFbcWAB4nVZCbV6X OMK4TW/buWoFAGmEoADBPqyZkRWQVRs5I0XPMR0biAM9sLxq2iIRK8E7LEUnYmEHSjsoZ37JlGpm KDLrgDtDYid15alWYqYwRgaWQfeOa/XRQTnSn79AfQgaP4Egt4MZiuPJx3XJpbmtuyPmceFL79n6 CZ+FZ1YAP60vvMLQgCrehegWMaA0ZeIFdKYYMPSqq7GpE7QUvGe6BaNLBjMMddjpGeb6MAW+toAT AhWotX6wgBG/S1Bh1HdZY5eAVQD72vNyOBZfXf26pykE59fqrmYuqUnavijshGAP8KwpoggzqCx5 BrssWuy6IGgnd08/0K2aM7srldMB3jdeKchVMgmPceVhkxmArpZk4F3Ql+mjKQQJD3poCLoyKqGv TAHifadiD7H/h7WAZZKK6Wb/KQW1jrnXDRv0U3wc7+prDPLm6KOY6fYPcJhx7dWP3wlz/PQpRUyy Nz3eWWrdfZQna3212OTsHQWil/eVOfS1htLSHkfgHzdVE9N0eJoRBBCcAQJtjg8TqGZ6Sy5PTfde bnJy6fTgqA3uxYOQheajkDcvZfLx2Ri/NY25tMTrXz4vdz4s2+ysaZiAd/WjCk+rJANK1ab9CEoI oW9iyn+eI3vEhkv5hKIrcKT4+gsmFKjKSVoIpIf8pXDs2fFAuMOrU4KNNK+QzIkbu11N3dGHj2oR VW1DFHWiK8U3rCUukHFpwpR2sRGmA8yDKS83ZOovK+AcmeK6ZDpBEIGNfKVjCHXqarjB3JESRO8u MlAdVFUJsHKN4Mgi5QHefVtRaL+M10zLXxLfiO4ggUzUr+PFpAEfgDv1ipFjr5WRsJXAiZ8Uz3Vn f3gND4HPHCSf1uD0ilTDGSqf4HgvXQm3Hg2q5zpjW+zlwSSnd42WZdMtVO1ML4gjtETveDFXOaMw MVpAeu9USrwQX891UpywDR46hfcupIQ21HSXgC67PHk0Qk4U6Ke7DojVQYkacJ6oawVS1b69T9Eq mjsjXnS2VmJL9mFtF5ozFGVrtMZSymHs21ii0W7iOyqLAfxUP4zXVydMN4FKssbxU/2ctp3epvRR 0xXOLzbfrZnowoFuWb9vzAtaRJoITRLDYKYbU/BHJQDIG0uLvk5nOHk2urOJqp4+SGy+ezMzwK13 zVpyBD2wbEGFpPLdaOwR9w01wxuASOtj0Zdc2aovEvz4/WQyJrl+jbBh2Ax/bn/a5oBQqvaAOMRg vwfjuhL020OzIOH8KrnTHUvch1iYMSFVB51d/Qfbrbs9Hu6w37JxB3gMK7hLExKOTopl/qEKCLf5 0H8bhiA+BG3Q0J/apPfLwVXqxYkoRmws1IjJy6Ghmbufhr+mEl8CE7esIE4rbfptRZm3F2SoTwJ5 fKPE/rIzNp+ledIi555TCkNpPKUCrqc8Yd/RojfX/mpRwAdaa+t9AUZM7qD3X2umLjUIVVY4r/Cz iO5r6UwYMYvTtAKHf62M6SsJwLvsd5ycVWVs5p5kTlTAaKXekXH0iW/7r9haiD5GOMrOIwLV9JF9 pHb3nNcrZMp0mw9ItL3WQsuueh4PzNzvmX+JV0Qnsq7Si/aWO8H6wDhL0zhNc4wvpmVEndtbrn9J BHVUcKha1shem/zQI7LaaDJOgKli2xERfm0zedKQTHBF+YwwbqR4OvELW47Sos37W03wUJQIcBWF 146HAS5505JWUJ6hqCAKZ/baS+VQ6p6tJhzjhKQe8a+ST3T67zbm7d+wxQVYeHe0y13gRH2u5GQf 9Smwa9FKObleSY61/qfwNYR2eMHtBeewt1Sy7Q5RZiHf5Qoj1GUD2negXU38Gxfx1a5XkrESV/CE WOTYDVg1AqZ+rs8IqAubf4iWx3oKbXf7WnoGK0HG/xMwXDcAwl4MndbiplxizIbaIhyJvkF2aY3y 1LJSPIiI5wmvX0HdltqvjV13eSpUBJ/5bwCoeXW9j8Vp5rMk1/F3oZYc6QOGPy5jZG4phV6ZQA4r c3zLmrE600fdX7JylZDQgls9NU1nQGH3cAOU/uDYJRlD2wf3GvJUs9aXQJqwxZIGmRrEgy2lBGLM pYr9nqHiqiZ22DelXRyeuTFfV390Jp808kmvNA7F9cpkEmiiby9MMo7VyET4qWHm52XBwQ/28lp8 Dle5oJOfbQgAvb6mRmMv4DDjHNZCRJN+9EEkVvliwEX9Q/uAoyklgif1jcuUqWKa6SOUsSGxop5C iMnArB1HsoMYX0Dm6mWw8DXctLXb1+UJIBf0mUBESNz6lzoWRtCq7T4EUhjq8r6i01WyNYEYqt7l LjN9jhmlZkrWdS3oq3GV3ZqZNT2xGfxK/viUdx5RFJFSsNPN4L6nvrIFlhdgnKDsinZW3382hTy8 GCZ/8tS5FJTHoOJVN8WC0xOTLUqQXggJudidgYUt776Ea3VxQC9gUOKWjyEjJef0z2Qvxim+r3RY 2xs/VYB/NJlVC8DtBgYqPeJqY2A9XcXMrlULQ3IxT+4VXOdMV4OVgYuAH49EXv7rtqilo3Ab345T Boy8Al2XBpF/delTw8jGFyuhGdanUdIZ+8ydTN54CoBTPJdGy0SvoRSKA3yeqI/9n9ZFkiNNnxHD z2YJngp0wq6onMRwUxjWGPUMaODI8Q39GXK7pG11/G7xv88CgkVWkXqzy9tSrMDlSD6pejWmaqLc qUNT78Vq8vZgrfxOF4QeHOITt0i0KkGO93ca0MDc4bJ61/dGCuStipzlCnPA1cdZOInCBsH5XJg4 xpJL80uGWzvwzk4lATA6+xfzEUfAUROQ1ZXoFTXG/79MH6u2yGg3rP7xB3Mi/0kj/NtjeQNEYFIt Fyzj1qaRkYNCF3rdB9vuxY3G4EcwG/MrPCTxgS+ilE+D7K8lXaPiuQl7PguH/RcZL7NsXKmJ+SAu REUbsMSTT6O1ZxdyHzVPiXBoZFrtTzVJU1C+pfolkQzM2Mg3Rv7EF9eZ7mP1apui4j/gOsru5d4C iY+RhtDWS4Yy2HXPkjMQQQtbTBzJDGABgTD/+H5W1YQlbjEue0tvuboRnVzJjHv5FN51YmxuwI4k sjc/NSwPPlXAgL+W47BcwtDF62fjuSULBDZoUWtj4Iy9HO/D35m8NYEYjRXWfUYlAS8we9gpkS36 rXTK2bH/xZf0LInObdWGDiCviHXdf/0styRl8ubDoVTvgR3wUwbSTPQXqdiDofUGyE2TKeQ/cQsf 3gt8pzIaKmbao1vfRqkqXqI5+GNnkDs3Nm1EWk5z/QghDYMM9m+PISOp/c7LGGppRFcu9oWuQQMH 3CDurosFAgkEDBLSmSPIi6xsUgk1EAc3DR/sAYrzxyK0zbFnhiuO2D0LETjHvE95Ncn6nokFybcW fR4M1kuMSL52oXFt/EwgWLkn4f0alllV0GSx6cCEuLDAu7gkHZtb2FAwd1+3GP33DbshPjAyLZYi AB4GqZlNumDXDhCV/dh8YGXBBBEslAAw0JxQ1u0awl8n7mgXKOrDXt4YBKsS5WYa1QFE28gMNxJ4 r0VBq6/26o4JSvvBqzVZL7r+8ydYr/6MiX7nrA7r7gPbN+lqPqlpeI6fqqzSumeFWAsM5Fgsv32J 5/4BNRIUTtGuJcIBzNtEzlGyjeNGsWZssmuqy7mY6zn4TP9b6WzVKw+1kLVDiydAvU+/7EkQp2i3 O9xQxtLqD4QU6mkB/VnR2U+cxBvUBhEF5QhRl6WeT1m2pai32JLgc2mfwrDebB94/GEm4S5uUwb9 NoA25k4ISvU29e73VJ+usgXI3p9grKFLDf90MWerxk+xXDWHPWFQPtrl/n0qL8l92gjd85ILZH8B sT9gaPumkvrnwZTwMKW5n3fuClfMJnkVKni2u1+tcm/5IVAsthWA3InAb6tWIN/+6Hyc7tB0jzwV QIlCOOj7JlBrNgZrvdE+5H3bnn8gHNE5Em2rASInIiqoYZdJBqt4x2BBZXtjoUpv5Yy7/kUQd4c2 6p8H/tqiNJs+SaCXX4t0J8uI+8OKYVvSKa0csGjKXU/2miMQff5tJiX5k73+/FXtQ1m7t/TzupW4 zh4qLEiWdSq/b8q4xlYyj2EtNg91WBdWIqnFvzFEzMLRM9QN2mDLX4B3Sjq/qhyGHB6mOUwqOSyy +MgBRs/zr/RD10ubl2mY9Tl6ROoVnAlkq2tE2aWXENr7lrGpdEDwVvrkks1MlBX3vv/gigwitEd3 o2Azbv4W7fiNkzK3L64ixl6AhX0ZwadCUPHyM67Oc8vfNkxXt4+2o12BZ6sgYIdiDYsSYqHL78xm r4eyq6xVbnlM9Gn6XqSlNrI0mDmSbQBHhBzn8qd+qROl+kqjVUHXaOuYXGQ3rpvUxTqtegdB81dF QmclzXo2QA4tcZ4M5Om0dsPzVCz9uie4cZ2wFT/dEkHgelthJD4G+lb7bYTuV3AdFUWe3B+mO36o 0PcJYEQCCSEFa7IiTmuY1CkgIdVlyutnxubyyS5JTbCg5P0I1jjGbApIHeKvZhUrWpAYm1aK5iz4 Dsu2w9iyDWELzgiwmmV6YLx0itfQXo2K6xgtRfMnqSSfaq9eG4co2o/VbB5dSMTUE3y5+OVAM9JQ DQEFp4epMeyXMLcv4D12iP7ovs3MUmlREdi1h0rY+DyGr7Zs1OEtiqSbq0hUU7n4rDL13geRLNGH rNO3uruIBUzyWPWLW2CoWWFBc4ARwRIzHO0eXKaJzTHq0Pw02uAlTSfYySngQygtF9k9bs9i0jqj QooBFbh3/Sk3uWhODB8X2W4zANmKDD29/Yf9L6jYmFG0Wvrvs5lR0xNf3frqW8owxUWCS86Yzhn0 2qR6jJ87xMM+0I1bdlMsUf5+Lv0d5P+GTrhPa7K99QUfwacMiWMJKZEprMWCVcDwctaEscUolUmT TsMSsiHPhWtIGD9QB4IZdAx3JUfX6jXTCx+NMSmLBWznntQlwYdJd2O/Ct7rGOOT2qA0PVpa3VRe 0SRiWUjO3tGt+9Ww7xVklWMgd9QydZVfwGPs9OIDCoXAtWzIL8v+UP8QiX5EqHtEODEQivOhoteN LPU0CfCACBMAsGWeAGrj0twLAbvoDpj+DAR0FirZEnsNw/EuSSmzoa1CYHkv/7xpjpbKEANn4CTX xj+M0iLh0sYZPBfsLputJZHaw6X8YVg8nI/bQ/UHfPxIS/lx6EqYFDJCBylRxmAnQQUPM93mfM+G QaqUUh7D4PlxVm8mD/vNH5XRx2z1oQdwR2J0LV3SXekR+s6aB5+eSFDI2nVuo9G0moIHNEiDm3D4 pJsKI6zhaskYdxBZxH0zzlh8Hq3QNhA5GcKqiY3F7j3doqm9ObusmUyoamX95d2sLDgUtt9pgVEG Yhh5h3//t1ldp4r31poRuYohm8VaeYIovCv4qANqp8Uc1BUTn1ybmZmE/JS8L773TwW1MFMYznxo btNlWg7dro0BA7e+SKBer00vKP/ZruIdRQ0VCWw+L5mn+sTKvcIQNK+Y49gJhXEPtToTuAQnQEfV eJ1uKDBAvgW6nTtAyNLyXI5/RWnQhRAUgGFDh3QSMfQTR0RKrcMRPCmq5lLy3SuhEoNYtyJeKW1a iOFfF5vA1pS7lVaLtbtgWep+2cHsJfnPE5a/XvGfRMAe2OpcQFhxDyPaWXW4CGsZiPEEwT57DhNf OhGRMjKqdCMDumbDBhO9iHJokQlsYL8gBSvlgsCgMT8oCsNTomftL2FheMroAQccq8DF+1HJI0++ wJ65H+0TsaizUronh2ZX3a2uK06c23Eph3i9dx3NxAGr/NJpLzt6lRfqyEBBENxiRYpY2vokXZK/ 6OoNcl/n5SUbRrErP9SnG04kjBVJGghg/nsUt+PfQsbO3BH9pWTyUQYwOAesrjG3asKuCjV+xy7X tn5YjGBAfB50BphbLgq9tBuHmiFCVdWMcLNHIL9JmmiiJhO23cHTRSh6KlrW+GAakx9nC+7XtRnd BHo88Dwsn4Sq+jeZmOYbM+puG1lfIxOUS1BD+/wai2e29bBO+2fVNR9oBAAl1n05gOm+Gx8B14gT 3X2LC2IpHsuXxtoG37Uq5WkrTqxOLBK+Bia5uUQjfPE3CXbRypb7tt+A1FJ7WDu7qAoNgBqvA6ds 9aLR99mTeYEyk04VxV5DXyJ4zu67D1UKQGAD/d+8ULtNBRyUulSRQuO/VvdZyZlxOZHurCKqlfz9 BuD4irLpZzvUnbSkv0eV8ICbvkWJPBGS4UPiQwliVK/LNfLFd5b2gwY93rzLzNES2wv8x9ED5wPS PIGtLaayUkXeQOJhBurYQ9FcJOv0GFxIOiub6UR1eT47ia/DQnJtMK0cjuf9ytKWy7vMQEuRj7b3 E6vQ2gayIoyOI1b++9CzNDBZoGJSDD3rVcnQZAarOoh1hgo1hre78yDn1b1swxjgIQELu4pjG9SH Qke6Px9rPQNfjSympt6lcfX6aV5DHU/9owDYX1hUoivrQu46huJRNM9IuIvFyh+zeR+LyospRUsI 1vG37QDhUT1O6GoCKvMrfWHySk72Fif8I3n20kUOGtZBQE+XO8lzOzvn2wy+yShS9Sf3PNDocAeI 5Ndh3f6swUXRroNWpVz47lo1YQKG2rWKV191a5BxZQkMKkmt7ZCwsEZpma1WZ8/ZAqDZ3cdCrhT9 mE9IM3DjpMMODUXn01eTRpo4IrdQ1StaIUZqM9iTZT+dy6arSMS7tLHd/M362iMqKYieUzupQrky jGr5X87AS+qjBYUuT5CPwntH7ivHavPt6MhUNFHrQOgesUcnoaoUBS1CfY+fwNLYNbtuDO9j07Gt vfpk0H86qoGODJhi/nwWLqY4eRlH34DxrDMUfm9OgbdxHUY7CwT0L5cDTy6NJYvlqzyktJ77Umn6 V83emxaZ4akwYyshh/W2BEjq45B/cFywM36SER6SFHTjqOxTKxI6AmfI2DoLILD8Sy6SPLyJB7MO nCR9qS4s0yNRCdaTZafIMCxRG40Dc2kcOkUp70NQ7X4lx0k+nEfVlSL2Xni6ykXkBUc0l/jtLZ4B CnQlLPWKp0zTq5XHWSIgxHQQC/+2YLPBumkrCdM6EJvwEMdVYaiOo1/rYvPevaACRorvxQwpT5II J9pyePd7/xOZrl7oKv1dNNS+WJ9TLLGKcWtaHczKmFN48CnJt4z2dfXAuahnT4xjdIRgrK9/wbO4 m/9VidpscKh3WFFKTfYxnyyGmmzFr2VJF1fgu596lo8QToiqI52iWaiBVS+fAjfF09ZST51AkZmH IrlDoUU63XbAAI0PNZJq8dqn018k7TGlT7nV55bYY3SuX+JP4x88PVXTJav/0S/czc8VOkamMaus FIvNlxWj/ugRwUdYv9sub9qQEMKHll8BMlJtynFCVX0qI/aJfFGBIzn5fSKSCqFLZrhPU0dTRDdC diAi8CDyT+3kUq2IwU785FKJaAQcMm5y02/JZYUDR+0aum9QwWOlUUwtA94+6qTNhotyUrvIGLMD G8u9gE/LMFTLiSGgkqWsSc1VtkOBP+hcWVF/fLRcBEAOyg7GNA7CDR/ZC1wC3o8F6CccJTJn8hN6 NOd/ZF30XrUJuQW1m65pQxUTN2OolZ4QJ02U/n5AMfixq2KuxsQkYUtzP2q+O4g1eiFBpdM5VmgT pRyntVorFKXxiNgYPUx57Y4TJj3LYGO795ei4oY2PKsP/bkXMzDVSQ9jv4SOXWvssbrpK8J7iTiq zgiDOO1YqTVgroBMbrbuEBL2rHijsC63o1eQ+ziDUN8MCVqqLi6aHk2qMvAcUS3d0//CveTEkGoc JNjZHgg6N3Zx+VS6//NsjqptF7oQDublWIqRwPgKGFlRouLmtuZULtchdzc8adQBbh21dK5cXiNB SFaDdtkwqFbjuo1PQ6WDge1jghiC+66vctwyn123h2trS7lBh2v/cW6BR9wvXUzPAzUOk4Xc3ocS j3+OkNlmHeMJB1bkZy8ZbT9jjRm1mEHsxka2BT1EnjwcXE6KScP6TgKpr0OSooueXxOanZPKQNB1 9LJGz1urcBjPjXkORO9e327QnU9/AKY6f8EkNyNwbODJMRyTSyxZYi+cprQ/0JmjM5oueUinMcMf OiyXQ0FKhsldYtZME3aoeRxUsNsbkS8n0lF4WoITLWurR3ik+zq4oBNu7x+CLyGWo5mB8N/AgmvT sK16NquOuSMXbU6hOl2pn0AhAixl0rvIn8i2rnjBjzxnwN2j3SC/ldLtHJvpPadK3jEdbfMJz2vg EpIOaxw0UXb/ileS12QqhR9i1z54ibWXaQD/UJyZ2/DwI8d/Y5O8BEqXec1fhgvSrBEPSF+dB4A0 yhx7+B3nWwHyr2q398Y2tcGr478lbTR8TmygOeiL/VfvK2wpxO7xc1TPQFXNQP0FFVyYmuHsJuvr Pv9yX07ZlWMeBSKp+jTIbIkFqXPtJ+aCWQM47npepu72kXCxZliPBdWeQz31MlT7B7nelE3J0eMs JZ4b3kUnxN68LZSZ3k3USZTqHxJbC4J4nb83ACCMM2MyK+nnTx0/6WJEWy13f4iANZXFPftpxbCi nxjpsCT8VRH36C/9d4ftEtl1SJiZWTE38JxndpoodlN6PodBw3okbVEM0zfFHKr1eun5nDqCBLYO 1ru8pi/wNlCq/sFknJpXHQ5aeYGP+UNPe9xItlMShDgaeiLr7u7C2rYNUSMhA0oJrwChpRpZgDhx wGW9M3p/r7Ck1gMaqXfeOkm4VWq/MVv1CH+mCvkAL1lMkbyZlF2OEeEjJHvNLJEB6DkTlgzqzym4 qo1GrI9d3NR2jpOIba2P9igEl/4ZNjOZYWqpasVHGCPbdq6haFFinfQqY8a1vUKsX53OvkekyjLg Sz50BM7B1tAjv1Q4nWwW2S8mDrcNl/JoMwczgveqgMX+3JM0/4pTT/wNBIPp34bP7vQt9wXijZNN 6M0SRZDuW7xUMreQVlhUTRzR0Eg9NHX/1voLpr5BjoUBmHx0ytEFWOE69nOA/sRD1UrE6y3uu08I MmUEB7jZ03WYktxb/v3X941T6ooPmQ1ouGguSW5Pskp1KyiH6KdKwZYeiqLEyvfy82ifn6sUcUKI rUUxePTrxaQyUplpVkOm6H/kgWo+Cl+GD+15noZob2Pk53l4jH2soMVlNAoj8GDTQh0w329ZLiuy /2RCEja6+1TjYLUqJtw6LqwFjfPOKPYKUV+N3DF/apP5saVuGNmwXwRmLRbguewZefnG3LsyBvz3 fdBZW1pM3A+ENVpIXuIrZiowD45DTcD2GuXXhhrKOqf41jnilhstgHuezxYhmJhSehbn+CxyyPqM uAKyqIBTvuVLabUtGFynywi3rYvEpzbPBRAprcFfZeKLmf/fAtDgw2t4SyCqyjuudjA5wdh3R06z ushOvy/3JKihbrN3Ke4+w+GWHA/BHG7/znF3iNBtey/N8lmJMqNpNOsmPRCMxFY+fcf65KOiWXEr xnH4/Kmf8G37u5Aud2M810w/o+133dm0gYjloIoTtBWXc0xBV/hg3K8YQGB14FmUbPHxTSR+JfD3 4jpPUHTPUVRPNSvGhQURcpxlwVr22U3xdwMnzX2W7sTOWZsuA4LsCVeQTmlWpaq8dEGLgin90mdF 4zZiiu1pW2ljjeNtCKSmyttesrT6v2dT9k+napCpEWz2oxRJXmofdQoPE3kgy0rvA/nZdo2KruxH 0AFmslZ/RJ6iza/BHUDRARC7l5VOyCbf4aHb2IIAHm66W1V/5GZfox1mtIvSgMXRJmjskN6PEbb1 8r2wnf0HtKZEYMscSbkojXerpsVZYx52RICkAtUaUejehqrsIoDFPdFxNQdFagj5e7rTBGanh+ZE 7Iq3wKFOXnNCgFNV//opPAzgTYr4cr/pXDs3XDaILgWhoO3HbeYQGLRrSRUsGloZMTA+wQvJ5RO9 lqGF/e9Oxrt2fQzpJyhfVCMK2ekejCnMdmsBBPWFIcqFLWkreie+5J+1fYLMio0HzvhJe6xfE1zL Lw/VcRe4Y8i1ORDqlXfk6lNBhBSZgs+zmHfLS6oh219qRD2RfgTQKE+SNR9kLcuaStlOrHcnzxg1 y8FKS4qYbjdWAXhZhW6zxEZqQFRJz8Bx0KWDD7rEzvoNFNwX9H2MslfrZitQGNtNqaYQsB7xS38/ B6K/5zyxnKgLjpDvH6nA+zL+SKMfWXIdeZ/i2uYPFSvmadIFFMvBJ5HJKf7fFi5U5/qFkmQ5T7Ke l1tHZWM4i3x7iWKO84h/93K8mumikuzI/LHzJBV1Z6r7hY93R9pFxTquaOucOWx9asWUjr2iHJ28 CraxtgX1oZPUPIE4xA9FMERXmmhtJDirz4KHxwCSrWSq2QRI7I76Avl6PQ1lHJja0wtJ2MZiPjzz jNc2HK6RDTIjDm4/RSaBuuIvXw9KwdyHFTIj67XOSCKr10nQkdEyJF8z2ofUwetHS1RBFLDdVTLG UXwsMbyJFLsJ1QIFdllqhPRABvoE6OOgHn4DdmjhvGGEKOY+8qtQHS8uMzJEGza5UCAt/MCcVBqM JfkaAet68KA8NhxQsPKn6ZYZB8ZLlTAxrfWOrlXstMZTtV2XSnytej5dSBLnf6bI3o2tYHizq6dN /i+9RObExyAD/qfDDNxA8QVFAg4mRgJXaZniPLrR/q6gou8xUtH09dwROm9XV+sPukVfMSumvWuk AXvnx092hfcFBVe0tIgV6vbr+08SNDAJDtYjeAkr8fHAuGmzcIJXTras0qfmRvufr3QiqbbSdQCN 1RWzlwJhoO98NMwDSVBSlUFHjlJMIWEMnnqsQIv3uCP8uTCDInZm8t/jZyv4OCTKILeWqScfBKE/ iNY7yDjdg8LBh4x84rLnHqE9KtH/yJshzQf278kL9Aq+1zw6OQs8nWNXXRb7JN6JK7V8EFtWLTqG l5dw9EOUs9jNjcDKMcCS5cF2r4b/jN0TM9xY4fvCuVtNjlCPu9EflET7108mth0/JCUz/5OfFOJY n2io6esXK+s8G3MAVIWVAsBwY2sEdQy33HnebAbyWHoY13KMZvFSUduWBVdDvFcJq+cGHR+2QPf1 8hrnZiZuhOTBesgeQq5jslyGzcSn8qqE15J+4Bz3NrYECSOqPRgTu9xQ6rM/aA6POlfvr1V5VipN Cwp/ixFhoFUe2rZlclpVQpuTFqUntJVRFgVtv1F9g2/3lstHeWi5S7YjCVQ88mBljDcb/pJg9eEE d9UsWPn8iZCfFaAF5hUErmNFRWEcbGrmCgtXlzi3ShrjAsKE3rXgugXW8Y41zax7/uoyNOZvlXfR gYpXp6l+c1VY7QS03iqDtVgp8HlToGFVyuuA+1/ZndTiHqomLMh+eNYs5nqP0UCSXhNGRusQhVIi VNUByfTq/G42vkg02upDmIoXc5mfhT1v6V48DdmxNbADE2O2mCe2V7wdNKYZgi5C8xMwjN1eZ+mO SFc2IStFhQAxJOPLaxL4AozsfUHVXoxmMJDeu1g+z7goU9GRQ6jgUu0Lq3dOMD5D3UyCDACU7d3N E5xrDQPzdpylx5Dan2/sK/hiF3Acoo8WpD8lDsrsJcC5iDs1wT0bQLxp3j2O2VGdD5k9kAHp4keP viUqc/Boudrnwr6PpvTKmBsyi4lIvC6cKQtljTreRlCptN/2P4KVCZ7jcUL+521ZIJdoCO0iQOV1 FkS06r41a0x1rBhcw34CChrFLC1PIK5ZfTyVfbEhXs9BkOZHHeEX/mj/ywSLS4d8VQ4FA063XBGo 6FYjmCp3U5jdDdKmkhHEv4U2OuGZnBKN2sDePtQ2tgw2rrSRtT8a8lZG4l1yGLa97AANKKGW0LND 5YPGn9qy8/Khello4lnoqeHy91h1wctKd/lCGtfpL2XdgjHPLc9XSuAHKtQ5rWYSF8b9jnwrR00F sVC2M/RpLkG9V7oPKNoU2Js1hPs6fGGSdefts2KVE/1u170hoAqOzObtDHkEN0lDR1PXqMDs9atd rWeKPH62U1PjSQxZo2y0F00TZrBrmDoHMREXtghMzmMPl9P4eLaljeEUhznYkYGJ4nkHBdwVZIIt bM6PCw4zJGMbk7Qozuy+VoT1u+fYfyzscNtWuR5ktVZOASQq8vaLd4/H5hFzDfgKbxta1nHKFXC5 p2KwD0RuD0G7SOEiXXVOop6qc38eNCAqJK4/Ew2+G0rvcLRE6LaUV9tvAZTm2n9WivlyGW5oP4ap VYJldyPVR2l+MOh27iRfrITzGe6Dup+pZmEo+5TA2srN48T4f0cfQ4pQs9u7jAknQcH6Q8AvYdnr RM32m8NI585e83OhoGSfBlrK8Yft0TIw+Oo/edyyhyL8Llp0WGah3mLPmRMpRR32avwCRuiHE8/9 WPCrtKHoNkEMlUCKOvjoyVTS98Fs/TbDdOmJc7SlsD9rVkJjN6mFx3guWGqfpryC3Vixz1BrmZkP wgS0xXM6LJ6N/2/+zzjl09rW7y2/UxxsmWWHh+BJwtWQ9I2aS/dD1X4ajqE5KkjlRgB+NtINFJt+ lEKV/c9sFSCIDRckr3aFJIz0tSCps+HxcGdJ2R3yao4nSRsPncJCZr5l2nx1IbTYNZPlG6WFmlud osDhU+6l7A3EQzEnc6yC8OYrCTmjo1HIkHLqjyNtHSDRBOtZES1+0UufOz3FAq2dqCrA4KGUkIVM GjSkxCx4gw0klqMnupCN5Rrp9VbiicdL5Q2TZnG2f3ZqnPGqNtPzzPTN712wlD0+qTCFqjnWhdI1 QwowUsh4T0ZnsQ5ixnzukMmiSmshDhvsq/F+a14p2XnExvdbIJf2E6upOR80lja3yATeON0VO2pZ otRORueQHhiagkHzXDlEO36rBhlymnJmTlTnaV7CiSiqZh+U+rxa8IlOGQwOXCVti9eM9rkSPfBQ uORSvePlh1FHx9p6FynOrzLLP9FsP2DApH+HcQBwdB+L+QzRL/cOpV59+q7ujD7PxV8Kg79rQPjL 8DLsK7WzV6zZHxlDeZhgbEwSxIVWaJ4zsSQSvx0GO/AgoIZxWxrQeK1mHtzzw7gKfcdKusxRBNxA GxaSsWMd8A3eMg0CE28Y2gahJqGHJDRAoFOthdUTe4Qm3jqli+XkL6+/PMNDNJS5sZro6G0PK5CW zfvyg8eSpfLTlqt1RPseZALEhMB9jYRqRrJ6L+cZs6oXHR9iZWTtI8R+E7s49C6Wocii9V3XkZFT iLJcV+X/bMLdY5y33br4nb2Shw6pIqUPbDGhcacsrV7SKVBfDIayU9tmctyuglhRW7Pm+wRyj/xC FBfu+9fBfvF3gRX4H+Wrxrmi7rszR5kJwqd/nBx9kVW6mRSKjY/7Dd8o6hLr7/KRrERZD5eVo7iT YNviEMLlUWfxQvKUyuKYM4EoeAQEyqP4QKg7fTFGHmQWC64sd2YOrFHp067erpekT8OS/dT6+fek MgVFrffGfWONSDRd3PpJ4QQsJN3RZzjyxJeNrI5AfPPtSZux7npHTy2+oUqk/tU9MZ8xUuvFT8ls a8Y5WPvEAhZ/H0akiIqv+Z7MNtnMDOMyDQt9cxvLgsB+4PoG1kXHmjT9cyrjjSsvDN8INnpJW9BA y/vnXQlJu1ADlQnFDXeNPj/kxSnOcXWBoEaN/ArNCFFI6CSUy9/S566GV3uF+5ta0K+eLj6kY88c eRYF4pC8X99X3Gx379zf2zjZzWGra0cLHIFR4FOM3AfE7Nf/Xt0RsGtPL/TURkbs1cftS09P6ymh VbzUpgnYsyIYD49nYTt4hAd7l0LSErGoHfAYfdpj7ociMam4fFNBcQBX91Rz8VgeFts+XgGY9FlF kOwBF6YzVbfyKJYcwhHs+WX97CMEZjt7dikJintvRmGsZ0L93Yd1KQzPX21xvb/CreVFQFuPv63f ejimCjX6w2KFboJDzhU7RHOE81HMP8Lmo2gaHCXTDAJ4fmtrOU0f1qVjNceauCI1y82u/gw8JDyi gSU/NofXgpFE0kqdSZytdsyUVHRig+mHLVi6BUC0pt/YPepOlsXS2llJTcft1s+2w/ydy4Ql1oQv Mtx0sK0qnsMfIpqgqGZQ3ESMunV5MAiJKKgw4ydB0oT8AHQ+MAT++uCK4X6clcB/ZKhV9va6OJKR Fb0Pkr9ZCskO/edmt/DZARKhx8bBpnhlraB7+9+UGyztFOw8GsXx24bCQlYdLK/yIr1fYDeiIrxY YxFtZHEC2odhCHtEGVdoV5i2v2xA+xWcsJcmsyFOGPztfoHbtW0ND9FmYGfZcIUJ3AiLyUZJIZbK c1G3wH+wOSYHr4BNA5QoJT/kESCzthidd2B9GsGDsFFHUUPT6vbJGa36jma8n5QO/gdLjeVkb86K yC0+Vz7YiDFf0rwycoTvDapfPTUPyaAStY1/ojR3iYXU4Fr9JG89acNkOtsVKhEo1ivBNtU8YC6x ICctfG+M1mYxnhICXkfXAAcHU1uGQqwHpwntiJnwCBYimQrkLZcofOMCZkdPh/sZhr1zaryIo3Ea rIBO+9V1eqYb9YGrT2s3n8ezCt472RlMnTPd/2e2TRTIBU+UTe2kOFhc1m3Xz4WIOkbnB7l+pQ3f qoZEyj3W4Imh7pfYmVJ4RCWsoG4ldOmIuyXX88ebjL10gSgdHKMri5A7SMdh1lcPsfR/V2b27qQf 2i2lNa7PickWTbdTz8FCPgXZ5jq0TpdbeXe2Ygy0C65YxZgkigrqQQjLef7bHroDhz4Mje3tPnt/ ne7glBfL72wy/XR0dNnlLP4lIl3E5rUT7Y4t6TEfLeJvxw3PpxgcyhaFRHrr1Wm63d+Cbu6jFrkk Bt+20/vOtM8OmqteIbcwV3FUt3ICDZhjm5HEp2hSsql+Cpl6UiSoRksNzDITaludVLuteC9mekyp +DFtuiP4eoF5SYeLbo5DTNm9cB7s8BBKHj5Dyn7Hc6HzPrr1NndrFbtLK0vqIoZxxjDP/gqdNL6a j0U+HRWj35XiInKvE8lhlKHjN+w2JoA62fcAgRKeq43BSWKlYbwQkn3Zj8B9yZrsWd5iaflMZDLx FwWvc8ohFL9y0DoK0qzlENqvkHTQDc/8IuaSysNxAjhIKMdjJz4iFWa8baKu2/JjVuq39mKqvd4S ZOlwvPbOPknOuCge1mbpKWtNcokZkZ0p4iHiaOSOl6qtGLsRjBEAZwyGBOzPXlnR//H8s8etG8jX E/TrVVgkIEJzA+JKclGE8O84zBXT+Pu0cGtQt2ayPkz+9DjMStz6JyE6Wzk0np9E9NcYu5wMkRxU +/2sDROQDGsykaIsB6yIwC9xGfrf1leeJCdCXPAVQjq+5Q575JCNIAgngUW7Hg+WgeM1ClpvZyLz XjYHqdcBy5TQ3Zzud2xNyXzGhlYVAi/p8R0NVWfGXSyI4mA+BSTVOZQaj3jXzaUvx9/6bI6U2jK7 LTDVKSStpXnAwvZCucu0JwNUpPmr++qYLvrRIHRONEyzUSagau6rfVQovskSFHViaoLrfOx/TqUJ TLzeG3mPWcXBVBPVc8MIbEpuitp8hjX6iSI4XhPl4aKh5Q+dFoS0rXgKIUwTLQdk6xHUI3QIBsu8 49T2fCk/M64pkEkCXf2eElhHI0kiKGCGLZWQII6Oii23y1Hk+ysBjfn7DcVHoiWfnxyC1O4AKmO2 tb/At2FZfoKumT0WTrFd4GXhbQzbltwO56TM0VI07J5nsIP0hvFMjurBHSI5Cqdk5HO1cxWn1adu ZUQaQzXm4JUXJUih40fDOwOSVdh8c70fU2ViNaOgAbrk6DQfx7M+GMC9IKsq4BctozGxn4dmCLIc jfoAffVAHfd0roqQa46ptFLh5DdSGCpePHad0chE24dar0e4Gi1TKv1ZGK0/JhZZ3Ge05f9n5tZq ikVg+o8//XX+yUHQGtFvrRZZfnL8Q0wjoGoNZ6B4QPs8WdgCqMzczpUUjMQSBNwl4YDJtC2UHYty tb8L8ixJ93zGOMBm2YVKIC5zsGLtM9T316774bnwypjFHmxDYoFg20mRKk55pIUFbnVi+cmXx2OS 4mONRGjVUh8J4BBV53WjR6Vj9B4fewXecLslyHjvd2iwR7J90OYtl29pE243ReMZBLTKmxn/oXc4 qwu+dOt42ZEIluffqjtLihv+z6iPL93hUCB57U7Qwfw1cwqmkOnBsknRbGxa3S4aZmgyg4gHwzz9 Snp0WiwEKFlxl3rl1dHoqIwEdGPRKL/KuZ+5WcBTiQlyhQNgQJJTpkd1KZ2ySpazZHP8Fc1wR5es Zfs+fTBoYYmzNr97Zdlt5itCOmoJgOOXfDLHfgwapOGZWtHa6fQ6+S5xGUjVZpJY33ZHySFSeQzh 8fl7V5bjPodN1cKnhX07m/UXmVRxWrZc+QwyTaljDsjRNrW4c8VmvY3rcxB9E5JMP7Wm8z6Ghwt1 r4h8Tivc4kUG0QdpgDs+MecsvrAtd44KC1TcC2OG3FL7QKHlclqo/9ya2I4CkcEKgH79RlS78hXG 0KuyNULalvUmOdOSa33S/c8gQEP/TNjARRhAgAP9n/e3MeE/C5wV3f/zjzEvKgCXQdRJ8R6NJ3it jNo1ZQLicgAfHAbIY5EryZxvHaeNNQGNNdcWDWcW3Kug8R0Bvbriic0J3ah49tr26vdMhd4vVpxy 9K+DbFjDk/NkUuy3nGWeUmnhLeqBTVuG0JH37Puk+Hpo/bXmAgmha6jH19N5JeJLCEvMV359j2eR THVVJok28/fppaO9vKuqP4NqPS61FGNx09PKdRsoPnW6NYqxFc/NeilPzQ8WjErvXVgrMOCOlkfM OtOjD70pojCXYs0Fq6At1SyGJ7XnotiuS5UgS+sp6YroOiRLFYC2EK5WvW+TdnNbNYdsdb50JHxZ giy8yita9hFfJ1y4nDsVmiTIpXuKSHdHe0mmrS/QpfrtS4bwPD8aoHz6UV3GvLeqLjd7X6lozZLq aEhphPpl/3hGP7XPBjRsZRK+tFP0OrBYO1vH4c1dLR7Ko+m8NTxnAAlTRwZNVtzAcC+f/TLaWukt SC/PBvT8OJ+7THUdBfE/XSPY8yPFyIWGvb5sZjdixx1K0YLAad0/Jb3mz4cPHy8VrTdP3JP3QMHp SFFfmPnXjew6ieaGsfRdNYTNTxzl+SOei+Xzc9pvtimtWvYtZ99QM59ebmEA5LJ4sAFq4ZrmQhpY sG+Iwv7j6EvGQXqU7g5HMQB4MtMd3TqnvGeyBEbPLjWAjRNTM7W/lJxTXfg4OPlqlLHLuuosQVlc GI/s5Bnuy5o5OBu+5gl339HDqIsFUlOAZKUJUSG06UR+7hqmdNbrY7FK8Ne6EFuOFcE91KXHwbBE R/eMmegTAjMNCUROfwhsh4sbuP1splunjG0NlxVgD3KABQ0NVLhOR0QAFkdaYRjYvFpVMErVKD3r LTNqNGMd4IWRy96WysFPth540GT3HJbLSNtu6+YJcNeZnuEy2m19wcrDFYz8atcSgXN+HFkcf8Br XquD9ERykcoNOd/wfUJODrfTYEIwO4wwBsoeb9VUWA4p3zprbRR4vY6ttD1ec/zQZg8mIIjk97rK +WUMfqARIX+vChZ7f9v1+lNHg7dIjt8f/pp9zi4ORJZz6oOAdA9k6okWAxIFQGQUdzBrT7EZvOJu DtqdTRVOKWBPunHnfeGa7l8pSXaKVGNEZJ/GvWm8IB8r6xQqfem9NSCKmLfdlkxCqGbW1ZFHOzkD NhdxxT+xd+6B/K310p9NT0yUKw6Pss7RiA0q6X6EDo6lmTIB1QZly8CATIF3/DrHrMCrA8OgBATK Kp7T+hdQJLSPixeufE2Cah3Ed0UTF7RfYRxpU1LMa0k2opfiLkjVN74mnCSIp09lxejYqD/LahA2 kcRS9vfbiokzDgRRI9O/jifm5Hrk7AE53jP12HUDAZ4+WmQpY0mMXwfU4GhFLg+fMHbIcl1ndwZc W4WUbsJ9nOJLMwpAENF/rAtWaeQZy/4vJfPCSOB80VdmT8+R9nJWkTtBi0nY6bsSEfXlH9eSJU/c IRZ288qRs7VVYANJHTfv8TSGwS4A0TNNrK278I0K7JF8uKx4eVC+TW/hUbmJvNHfbjpb3BjmWx6N 4GbevyCRYgj+LuejoVBnNRV9QK7UZz0EKBLQ82FptBYbwzrDz1V9HQlfx9LeLDoM3gcZgDpHLPqo aq84gpsIWZXwbz+QJgHrv4iP9aGO2kwYOoLQFskPb7CZ97DwisJnARgdJdiY7BC3cNtLlc6jwkkE ffHyGeR0iOq+2YHow2VTgjO3yTG6S3ubxzvXqtmlM2xe5WOHa05iniKBZupt1NT3ydhed3hU+mhO zfgeNqO0wL3USfSBK3A0cTCipRbHydMTHlijfWtGtQAnoIzZtIYKOw0gmBQZ5Br7/HMVqqOiWX5U /sf8l+7djW+lfLHugZ4uL7Y3Cx74ickoNMIBbgPbFhMm+tlQpA0kFYnNz4v5UHB1aP6iZGPaQWhA rat0tpnTx8mX2rMPSYwoFa2021DbAcqywvEuYtEcVvlpwf2RQgW9wzru05q5UqybdnjbQVhZvqNU 5O9gRCB+5zTVYyuhEFc4jgTt3F0HTpSg7Y48rdxv6N8T8feuuWnPiQe8xtlNM1BA9J2xCdoDDnkS 9PsgQb4jI1/KuycdFpQigpBG/ePNfhFLGN7D7QTmln5rnaqIo85HyL1c+6iw9+m3LgW1HuoJIWv+ RX3PFnh18pZNUJ3GvT1u02LLhjp1TehFQUilqJMlddKjH61LTO0WWgl+DvwKk0S7lUj3yZfrTSuI 7k2EJE30/Cr6gyAjZIqdGCR/Bqe2h/eL7evPL5eTreWLhD7BIDZ5eZiXnT5sXXwWVi921gNtlTzb SfywLMQ8YQSzi0vzYkfgfet20EOrwbTgDvRf68hutpZW3RiEFJmnley+b4zolFTfVySMu66h9OIQ 79Q+7hl0C4Er9w6wwtNW5g4Wbm6fTCERugv68EDgROb02O/R+iPV/Y+hegtVlda1wT8DfbSvONNj KG3vZEcJoe1CTYxITHsCFN/p1Ge8NRw/GyQPCtbwDHNkvtBTb0llRf6prCTyN+N4QjXqxjoACVf6 ZG4kKRLobIgc5WhQf1fEhT5K6xbDyoTVnr00hpnc2Wp35RyB3z6ltesTelQybxxiBt1Adpm7n4sp Zhz/Mggx/zaOq2GT6x/fXSitpCSWQfawRzSqRwsnhlpkRwrDJ+Xcl8XVbjcbZlq9tT9dRAPO6EdD 6VSD0/cqeZn966o0aQ7xDg18XCDYVy9k+BAQcFIjFoYbrHKCC8KShu6Aaq1KdnyGX5tfdIUIDTCV CpMlYy+HADTt3Pd2CNZbVfIPydrkOZ1d8UozpMpReCsyf1h9B3UFdw97XOb3gVvmVV8ijQTphPfX 8MK1QI2PGeCjyVY9A6zgCn4L5Flkv2YS/ylaYNnwWHlLarCVcXcKDrB8CNjba/lAx0PXqSDx10ah 9e5gAxxLvmFVuB8YbooKoB7b0f+Wp8ylhTBr2Jo48W0T0Zau2F6YwwvBJz3f8MrL/w/DG9LXQhLa BslZneArFS+dmwSC6KSHGBmHNNQ8uMzgXTuTkjQGJg38cl20T/oUZwrT5nTrNwkO2mDBseGMyHl0 aWzXAnuV7FhbOBe6lUauDf2lRP5GgWN1AZiu9Tbo8LkJC7fTaKVPZ73A9A78tUj4NpnSX2+O4fLl 00PVxU0Wsxb46VqbaHDlo5XJBqffuh5OCX3r6pRVgMDmWEXdK8HaN+EAj2lA7C4RFZpCmGnhMRV6 4KzxDMHvEoFZTyBBJsc/QVIMFWzopsycXE4tHO+b6vKKHYNKG5ckK3PF8spaXbAsKS6Za7KRWIlJ V5AW1C13EHB/iI6+tzjdcW9YxSmOQPIP4X1FvE0/JXGgkGtVmCh/Ksy5r9W1GXcJPQb861Ll+PEG SldVkMEs9XuVhkTEbu1qva/OXuOiOwHan7nQm89avTI648jf00eN1t+6ESrxgKC5sah5R1nv41Gj Gc365JxBMrXHYjtEnw0G8SgOcALhQaGYLPMR1QygEwTMNzglINQwasqBVxBkDZXTfk7bUAAS7v+z XW894EA6UQ3Jwes+5XnFALx7kbLl8ym3RmVapR9cZPsCf4DVc0mCeeidWAgQenxwfe3G7my50hlF cP+zYzjvBcgfbvJH2BwPn3wysmEg3EJgVwqp79w7WFT2EPC/vTt+oWXQA7+wHDkP05OLTleujIqv +iHaVboq8jZFYzxjlAChu7I0fSNp/uKb5KZ8SWf+9Bh1DUuSCx8HRSW7HEZGKe6StTe+4LmvsnN/ b5E8kA3t3ua2rjVe/FqS/Jy7wNx+ZCuzZ9X1yM/B5PxvIMdISAFwMswSEfBVPzQ/WrYSIVs8eDZQ Y1dmFQ7iuUmJew+4caT4U5iHcjpeBz1l77kUGLn7WpZIk8LiUjarIqjxcF8ofDfTltLrx0ll/Tky IwbUWEjqbHEn3cnZEpyMcQdC6KNDyMacFGWoMu/Yu50xEfvsVfXCRk10dh9Z24xeLFyMW+GvGRKg NSfAShG9dzXG8CKq5nWQ83yIrILH+CNY+tf/497ilvFfUAwRvqe6k/ybY6ZtjwekgWsvIcGHdM3n PLDJpPCGhgSyqbAIzWQabN8nrNQsjTTshAEzn5p5gTmz4B5g1eBHYlym6bZlYVYWArQ7UA57x1xP 8pK4quJosu2rtGhNfeNAaQI6uguNU/7gm7mRYEuTlbSmfoc2wdLZUqz5NhAu0V7iFSKBiHETsu33 3si1uJB59OTDv+iiVk8n/hJMTRZmO7sQm8Li0+4ExzbgkR58tUIq/IVq9za0G8HMlqo7iTKvOfLw Cyj87WsSlE9bnKysupHkBYOCnRtm6KAQPxaXgJloPcxk3hMdvP6875o24nhMYNGyIBvPVVLa9qY3 77Zfl7j9XYqEETucThbnkEDkSuuAGyZwyr7b3iyX5GoYH0E6qgDeFZGgGGST9HCwW1/YG6wJoJHt 7EEG7H4GV09U+zSVhUbWb5elilHEb8GqmwhTTTjQLImeZYiwQfGqtfeZqfBjpxCs5lkNhEYKEDYm 1Ida0stkcA6d465j0Ei/mw4YdmsYKrEt1DiduoEydnqnLdhSy9jzh+zPQw6A8Cx/rnU+r76DEIbX QaZnRkrBtYLw5tz5Rxx8DE/j0xp3sq0nJvTiXEnYYljj84tphFlSrtM1nD9MGnfJ8YO9uzgYNdN/ 0oRro9xAq0qMEaYrmQSLfrhlcqyt80wkmw8dItQY6r9F63su6ajKEhT3BG3Jv05Vi0M+lpXVjCBs U4SnQIHmAeGyd9h+JtEedvF5XgD4OkwaQvs/BuBfn5sxkzWM6dYGI9K4F1HZGUv+dnj7w7KCaw8X pg9gy+mi/XYYjMweDd576cR8/7EmbtedlD7TeAkPfLILo8Vh1aGK2kouMkCy7AA+2SBVRXvC4Ibs 04IMabgJEturRrwsdU1QVrfA8MIhWUS+B2cfpzH0s0cEUe24Nofk3Xp5i5UDDbYwS7TntSzJaG+U Kry5DDYyjKrtl7opSwetRGneo7q1q2lCP3H4zt3j1aHWP+AWU+if0qVnwoL7YzcQz5/ZRNtbUc39 i6DYuHA8ubICogzjmKrzMhL6zRHsq1/sjlDN2QUB9hYdoCHnzSkEDFBIIhkzisshLZvWJVZdZ0HZ VeiQXJqOs5xNak/AxmWGZ6PEC5zcvbe8PjgCGLdNqIrD15+WF3IUt3H8/Hfv3aP3wn+82kSkLVtk kExORK+pLBsPZfap3JOJpIjAPNI993MBLL71O3DamENuKHE1KoTuIW/0zdC7/g/2yVRvDwRT/LdE K1OfG6kL1o4QCgScQHE+x3ySH3ruADGMzOrMP3z8tXzoq/ilPdO8QngFAtOALKGGy6ejd3XIY+bP caLxYg2GTdglhLPcdq4/MPRbpaWkwJgak1PT2rW6GJiABzRuZILYkVUJOtxbs0HlUmAmXFI2Xtxv bVXHpkqEysJrqGMz7knDdDLmcVSUst1QAGp6/gBYQcPAZcfLTpUwXPGXq+kwxPagbKH4p+PTNUkD SFnGLtwPa+a8U0UZM1XVaKxEeeDGSHqqIXvD4pku6Eh+2qaHqI3zfFdiA8Lwa4inLzspbbF8Nz9a HrQigDqKJL1ixEwsXLN9I5ILaNcRz3mnFxAihGKJScL1hL0aCnRlqvhfz3YRR54IUAhoaQp0UtAm 4OHo0GGRjzTWq6oqM5cTipmoVNuNG57KQe6HgjtLpWDax+M/Tk0hdPElA7qeD2Q6fl/d5qrii3Mq scyVepECTw+WSAm7WsAzOC2kofHUt5OcacGJIz1yM9mGVkvYzhEFmhBgWNJZdp9eWxQfbJ69kJ4i ejHxGkG+9aO+zTC3/q74I76udngupwZzdY0gOVDGjHhfxUZlakVjfJuu3oLMshCruTd6SKvX1O7I XCrW9FOrtA2qyqhLGdj36i2CtSO9NWgLvvTFu+vdq1tlfLluclGH+JfrlJjtg/hQJ4c/nigY8f0P lJcxUU+lkPrjis3jHaF/qQ5oKdM7E27qlfILRwaPOq6LuVCXiku17D5pdRz3N94qZPm1zRK0l4Vi zsBQuDBw/BtcTwBvayXiaftH9L9zqxyZHVfsIf4Kaiek+UR098qfZcDjTDPDrtAfUMENsRJjd6FQ 5RRbfMgEOnkOG9cq74Uu3NY9Js+mIhnnn8kbb4QDshPBPtmhP03XS7lA78UzfDhRZ9MQPpcDiicl OT2ktLyU4wzIUjqVFMp/hQu6mu23L30yeDWsKZ8UyvsXjXvo6H2bREpxyemegy9KFaqfuVgyM7Ca 10Du7dkhhRrQ9wDaq6uESKDtEe4pe49YKpqDKfFh/MlUgm/WAqjdpQDVbDotL9G/F8lwq9ScB58k w/exfGirTIRm8BPHBXAYy2xNXWBes7G9WIPRKOLsT9ABVGDXGk1HqpZ4DX7EeuEUejcuyszyi5Ul 37ZfkXqOJqeo9YtRXTd8RZ5h2JsSspjnEg4TuwtZmWtqrqSw4VhbxH9NbQ3FVaa7oZf5pGT+FK2J Uz6sxP/tmyqykpD2tmUnIpbFh++ExeKWUS+294atgqw95uvaCRUDu6TQ75E+hcYacEfI7RuVu1XL asy1H+stWVEzo14yYKJXDgwi4Llr2EF97YONPjV/OCR+A4gVFdM/MU2gw1yqWb0WvpL/gLYK39AC Tv/yv7RIQiib6Ys+JQgaLYHF+Tl978Mvsm8oG3GOdARg9cpfb8TkS0oqTD43PjYrG1EkS7n9Pkg4 ZJWeLia35NdIa+tstqhiutLfc8SMawyOoeA7vbzzKX6HohtP12Zhd8JL8rfasugxYEGizLuAEmPq wHBYNe/U48bRXvEp/XdndXqQkCrieJenzB/UjCgq4rH1KG/qz8YqCJbFz0LuWG7C2p1MbPbqQBrA 5MSAZSuIHzz8O+b2j6lLI5Mt/OENa9/FN+qjkCRNntStY1Q5InvyA+ZFmcK8FJ2g0seVQk1Hwcwt nIxttvQjI2+TqByPqakoF/MDbbiGh1wrVwl54XvupFCrYWomBx+MKYVL98DwuEMB69l5BJzkyggo cDsXILoif77yW+zWkLE8lVzHOwIZLT8yE927ZwV6ODmRCcC4Grul2R039dNXKQOMQRDZpUwf/FXb RejoMHO9A6+g2/Vs3j4nAMgD3196sd2OX9/Fxign045j+HM5/IV4H/mlG9ixWpKmklo5+GCgIiqZ yCwmNx2T6Duk9Sz66rnosD5AYXgpP2oOGSQscFnRH84gATfEhLxwFNaQ5WuJXPaSmeNpVPXMdq6D 7nPRfAQmqwZZh1+0mx9MWnXZeH2arh7B28jw+woXogSG1z6KEv4Y4EzmBBDt1XrRrHuyBahzGrvz New1Ca/4MzMlYLzccKiRM5FWDdmDyikQBOfmD+8nrEBcno3vnRZ3k5FZNcZpxhef0R4qQHB1vxsu EYJlueLfeZGZm4cZkVPyfmBmcTefX2Ab80IJSGQAY0Rc5c9057F49waypR2B7ro8yzjZ+Xnw8TmX GoZY4OI4299qdfB0qPvHZ3SS+TXia+SVXipjcUIYFc0flqRx0OQhq71KS9FkEOBy3QnLNRNO6t+A vN7Iq2tgxKyVURuZKwnQW3txYoYuwmW1zlRwxhfIy0O4eRiGN7tbc43SJZOvQjdOGjNQq5ak9wUH a0ImqzopWJdFRquIHh0tazfOvWx8SXeYnzfFBdq+WrtGkW+L9M7nt6ok5+9k59Z9ewEPxwPMVn/C 3cXd2iSzvOtZZnz0t/H7tQ1rSMsy1Dq8p1pFUKkAmuvc4G8f2QctrC0V1/HXi321M0Ac1Htr5csS a6pUpAWYjWvTK5OuMVWCIZJ7XAftIEhNMH4PEnH/o4fUX6Svlp/K2WLessm94oVvQnMB/MvFT7FF CjwypAV0zWZbffsuujtNKLIYlN5dWFlVj134VN+nUBeSlt6LYVCviTYNfwiLbBXHDXeP0yMYmsTM 1aVhQYMI4NuZLDJRx8OiE5ItOAo8rSYajxPBhKNO9NrApvTIY1Mbq+GlnOzKNejXhJgOADoNaWBs GMYOyoiMHABvkY4uQVuWTqcFwvnbo4E1ECehUwpHn67x/sxKcyHQ6vDh4HjfPFWoW2TYnRaGr8M+ Wq2vrJxpG7+vzMB3Ys7vfEyha1h0q36Bx1+5uM+5R4R1X+wjjcSo4jlzrqe9lfCi6gNi8k4J1YMA DKxj11j7OTCHqneuIbNFB2+zawf1AJ5l0UnlFzyO3S27DmSLcLkevzq/ua7shzLygRqakpG4MYJg Bj3kSRpWz4zALYTn7JivesFRPtXpR/qXrJezmWUMm7lkSFSnHwQrfkjXDltwHpp0PgxmIwfKAQux PWJR1ymGf0SFvrtzcFjagyLppYAgWsC6nec+1u+eV1qK4v8Yygj53pJNrRFPLFW9gzKD7WetViJt 9q8XSo6o6LUJUhYN+GOUNNu3u0V9uu3tbAgOHeIjTZlYDFAn4RiCgAVdmzF8WB54Yc7duj9Qp9eK G0f4M3r6d+9Cr+f0tFNuPeJLHAKFLrFsG+WAKUY6//qc0f0bWBGBbXoX63WdaTTOn/SKg+uAUyhz 4vxx3sEVwhbYEYbMHklaIJBPYO6Cp2SaelMJ8NPTK3Zl0pxco1qO/keGy7cnp3kWlWpB9sfCuAOV fLWBQu39kU23ubijkzhoc7983xCHSe3epJCzyreJdaz/aZrVnQ0bamRiqTooa4ukP5NMayAU/oBj 2uYv/2lDVo5FLstZBVeKxiIXC8zUMQm/JAesuaUfM8QjXkW4Ugx1Pr+Knv6R1YmvbwWW49/PPQ7Y FdokKXsO7E04crYkyXpfk8DHaN368P8HYfONnrv9kDgV6AUmvLyCCcCl451jlCJ0HIPBlojUk1SM cJFTPuxgm7VlwHk7IT8wS+xT+9P5RA0JnWYO2ruHW6vE3/HsomJk/kdI2Tkdo9jRJeUKIWFbIr4m UaSAgm9aIuUxptYYSeWWGKv2saj7spEMkQa43BUhqWaEhQCN3PPlzxStdGBHH4KYER/cclcxslSK 90DTsk6VFmDbte8tRGsqO7AJRhXpWgSqvGBjlXhPLJEWnmcEQsEeQQVcP1jzQNEjfF4QmHbGDJRv N28Tym//LijBMutVMm0tf5luiJHFYfkU8rN7GYlGMWL351MINVgeuiUu8qAjWU4w275miCjISJxH aein2/kfWOqNT1B9+Lh+A/Uqr56onSbAuJfUwx6qaSKx2bR79d5cMJu3C6BkM0Zj96RncQJvDEL4 n4tG82VmezoE+lX0WcaD9poMyDIv5338IfvS0R5YTpWGSbbuMEg7knYKs4wW+xmZRw6JeO2rXCQ+ c+RVTy9yCkh8hxzWJPVGo8HyiiYWOpodp0/epvkXlduu3WV66m0LdTp6TPt8zGDxAaTnjyX0pyU1 sAIEG7AFvcwMDSV4bwtnJ8pxpSEP2SJDR/7nfy/Nw6ogPDuqPjl1fgXXaD6rZPLHqFDKdUvEgYaE DBDZdUFpCA08dbtV4Y+Sonvd8mEvQvnKJM/MZxILkut7380cweUGiRs0mUy3oboGh6Q5gcnqCWTR gZyKqpmBpVIbsf2StYkDKdrXGt2FeK2E6ilRtlezVTg33INYg3E/AftAhqum0a4yu+cvJ3FfryiW PwUXC5bw1pfArCqP7BlqOqpUYPxhVWT/cbsGjtO+SRntE9vEQSAcjgxb1LV6ZGYU6ll9GIcVYbIs jzr9VLvu6jZTctrYlkM7ZIJCMIedfsWbc63ohA2wH+2I6yVHSvr+J4CQX1A5rjpMpm+MWXlEu3JC EmiNTTieG0hQNSr0vHfq2yymVjzIgHebOUBNaAdbEsQjAB67YSBSjM6lacpbaWIUi1salGKf0lzd zVpuG0cOGJuhdY12HCHsLEj0XqJZp7uVbqQ/1/1sXotOOi50cReVTzjPYeRlQZN8dfxMgQKTabAX R9QTv5eiJAoorzbP+5gBIr+lmgMX5tHJtLVveaPyECrN+Hwit2p6MsFIV3DbsajVxnGi7Ny+4OAd BsukVB7q2Ftu0zQWsrTHOmHZd2735/AoDHdqvNG3FHQUO7QOrVe0YzJzn2TdW6c5O2nCW6fYYpDW CxVLTRlE0NaRGS7RCc+3f/Bl1GV5FCbfZNKFIboCqA183stIFulhybNiM3uDOFoDBB5XryUczEdQ UPhzcIM5F/MgTQTZuHPqa2LC6t0vl4U/lv4rLWggBLwxML/CjbLQvuzoMz3RpHwz144D+3ytY2vu 4rSRYIL9byVgeiaz5gBi7LpTuWEZ4jGmbXTviencnL5ESX1F2J22HWYmBhr09IKFodwoa3Ach+t6 eowYv9QAWHmwex/VbrvwfY25DB7jgcPcDh870qNy6SWnme4JjlvYcuyiKLDt3Ad4QR0HomSDsuvx mKjbn1/lXsJ2TXL1Krv0YN2cC4Y/2nFEsjglMvDNeyw1hfgeSpcLrQvl3hTQThTqOzXqW0Hm9FPF HNitfH7KTUv61oBGLWLaj7HHUuK5td10+5l6yecdRiPkRu71oZK7GwMOMi+9qyNtQFlhjMvX5ym6 h4TL5JIIw1te8KxFrzufR9cjlwt1Xyb13lx1mkscv2SxrN/UnAb6ekRZNCH9ZkycG9yLoKuN0gUB D78aO8u880ih7pi5/6pzuUjudK7nOjJIfrlBsUbnqhUWUc6v4//MY0TiXah1HkzmHmzrU7/1sYQt RkO07dVqPo8gUmtS0yEwhJYLfPfNf/xuZxlJ/tssftgX/jz2QLVIb+087W6WT8svEBMV/qXidk5b RqLkZd8/+OT1eyk9QUnGbr1l9TJy1gLVjVI+SSF8CTC3FSzKomHsJfuRU+KU60qcB6nNg9zM3p1V nyN7u4YKUqQP8bPfrGU63y+mYRkbI7E7dy6aomwPyBo6Hc3jUqTkV/wQtqOhQtKq5jsIBfpv8bcv IiPWkXzGj4cSGc2pGpPPX2BHGNkao+Gxv+MQxu3ohAPONypfSUj+VVehjto9wf560G50Js7ZIZET CMduag0c/6Ga9HDuKuRGqaPFoxw+Hl4Lp660WX5bzp8GwaDvOjjlZtWsDAELVyleO8RsDyM/3dUX xCovHpUtV9MNHOuZjzWxqrUwFUpsAIIlWNIMSKQGeF5JvjMxonQy1iOKcoV7WTrlgyroZM/ce441 v+ap7HbBEiSHplyMMCkL001in5pHDygxZibYitUAI5nMji04BSual1DupzpoqiDyVrZzOqZNB8qV pmvGX5XSirNd+42E1mM3bCjmCiFLCUwFm+oPJVKX7wh0NeCFS9nKVvFvDVHB3JRS/DrMyUg0E43K zPGKohCigtcKMY2ABwCEzAmxxSSye50MzrVurUXpAE/c/3kdkGw4CLtdnrh+n+Wa+24+J4aOFuJg F5WkRd0Xh3gc7NHbb0f+zEpnGnfPPbXrqsyz1ppnA9nMT//UBuBDl7M3hpAwBxcTNwDGsXnX9iWl uPGergKJFrldESFlP4zLTZfY/7boXnujSvj3DM7BxAq0DF94LbV9Lrb6agjb2JbgaJqxdBYGf3tz qn0Ksx+UYr1zcqXYhlvRjg+PxqiHRmyvOwappukRehdC5/nRfB9bAY6FHpmIIxFTopr60kwh/VIX IMy8kVBbVOLkPfmeV6zC5BakTcd+lg6cYzYT4xoa/LRjx1eG+E9+PWR0jrq48NRrcE3P7pZcl3OE 6+RorR5modvyWvRGFayjmhyxKG79z0RI01WKGHeh5bjKw1xZioDhE13dYmk2V2Hjd7fF4UuRw+IO qvecJVFebY2l8Te4qlvs5Y9cuX5ShYt977DntSLf0KCc1Lm7/NzDzmtD1j6ME8HskAddNUleooFc nUvX2mtkMn+na4w9a96HVcUf1pJO/j38K7fqpR3ryDBY5D/taDBXZrqXMNBJsJxmOaSOsfqYvFk6 EqiLjL6cqmSerpdBoCHn4rfZZqHt5vjtSGges3sX/JYIT/SXZwkzaXdBH3UG9qrRx/ISM28/SzsE q642L62iHofQX7uk+YwlHisZpAUY2BSGyKyoiotWT8fmIQrXdsUyNNdOJzLREXGAKzAfsu7ozIyS 6Xi04aLNzlG2r07cHJOuRmr92zIdjhfjV4VEykiPP6yoAoIYfL7l7DqMZN/GlfYdR9zd4CBgFfFL gvDv6FeEDF6reksEEMQ5g2O2tPB6Gc354pKrQR9kNsEmASReNmRSdY9lUMnC95lUYq+B/xvEVh3U F1rX3oY6ElH8tV0OtKRcexwfsT4xXMvT0eNx8CkzYYLbiAZ/RqGLUv297nwMnBc6hEfDV1OfsAM7 QBrYyZMq7NiL7UNHQFBZfe3f1ddJBF98XUJjtz/Ghah5TI7nY6kOBQjzdBEecBY4ogYaYqLkaqQR eFhrPa8sO8IuI3yjIkY4DD/Kxx9ofvc7i1xnSeD8eoj1yojRVBdqPMGyvOVS2lQgVCFGnMXClNU0 jfgTEx9loMhuFYkqtfO5F3hNCh6sE5Vq4WYK1eGHzN3ikIQyXc8OpIYrk4lHUTqx8EmH5+7v7K2N CPb6RmaHRJKU9oeVmptSDsVQw1w83v7RD6yU9qvEn1D+N29H1LyWq6C6Dcq9FLdkhCeC5MANQhB4 rXhSG2P+BYeQIcQA/3k+nMtOg2jia1f4w1q9Z0VVKFd3aakiTMdsJSm2HWYhODivwdPMjUBplMtl mRio4T1RNp/APbBKuP2c1wVEirV0s9DqpDvu0xsUXpqjEP4jrheHaT1jVZUHQYFPos3gLQQ/jpUH m/xAmEPVEPW0asxXlzP6JJ+d9JSgFewkwyuuLSQKQwqrRny/Ea+51JYtp5vPiPtAWpdgPaN1vGOQ 8xLmw7Y6CemLLwFgEcyCk7u0/xSaFPfVRkt3t6XQOm5nFMEC3p5YkzbU4UTvzRgVkYO3L+DuT88y 2nofSGvAb+r+A784xnJv8dyb6ciAkqUPjN0V5Nqn/0cBzjJbVMU2QJopyRhQqB86L+acpNQLZH/f ucx+tr+DncE6r/+A1bnE7+NjzL6tMiQYuwqDS9R3YkbAiBSSsKAJPgMrc21xOsAOEywlvxp50+FK yFH/vJPapJfcARPNPXNQj/zx6t2JIJpqXl63QKI3fN+mEAb3JkxXQRDtkTYRizOB9sJ6XfE552F1 AG4GfpG1VvYaQW0Ge+uO8w1uuSY5I0EiY46mj2LP8CqIcXRD0LreoKLI0B7+u3a/3SywrmOivQZS AFEESmJJAStJhBe9z3YKRIcdlD5fPkH9bOSf08PHEepWmGO5yLOcqlO/aMkGE0TEmnfRTbZXZkf6 3KNrZPEK/uTnEmnGE+lS8ii7ovmEzfaM9i9Xfl/okO4qCiCUewX0T/QRrvAaYu/uxuVB2LEXFSvx yOb1FHQdCTHHfWU6c3x92qDmMVKi/iMxJJFI7NPErYVDVvcgZROdZt8bpQzbe2ci7E3D0CBCZfsD b2E7PVXz1hXh1yQQZcngUc92h5vScdiyFP1BYy0jqiNfrWvUZ/VC6LwjY8RcyXM/oqWfoQcSGbE5 0o2nYwkpnyAWaRf/f2TH7exer3kJchCQMyGXI2N3x0TkyIwbWA9eiTCDSXZrs662ubp7S+JW8wUt +O3mUOcKgiFRyOs9h1Ozh8GBGURGIFbM9EhffFAWWCwjQ4JV4f2NBEJ/Lcs15JvEQFbEHyILBcKE ZNgQUo2hjURQ3OLLeuPdDsq9X/CBMjoeHZr5yGc+2nEgBrzKRu7O96Qs9qTgzXgq1e84TQJY52/o AKsAu8ObXuJTrRc7xNOqyW714H/1+QRd1Dx1sp4yMzZIj+LAvun0ScKG31Q3GPfV436cRih9ZSNk xMK4daco291fdj0vnfVfoJK2B4JV/sb74iX0VdAs100dIgRjWZ72avic+ZUp6rfq7rYtdixZLy9R 0TdI10/Yi5UDhLOolQ1XSFX1+zKWeOdyECFLLuMwaNymjnQe4ST15kV6kMm31lfdhERMcjngpfZB M1JkCOvnTUgb2AZ4dw7xcPEoweUqp2rTsGIAaFWLINsrrKf5+8Um7VZkDVFkbFJstBFmPhd8EwdA TZJ4U+rcQVtpU3wLGVYYs6EkXKkQ24gTb8w97YyjdgsOuPLB+UI68IvFft/PBvcKSh1+xrujAAIY kvQ4D30x7HFC6CNMI45Vu+ECVJTzmJH8km8srv47yVKnL5deHNLhxAWwUh4odbeKnD8Up+CCdyPz N5JdhV84/XmX2DMqB4kiUWGao4UuLHUJkvjVoSe62snIa/LA7ZDXZwsd6xJf2i1QKJyLyPUdX6Nh iI2BwHBJyDrSw9EpIQZsQHtXAU2F169gvzDXB05NEoWwgjcD7Vqyj19NxqOF8nMiqi3V4J1m1RW/ 74mj7OMuZ5PcFaWJnf8I3K2aOJ0p/DBEbwNdhHVPebH05mFqG/552bh3/eVg3QeJ75dj5EQWrkQn TjZjSzZUx1RBnSOdJyI57xCprkxC0hrCksFbYltBKj0xUgzBco2EWDvXJ1XyhYmTpjbBCqzy1Fl4 jEAiS9DpiyVpqp0l30G1ZtXaEVmPwnV4xHuzzW50sGpzt6tOyWWSNzylRrYMEohaAMz6bSgaeJ8/ rV5I9EfvI52vHRKiePrUvh/QV0snYUtPrQxL5KOrGgGfJ+/AKPmbsTyP550uppxPMHwl6dneYakc 0Zd84B93ZhT4w5Wc6BDRl8BKgujNaGt3yxakDUhbXfD9tqg+jjwDYDIA9gOsAkm6JR8uZW6aHTCB tjME0VJ64+SeOGslJ+P9yqysfQ32RAEQMJs1eLD7UHX01n/O+WOHAGq7nSV4laUW9svJUBSHqH3/ BFuPDTDCDfKDPdIdVPJUeI7wrgM3jy1Jx1kiFG3S0I0+v+7i4dRRudabTiyIF/9FQcHROYPIazRt HWZWtozBfTSbXjCi88cSIhifF3Z5OzpBoLG80FGObQ4RwN42OFRp3jpXDqNraH8UFxxiTjevIfbY /LjvLVhTJE6MwfpNUm4xnWKGoR4mLujkdz7lf4y/Ose9Cq3EXpGWZ1eXAcltut9sVFjr7vfy/wP/ b8nWyN8Q8LP3a9AJNDUaqTG5ysWfMqspuMpNppbIrb0WB1Xi823ZtX7LMWXEumOcOgenCDPBhaJy atGa+BNWpcINR3kKn9feAyxlk4X0tRJh5i6Si3nAbj5atr9Hkb0aSYsmX+obXoLQ/xYleZOAfQ78 3jinAqQbWhLk7iaRRQBO3XFus6EPSykbJ6zA2LMUvGeM2o8KZ6mNlmk19CDvOYzxdReBxQkH7U4R GS0mb0Nw1bXEtqkIylRUgFT8FjmrxrB7jFnMkAf8ERxhXM1vakqyi13YPhcfZVRtAy4J6k5syFDz jO+5W6BuYVECfXyNYtL6BOLmsAkoTAYT+vAazP3yx2mchYC42otT4yqhsqnuvf0+EbdiNmH5DFew gje+mlBGBJvQVhOYcTHaXGbwY20GmA3C7AfgkazyJEJeYJWnTyOXnHFLj5iOZbCo4sW5dPLokqqa QzLkVB93mbRiNubRxtdQvpuFND6p7tpqpQ0BQOVnY2QVpbVO7mE08BfyT7/LAzYSdWUkLmtlFTNd wHfWxuHfJ1ZIqL5FJ0BoGprAE0p6PP5K+mk1HFXSeI2QQ8OcLXGx41F7EXKXOzozXPbY8QvgXB4F ZhicF5fivyMfYoVKvkPplkxOAIBJihlS9Wk9wOQ0Smuo9Ggx9RBplkeCChB1uW6V6DdQEqkdqcer vMyubBQQA1CnbwbSlAY3igdbt7WBBFrdE8JK+AosBVAcZZXzSdiWvns4dE0JHiBn6hx3lrlPeXkl BEES+XsDvbW41wS6kxWO8xrzOAwME/c77B8mCY3Cs9X6Y/3U6aCGoZdRN4ASXY1IeRO20YeG/dH1 pcUi8Z0nzTezSII3pcGXp57InA5yAGxKpDjrZpWquG75rXm4jLRavJmhYez7a1EdOCgjMcTiEXqL noLnMJoVV0nOWc5IVZJ18bVy0p/nV/fadX125UD3YoEHU1YOIuZlPIxnRYp9CJVU7ljOyuhyQR5J ZBWdJoNB38SmsS7nEYZRBNdbZya3H5jGCVDqI69MLa2Pw3quEV/QWBUtnIcAOAEQOyPPpNyyPvBj jup03IjmS8F2U5GLKhL3N+L/tZyOJG3OhfkiDeUl6ZW1o5k44/dgsehcTt2auQQI9i+VEabeN7c/ n+gf9fKsUGHzjk3PcE+LqkklUk7JW+mmVg+vWKiqgOWStekEv/tFoM7dmEAzTdPZV5PlNbQNdIvj 9/Qke5JoYngaUD/h0q6hXi4Fz0AXohBxa7JdA1gu8HR1DRYERjBJA5JRMR0QisprV+Srq5cQ3IdT d5qJ1HWbr8+oaMR3zfgfBQiETrxd6iy8Z473tN5T0r3vJ+mi92jQfEG7OxXzuAcjv+l1gNfmQpkI HzBaW5/i3VbnJLdWKGr6Z+6448Ao01hNxaBtqWq9AZW7ZePA7TKhFvs7Txowy+oURiFxFeM2eMUE e+YQregaxaLH0tHOkfXiL8BPhQObZ9OTjlx66bl0EkR85yuQeNueZtIK2yj120Fr1t2H4JkMCo1s +29aqUb9g4HZHoD+KhD6RduV7NqPNSlanGRdq2+JRmm2HdAKAzfC+PPphnaVsEmmLjdSEf+T6rt2 x36tcuKJfzZF4jw19yiZmsG7K1soYtr4Y0MQ7obU9l0ZiAuP4lGSykUwpfHHl3QsWx4XBs6KRr1z NTm70FOxilCO9KhmQPO0lueJglIijh1E+sM4HFoOdtlfXvflWhuKDOSlGlpJ+QnM8f43MdKuQgq0 di5uWzaMoBnrdMExK7Dc/MAB7IP5j1KU1iHnMlTuXIEzI4wZUgzI+1+W0wtc5G5n8aG9IoOhAPiS jVie95Tg1T0ScsLuBzi/xJamRhDcbCApN2l4jtltA+3XMZ4wKptJGFqFn+ctFtomjN/kPh2w7wGe wKJfrFmpJcWeCAeEeUMre0Jae/dhFU/k6IY0FeLbgkjtj5s5Y5EsHUXEYR2F8+pXhAbcY4scYjEi ol+nspvoHOY3n7oVI06GaTyIR+A0F8CE3fCO67nXL7TWCQenV0js/TfIv/Tk3i4BCtB+sO2U/Pm/ ju45IY/8T9BPwfgia4zzd405DFqDff7VKdN0YQ4StmS0PqlbiGInA3s8xdHoXyCon2XjH17wG2DT 1tjxs6BxwX304J1RTN0i/w1KGKfaB9MxB2Orw3VXjiCKF/V3Bd7TIh7PwOXnnuysEj5jQ7xlJqRU v3hj3yZtceZe23fHxMMIZQcDv6j80kh8eaSxDSHtgXFTMVBvGoRim698VPW9uF/R+uv3Kj+mghG5 jf+Z0d1blC2ex36d/2VwaPdx6j09Y9AI1mztUMPZcJuANO5Gvv8r98Aky2iRIkz57bLx+Sixr5M2 rI/lf87cDMVx8Jmt7MWS+Q8QYl2hRT/WQTVZUaL6k/oPEDAfbAKIHwY8icZDnBSbjTyuDIBtDOTc 4ccC5BWC7+3VOnUsTe8KpIOA4rYcDVmbouWAU9XYtgVZa7bKW1ZT0iMUw3Vkdo9INQN8/3bm2ykK qKxK0bEM6D/4gFvuEj4WyDsj1nDLRAWLUjsKquLm5wlFkkD6+HZkT/7R7HFlS6iLInuLzp5M0r+b DYsIHWCOzVOVotPyrXlOBgDj/1caKjkwoPNhNkDEoK/P+kCuo2GvPmFnovYJBpS6lPji2gJ2qaj7 4usOm3JMU23pY2x+DaLkE+NbYNO8TIVxyYe8p7O2f+2nHOffOsIxFVXEawl2rhARjlAmkcVRlM11 Mjskq1goLSASxWAAgqbSMfFl14WHNWbXrfawsDIvIc++Esu+Zfe7wUINO/xq2zOiMuGF4ZF17sBk GxdYNwjaz4UN9RjSYKNwx3aba9zM5okSQXJ+n7XWhDVQKnM0dlt+9USSPGfe4f7Ju5fVDQaLfXun GoBV6/MJA2MD8Z1fznaFVPyjglx4tWE0HPnbyvY4/Nn5144X8gAXVNgvAC2hFo4HYF2f+vmxrRpM qCp2CL+BWMYR9p+gxdDDofauBfiPcCYMeY+rKQSaZ2WNI2Y9LUgMGVl1iVLe7/bipOWdRFAJtCi2 MXlIYWlPIpLcPGnc9ChZIEVfr00YI1iloEwDAPNISrReerfT+it9uX7btzZLyeurCKz7rw0kSz96 FthZimKkGfezLL/xKgEPxFJ9VS46djbRaz2k5RUsHH4kX1f1Tag35q0w6DaFKPoce2h9vkPp9wTo Y+znVU+m7pRS199v/hJAvozaBcONW+lubzyx/HyYwab7xlBRGdlw+HzBprDVnC1KzEPq4yJ38y0O p7BLPcfuZR9F0SRQzhswiFzXMflDaXbTqljHpvdg8ljFRJGHW4Hen4xozVSdxnKU74ai9B6NyCwV 8A9vTldghXOdtYAUB6Ql+uYRClEYnCRW9pTeAosYY7hxeIIghnQouqlmaC5VAHu1b3r6aHn4BGYN oy+PzhwpncUwRUfSBDZ8ce54rJ8H3cEQW1ulwhOUmb6lltWXjLfvBffoegCHwy/NJUgBg8dbRZZp jgZQBzIbD6Xw9WJCz++sH7yRtsj8wtSTvSPbhB2LwGvMKoBx8nvqL2msth8XP84ftkAH62b2qah/ lpMH41Fq8AjxtP2g3mGUxAzHaqNctOkkyueVRps1E/nSgehTeZ9K78xWbfoFmoqZiCCYjbMQX+DX hfBk0C+nER5donxQcaUGpvL5qWTXZB8SuTJVrmKeykRkeap4EQxr/EFsjuCtKRvw3u+E8FeisIi9 vSu6RxjwiTyS3b4hTDcvZpH2x9eEuMbzvcs5Bmg1GOnuc7pFlN69++FDJ8aELthJ3R7asCb1zQyw qRvKopM03VRhxc94yfSXe+z923XsQC68oT9Whpw4hAKATJZPAsBoyGYJZfJO5pzSZ1+kzhWsk4Oz 7RtqUP4ggz6LRHwSgGUD//xI2BpcIPNFJg9SD7hDKvQ8IlzsOdXNdq2++S2/G0UdEHP0iutObBkw vSxszwCLNrfSFWxmFQfKgNxmnw1yx6/6Fbyp2meYvhu9MfZXxPlptJ2Pby9t25jCociE2x0lFIHC XvSDTwgYEkJ9JvCIuQl3uZEKGOrLuTr1suZhohyL5mtZoH+stVpfvGtcp7Rtzwsqc95q9GIY5boF l0IvK14ZYcGMmXtGoOXQAvtAKt1ccxxpn7TgfSSBy+ZS8H0wQULqQGkTk2geG922HuuPXSQyuebp YZjjwkR9AiboBi14T16kA+1EStdrLUsShdiNImvDIwwSp52CMiQsZLbPPu3AyKF1mx5qIQkt+bFk jkzG/GaizReFSD1xEpx9iux0Cd0eqLugxVGwFiSWLrtLHIGP2AH4xdDYSEZ4nSLT/2AE6rRgnec3 G7Ad83oE/zl5oRD1I8MIDRGwrOPjaqjaAKb1ljiSZ0v/aXZw+ZqnWanXduObDt6nVH+zKxs5H2Cn 3oMuae23Ej/FFrZDBA/JmPMX0ADHo8d53Rh6UFki2b+ZbR+Qq9F2KPsALkD7MV6uoXcndvDqqpUF xouvGvR7vGHVJDr666tm0VDlMCltXeBFlYjYT4QIpiUMce+cuVlvqrvTCYo4mH010/avR/xWK+RH aPbezMZdKgzcAmeuwS7P5Ij/+9vrKGZFMYUW9uhn0Eia0hSAZ9W+dO3oDUsjSAwrjLgfo+4vhwGW GnEw2B+Hwpy81iPrIM0nPc08vNfBxfGIWYsvIf4xNG1OS83Mljw+CK5J1ui7se4rn4HyDIYYe8HW gZWBFG7DeWL5cvBjrOlq+x2MPu1bL7aD89gs5Lozdk3tuKht4WKBbMlp7R6qHxhNZFJQSq2y6/fX KYtZ5ElxI9CUw5gJC4iRgUiaWCdmiImLvnhwfidIozwZOTyXRseKhQJKirrUR8cZj79JmheEKB+F WLEF8rXuqqWFXMKrGphE4A32LcoZE6Xf7IeruMwVURIIpavfa9NKNYkv9apnBV3Hgb5wVV5cXGxh cGmL5iLfWyowD5tcfu6z2OdnR1RC8vGqRXBYNsvpqQ5xbZ3n2/m/+UlCeGYdPc5UxOkG/3hhJpA3 zsahFpsSiV2Yb8nUPorSgUobrObwRnAOgV9dOk4kzo1h+mywR0AZ0enw6Q2X37CINOYEacvEMToC zAbfHv5HKP+G6kzwclJpJe6m11jDyWsx/83+D6YXq4XiPbQ1UMM49w7rJej6L4/Z7XG3Y5mJ9inQ vFlN9FFi/8u+AOKVfra1hKJM2rnbZ6lwQv7yQr93lWJY2oqIG96mTVZUFbWT03pUNNH2/UKxeyxM AZJLr6sxN2Ebl4Ivs+InKUwEoTCHhTmTIBHHbBzGWiwEEggbQf/As6+3ciC0zpBgplTZ/w9RuOxn 3g20JN2z5EQNlWnB+6RsBIm+AEcIYPjmzIuxB9JfwvZ26oRDLQcLPvz8CDEiNIU7/WZa0kqUUvwN oSH/BFfCOtGhUkwkU/vQlbNruVdSh2RcE9FAU38JsTEyuTANotIFUhSC/eU0tNsc3nIjT57eG8kK 2fKxPiXRFqieB8CuuswoYOGc6z6eRqfXBOiCrutQVvdOR7sCDjoPRFdQ3FK/q4FkohKhokU6GJCk GGo3XZYfTwIeLpV/vIDybGaGRdEdJvfr0SOIg+tG62ceA1hQxBreXQmca3KKimIZ1orJQA2kMnrJ nlkWckwekbbxY548C4sNGG6Mb/kR29HG1KsXK2inkSFhErFgNoSn4cXTGBxMSZ6aROEKGx9JQXJZ n+TX7R7Jr4mxPo+K68c4Yl5ALaZQAgWJmIS5seND0u5wsN2GuJCN3Nkp7f7PP3Z35H8FptFfmccx YM8ISFYNa0jEgdRYbXeiotbTXlLzXShMPvY2mQ5IGfFJ1VPATl/j7OhCB3Oqb+3PPOb2PPILi8Hm Rn9uSF7tKwXqW5/VgXmeUfGmpfZcEjEA8J3uCXy7JJNPlhOp2YqciRXde2lCXn1lP9tjpNO1QWRV ZYrsM+VokrqA8fOGA6Ogtt97bDV0hVMOt8XTlGoofrXOTzfdLopANYHCPy5Aeeaqy7wiBeL+KhfQ 3fn9E93PemP+r+HhzfYqAQkFb32ezEpKN9mhz8ue0msnkKO9OA6iQpkaTr55Hp48IOnfyV64VRGn 3/+uhT2+LK2YoWoSKHtfCSn9e2akM/B3RqSQRMx3LRBH1ez2jxOoV0HbPIKS3p5Co7wvBnaKUqHe STxHYl87bDebxFW/oeW/On17+AtTcci6suPIUZ+iaj54mNtXY7DoK7m2LkPMWUqIuI57vny/r5GU rBZy71HNNducyphUmeyxi8IyZkh3Nq4IEMkGrf9hsB9OTsBDPXUJ9B/n/U+bF5Gj15PXEXhBoN0d 3Neih226OFH636VUlMgCy5G0vfXG9VUjJnscehGVj62iJsNDaFA2qL0pRn68OPUjfkAYacGMwtPh +McreMn2nCN4aT8ZPH06zjFfJgKGu+hMp0c3YuCeD6i9xUF5hOPvHv/+sqhL3fVxiirPs8Jv/zwt +yE0Kx0X2Ebs07kmZ/2RwjBrHsqdj9mMXGtYEUAXosS2E5k90xYldCNqm4ADYl1yoMuFwRFy3/mV +Rln+xNvqcxfLNHfGsD45tS/tQ0TwHJdu8v+T9MvpsSpLIs44o8+jqc7mUnHQdMlJx3txmbms+Es L0RenJPXvgKxYj9tjWv3haTZQBZTn4QRr23Hy6BGhlbjjte0IQyTsKQMA5yvCWz9mXyo83mLlFOG jBazAYMGaLg0OUDC1xIG63KESR0EuYgf+DPPMLPqpuGp/rgXDHH25yZ7TELyspz47kbLJsat8Unf 9Y2uY8r+bQCogRJAyDwJan1dbV5mEmjXVoZ5TsrVrOgDvDnN7fks14mQvzdJoeDp0Og/9AeL4p/V +zR6fLGa1n0uv3mrke3+lL2mPy6Vj9VXg6caGkm1N2CKrWKoBWwnPHXaPcyoVQCDqrrDscYNoa1h 1CVJFgAmROviXg8CTtMgflpDGPUi17ovQ9fMh+BixiYLIhiUbRMjTCHncQ9bnL2iyf1ygEI3iJIJ 5OO2jaTnFokigOyEfhJw8mvcOLRE9hAFaVj6MLkhuOZYWspEfJFFtTqny87sg4sgDRGJYwnrpPn4 w87TSnG7A+XSHphJFzjY+kDjKSdT98CwPPOFQPR2PKnZDxWcn/VCnoyANNd9t1WM76Ndl+aPCRvv 2BhFQUmuOIn6yTHfxrhwnGwDqhNQkoj8/i00oJeQGCNR7mhl5yUP0c/MJXVNk7CEUzfjH7Lgskf9 QvxCcZLwnUPaOVvIJ7c2cfzOCgzkUR6LTPrtmqAan2U5FGC1mktwxwFmJT5HaVu01u3UXS5Rh1Rh u+6o1uoZKZfIK/98WTwoFAIh/1Q/7Civ9EndgEbVk1OcV4Ica1gJHgwZDQbtewv7642CFLOFczUY tPUnQq8Xa88Wvm4kf6E/QUGr/u2cieeyIzppJ2w+xg2q460SB/PeASdmmOUtrm6zATy6JuawN0U/ Yv9Fla/JbUrs/ywZMwfWcTLIJoSm965aLUzffPqFMUk7dIX1Iszs0BQ4Pn/gJ2y546nD7EQuhZAk XjhtkeBCJzypl5HVUCfv7y6kPVVe1mrOAuLTTXRQcDwg3WsSWYMBVt8cKFsHFLF5flulUCJ6F0GW UB2FwEWUX95/6BCiJpirEzbRpYpkjJc34gJ3NlC+KZuTLMHM+g5bd8j/C8VA3I0LDt/Gh0+hY/uu nQfN+5sV6PCesfsoNibZu7agCz6d54q8rxO39voWMrY684EBLtyvd3f/D1n3SKWc1qQfmG9Hj/Mh MArnwirKfjjN0UBofOvyeDoweHD9IBbNjWTeCdfTUILBCaQujEOrfOwmaQdlFenCyhMuhjeU/+xd Tl3Mjl936SdZZd8hekcH48R7/Sh0y+ywVxaApyOkoOhVif4Q0JTL5QJ9HoAc443uhQSlTkHtWV/c YKbXj9/oEJ/5PUb/xTND5Eb/dXP9oNSl0gXKxPaOfiIwU+K7XcSRs5Ecmm3aRCIoK4WlAcVPzihE Xx1aUq7ALzhOcqmmyNzRUXGEZInL/q6BLqYHvdC7NQhRGkYbkDF33N9hPjHx9aoUo83UJcfAHh9v gVIb+MEi8K1KIjTdlr0zp+MVgS1l97AJdZnR0Q7LxfGQTEVZzP+zQtOsL/REKJHDUUhE70Y+x6gN BaUcdmnxZFBhIEFZcIp12kGI99OEE6PoLESZwzMWXIgn36cvpHlX525tqtVVK25A94IIMtxHhgwB 3xA9/mB/T1LmbNLIUE9RnHCcvT/Z2P9OnUbJKHDK9h1EvDXgqVyQk/zrND8vpwJnh+AJjnaZxMkB fYNGPfoaXSsz5MH+zSsMJRfz+T/fcOZy7piiTllnBjEWv/BneWNsii+aDWH4kJ+PxVAkIgYdWyc/ 9su4ki+f2lSOPAFJujWsYf7RcNKY0jsp3s8fosgAmCfYOV5u4iDbCa5ked607DmCwkqq0udyocdW NGSSgrzwaYLCmBnFlOy2/x8OUCdvLQEo1yCiu7hxzuz+UiwiP0bDL3U6UdsDDvZCJpZ2h2+7nKG4 +N1VKGO09UPwLfbm+G/o7KY2hxfXTfbTuJCU4wwTK5M4o1VI5LWYAh4EhgWmsaQR2q8JF+bApFka pK1Gh7B9ul5gyfxL56T5Yu3EM5935iV1y+RKRFHb89TbKVEHrbdnCuB9lLztNwot/LWGmEMGhpIS QdRFNks42WxjVdzjOq4ExNeTj77CqvM2PNuSTelpaDh1jqycDyiCYqHQyG5xpoLNw5oq8hsHp/cq AgximN0ISpgc9zAuejsHZZ+GwwZgDRraniCXmM5iIL95nC//Egq6P8H/Qf7pgM/2YMCHOTYmT9ZN Qm1qxs+98rxQvnLqRZYwP4mhLpTONjLetUpBRVGWYkNimxDhj6astwhxLMD/mFRrTJtNzvnuUkFC ZHE6I4RwjTDCDmL40iL8jMBTu4C+SRGYkHbgAVl7f21cqum7H6nE+5HSotZNIPbnIxw2p1m7gnJC /W3xSxMPGejzaFffLbG3wLG3SNfKlMvlcI5BJK8PqCVU1o3y4XR47UMf8yuXgLqZmH38geQvqNuz bDPZJ+bBEi2DtaErEgmAIEaIDiOLp2RzzuuwV9Nfw63EhsogX3iFNrfwY3KrUqG/938p8eWDzGC/ JRjzcJXd0cQRflaKOTsmFUTt1NO/KoMgHg8GzzNZQg3mPbpy6d3DD+8CFTII9A3rz47tkfdgjxI0 zi7tbTvUejdqIrXkmBaejR37OzsWwjs8Hm/uihdlTGyHSl+6HbQ3pE45tu50Ovu8emGLc6RfhgRg Ppprw3H9opHGB0ZjFV6bxFrd9AhNk0PKrFeKRfyg/t3pCAIHYPi+VSEa6/ip32C13qqQLzdCSW3H r28Ve1Rs5SqEvtR7orr+WHD7zM44k6rYoH7XUTEia76xc7fsOeWOAlbvbOJe19al62MemhU5y8qL 7ajgnavIjym55cKMaAc3TPXCMq0cri9em7OpkRvxa/YbcRCfS/OlLJQvaPlBt/PJpQKF3JP/Ex9i xIsuKN7VZT6UZNvkz0CwvO5R9m/c9Rtx/2IlKT4HqUvEgxRlQK+Mcy7obB0DbP5naVTA2Rhi00eG SsWQIa3Y2n4886kWvRcObJvkqdMhrWOF1qQsW41tW/sEwrPesuyGfkLVtlsOh8JEVZdLiwYX/tZT 2h39GxrtRjfQxr4a2nFcrTWTaW5aSn8lsicjoYSAu438a+63rzhsR3k1ya3GdbSCDlNxkle7Tdx1 VaNDwBcvCyugj5zCH12W7eOfvu/UYRdIppmIYspDmiE7uKe0I+1/+QvWd++zGJthjrxnldidpQEr CulOUVRehWOxzXWDpIKr7+ZTQ19LU2mo1YJmdjF36c1MPpDBo9BJmNElLMDQJ+EJwBJksIo5RNa/ AvMA5EvWM35T+DIzdNerhTA1cJLWZ8f0kag8l/G9wJ+WS03l5EeKgPRJkxB4eUJ4hFpUYo/aVT1/ PNCmUm+elxfhWtiiL36OohyHxNJEYX2vI2uhUcG01oUxA7Gt/+t8e+gLrwE12jOa1bjkKWv6fUB2 qMKM9XarAdX3E1XZqNdMahzl6X0U1wxM8YtlcAt7aL0Vkcz+5Dkw3aihEb8vRMF2IdF9C2v6K2uU /VMFmm0afLgbv+Nm90j+Gy2s3vuTHDHRgD/zA0JNQsT99wWNDzwZ6Mc1X7vVx/cuY+hCBRjBE1ve oXugi/UV9jpemjUxbLCafO5gSNcnu7iK3tR2KdjDt6WMShyNUhwsz3BZKpdvLapDkODWq14902pN v44zp0GF6fFfVsybRuhZ3DIhLwm7Flw8DGmPH5Gd89drA8CPMEtN0S4XTyuxACLfHkdMkAuckKa/ DCnrNQjqR00mAHDex28atZ5W4hCgSDEQHpKG/BaBYbMhL0EGN4wswOZGfyhn2i1Ydl5JtihG2D2h r86Zqdfz/6MUJpQUh82bFfvVl+Ledni0fhd70xz14gmlAIj0MdhUbzKiJZDV/3Iug3h5OXbOudER /HQdkYNwWqw5iigFtrg04qaSnGyM6+EBGGMWZc67Vs77Sbbz0GG37MfHGWzL4sBtfzNBF4W/EKWs gGlt5QHOwL08Zh2OKNQ1pA5oXEjZUYsp4/auQIzVa3E4zxH8kkbGZ36a7Z49F068dPPxdTeQYMrt phbLL8T//99cchGXSMLAKjgGz2c3LIzKt/e+SkND/2NP+k7dSKiwfq1ABUGR1rfRjmJhD51nQfCt Kn1k9ipKyoiuxMdZp2jelmhTAWeODMVijIxQtR8N/lIOhs+PhgxpTvzmd0ISCK69Vq6+VpHPB7RI UNryyNm5vcgowuTut3sXWKFtdHeFE428+K4EE23N0K/3yT/w1U7yBwPkxoas1AIeqGUkrMEA1eAh daL3TDw5xyWuj+BLgntLRiNS9sgxbet3VNeTSUaBJb0dzVKlezo1dq9Tu7b+Up6bw9DNjljm3NfU mhEOKA/q+vahmed+iDtgt6Rj9I3SCbHIAcFQ0aYaMlw+t+c6qmsHa0wFApujFxC3bAZ8C7togW0c gmEuLrl1jMrdB/vOHMrghlNNeQ6XOoKEaJ0iXvnjd2azL7RRLzRwD6IjFBjyzsW2zed/emEeR86M FneMk5Ca+QaFeMdyWCbt4u2pBwnF0D15f8csgSylm9/wFaSx3Oz5ZTo2DsPO+DRDx7QvYOqNORHH VzAUe1EbK+Ba7yAwtK0pyE1J43X9+XWqd/JWVxG991scrbsV4ivHb1lCJB7fBPYvzGgry2GIAUMS 8MXCOImjaUAQL9goeNJfD5ju8YgQhBz49ju1wAUMEspnGIjMn77lZPnaY81GPI65N4sfPe32OD/H w++F0gM4K1ApO+WDM+x/Z6swrmHMhJSDTcB7AXo4d22Ik7h95lrnvR/5z04XqAjY7ywE/7VUbIMt 3ERpUtCJyOy3TA9aJYlXZhFfiVERNzV0GvhjoxA5cjm7K/CcRFZZXBnRIFNIpraflIKiLxsm0o4i ztx+MxfUlgHy5BVycdQqAieLIqgsOEL6nQFfd57SH0R+gJ1KaDm0xWdmsZEYZaB1PaTW3J4VDOgg Y0QafZfdIWtENSsP5pDC7D6aSY8HrfMtXhzKLed2RxV0p4ScMGq6pAOUAkimEMz2zXbUbn9QSqNH DKyg3OXJuG+6Hflmua1Hid575Ndje2onx5iDkgmnI+Pn0e3SWcNYOUtqUNjQtZELTGLVvV60eRkb nACZYcs09mmrqR++n15E6dNGVjBxZo8ws3FZMiIBvL8u7LuRUhZQYQx6TeqxI1ohsTKxZtDIDxBc cXBKo85DcNKrFaGbyxrgTyI2e/M1KtAIbYEtcNtnP83q9heJITzDjQ7A8SnyOYXB4znFXy8souAr 05ediCsEDAQnxJUh1b3NKCL3Wgnl+O0pvNG1G+7sV+WQVkjwVOcZl7mdIy2Ky4Af7UG+FIC+Znx2 6B/vrOVckMbqg24HlPXR/L7eh28yOjqhkxZ9ROlHsSbWA987K1yvrkSU+Adx+mpkfYLMvCZbdwcX W8EyXsp/37Ed0NgfVwyV4wh9w4J6KVbt3Y7PcF70d2q3tBV+oraLofNWTnDISwBQdzigxL+yGI/3 QciH4dCcM2bU52K/Wbhu3JVGVLuLB2kT6K+/yZWwS+Y/o/xjEMw4+SRaYSVfiQRgrmiGldLoN40f VUZL2B1VTQXmJBFz4eC/GGvPQsBu4VSwM6Jnt0JfLqRfqVyE8+XKZYJtccVjReQGTNROBuWPHoLY MsdFyTIbX4Pk8rQIJjIqsIBGvF7eTNJNTmD7s9ieq8J+C5k1/LloxvnOpgZie/qW2Mrrtlh4Tc5r t0SpcToBpXW+LVVT4n+QM+PBYHh7wTyL4x+db4ZVpxLdrpMpKqzhrWnnYW8WlpmxhVZAWfsu/t4H CVB3ja9C+9DA/u25UXZiRA+EFp7u6gFtbg4Xxhsm319wU+RP/XJWZUKUbyB6+qZS7ZxtnWDg+Dga LWSoiPwJ/9tTfpKtxQ/chWOokd0UNRT46pmNfBm+07sVaRyCgAX6e82k0ZCraMpbM/CSLYwwG1q8 Jp01p13H5qbBzIBeaInHTPsuKyqw7SZZJSeYZzySotbnlaTJmxxNBKXMPo1MPVwiQptaFpNva0gB pjQyFR1+AOXuM0wGDr4sRtTTDx3iK/NPvGCGGxpGMPGcoxB/NuA+VxX3bYDn/HUHA2Wft1Tq/8g5 17ufDTa6JGp0LygO8npvcR5AD1o3FZpOTij9CD/3NhW3C19BW6snDRMv7N9IkgQTMobrAIz9CEPg kd2Wo7Gx8jgYLq6IiGl4P5puVdeiCCV702KEMcZEo7kGl78JBB8gcZiPt7RMAR+gstiim1v8zvjw VM9GsHYeKKWBIGJqgUVMOqqGA9TO9/2KK478kWAfPi4v3R0HhW6h7sv4otu03hed3KCas2wDRhnv lPl7hNruyiN2D3oKOSrweFXUfEDYm1yZlB4dZg76QVfm+r74x2ZnC0PnRki5m7Wl9mfBtR3/ZePG couY8+pCGvC0O4uw41GWXilwsltnE/LLLgfcefixirGoH+uhSc5j8GqS0yQOiKB0Sp7yLiXc51zY fmA3p4qk09cnYku3epIulJvBYQUHH2il0Rz0Hxz0oYHdkBrkHPj/19Vtwq9Rz8g+OcTF1/YUxHP9 OfxMdh0A+kvDAzLEuTByFUn1A/uEvD4z9NfIJFJhI7DYFd3CuL+4bs6ERfclHJdom3j9YztBvLMb bAYAGlKSV4TpEw9mxBG04A+MigLpSFOoMSny7BfB64e2EIu4HRtAdHHC/0+6gB+sxnX6G0u7EidY YzrwK+8Rq6DrSsU4Z8rqoPWLcZ+MzCWsm95sSmx0yPlXg/u5uFpuOyhxv7lXGKCdIHHqksZzAjXu nWeD7RqtJ5a/nkr8fULFFgxKwr2XMx6PRm29/3VO4UK+MMTFt943inChKYevnf6hQ6koK/0KYjeH cephtqIEwSTUAnT6TxF5bt6a8vTQFWRKTv7sWP2ak4/5Da53g+Gx42eoJDmOrq1yWGqrNNISo+kj iRXF5960zR+ZQSrzfXNMVd7H79j1ImH6s0kXdeKg7W42TbO8MYASZCE1bmQGOqBQqy5tFWQcA5Ll gjWR7iFxj+GuLVD2t6TDtwuozpM1LGy7RNzECLcQiwLFWcggN3aNsA1HjDTmy+9inqsDXbQewezm 1lI3h9SZJDI8wrC1CkMiDD+kxJQiRVe+cCzLbJPxi8t4Wx+vbP+yPicmdjE9sjuCIkPS+A2jtAuv 6S2ut0n4i05/X2Idr9osWZHcvDS18jiS4teY4M+tS78KRHxlHg+QmRoO5+BUKTuoXvOQMdOBcgIQ ZuqRWwoeyzUUJApPapRA5fszeXatnqvd2PsywmbyPyxN8w0ntB/WCljuQThIHXdBnh6OpNeqV/41 uscLPLGVqIoAKRDqgMbrpV2r7IzJsL0zbFvUjUofDqvmuaChsYzjmvsyKgNPCaTU5Mr6oOLs46u8 kN2J0P5TD7TXkim68l9JiWPt2p4niskruw9W0eL5y7IPX1mVvmo7kavqh2TP+E5rzSsgoWBq/YHo LT5K1W5AV/49VqXtQYI4uEUT5BHirHpWI8rYPDe9NQ5Rem7YZyvfGpUWcOjRt734QLVqXC1MLThl 8ukhywTR8KPQ8zdzPowIITzG/goMuOku2Z54Lq9byoDXMjHLMWsTKeTJSD7MFtpEyYLTL6AA97o7 e0JGLptTwIJOfVM/cwhkY2uGJTv96tlZ4H7BzeR13XvMGsWGMDKGhXGvq+HePn3z1XfS28c6HhOb EorS4ktTP0pb9c8nDNeJhxdLPvluSounf6MtCfF4ZOkSHIvqSQz+DvQGWe/qzbKgnzdYoKZajz+Z p3cm6ScRl+29hQd6X6auPcpiodB+4puFQ3Jy6jj53csrUTiGYbGoTFvd/laxbABYHKjz/QkxanJL GvONAd4jwaIxuquBrJTE2lfjWRLfu4gNIiCMoZ8ggKw7LkOJwmTZ/hQaTicbr9jSBy4Nw/U3VWI8 jDJf6Vlu7JZmnhwUcTb4hjQA1QgMwdELaFHG+JA+eWBteSRzsNo1mhhf/e3m/fYSEiJKGe5uM+AA gET2jTNWDJhGPk2ZsquhTVgQRVkkvZdTt7mO/jFRvM0IkIbmmxysuVlDPcKoFM/aPFY3pntQ8mEz i5Pjn/fHYOyb1FgGLOGacELIqkc6NZua0/w52Oe04ij5dWIfyCt3uY6qXbpye6AAr/iyhiLp1s/f 3xhBVS+vyRrKintSl1qhowD4tE5RBYa3R8oBKUnUtKJSkC++6th02t71G/fv6AHYmZayvhZNQoaC ybUw/MGZDsIY6VYsCzzJOwTeLmOS04AS9bxUlEB6iYWdXNIX4hI+qgYBRfZUVyNKtWxveMW/Zk3h wq5lYMrCnTpcJLse4jsLD8gUnNXzQ8UYXO27yOqU4sPRwja2NkHUTnaeV09C/COZyvBKmF7j9tBs uM0EYo/ppS7SD7EvQAXIwrRjNpabd2ahGZ5esrFKFy+l+RYIo/pF2RZuOJRK6OL7Vrk6o0FKJifW 7LQc2f5pYf52/jJGvYREq4s+cgWb95gRJBnknATWDC2KlUxp+95uMS+/L1Ka9lCWqik6PszfHhfY jKEQ6t8p5qaE1UjXYEo8vWUcKzfqQexdW5UUr6k2dmzJtZBkBB93ZeNnyCONgMpwJVwYenmRGY3b 19uZZct4t0FSLOVZ7OL8GtwIvI19fz4bkLwZwIIaEQjSDZN/egG6aUzDQ1Sc4fTO7EeGLgpK98Op EDiDi9BL5Pfjv3p2qjD9TFg0UsvBTUwMhd5NfIg+nU+FC2cWLsrRjKUVOwlI2w2CJ8MKws4p5QD0 6WVuPJBN/XlMyAvgptf2M4YhBPz1qQrah/pKAnFHRGJKFSs5fyeZxY4O9K/1SbUdYs+oyX1aKsas /NbTfyo1cw1wSvhhWrJtEDnWbW0hJs/rui5p4ItpSWCBpfcj29IcQqerMkZxkk3U6QULzEvjFCry 81N9pJ9+EgHZu2xDTjqzE28qQ1ALsd5SSp04oVAp41nPFpeER9Aiy14zUN9bHXYWz5b5yc0pN8wg Wljl6FhlLVQfEvNuUpZESdjjD8DZBJORsV/Oo7TUVLLlm8aJhjnGWqNpCUvBKmiyaf44u36hBh/9 ViFcdA2D2AteTDwsC+2apVm8NXbI8l+fIpSI60bflelDJSUyJR3oqOXWye1G3WhaNLq8OztFPD// NKSmCuYZp3++iJfTO24WEOWsqg4DlFZj7tbjpfpcJaREs4NR3aeHYeihWDloHJt0vrM31LXbjIYK DezkZW6WAWezdHEVey/2qmhzYk/XOyVzdpiynYydc4O2h7u3S34KXsLqDZyU312h4+WS6KbARWRB RA4CLurggUfEPqxtXN2RPDr5wSQ7dCuewx+bbNOrxQb+1GKg8OOLoubwtzfNygqOpptg8LMD80p4 hU1iQ7NwQiuUkIAaeOThqp5gUb4juoDF8z5fBEA6Zmyn0Td4Yz3fx3nrJEOHcaeogRp/euLnu2xB Rrt/N0De7f27nKO++iJv4gUHyIWW9v+Esyk4g/Z73XJxYy/AYNEmiErDahG0g+N5rJbkBktJbcUe 1sSAFAMbKrbodrvU3MFUO3AtCyvAAA9Y4Qi8iG/Uw0pUlUc/JLkIuFLDP6rovgxlZ18OFgc232rT HjTTXjgsCgy8iuVeHVuTa/jeuWT2HjumYAA+K3IbGTuSNAPAmUNmE8xPMmlgPDiQ3t/r/VXFyhjX qusbJ/Ves/vkbrIe82u9fu1tXnCPztAU01OCH+L284VriRUbGIdTc51u/jn+19VfiUyuYaYYjaJ+ dEW+ch50TiNnrsqFPPGjWBMKHLL0hlOeogOLjKOavst7dFsPYu8Nk2q/qAunClo3wc1RMGaKVFOh dG69Ol5JmVCJjfb1VOxG8++EqV4Kt9evIgj0YIZ5aR6VJlDxjZDCMV0LL6jej0Tdp3Ml8fxb1kwW rcKxB0EKgQKG+2FFxTwOcy22fHRBqZCw+v2Ur6vBbieubsKSeCCJ7y3ft1ocaUW3tSqEkW1lkDv/ w+qofATFklbozhfg/l1olYUUUVvYXvFc0Y6VcZbWTS89JN6hPrZEpDNRPbh1F+03NroF79d5MbyX 80ZFRECS0RrAv+m5++Nwe23fRyBJ9PYJYgtWydJYiof4yaG9V0TRdo05kP3qE5ThdUdr+DR4YfcR 3FLHVbMeACwcR9UeP3iWTiMHa71py20gvUTmB569hoHQGB4qxGcJ48jC6csd7Hlv6y6JEXG3g/yC MnVsgWTjSCLEpX0ZCLjZB4GxXGf8bwFG25Ipfjps1VzJjfq7uXrzWOfFOYegvNfoGNvysZTCdq6L bniKYw9FPnM5EwoIsM6uCVPanUV5DJBnbRJXKB1eCeiMgV9gsLZnYt9a/S+kS7R20RkUsVEalNaH s0sjLQi1tfMTCHU2umPNkpNNGjodQO34BOQWB/3smqvQUkKNdkYApobQg3RM0hbb5BrGS68ej5rQ qad0MQJ/qUxxM+w6DZwTpkEenAvLNfR3TKO596bI7TQQneBBhAZjSM5t2yEKX98OeIRn++lfFxy4 X4ewbkf3xFjIkN8IZ344UOHuXes7de2RdquccdRu2Oa7TxpmNysMCFpwRESUWlR2n2VRIu+Kz4kE rlPTdy1coc+Awvt/kKccV3CItmTbWg8QJJVoLwELkRHc8fVGYLcato2UPALOHfklMaQn7oI+Ra+n H+YN8KxmSokrXfEJ8nii92GsNEoC4fAhKBokB+rK26B04xgZEGz5EW7SHGLKNFItKCeWNWJguQcX RuAo5f83nFtOT2OE80fV1n0Hc/YhZigRwqYSng9in6+hTgj13/u28EH98O96PeBAVOa2RbWGCo3j fvU37HcgvA+y27FDxiewVWrwhE6Q5zpPZJxcpWYhKXJYfv33naPSyi47hKMWvD1ymdl30PmpbBEr c+DrD/sn4Rcdx5kcHo+7ccazRW/cdmz71CsJZAzezA3q4/11xBuve8t1mRizrVQ/PKsQPBP0gGao iRN7bXG0aTio/YrxzMCqMu77RldURoej0tmUp7EbY0DOZb24sHOv2rI3jClCe8njT8aKGMexi0Dz QiHjmx6SEv9DuTVjd6HqdfD/YUnLtD0lde6V0xlaETki1XM9gmN+HhfHwjmBhtORN1Cqve+gEYAS Js4wVSjtCu1qpnW7MUQcnCcoJvF2KbfqDzmRutG8jBRqrf+ljfpZHtifHv9Zkq02BLxxA7FlMJlv hNJ/Z38nbYBYRmeza0PX4YL/vtDcS7sP8ExCWIJgbSyDGRiG6aaUPF2pdn3cmy6TmiMRS/AwXfET PFJQc+dOl0LoNkqXM4w88luRnhb/nbG5eeVf3s/lZAL81aJhjfi+tVTZ5iA1fKJiSqTOid4e0Kww ia81aSrGzbRRwIwoQBHxZEHFpip4dvaSOzOdv0UKcW6Vm2n+px0Flo385LuKRcqZtmoRPDG5dJR2 JOsFZ6G+3jM/WvTnsnedaBNswBhCBMgpb3FdmWQWd0MUjitYBVgDV1cQPP04gmWSbJqhm0RDqsJd Mnr0PPAbopZ82JRzv5WnY3uZ8HaCWACh8XHd6ACni1+UKOX44YmTfddTlgmiVX++PX8iD+kw4dlG qb6YH8qGXNgaNkPoVY+SwnhLkQ9Z7CVxij5t8c8lyi8LgZZsoaU4MpZ0KWNPM8gqN0RvDlwHFTrD on4aN3BzVFxi2P3RGl7an9tt4jzc0H0XTWn++2tlaR4fSusPM78XqmdRlhuAeyb0jFLYV30pZc05 Y/ZPI81aT3cflri4Zv+sHND39/UGNxNcFIDAu81nUGmnWJw6MrnLy4y4nLZ1RFlkjocA/2sq2OKc YApf+LJwLxYhLw3UEvO1wwnFfPa++Sh2cVJSDMST2d1cJpBcRNGWFxJO1gMBOjeu2TBfpb9P6o0J 6jGUldSrRDJMtBKh0YQb2bZlfB6zJm+S3pTwLtkylZYgQmpNjo0EpPofmj8KYxUJm+ZcH+3vzk/8 dRV0DbiH2ZmiQ08WbjyLcN/oa4CFWXDZJiRQVp2mHMjPah46/z5Ndzd9ck8F8okM22A23tsdc/Bg QHgBZ5BuzswXEUMOYZBXrPqB/phtark2vYzHIiOf+NbrnQ078sN7fGLplwWAJ4FXV/aq+pWX8KX4 Yh4dL4ZjHUi5rJfpae+y810OVQ6KEwJT2Oqnnho/904bKCyPmYLahqneGqQTx6i8zR/HWBFZTI0U 9AXYN2xO3RfpUB3sXihp7HJ58Pr6KFvRryjqBRY+bwxErMcXX9vt5iNrIpStjdkGUFG6LQUEZyRd +Nrhz4BaGF19OxYDvI16KC2R8HPTLlDMlriGX3Aot/ES1JMctdSWTC5bvdDVXLG3/kdz4l+h++Hp 4eUo30iP4bylBXwmnFBBi4EoJ4FlKbmMF0AIsuCO1jRzCbWtSTrGp67Iiy5Nj1uQ2O9uWCg0Cfat 0Rqp0zVH3nwGURX2JJ1eiBT83k+ZMzOjKYFZ1FNoLdmkx+BMl16vK95GHct0+VmAHWEJRDFFYnzt OL5S2wXsQKpEdZD7DSW4H5cTUwhCeCiaBamSqxP+AlM9i0JI7QcSGQSvNt/3X03LqAjrIvGv0WmW UzJEiohynelyfppq2GGOEYY/TaRPNLX417wAlo+ysrpjNjVrK2JvrYyVK/TrZddTrdja65U+Zcxe p3OOoDnTvWdynlCwZCBXRI/FE96o7CF/VfqKIZf+Gb02E2g03FGADVYS8wQqF0YvA6Z9hEJJhDS3 HkhGqQwXctAgqFsyLKoMkkydWxlNs8+Myuf8a3bJvg5lESg1xU9VOLc1ktumv6J8kgsxp3pJSDHw LGhSFR8kbL/G/Ro93QIIKov7YfXPKbKFsYarPvEI54zBDpiFN750/SWARTI2lUIcVU4r5A9yqFNI 2FbyPZtjbhQKj61iXS1q5dir2Z1CQOBSuwNN0JmOnzNJP1g6GigBmQN+ohTLm9ABQZKYCON3H1Jo ROM+zwVOEEGqXqJz7vtkmwpJvLNrOG52ceHu9aNN91x8sVY2UECQTfwGsdojaYxjCpJpiHZOcfIu eUZ+BJmMu1tVM4ixFD4nANhmKfgYZoEwlAuVy1V6Qnq7sCfeI+dWN3YK+FMSHg4uA9GEDV87fqa6 Tk8WqRt3KmC2b3SNGJ+2A9wgXCsW1Z3NVFXlBObSYbR0jhaKd3m9Pn9AH0EiHHCoN0wghAjZzCsi L/RYHmNUZNBdU2u91qwEaWVVnhudh7VZ4eJWUyRC7bZT+5vcz72Ft0pg3XDg/aPGv1+4pjM2LRyE V0T0+POf+Oa4sLts7TdyLRiHmzAHqpBDcJr7w+OlgPNcidQ9/gZs0Q74Ef57LyumIy1CZjRTJoSr MgX1DQpGjNC9FkipnNkaDCTPDSlYTXmTdQcfjY9CD/dymHv4T8YtVwqf19ut4zxwoucrUzyaujHQ v9Z3M18tJIKWYl/UipacSdrR3TJ4NzkfHnE7ZWKR5kySwQw7KPFD2rxsZmP2H4ZJPERarAwgNoGL Y2aRg49VhJs5EEYjE1/xN3Rt0n4vAgU2OSxNnXzcO60plWI6LxvGbYTV9SzCf3GjA7A4A8TdNmPh o5LCQlYaMa7CPKJll7vf0qr5HXDkhhFxJOeDAAGXj6hK9vV1N8MFHg+G1hHakJnBe55Lzp8ZEiCh w+hwSwe1Q+iYW8KbPNO5Y8D8MpUo3iF/gFhGqwQmFhRZghoP25RnU/F8fkjsxjhfWYz3+Kiqu49B 9b1SXLzdq8FSlXP0Tc6zMsQ9AX/1blqeZVheoDHb8JMv5dze0qBi1c7i9MUvKeMMY5ax66zEc39b O3+yfyGyBfejjPVI3QwVdE9QkFiBUaGd+KDF1SWiEz1n+T6ZVUjPWcgxDpWnwv3Ssv0R/AbHEooo xKhE6PoB7gqyI+Eeq7BsWA3yeVtwLK79EwO+6LDlp8g7oKfwZbH7KGI8so9T8DUd6X8ze+MdcJ/x HBqmqmiz4z5n1TXj3bPlwQAxDhm3xqep46qKjO0PfoBXkfY9DenOjYXftoEVQgBq+Xphn9D3RN9K x0AavAJcnms1qFrtXBV5oTFhoI8v+54RMUmi4u3HYFMIY/NFow9XvG6Il2AyhSYA+W+amsaPMYdd 4JbuyVVRLgpd5AWCc5nutrJx4zXmIZneX/Ct89aptkDpQMrprwUQ4RqrQoXlFuHFjHAXSmGePBRY 0s5xgUSmsdbXWKBvtrWp91uOMS3/c3W2wUy7aQ8eypbJCymODmB4z1tDbt/O//eUjzAOPID84nHo 9vjsJGsWPanwVljeX0pg9fPtdig/78WpVz1pSkKgtaOxLd0Lm+bpiMuJmH8zsAVvJBmnMAbpbYng WVBbHmTlINXCflI/R2PDV6qK2VLyBW/GCJ9g06aKRVLaDJ09EhqpdSZKlrea3HtMVvoIkytzUcIA HW/gPoYqDfE0tUemH+ii+lRjWzB7Namgwjj9Dn0vpSzFgfpAmoxzHwIEyEov9U7YOkyBhI0CXv4m ZFp6L3cQoDlVU2Hy00DgPGb2Fueyp3txYuVCKc6M+ZOG57hXou2D6n9uaTTIM1HK6SUEG3uVf08/ rWSiwwk0ys7EOtQqCLM0+OKR1FHM8WRydiRc2BbQPvEheBlu/1smq059DJNcGh5av7qVdqgjV/8s RCKWk+G1M6ad65irq7fZjaBHih8FKYy4fhciOQrgfI9CgmZSEEHGRcGFf/Q6g1RQFYyggZOnen+/ /OCt/1PvpnmFzE8VuzWDoS2GCP+jIW0ks+mAiZ2Av18w/QGm22hO7RswFIIaeMCPwQrsEznNneFv 0dfA9ay6AmSWxN2jcjvFEnx4IirXXcWHb2fSBa/SzHlOtwgyPqe1udGtUHPeamrdJBv1mLa9v9+F 4UvJ4mRiG48laYrRisVnzV1itvQFZibWdC6dIigFDnKKMj/hrdWzTQKCxhP7iK5PjPvQNBC7jG8y 99iG7RW122OfUf1h2qLRzlvWAdpPDNJRZrulTEwXw12zOfyU7ZcabTzo4Hho78OgkWEH+A2Z1pmb HKnmnp4WudP1cbxiMLK20jHFTU9baH7cpO/wXvmvgnb5/H7xYBWGIKSQ7EgaWi54XjVf78s6wlAy XhD2ZXwDmO57HL2kUfsNxFY0o9Vff1nZPZUDcO1/N/T7r/P4idQYXiT52Mu2LjYhuzJ9zge2FX7j XrrodhUG7C2krAUfgJ7PFYiHwxl0nXtIvVK+IUYL4fw/zLL1Uhs5EsmyVLfiWxJ+2gYq8hXBH5M7 NPuIfGbydnNSYck+A2XMsZR21pOWGv+2i8w3s2bnxFqkk7utihP9BQlkWdiAqc4unX5QR6KmFg3Z XsYlE3CciZSMa+VJMg2xOteRq4a1gJgD+Yny4E8GjRnFMjphzQzqer0GAUJJlHnhBR5CyBw8uoA2 CB+a4v1Ztugi99t5R1T9dGRP/JiVQz9OUJcjUqHykAERhsOeEc9jh86aMkYfl3xJnezYPXgcIqao dGjEUtenDmZP/A+jnjgclyN9W6TpY35rkQtsVxejoTmzk77QTIcsZoYLqJMyvm61mU1W/F+Onp3I 9DAYSQAB3qaQDD6F4QEWH23KQhL9tI1SXZUawOv1vY3qyebLbDmjSf3iy3xnncLsWdraPiFk5/Pj gHKmzI5bD6yrLr3RyIAVxJis8NeFolTziaP3v4QiReAm7a1ITfmPQ3+KvUTmKfwsSClxhZ6iAwSn 8EMDYUrcNhfYAlNhhmr2ISMK2jlsiLaKjgmrVIGatURnjZ7NVuoyN9jbyBFPd4UiFLimAtMymRD2 ErQ4cI/44Ugox+z6+aFL2OO7awGFbPyAaF+49sqEGR8E3v5BGh5+AeeebKkLp960UInx5CcuGw4u r1giTdTrizTFTFMcpr5PvsTfIg2ao/h0apyoLeuKw5LxkuHxNE8dsOaSZeWX/Yg0JdDhyjtaYY9u AxJv2+jyOn/gHIf3IC5Qq2MA5EnmwDk/KX9066GiqVoxdK/ihc7aAEnGuA2cXe3ilk4OFrBakB2e qnYXcQ7EKgl/dgsLsDh3B7ruGlIq7F+yzTupH33mSZQimD/oP2H/yaBMoTXqGxVTCoqUrk+vWj4z T2tIkih8dqFOmHbUgXbCeRCddPvAdpLINmFByE3zYXeulUrJHOoyJQhAJh0Xs8PiCh5ZN3w33ef1 XMvWm/z4HLTPLMoO7Q95ynw0TjMxnC+Wn7AuFGoON+47i9Tgik2ffZn+J/ncN1Yi1Xdc6iZDuQVA N5Ulgtg24ffFSnKdveuCj2pkQ6H1FP15rJaU+2DPsHrKyU8g5RmtdAK4YVahexymwkmLIpVbxVLC xO/GZibNo71aYLpgIiyoTdkI6X4TYCU8CCJlBKnsqSbuGRWMbuUluZbq16unCvH6cpiM89n2RRkr phpHXP/OSxZ/uAVuMXcIjqu/o0TXjv4Kcybf89OwXzUmf88k51wPNFQv/s4rRkN0IU344StzT6o6 cHT6bmFk9e3WrFhbk1zSyT1aaaW/ibivCU/BVPaaSO5VMSZuNdnRkbC1ffF0Ca5jsj478TbYIM4w u49WgWnhKMJYfNM0e2MrjOUbwm/OOqItXhvzIXJOnb8z5fRMIn1zDHGWnLyBK/xSzrLtVJw7Wqug XZlpdTu00j7hjXpjx8C5QA35dqMUCPnM8ZzQd9RENl22gdPBTMLS4rvfCoauyTu6UJW8HA+jJqE3 NIrZDalERU7LNbZJ3SwvgQSUl5cYy0KcWb63+jzjrliA2qNUyF/3gFJNiH621yRSL5u9zT7eaoDS wkYuSfG2T/gbpXKnASsX2CzajoisHhSMWNDLLHJTanFvGmsuvAGMgu6rCHdBsO5I9/XFcsB3lTRg wnNo+c8xgBMaa7Alsfutzzthrg8/ATLVwGH5vMVhVRHELA4WBCITQtsd6ifS2SSRoqsfznvlLj4Q KdCAat7So8F4mRjkRzgPAYHoejdSsHrtvsAM0oarUdF39hKpn9hMaxhKKKImRe/hW8/E7TDBmVhp GwrLQWKlKiu0rZV+LdRuTlJe4Wua+qGj7bCeoxe30GLd+0yD/i/jt2VDpA/j133PessIh7UCNCim 2uszqoA1UJFzvQDjKpGlBD2CSlLp6piyLVEm1DIXr9UgEGK8WC2Zr0vC2nbjaqlPjSxYmHQmJKmn arwf+lUSD/pEvkDYkLCQ2JcYuTzRU4vrVKMCimfTDFNiC2M05KZWHuoR8EibonKxE8P4mq7IpEwG gnCjzytycfvMgOjcSE846ltm7sBo75wspj7k9DFkzVW8mAzev+EQ9H7zZRw4JsqdUT6SuFOiVItH fY0bTeGalJh3ng+Cr2WzrwSwXyyDdYcKgtkd5atW6DT810/sIUTH2xrHp4vIB1GOKatN02NH4K8m hYLgqMMjou1b9x3kCE94KvwrDU5BN2fFtxplLgHIaE9wNS8juA4QtKh836PjvasaNolm0Gr2K7kU amdZm0QFmQtfqUoOGBmZRe6IYYcVnl9gHKQ3YqitxANFybYoiypvPl0w63pVYk5vf5v3HaW9hfwr Cqn71R0W4Wll5d6mZDpP6UrWfirF40C6I44B0lc9RU0hKx5drhDGA2Eb2yVB+wx/ypQbd6pDvGcU T74cKxFSJey4A1/0w7yQXH4wmXRu+j17E6BJvj+CwkbQqOzZdmG3orBU1x5mPHX4nk3knFzLEB4J N3hEucrGhRGpIChEZ8ocn4hPBCqa8oJUsDlqNtvFd4bGjOYxllZwGgTN7rcMD5po0Ke7TXJMl/if KyKFOZZsUvf2iQZPB76RG0O9cr5izrffclnxT0QJ/e+vamWeloZSEhTmREcmZ99ujRljuQRX5vIP lScbKS3cLySBWfDFMP2hten+ywZN33hwdqE4gXX9r83StPbSlKsGR/GXs++/bq/OhQBpxkVu58GN due/qX6+Pg/BTadxfbS1durlIxJs1AN/ctjmZQlP+KGqmNmBKxcfEqGwcUDh+hzxqBkfffOQxeOF 3BiNzsKAJl1Y+UfUjmNWJVh21Z8G4lxJ7+3/U+j0T1U5Nf+5jSCmmMGQtFaEWlwU6pY5SKHY6zAz 9mXRtqI8IaF/Onn9P7E3cffYoGZAw6FxgpJDq6X3CK+1tu6U3ZOwPqiWNKnSsnl45IXDgttHja6d vX/viKbNBQoKcQzQ7LBP3I7uCAzp5d+NmEML3/+WTlTL7/YMO90jnSHtGs4hGEf2oPNDho3J5NPi 9GzNRZWAOOKP++s7sIlWnKCmfgRsqfpOZ3XKpxqxJNqUxzsY+VvHNcmnIptxX+RGB0kai26jdCOh EOFPo1gHmETgFG2uO5pSFeVX5BBuMW9P54+YZq8EUp/naIVqPUi8IMioifDdK0HLVBgj5PrWHmeh erhXQ16sPQlQmmNOzqWKA23wDPFvzcOmx8xseHKIQB460oE3U/R8K4qiJEj/usUn90VmLMY9p7ip GQPtf/Bh9VZsKX3rddxCwv7fm40qAL7Dlg9/FyZqBRG4Wtoe3SIUDrnUwi/UNDcpTo9jyDEE0OL5 lACY0qcEhryWdQoxqmXvefbKG1YydAPOS9pDY4DsInHC9TAPqyEkVPF3MgdO+g1iBJ7If5fJe1VD 4recDKQh+FQgXsvNnoWXoYHBuKlkQCwbnzHPOztCmG6eK+/Cax9qLv4RNPwdSTLsa57eu6WOqM1T +wD2t2oNDdBKSUt9BZcaxoQScDleVEWejLQg7H/1omuyeY2svc+fRRF5HdYb4vam38SMQrt+dhG5 io60HTHx3h66tXDpnVyUnRPgsV9TamLtIeKLbtYBWm8N8pUQzzL1us/MeNk6PgeVSXbVhMEDU1oa pQtX8ERbri2y7cz1oOM0ocXyj0fhk08vnK2GUpLC+LAJ8DzUh7QKIp66xXlzVSiUzFiBsomrCQAI zmySIQiByLdV7IK3pf8P0GXk1LYH+f0Dp4DSv9+hZwXpJkhEwTKTQSzQPhVG4eLjZ0N1yAPkn1vw ViQmZ7l01zi43IF0263HXp1YovZ20ZwkrGKK5u9/6rxpT/8+zDguzGAoJjHDyPi9iwek7hBEc07h hONfzbwlwJKbJGeE4q6jCosFaWSneWWnmJUnH+3WQ7oSYSuQ0wTGulJsi8CiXoPkkPb9ObmjfYKF OwRefxlkDtjOFbSAJMPPPhdhk0d699oNZnP43pu3nCXhayqVDdm93QjCJ7IZJbmsrMGjztMtvIuX +OPWGgZoA7JG5063XHhvjf03m2duQi1H0sJPw1ko1IF44OR5MXRuYjqP0L7sbbI7Kme5NnVMzc+K ySM9Xun+e8IXhgIkRp7KEuk3ONTfsZpvk+V9q/RBZnrN5QKaC8iWbClm1r7yqpljLOrXV6Zwy9PR SQXHBP5FsHOjK1Iwjjij2jSPzppzfLpg9b2QKPDMlFv1puWVjHIjB1KdT/EIFOlIw3LwbupXompT gPn/EiPigaiFQIj5LKgatlV6OWXXdUCHhMaN8JLhrWsyncB3aWsYZuvb2A9jlzC/paavVyOLebq+ rm2/QdbRwU0aVOhyTVkEK6FDO+PQdnyUP9777pJzQ65IKAwLHOJsysg63Pj8JYUzyRJb/ijxGQdP kj8hibBHbsWlrbRgWUjrb1gO1JFEJGZB/x8ryh0Q4b06Xcfp5jBvbbMyAgNThBiEyjlbP4vyf40t a1KyEAiNvGD4xTY0o/yMZs66Hs95qhKKzcoYIEhH7FkVn55V4Au44XiONG7wCU1AgbTg+BRRWB0D y4kPXhtTGupFfKbErCyTBXm/n4UQ1zB8Y9Mf35er84O5kvSEWHZihIsne2worc9R1UWulx8anQDl e2Y46lSMxNA7b9qMFA5fDpXDSndGPKkOfkG5aNJkKPATQJkL3AwTHbCzKYShLp1ddGlvWxTcksUd ekKGDk3971zZU/XueRdvekVREtGy8OCEseaRnpJ8VQUpjYAmoulEtZVL6a8YJu8cL4Q5TcgQdl7A IC4AChCwAgrq2WmDM7Y3bNGVTSPxDCPk6NrE5gFG1Sao2HuzfCDwJRgSWhQGtoSiXjMrm+ABu8Eg x0/RhSwZcxk5XHyGh/PxzFo4/EzJ6KXnKRTVJsMavIvckJJ4iDV7O8zZkDsMeEvCCOKJ4cWOnH/0 xSTibJfqDWMSoAk+M8ZY+hA2d41SLxgJSmZ0CWUe1F0Jr4CuX5Et5f+k/KYmc3T3hYXUApj0Lds8 YVOiyIDBxH1VvA9VASnGxCaunruSpLXksyO3KXmi2rZqyiEdvQyV2ebHYI0lI5Uh2O9Hl0ao4h2l y9OLT54TGFqI8LKLsZPY0SpGltV0NY9BeddZl9TpzgjgqCauCUpqUo726GtCHJ682PrUupZi9zTP y05kWcYnqHaVco1G28/fqW+p8x1UJQE6a6Rr8JswMmH6EtbNjLKhGyZc3UrMvJq9mzxGOxGOCASM sbmMPv3Eb5ZNfmaEjklTZDr0sHto2FSGcMCVfUvqzDoaqf4QgeB+DXB32BUWJDK3ieKU3g5hihOK m9wTs674x1G/Z6CL133OZ7fcf2SrVJW0iAT0f8LFAHNPhTo6v/FYiBwjd8hGYmtZhXw8muZzIwKs AI8pXfz50O3KZhWId2SvbvGzYLE0SQv64V480PF75lpUw4cpwAMFyMUv618IsmYXo4wlky3z9Xro NTnwNhdyXGOLHt2tK85w0/E1Qls/b1foDdqwwqFXVZrvS/lcEiqLdoIRGEABSsjWua7VkmBkBV4p guv+7Y9evczqA+Dcj6Z6uUbiLzv2dAXXcTrfsc8Y48O8Ie73CFL13BDmE9/imGYJKZN0hIAnkBje cd/qn+o7DNCSH3x4lyfkuybeEjaZv6k/YWEn4BT6pfO3sb4OqHmNSZ96rhdffVS/T7DhiM5/ZQAc W3qzyZW2T06nD3on3Qsm65wUQDRxearos5VazvYc6e3Y17TUnssGn06uBNqXdDk3twrrkw2AAWCD WtMaHr0LJtkZcq36GRQOpPTqYQLQf3dpHWxvZ1G4pb40qZMBj84V/xksY3ZX3sRuX2lhlM8e/aUP Mf0WnSJbO84cfWKnwDOFTF73EnbhUFQjP9FeZlFl8zPVj0vdk7NTUFOnHqdDmp5+O92JWFl85Gu5 dYamLtVlRw1UIH0TUM6RojvUpR0HFYz9naUovIcZrmsT8oskGbjJMGVlqcCspVyJvk9FTWsN+RyN PermmEE9nGR3c6yIrUoZDuzEBzOvLRVWWI9n1+QpHV5S3xMBLLqY+Ne3bQegkCXlwYN2AqaSgVbn QtAVKnaE3FvYgdsNqQUCIaseOlmiUq27Eph5GwjpPzv9ITe51YEEna9kS5X+w3E9WUma6pRYAe8o xw4ZfnFX5wrxtscn47nzDImjx+zEqK+Ov+o2Mo97pYJFGUCubHZ8xCt0W+IZZvW2XIBPOWnTm7Yw h4Iz68vFeS4fsfKnWaKJuUZYlER2Adk1OgZ9Bh0OHhh+eQi1B6z5LlDMG9pdmsmpwiqiCiqz9xXE EunUij/iLfKysJ4jMPjEgfQC7tfv6E0jotPSUvlRDWRZ214+/hCwzQceWyW3O2TXMDt2tM9u7E5q eOuc9hqZCvH02400DYPjDCwKNi6G8B9hY3RPmWXbYYxieG0TbYLsP3nOgVUh3EYkkswak7tFGgSf TPTVxJmgHHJUAZH1tUa26P/Wl1iIdOEMorM2r+FSZhoWV6BP3tzPRMISa2SM/auUq+vEBohe1hDQ WrGDRix2TDGPbNbIvb8Bl+vFb2C7j8sDJPdoE0cJMlmrgpqvdvUowEoS4rmh5hEcfB2qm5TZXg/6 KeCpD+iEBhAkPUCoKv5QgIIYitdT/PrERnqAoHlrHDTUTIh7mtVd9SeZ+1pj5WVK7M9BQ9ZkIvJt KujqiE2AhIBNHMV253uyKADhZREyZWlnzcgOt/hxcy/I3gaGrar1DY/fGdCTOHILslsjYZC1Np9K +6NgAFskYsDtAbftqqY/5CGLQpPXhU+023Fz7HOIKJ07kOuCA9dkI/7X3OfAEX6T/1twGVNWG+jj of+sUcyxvNPsp17+oH26pK7vizjV3TDrj+DggedIuePHeGNlkefN79kqzjdBW1q8PFgciOveKSUM 4rfOKS0MqEdacAc5X0ciacWfIszHm1X4g4kqTS2qay71w/avHNCtanbv5MxgJ/R/f0eR4se96fpo sYBh3cBGNF5ZqogkY2ivuj9nW2D4mXxbWoJZCPfNkYY2XWvfC5EgBOm/XzJilYyMXV5SBAojtzMk DzrGIg4yQ8Zk9nmVno2LDrZ9l5x5sKbU8pgzUKl3IKqUrL7KXW+nIK1NMB7Ih8CjqzLdm+zLI6u4 S3gq2EZQlOm3kS6dIOqzVrxYp024vaZpWx5/0953m/JfdBFTPfGolhfmHh3EFivvIRkGIdbTrdzc OedpuBReLsXryYqNQIxPzNLvpE6VQqBleKtoTG4Ygw/7LwmFk8qT3qWOekmPJdL/1uhomb2OhWvA MR4FXKRQ51snHbF7Xb6cTx48h/uNpbPUtbjBhkAq5VELJlsN5LV6kmG4auS97gUJT24y7VjX5L8b nUWDpjGDktvqpaCt1ZPS0UcQS+ZIxXeKqOtKpU+Iy5HVTUeYyXt7NgQyeT/QMeL1VRyjuKJ7e6e0 jHpnCLi0T7Ad0EH3NXZY8daW+d+gLsQ+Rq7r8wmcZ7F60XoF32Lchqo+EMdO+nr2rKY0ODjgd70e xXZqaaX5Y4LZoanql7NwtfN7zGbwiJfFFhb7jb0aw+o04cksbV/yCr3gCIBfnVdEv2c9EliuZnBd qWxvaPnuw+wvga/a9hhga4n0BO+8GjxusF64KeSM+ssKXd7sgK2jXQq0WKcLo7SSgGTzp+a9KqRY gY3dpTWgLdkksCCL7rqxY4bppwVkzdCqPUdP3+UqdbVPu1N2vxR7MTQfHbnnE+bCWrvOR9/mRw1u vAq4qmIwoBgpU3+DuZ6fkYIRZqqA2RGf8xhHVDUpmnoT6+d6ZxGR9eWOcOKtw1HI7aRWMmWJ44Pm yKDmpvikwElJa9dfclw2y+7oU1FFGNBgdvAkGbgbeVJAPXN9j8MBxVN9gH69gGd/A4raRUSOsRD1 YcfUPDeyabdFP1i0hmADJmKaYwOwdB+gRGrEyCJ1HsQ+gKcfpL3dwpqNUYy6zKQlyauhWJhNAB42 vahzMjwnvO+XG3Br6rC4vPYHr2ULFWp1CnHR7giNhtXOBP8FxCdOlG9d66hf8Bj/RRy175QPQwrd 3oPOqAcmQovN0ejbmxgCQXRVL8bR21YR66RLT3VsPUlLAcquNWK8EcWKdyDB5ZdJM2gTPTm5FcZS EPlitCheDeAkutFLdY6oQh0Oqnkjn3lo9+pQcH400j2vW3IJxTIB/x7Hj9yk/tPSxQ5jithHrSvq Pec5LkB2sSOqjm5uBjWmPmh2FsHKF+T1FJn6NsGoETpZ66rjzzfxnZ6SoL1KW52PvHpo5z+FZRYV NppZH0ya9JAhimoE9mSQ0rIR/nJrq4jj/DzG+7dZNqRdcb7GgupFAXlY2mkitQsRZ/LC3pdIrYYi /9pBe3Ora/T/rw9N1pk5hnVb14G2G3LaHhLrxlCmOJ6a2hnvFyeSkgp37XpTL5gX6ZvewCS2iIWC 1WbbwYVGx9OIag99oK7VygM79tNuZBXnxY0QLqVClSd/0hGgwuyfixaR8VnZHxftQCOvIuIf5k58 /ZAH/XgiqiAw493KwVvLPYH8jLitozpAK9A8cSdVWP2wp2Qj3GGodgDgN5tlakLRtBakJUsJWOo9 2h7flWWTgbn9yQQ5BALcOqyj7s8uIwf2sjQ8zLlfiuwFS6Rgy5B0jSNkC3aOYJGasXoC/RRB/FPy eisUID+6Rkhhmmq2plXsMWWEfy/I8KeHEehy5Io+yGKhmwXQlgQ9W60sXNv/S+pwEkyyfRTHKtoD nbsi+L0eDzFD885cNhDM7eQrprsMGW1HC/khmB4ke/usBzsjtcgcNRIhnYIy2sHNipQ3roUKt9+q y97YXhMGfLOEBugFOQRZrHCdI4RsX21qpkp5IS2pEN6ScspWA9HKYaA+u1MHgFnSMewqQ9r+bqEb 7abiNVjs7lwYA5aikxMP4dAi04m3v6ie8WIfIphj4f0g3lYKJhTjScEYpWESdKHblZzwKsZbp8dR Sav6Jc8TT7WFn26pukvTMIFeDvShrokKtijzrM/KNjLOJ/eT25MpStzHUYikewDr+2ouLKQqVYPI 1dEhBD+fp8e9geWHPyCdwDoeZQBXvfFVBs7Gv/X/gKm5j0XaH86u4HHbtCQiKZrJXLFLOYyqZYHf u+WiWHfXYyOQVmPt0P7fU0T3S5Cm8uAZvnvJ4I2Hj36VQvg6yUAQBRRO3vJOiBltdQAimJLnbmym 3H/HvVovc7xr2i5Y7ZhC5sEjjjzA0GGu+6tgnFEU8/SeDzIrNX3HbY0DRtj6wLxf6Y4JwvE1zAas 4WUDcYua2q3CBj3VCc3D/alqUsuCm67/LqXBwNqG1s81BiXK/mvrtxtrgLWZjwlEneAFcHYkIjB1 2Ch0FZvB14i1n/etawbkO75WfBHU9aQsemds/xHEGafhaRPmdqn/e29LajpJxEZ4gILcjX/nayaX a4QIjdVjMH4IVVBsD5mOUveprjjhoIjPDP6470Di9/TJH8wvGzRiZwyd6kDa2ho/fBM217EAHgxh VbP8T+fov2FlaVQRHxucAdj7tsoKYJ/Rc5Z8ddf6qesk89w0riHexdCib3HOZX4qDzdoGmu80tby OLGvsqp759GSDlBclFlk64w4SbQ3bEYNGdg6qSR7ZR44Xk5g+hXDnodSqQ2lcP2Lpu3OScpOxiW6 7KmfnJjlVf4tG+XnDmeRYvgeGIRuOagnx1Ei3UWHWB7OOSMsSEDA8pH9qGs0SPKBrzlMbi6xwb1G JJC91YHGdv/q1iJNIlsZ3Ozz3h0kATFBbwnxusAJZzFjSCaTUcZnTtBNO7fsBbZy8Xh+M01Hw1yM ZdBxQAWXh/FwSX5QnHPrePk0S6UYvPU72DjT5uxVU4gwQ0bKT/pz10Yl+VtQhvP6BEuwQTSoVPZk xkM+r0X4p4fnhFvP8aP7oTCnWkLYlZPaYB0EP1d0PjlknqkL419AsdwHC7OC6WP6ufwTfIjg25eZ 2SE/EniaX8Y+VTAFDgaRkaIAG6sv91cr4wpQ64E1IQN26wjf6pKBQCuqHHgr3bHhSV4IzLbjQLdi hCcRycn4e3q4kGhp3GL4+gF9HSJfb9cklwf9KrVnH/urp+N9wSuANht8zF2hnpcXDWS+eyUGNl+Y C9D+OgGE71Orzfv4HH1acxa413PQQGZ8u+Mck9YUNWm2Xt2Ydmgnv/KECAVIk0P6oJno5Pai2Lla vpW0DwBD60wNj2c5h3vNFxxXstOcjhdFPpiMEoiyW6c9LlPPuuzrkPxI6z8cwP4zUZVDWCFGXW9Z Zg/xnQFnVXKdkFPpOx2ibfsli/2dEMr5iQBqvpT82oCfGBVSlqEVQL2YqRSZRqFZDWg0Km5AisIT F9/SX/FVikJc3KGVP/XWKj0XJ/4r139OII2eloL2fmqfYTc0+8Ox+/f9F/0v4poZ++z1eZlBwP7S NbFxgk3ymeOjgrVAO0eTL4pKVLXb49psFTSEQsdVilPSC92voFzDRF+X7/+mlwTOXu/crAV/M9Pw HGLYHXCaNFQ7YgQRfwJpvDXb6T7bQicTbqPlIAbf8nOfihTe2Lz/X6w5fmlEb+BEO9ZUH7RePjPD ddHktmiBv47kDhLFBecFu0iD280duHJkoEUqvlUihMgyiE35yBTnOqRz42Xm3O4FEwov1Mh01e9K t8Vd4qvUIF8WCVfofUvTpTTBYjtkGmLcvm5f0A2h1N6V16YaqpfwgfRZLLVf6NhQPsMJ+owmHQQ5 nRN19F4KMtw0xrk3agYJVkPhLgHEueYUia1tSuoW1UE6Wr9AIJcexFPwe1Cs4L2kPB0WW5Qha6y2 8cYQhW1N4XS0h4U3CurXvD9alr73nKd+r4mIhUotiKe3ruN801bOMeH7RvSjI2NvvZ5udKHkZEJ2 5opj5zt0Uh3r/qqnRjGzrBaZSFemgz9Yj7DgQSBvGfhtFvIbluigrhqHQFvWoFf68cjP8S24lSIw gy9/doBRq4RHqEtfdDUfva54EzBCM93Md7g3zZXNLqwDSNZi/ax5Owz6GbjH6++YDRns3iCWNUJC 9Vw8CCBF+rDHuYj9ohI6r2m1kg1AQXodEryfncKHi9wcpY4MQg3dW5LmhG/S1FP+4KNSz0SU3dfG kq5PgKYJOZ1Pzj/yKYGWXGaNGIHCMnvR8dhNInivJ4NREtwZZhGF3KyxyPtRixmWEeX+zIvWnRZg PTRynWIk3z1eg2NF85zxbzQv7Ic+Tl+58lebLtL0QNrTlMv63zPx+5RhdojvY8mRLNGfdomMKDUq FhmzVRWx4Y9hpMnjOlzhToy7n/kJ7MHGAz9YpXEogAu9YGSagnH4QNdQML4M5T1G+8tZoJwAlDZU gLfKmpouD5sjDDCew26CLs9HM/uhWQVu3z03nAf94HgPcL7pxGRswH7yoG7n8KYmh4uqXRM+YBnq hyZlkvGvWUZSMFWjWZSeo0b16iiHyTxqYodDV8hhPiVVyJILCrayY0gJPBxTD4iXpTpn+ppbhF4b wxnwzFVwt8JjPoo6qsqXYIQvb7UhZSwA/QngAaOmGaxMVXONt4/XxZktuMEp6c5ZVfRM2NVVYvcD 5X170mGmNA+X6sn5IAdnBgo0WMYCIQZ8VVrjcx2H60FtX+16Z6IjFQ3b6YPV/SUfkeH9FvElRM1b v7kvC5qdCp4dNwjzo5A/lx0f4tIwuvlueuZXqDaDlTKOvil7ei08+5SUDW+2KVaY24tQZoip/Rg/ PMB16/gzuX+WfZy8KsBjD1Lxrr5yM83OyqBG3E7FjEr9CLveN8I+CSHzNjWyZPDgsQ/CWQTyn1O2 yNHw0NesihhTacLwlXlwPuXASVxdfy58prMaYizFqkmFj3eVqORFWgtTLZqd5TdS05Z7PSPbVQ9x nofdfORSWjbJNH61+pRuFFioiKXJWK2WZ+NCiA1Ai/KRcJe9HQleINNIfJDqQ+nCxZbIbyw0lulg 8agr0IEyCgexpoYW9WaaMbXmYpuWDmLoIYSV2GNGcORlQdr1YUM2h2zOxAQAn4Y0nKHuYuPD/mtv uQuM7SOYpv28+DW3ZOK/U2FFhlxvXu1GEIlJ+SjicYzvwVDeNlYpFF47gBpeE07Eo3FbWhia5oEI EIk37oBepwcZTop3jOyDFQVMO9m3CNOmHm2kqVArZ+QtbLPnFolMr/FHp9IQ4BnuhSWTwit7T2MP M4ZK2YhmjA+eI+DnRvYbExhNLCLYvJC9G6a5tfQIivNR10Wtakvun6Mm5tQEGc28dqncnY/MsDH+ 3ra8w2ls5tS4M74L6DWHXcGRxlvbxu5qYWIBH9yN1+OhtaM+08QL/BQAh4akua2HhTrFa6E4v9rx bbP84/U5YABH8a+GmK0MyW7U9zSyOdT4jiqc1/bU05xWoFG2HpjAB9WzjyhKLiOmA3NxgQpJT9w7 8oM4BDvlGcBtsw2nIrckXcZvzUhSSMy/kEQK4yzh6AfVKIVA7jDYfl1EJT0gPgtTP31J2xFHEHru 3MDUi0wA8xcf1QhFqBbXxMFgDFx0An4w9q5hfJJcBHKsxwfhWDbnU9bgGNjx/W8IeSnvG7xTzFbA bBahOiExROrX28IcSj06NlU2PYx6e9V7ntJ07I4HqiNyZ6ZElttIjyjuD/Xh4tIGMV9IpC9zpons NchKkHIBU7ZvH2di6XhuEqLYRcbEmUBkP7IbVQG05ZST6h4MVY1+h3h4WZk4hn9HEtHyKR0SkXay dNzATvlJPPsjyytDTkw0xqFhY9g/+OlpShBP9DRzuhivBMllWyPcnVaHPW0hEZ5DBAN4iwbrnadp m973dgXHOixerDyGqBqv8E9fWcF9LLg6SRWjk353WcNXAczNPkrg+R1xL5q27vMGPJBWjm6d/1lL hLrhjGtUsffZ02N3qQdI9bPgH2m7zuE1M+oG8lBMdhng7HnR+JFK5UI7t+juKwWACh7Cxwya39uW ZroFPOgm4yL2GJUOgicP69kD1Zt+6GB5UW/Z8+pHzcOfzWJj+ZhqZ6QJI8F1Wj1YdKXkl+j3PBNi QmG+gx4iqCJEzNcfhz/ZiIvB90ZPQTfF9YpmpovKxg8X/e66nPGqH/XnuiYQZSNv0JIoNFC37dem z27NYGW6j08H/J2M5APa+zY9uPVyKG2BPSpPzDrIdNaWDf2lE1C2SxQEJ472iysIp2EnvrWnYJCl UInOFPetUVeLtDTvxD65TDzU2bwJeh7SadLGzjsBGsWd591rUHH5qS8h7+n0vhDisUe4MoS2ZiAg jRg7sf+mmioMQvm0wZ1yFQCunEE0GXdQdyxjr/OhelMdLFXDyLPeHoTQR85OaTsLlPDb0cTMIOsm 5WKnJ8nvcFGlKZgUB1SL8DtNAYGreGKaSxaZl2RO6yTP/L9tT5nP/cHC8VadYTa8ZvgNXjsMcmk3 VlUBDtOCuKgm1yJLMwuPK9nN3RRt3J2T9EHZCiJ22nNzEyrXvxtjjDQGJoA7KMISoBoCvjc5zeqA H02DgD/TbyGnop4mNmwBLd9p76pUc76VlT0DHuFY2xXHIU1T5vCfBEtJYwhC9qFZM1+HuPtMlpNh PluKi8qMf7FH8wThyT8vfsuyXHV7GE2jMDuH/gKiVKHw8VQFjZI/HgETJ5iIa8uAGsjJawCjfRLg dB5I2qsD/OdNX8bbVyalVUSzIx2+RAZztqqre/hWi26aMfwgScuBg4pcu4UM+11F++/sDvucxpsh gdyhIDpJhQva7vYiN+MMj5poXBK41phvKSMEY9HSaHxFZS7WxCCF1Gc/ErPg9VMJYPY0wb/JiM0/ SiGA7qSwQdXOjX5vvnu8BRwVWY2TpECGIQKYY78KwxmQtomglyt1pRN32Tt5S9QPYBUyKyA++Unp CGi2k5QV7lCpUtSqWowVmT72uN0vh366/zNI7UT+DL5t+wtSj8dLhu8AMRyZuLnclpzGtkAESV3Z Z7LkNp1/3ehUYqEWq6SEK5TIEDWXowZlDxF9X4Hluo69jOi95GmERaX9F3RcPs+L3vLHfnnIo3bp u+cLw/Jz7FrHenOvZ3XD82cVOesE3zyv5QLK8Y2L+9bkhfd1hkCmmcef6Ca5pi5nQz3o2vTWMM8e YOULVhx1YacV/JSG3+o6FbDbJeVvR6yZyzwynlNjcTZ3Sx21oCRWvOwcuJ+xfNoUYVe8lwEIlUHb u8PuxZxUtIx7H4bYZ0ToXdVfXMeXVwZhs3HIS72QgZl8O3Ee5ckK6p8oJnM3IQkDHsHhezKr5ZNk 5caBWIttN7XF0Y8JlaoL+cDe1z9cwKjVtZ5WmeZa4Ym89ooSsN7vzYy5vJ530dh4rNuU+5dy41Vm JBp7z1d+CW/TNYYfsHYdA1FghRYGTCe3Kw+mY+ltpW0Gwai7VvmEmbHC06XG1PWaYgyFix72A8Tg b4hwK3VRSIeuASNrKaCAw1895IE7rqQJIcqxdMfW+HD+A5f1DKTlG3ZO0nfhgXXaQLNG8J4rQ5IM 456OmhaBXJ87csZzlERW3tPOgbBUmbxTouaER5wrWuGGAUyse2X9ghFzeQp9TaKXJ69yFpoaCGvT Rs6KAaW2D5oYTY8ANPOzWBdW0KMEQI04cJ4x9xfPUiFvwoT/hFrLnHM3iuJx3Z/C6gW7azH+sfos 6RnKmI2km254YYHdFY9iQh3svwmswp9Lb8aWi6NyThdq+sdiTEGmK/hfsIe/mnG+49ZuAlxwua2X xFctWjGJh9yD0da8BA6rgCJwEsMIybM7TMKkfwR4D+4x0JyXrzrNW8ro3Q+XyhkZ5aKNGOuKadtg rdpACMpNjOTE+GnL3igS6G5vKBdOIV71eMYA1N9wZhP5xbXuANAJU1mLPyPzEbnnDAol6hkjzn7B 4hUKsEJFK9kFLMyEys4ZOkrk9L/wCZuabKUXCfu9+YrXvyUz5shy/xGbW5bGMuYkxIIMMBrLxJ3h 3WbSldTG3j0tnYlbdCr6f2Yb7h8z864GcC4F00KeRikBJZEQljbp9v4BNrlejboRl0EmGcWDgfX7 E9kS6hBjjJxPuYsK+Cv19aiS/oEuEER8kDjs21iblz8DH4r7RUBZSSlXERf+8Ps4m0dArch0iGUr 1xqLKCRBF/NpwQiiG+mxdZw+6AQGQuzyHlAKUbGWYr4GuQCDFA+r8BmSwMIZzVEnE/HY7Z39j6bN vr8Q9lRAEqWXg1VFlsu2XrWuNftxEyuM7pfP5UFHRnxD6AGu42T73Se5IMj2bIyiRI5LTLaAugxQ 1lv5w5/k7c7JLdUVzQgi3iv3st/X7oIRD36zRjOEblwBCs17w7cbtmWT5fuGGpsl5+7R5KZmo90x bvGExkCJc/X6+xsS6/RVKl/sykRlBrci6nFWquCOgIwGzNM2pTZxYkYpuAFtoV2a1ahNazTQaBr3 p9JjU1NuiSinlRiIPVyndqG6dhBPVdZmc4V2vLnctpgkXtm8mnrciYeUvwDPcsCpOVMFv4lv4mB8 XxeiXpSo4mRurp1kzoqCHHSAscewGalmW88N7Vm+gcb8fJAKjHahjYwIoOn5L5T8UtxlMQI1fJwt 8QkUIix+KPJkQjQZcfIpU4ZGCbPUtEPg1Cbljn95cAKldvtErYNiNJX7mdif4s0HQZKNgi6+hvOJ frq95NtqIH++FzJF7TO6OHFzas3f2ACISS8rVHQh8+hflRmYFz7hOhoi/1E6FwYlmxvnkhii9FYJ 33g4I41JqQBWN5Oz4sZbIPUI9J7LmIxk2hdmqYhWf7WOmm2/I4NAPBUc5+YZwOdWRLOprySuO+0e IjNXsC1W43Gc+4LfMRqqox2ABKgrCBj30FCoZEX1EQ7/xUDJpaVHxNNp6MZsiN/m2TC2GnGYYJ3V y8YsTXIFsJrUvhp7aLwf0yibgtzkev5o+BvJhbu3A99OT5hhd3DMVzmOmTQaadjrQvkiRrA0WSO8 DV5IKP79A8xjuOiphDSFmXN0LlrAtywapDTI4MY9XnQa7Dv7hMS/Fmk7l/peaSPdDDuQdXn77HAW g9HU3tippJgCm+H/gyJR91Kw0AnqJ5C+wkSi5yOgsi2t/JZR6lIb6zF5219z1kJgKtvzHKbHYkNf ryF9XVQpnfKV+NNkjJW73aiA35PVnWR/e18sWW00hryYq50jUaHx5rnPH8GEb7AMdsPUOgiYU3DR Yd0zVMEgyN1nUfKQz4PJsQ7NLkO7ndne7Igp8+IrahOm/xdZTWf9zfXPaxwbKs282FXdjlqAMWj4 VKmg47WxAH5/2UFi4DFElzfPlqMwE5jfUpSmW9897aEZjGga/56ZZest29gBgmNj0fua5yvGkbQR j9CwqJEIFWqXeYJ7mSskFhqr9fOFxt9idxYFDg1TGBc2mpKtTzijJ7MUCfVq7unNbeTZG9wAK0fn pQAHuWq8tmWS4VsU84O5KYMJAlniAWw4Nlc8nX8v1UMKqOcAgg4Ev4BHut+md+8Hpw7SfRxcjf+D UxuPVRpg9qMdvVbjf2oqDrqYHUylsL1YmErEucYMgg45rj7vkR7SaUo8rAquhCuJjXa6jhePnzwM h11DHS84Wu38kp5m7dqf8WJeKwn6C8+NQ49vcv6gIYeVsCLG5m3cuA1mR95dyc1Y0IySODzQKJ/5 tp1PeAPiDu/zzVSBdXbcuDuwQkiIi3fZQRg/th11J6HH460EjzwC8I5C7KtW3QoIrw60aEWm2mmt j3ZFhCJ+aEoU/69skq6EwTnP0qiXzf2+cylqrx/BjMouBZLT1nXnqU0VN/r6MGzcDcD43Cgxeos+ BAgCape8T9xICRFwQ6X+2Cmr1a06zXC6950jEwPUO+KbR2bYIOGwh1g/rk6jVsErDf1ngzwg4OGS 9OJepVRD/48QmErXLCvCFx9hr29W7GEkFCMYxAZOvbcVHlxvK7rTB5BJ2oUkUp+BQHMnjs/dDc4M th+SqogJOmfs2zy//RvxwGcJmTDBwSzjzLzT0Rz2X2lC86XYa0EvewUhzX48Xn6nmnV+hdWg2iUW 7rdSD5mQrY+7XVJ4GH9oZNF1mi5MHSjQFMfeR0/Vg5WP83/twzVzLq1iucFplEiLGrRBuNgNmjvu YaS5ztzyZ/AFnJMLHBikFWxLIPamfN2Kb47oCv9OIJaakfjVddeQpkusSV+Hyx0Kf/GLC17IVtjR 80T5BOKBKmQDTG+yYaDFPV3B7jTr/uAR9ymZ5cnoO8n4E+v6DUtipWhlLcUuxHE/JjxoLHpTb5Jl Q7uARcnkSRmTCcZhmIu/1yTHw9rN9tHqIPil/AKe/Mfxky1I9145YOGtupFcaJxEISm8mQ4VFhCQ lbThvxlQo7uY6wyosX0B2rIAtRRh1Emu8rtwCo7H9SjkPEAwq6/JXGn8+X3VqRjV4wPv++yGN7PQ 603AnE0B4kF6MLb88VwK1RzJAahHxhECKD9p82BnJFesz6Vq4kdWNfR9jaIm1ShbWoPssZ4xVJ2I 51LeD3s1+zd7Ga59HBBy4Smr762VOiVYL8tXsNPq/WbxZaMr1yZnUOtikfUwUjhMWwBoUAAOZ7JB FuDFLIfFdd46o4a96t8YZuPvqePEMCl+RzAixjFnkP5E4dvQpX0dI/mtoZLhnImSle2De/zBNL9n hDWogemS7b5TuXI+eQzcpQdj16gE4zwWK2Lq6JTEU7h8n2ipi5rQ+OnwQZ5pxsufNggFUUQXw2EZ U/ZAT1IsNmpIexWF78UcchIaPSF+orlHHZO7DwKZAmoE1vr5TZEQSfbM4caBT2tzd5EtJ2ZyxtJN L+ElJNfPExj6pSP2fOTLox4GVprnqczws/8uyT7BVIaXiN4XF8rhG8ZK1XWsWJhWbnXVXkAfOPh2 WkBWiWklxdrtLhM6b/1Dq99zXc8yWqWthQZTLB8jzoBmS/KSP5Mzi7cPkaKeP16w3SEZsUbwWR4t PdQZF1rQ1GouPw5xoeIska3+1qTapAKCHkXosNNcxQkTr8KmrQI1lYTfPLu8COwvNqt3GlFdT9LN +0+BJ2xKMLgeRo78iUXVG92EBPVm0StSHGFpmde3SgZsbkGQmRnJM5N+ZjHt/JCe5jJ0wWhadvw3 qaSScGv+79wXRN2hTcKpYr1OOaUNuhs6CUF6damQCC00JVRhM5Gv0uU5j1+OqnSF8X2cSEc/6jfg 3r2DI6C8LCi0+s9bKW6wF1bTfjXAkuudFOH86Zuu86EtE24S6z/ZIf42z7yaCea77E9QLprn0ai3 dtuX7sluyo7Nwbintff2exhpbVgJWtNqt7t00sOmuWBJjHyQaUUnGPTmr5h823WHnb5TiqHZIrb6 6Kmuwll3pRAoOU43WKguCt4Y5VndNAcF6OyRQgM9TCXkIvynRBqBOTq51z6DkEmbPrA/uqFny6P7 rmTyk8Llu3D33c7BxDrwx/SMXJt9qfzD0pHzgL9cCsUziNo/NvsPvoYW0ooxwUCdCI5YsTnC/FP7 iKoLopnw2VTGu67cuNTuTZGhcc0+j6PbP+d7mzRn1jfcky5jsfKXWKtRKff9241cHA5v0qNJzSb9 OdY1bIzCy8WUXQUkQLvoCP86Q+cd5Oldi+BkKRinGCcMnOpgV6CqoiQlxm6vUZIZPLMet95TLEYC 6daPs2DLyrTcTwPQq5fwK3PNtExPabRkJhcIR2uF4Wv9N12PynAzBnUsHbs0luisyl9kR9MMK3GY 2CnD4KHQJ1rA5Ts95HzXtEGBIVx/e0+L8fUtU06rPf31LSOASCfPyW2LFG768C/mh5JDu55CZ+4i PWpYyXrFJO/x2uOqwzgSgyfdC3RLIAmswntKD5kYqWR/Ql8WWScuY7kKrU/DxkIJC4vZfRAMFiB+ Bo/4Hlv6RzFH4wS8PMrj+4GQ2xifpe8CsuC63wWntci3JHw6BM0NLdebcJaouEREKJeiCpLYsZeY Yqy5gWmB7tseg23sAOhK4AS51hA95CkxJoE4QW5unk2zl+Y7+h6+CTtEAxiPnNdAhUeuKmOEumDN bQvlBOi24xhXhB0kWpaMsZJEg5bYkgkR8SwhAwIUwKwqDZZwbcy2xYa1Gd7qE/3CS6BXTxeaa7kM jdlLZPvsCxNVoRL3n1B92T2DlpvuFz91WjhmEHfuamXl/tt4QPIfLQ63RLUcImyohhYtGOSSgyfd b+nhQdK27cUlGNUzceJM82UstyLWI4O9ftudbKt0nNpOO1JPCdgB2z5DVYj6G+Xv5v4ZNMJDjOvj CzWB56G6YGibfMdlV47erDb91V0KUcji5J5sacGP2OUkHJH7NOHb/baAWzXKLlqv32T8SMDOmuHe xPvRAnDWNmIuVrG5d6KhsvuOtWqVSVNfazYYj9FMSi5QtvAZEDCymvTkNfuF4wXB0jWFIfA8JWvN 5ox9ggiknMCC8jEmpbb3jXmIptRqbMAXaVmqVUDv5YVulaBNrlD9tlneGTpI/m/+DE7sw48oRxBi TmO2HF4vR5URqjl93b4UuUdjSag5XaiueWcF7bDwTgr8H7CHUzzdh/Sslxq3hqdaHHrE2PpYGmtb WbAjz6E6vz0UGx7oNGQIqkZOZNuwyRvDN9gvQhJNkAyVFiev9S0x+WqJyYAQIZnbj65/iaBxaczY wq+13mVsBZHzBds1qC9oE8SIHJ9PZ0B0uzeq1goKnsKbUGdHbtNw+QSNuN33ItjB7qtnRuNCSHdu 9wTxDHGB2Nzd1xcNVoePYKYDUV+nWhfYA9XbWHWKw2+l0G+I4JowAcEG6L/ugevntrcU8InTf5BB uMxnznbAIhK4QhYg4dPA69OL47FOsgyybo7tlT7cKYgNYSSOE5OMTCyyUewyyJbcGw2A/5DBVZx8 g4H/FnbQuYKc8oOkzypZ+mU+c0GQaVv0RKdABTT4a3Gn4aaedhSM/rqx0vZ5djxdmLfQWBQZEmCh YRjVZwr1Ufy9oxaSBooGTyfp+tb7fEmLHkEK2R395IUR6w7gfl7vbmZeW2DjlhERaCSRnZGQhPlP kN7W2iFEuS3UPNupviu6DbOUUAW6NhdWRYAgouNtzPLD354ckaH9hNvUETL1fOt2KmPz0nAw/VQ3 6OzA6KdP1/XKAGWQ+HEKW8RQ24FAToTPCUq/o5YyZq486i1Nyb7fEdEqpeHQ0SflvnfztAGVRxyG xNNXy4pJtZF7n+ZBiAIjyGrB/p/V+VHziFmr2Bmk/7128Us64TPLKCxxyaozeZj1uoHUW4yZlAZQ E5IDaNz2l2pQDGsJAAqjmEPenHPximejX6kDj46AWPNtA+vv/KSPMNEiPDGR4iHQ4kkCeJLxHmoL gfYaoxpL+F3FQ+fIJietUg/VIE3erOgt0tUQMCxcis3rNzaZLgC8EG56BDNIYV9BPE8on6htspV2 qU8YIoEN3qdUkhL9sZkYkRUppH04WL15Cl92Kl+/CDwFopA4xqo6iNpYnmgGt0p0bhOqk+kXGAbl Hx1iWiPqDc+9kU7Dxjqo23m9AAmnUr0wjpTIeF4m72b3HCvHf7J4wPYhOKKvEF/FOEWzMbml727w ojFMopGCZLH2Qw6rhL/6fHwWbVEvHIU+2EzY54RzJgS/5rT6v1G22FZWMUzt99wtn04ntaOjBnEF a16F6nKrckPI1jHTfu8CfXrxj16cWX0AUu9CF8klzoEBQFeiQU2yOzb8tqrSUhVnPLReZTMlZk1+ 2Use+J8OKb54LDEUCZ+q/ctFzh+4hQ+zqvIs2JOU2cfIpHHypeswY0vc/DC7j88xM7RVjsQo5ndd uE/qE/KwJ1NnX3BLAoUinWN6CenL27HiWgPCRnLoFX5pFrbkQ+PMLXn+glZwFx1R6mOL1cfGXpon 7lfvFgt5SeDsAEnBN4Dna/dPHCyZLly477S2+u42QW/BiHj9nuFUMkLOrTS3Mka0G4JtEDmuWJQO n/thyDCA1IatFzYQu9lOEw3UFtycGoMkZK2iGvcPcPjLO+M06KysXx5tsGO3AiTjTH/GeMZFH11r Kcj215IdEiw/r0Sq2YSOHtWK7P/Eqi1e2LSa0Vc60Aml5psgoCO0lFuNI6ahm5maYCwII/j5U2+j aqBz0WYuajk/u+MjQe3Y5SGNe5hNc/yUqRTYscwqxc5lECuurarGXezE38j9FobhQ9QgglfCekzs 0kL/Fmoz2+gaikjbBu2aEMrc9Ak0aVgCqCGwxx4e6EO6jswJ0DW3PD1tkESfOcMaYipY+WQaXFVv 3seR8pagJbbPealoUpoDHdpsrSQ/YNPtSpTUHuKkG/W4qj+tJNhzNTr3Q7EVPanK0bGFJB13Z3VH hbBNGIDavr4y2oc6D+0CvDvPOiS71r1Fa48Ku4TSgc38xikAAFjjwKkrrNkN4lca6CysxaqaEM9l 4fvjymQGFpbuCr6K1zJWH7aM/ewDyuR0/WBmcV+9rUlOL24C3NCkuSU+MRt9AOJ7CXwTxiicWl9e iG/6I1I8iD9nJA+lSEQSNQs7x3a5LXaSOb0Kb4xBJdVFVKiUQzKFgcHePTL2PKzJZ9FZU9HqD+dF p9gRrXIMiASx5IMxlfzGrYx5quh45IP5DNmvTEEHUkW462ZVjza+W0gW/3Z7Me8WZcR+6531Hfmz QS0iMD3iPZoUGkoeLyh8w/CCWbnq6DXdRFi6vmtxZmMnAOjzzXCcCSaTTukeoeX+i9lvDQ7CNQGw y2VEWbjCLHF6dbyBvGCmbTwCAkGKrqqxxWu2wxfwEaFom25g5Bx9++E4nHE47XClml5IVhtjrAMI mieCbgxj1rgvcr59o/uDa/2nXA4GbWJ3JZEyMWc6GwQ8k6AuQZKLPeYks15liEuLmFyMyzMurXdq pkG3xRwXRQP+KF9JrDmYwuCtFkIyJuyz41iYcbbSnViOq1614x1RIO2VIh0hSypgyxZLyorAd6Ij XNG67870ECvjYeA1rQXdM66JmKf2WE5I8EB9aLADf/AkdG1+eQMf1+fcsxvSEbtsz03JcxLBtEAV 050Q2kUN7EJ7KBOdCvWQRgQbDu4bIYYUkjfKEsmwyhDLEgchgHr888+7t6DET6S9JR9du/7X8YFf RnOhSzcmBdnvyZvioC3boesvf9TNoob57UmKFH1OKKpJzpE2cXQD0I65+qp5X4XwReJ/9CtyqV0Z NQANP0JDpNbT8ztgTaUMe+tqhMG9K3HlqbmWbubdXLq5W6Fo2mUTdQQX0iqd26YrovmE3WTbCunZ v7ad/vkQUbtVP8edOwRgT1QAPxJ5h8GhLTlFGVss0cmJLhMUFGvzoT1XbPzxbD0LiEdsR29BJr0X fCx4v6vxHGxmQFjzrwB9kGHAl7+wr1S9m7KpCGohR5+DYo/3m1G//WOuLfQSteL8QCyhKwE59rEG /TTtzodRqqbRT6Vup7OPUl8jjg1Ja5owjd1rdjnfoBECC11D24USy8FLQ7348LwfrZt97q+N1B+Q AORMvxWIAAWcTBCOCI/ptkr5Kwpma9m8kuId1sZaJSIwL/Z7KGgLjElMI4gIjZPMX9SnE127Qp3D dxDQbqbO6tIhv4C5bGLGy8SZ5fPcvCwfVWkrVoLt8N6YnSIXUrogo0QGa+5k99LR/THDhGZ1bFPF fShcMoZXeu+ZNeV9net78YDOEGbPII1PQZrmFgciVs0HPqOyLAxD+d8FVPPOVgIpUsaRHSnVoibj TAKBeFxgFPZl4OsIt3mTgKgNbFheQ22CWZvR6Xtnggsvie8DvdB2NdYIgi0lqR9LxyCv4Od4oLX7 UNDN3wobJ1yTxoS98WDt8xaVDzIwskUCBfzO8TspGtzAfgc3LpqeIo3gItAqCGcJW6bekIw3yuuU mpus33XhOIcBXBUT2nKm6xe74tCXQQIoGc5B6fd+UmPxf/ma44/McI0rBBB/O9D/FumeepjC1vVd XptbiewGqYO8gOdwYbxj0h6x6qUpLnud4d5Nnv2g9Vom6/F8Pcw86gd7lAlR5xkApUaC17Zt7skp U9IAmEAdvdioBzHm+ePbMbtNz3A/OFWJ/P3LEDhZq5o7aibezH5WhySJyMa34PXUhj6Qm7gJ6iMf amMOpZzIlcg5L6zz8krtbmhiBhwtns9K/A8BfJKxGbO9PQR12pXmYxfFULpb8A8YHoNRJV5/KyD3 YGQ1IR2Lsk0XZxJZYdpywieyAwBoi2eYc6fQ9MhjqGcUfSFcBomr0m3jRRR1kxFXRQQiK9hB8PmS iSIZUWQ2zIlRv+lvitEA8mGuuJV07HXFv0ufGSyQkde3Q+Zo8FyyD6P/ziHnFo2reBEC1AwvIqmB glaL5ZBs9wHfHkNe+PaJT6sm2cDb3oekFZ8s/RHp8/vhhT7qkX9fdpVIK8jK+HDfVuPbMVeMjiC3 Y9Y5GlaDCmrdbENg4uI3nOJv73FhpZ0xgyik8J06w7Mu/FKcMX45HzPCttAdXj+bX095B0tCb60n hGcdN8SR39yI3c/FDBmnRaz2rovmnv43upcSkpE0g48r+Msy+eKRaeu1WoIu0QVjgcWAqrFN2W4Q 181CczBwba+rk90WQhlNL4kfiIoEkQ4Bl7KEdo/BxQS86IMPw94Y6sYn4MmbgrkayRd0V3W4XK8w BhlltI9sPB11uP8YpW+0mm/AjN6uqNahl4oZ1SN2JQnRyKaYjs3lWONl6mPbvF3zLDf5pmui8GAd Woyt99dKk6rKJz/sWRr+KrKIBUszfo3V7KqjQqKc4xACuOAOvMWrhy1+iCSX7fxRvd9L17NB6+rA 2gVTiiTnpAK/XCngIZAgh8IT9flUTpI6oIq19sHg9rFb5W7Mh91AVa2jImLEMHF4luegqQUgiQE3 5ddJDaWPlPxsuP12D9lSUc6x9xyAuLqVbC1SmMfrUm20qaSwirLt5iC2DLREOXBoAbfWl1Ugv/2b uDuEuvcs06zfwGpriJwCwOH4PDngTSYzYbD3kFqFBQfXswlLsCxE4LIexOvD0J02R/+cPynOQgeC 3RXQYIO6mHqkkk3mYf/GJpw+v22vnyk8U3A10vfNDvsWbFw4U1eXKmr6ctvlEYNBhhSeIQChKaLc p/I1TZqdaocs7d8PdGLqaAIo5qc0gJS9LZVpp2eIR7YCDO5w1uJdSa4hLUTgc+7ytd6B7ddbM1iU Bgbjp8wwQvFhXrB8k0JEJkvHaNsbxZA/OvLCy8ylGq/HoVfx1mNsZHsZevTDy6H295RI8cP7mISj CBp+TgkBZW95rwfWUY6vOwnGVJ6n7PEYlPrIo1OfODv65dmyC71MgQXGeLmTBuDRcrZKkf3RVOoH qlFRIHCLkVdDq8Yp/Q26XKyIxdT0yz2HyUWiEtu+M4Wg2aqBfEzRbXgq/Tl2Ls9CX7k3j/+5YUyH CiTgoEP/DizXBb1YbyUarkJ2hEbh2qUHLn9Z/1lweFaXnDxLijvurw2iKY/PQJllmY6ReeLFinAo //keXSxsacGIg9vjw3Z1ZYk0ZAm3WXW98rjnBeki4NJtMncudvrt6MaI/aqHN4x2EW5Deb0JTGL4 +lBdw3ZVOKsgyUONxfB9EsavBfzVHQKIjYwipYrh+3mJaWUj7N/Z8geJISgRLZjkzFKTfkvio8Hr T58wYtZT03Cz2+LL5WvTmOASoOpvN73TGLNQ9ZmeRTuTMhYtP0QIOkDr8XMNZrCWhsTsW9tZ8MLs 23YAPgAhbyZDlXDJg4FxJpWhz1dysrTDSbHUuTV/4sG4EJAf7aEmhOYEsc7MTX1v09efpUjT6Elz UkE97xsTKVf+O92Z2jY5qQSTcSMZ//gC8glnE+2xuWMqbr1PXOuEL6htpQXPu8kNUgY03K4I3xKS mMF9Diyy7a5I9s+sUi2+rtIJqEeA+O2d7enfUsKgq5azMj24P9u5S2HT7aQs7ik2NHoxGVQ45NTq OEkfjmZ7+uEu7p4nfSZVZ39SAuz/83LBaf44/aLPLhTarLP6fAY5LG/8bUCC6cXjrx+zb9y6KG5J 28LDO7v5yyG6TxPxonegSX02Ymm9qE6ltiEjMUYPHbXt+DncS7osgAiPUHWjWkZHnTG5Z5kdk1br SeciYpXDexT9Owj6JzacSR5fPUdKoglWKSNeCl4SAzYOzZhhmqJbYxSdZ6TfTNe99WKL8LDdTdWM 9mp7TzkWoPv/IIN0dgFBlUYPAWHUcSR5KfXhd/G4Id0rHuAC0Dn4rQYdyqMkIiFdjHDgIen1NhKN jBE0JgIC/CWK+jZW77TcfF8x3Od7LBvSPa9bG4rsEt3gR08qBTSysae26fLhMwOmgvtZDEQELhtT tgyyCN9E3zhcHzPEmhE968DSSHJNQjK9mTYPWQIswsVY28wRVVoh4UTMVb/Quds5TJIqyF08hY0P +oBcZxEifjDHGex9jWoUlH5LY2EU/gnOOtrdDj252yCVQ/Ac2IHBia+lysdbcBWEqUiRS0IB8Dji 7iNDWukLTVuoueYlFyXGrKLReGFROGhqkStlQgXBLbWzSRoWmhTQkxR1g3aHN7mFUKoA2kINiZ2p RYejnrLCX53I2vrhfcmuZLI6tOsMERPuXIpcYzh2nU2R787lMDaNZ54CVU3zjymACnCFOHjIiFyz j4Uqc5BW/VQXSog1mxaq3NNJRP9vCd/SqPE1BqE53hfmT7n41HxZnuR4c01q7bbKnbmmYrScGhIC YA04wPKWZu7vlfMMWQkOAiajcmc8azPCrS+OcOPrJwzHHRRYoA2Dbhl1xHAZMGwI9iSE5CNeDLe+ c0wRvR29TdY6L+rD8nEWITwj0yAnt4B70PyIW1aKbP60jQiDOS5to6oj/Ij0L9nm9+cWTxiMHSY4 oHmlBJASiGTgkGX92tXgAe41SDntmY0OUlVwfHevQanYiniARwXPBdY1C6LJAXbGDNKMi8Term3L qTL6KOkTOcqQe6CD2eTiXcjubwJcM/kvKwGNRIwIdPWrmyXYBVUrn7mDHCQpugZFMtME5+rO5plW ERO/8mKBVFKj4Z+HIg5iR9VysugEA7remcyPgF1tIAJZmLpuyWyHbmInREGs4Tc9AxTev0qw9afr pT6mR9iqXTMEsbaKly64cOQdhXtl+9hyS+RwYc1OegEFt+g3xMmwP+5+zudS0NJb1nmB0/RHx5Mh C2f262x/hUrMdNmHyBzRZENz46cBM5LGneCnm6IjkXpPJS4rnum00D1evn6nnrI5GR7MzpFGZqcM I/b4unD/maRXrQaDwg2HMCxg2PjdYzYheA6l578Y5TWkXXEFfWsIjtwQZTKRaPo1KrJkp0nFtdJ1 pA5TK67cvpTezjO1t+SSTiRR8T5CMt9w/IuSaeSeL64lIft9U3Q8VZ2mz7WPVaIndS4waF78KuRj iriSUVDW1n00DOOyEJydB4tynf+XIh/AJ+7QrtViTwTD0Wgjdwe0tDwNdrUoM5aZFjilB3ODjfHw kca71oJD/g6VUTrC5Tj7zScnv3GssPLxPcSk2kjPpCh/b0vYI43xbxY+DjV4+utY3WLcu/juSMb6 CJMUgBG0FXpKZQZ7JipMqh4KPk6QfyfqjSSKmlXjRUcCiUo5C0cVUbTIyx4IzWK0AWTWl8uPJ81N 7qJO1SxvAqcl+UVT9kN+QKg+ddj0KX2u2g2WWap3x8gTAJrc2fr4CEqhNh7zDBG+lx1pRAuvx1sG 0EgiykI3mBB9hiipAD7SbVZIypyJiIH12+NodkIlZdoJrml/ksjy/CungKpmj84f5+nR7/TJtpqP yO4FbIrspKGv8e7FV+gyHyGcP4tUVLBHL/nRklJdXWyYjBySuUuZyTRNKHQ07ybmpbPAHVhVBRai CCD/ImwiDIRQLWMa2eUbqyGeHOaiEjb/A5m6eEq3D8AhGvuAiFRLizw9aYL3QuNIpiqzwl4Shg/u sXpBz3OUYyA31K6GH+1e+w+SZmE3SkG4WwQpV0QjIZuhusq2NwY7SmBDi0ff1jRsQ/okIJ1AZENI 6BsTbHs/BKVlisZJa+uWWjdwdyWlqF0XX+vKY0DN6brkJ4944P5a2lt8F/O42UUCClSACFuIsR/x caMVDJ16RlQDZUgnP2GUxXcASzK3DfGwPcuVUsjrMJytWvWyFQ7Z8TtK8N3A2vXbZuvxdUKUqkjO mP4XTNu9TJqYxCI+bQYy8Vvlw0f9RKn6r9XNJaKaOvJbgtAblva2EOUNJuHrYVGcxBuEjfH9GAtw VHvIP+VTfjguyROv2OfZnwZpKo9XUiZ73z+TnjM1l69fznbTwnsnJ3gG6sKFViPdWz+z1hx1SwAv ikUGTGVWir9jwNSQZGzqKBaW6ehuwZxCa3yD5jTz46J8nX7lCvDFaYTq6mC1EZz68bcQc/2iajy5 CwMGdnBey8yWiAq70PTy7OQmUCy6HmEMlpKxO39Z6JgKcXarqPRCx4ZrrXHxG/vhkkF7rM71hG/5 PN53ItfvsfKcCGuV8qK3gyztGFuwg7dYAOu43bkc2EhG5QZ931yItkXssZkJjrzohtJ9L/zskZ8x jgm5hr9k5UtMQTmZJ7AybyyNCsfFacLM8yVwDC2wH8oveB3QKITbsrAUppqY2Is5TUfHD2x5g1xr Q62cx7YpQnslEcScdQMKRZh18dhleQZLGiTyPLZQkMo8jOdoN1if34EVLd4Yl0VNxUBx9XDyOIAI GzulILOUb8PQmxiTjUv1KtEcFB4tvUtdybo8xDhpfq2Ame7jvSVzQSumnkQu1haOP2Xuxa8wGUlz UzL21wsi4STowA4V7sYnogmRW2fUCkKmaUAQqAyLtrwLRjESMn0kiwv7gCFCNpmmvZ3ZSjfgn9sg 9SgECiQiETtys98fwJ/u3o+UksSuU4OjV2vR6m8prFof4Zl0YnUqT1JQ/j6c4B6QlbsqvKyKACAo LpCAB7gHItgAJNUL4wTTIo0AFkHNxal8rsDIQxWl+MPS0c7iwD5fWT0iJftciNpqJF9EnTHsGmgj YgmsMVnPTpMHPITwk1WOIyyz7sltAfpg6Y5iGUzKv3N+ermbe9BuP7iOpWSXWYTar3WPJ4OupYVy fR1w3KDfPxEcgd0bdd8HJ9qSA+cgBYt6EVTld9gBD3loWSl/8hrMxXTZ71MCJDF+ZDulQUcSGhK3 32nbhKb89mQXJ0gVlUi0WaaLXvBYM87Ebas97oyiIHHrEkPkKMFH5rDqlWJTqE6YbEIzzjbVQZHd +bkrTz5Cloz1Jz8i+EVjguiQxwWv6KydYjQlsHQL3z3LHSbGFm8kMr64y74ZXrreix2Q8eVlhNU9 BL1Ph21Qv8K07L04jR+l3xmsX+xZfgdYEHlaDnRc+HB6hb21r3n9zM6EZNwqtB2qDtPlAYi2JVa7 1JBC89xOs0LLTic/vazXUnjxMBlKHacVPLiwwsocietLC0kjWFJTQ3Y/adZ7rFWNlWBpCMcbMJE2 Xpp35uDlwIPl7yresfcbFDV2GjYryFth6FK6QaQvfmJlBp1ViNKeFZ3e7MHeRn6xbXDXAb0wsIVk q/TiuGT2I4NTo2VUERVLXAxdRHtRgGNbk5YKdtH/MvunW8wZ3nAiHilzyu6svbU4nmrWdeMiCgYd TVB/r8fUifKNLKO0xCYyCiYdLetwDjHesqlaybhzTI53qgT2Fx0FBHB4g/IG5wZw9J5XfeWSJaf6 4A4W84w/8U87aWaAmZnET1CWyWLnoXuY0zuk7sx7bLH2ZATkQ+hZrP1i2KY07cwunhG5gY9SVALn 8rkyaJWJdmvK4zqefnpQxZgvFawbLEI46KUVcfo9k3LZGWH/n+OuHa4BxUpUXgS/4AsNLqpkR+Q2 cifXVlPWR45v2kzV9fn2nMe8mXE2CxacUupqshJqLEXQlP/NLUi5A27/dGcDf5UOTwwIgvz5Q4FT f2ut6MC0US9Gri4jVY06dPM2I7XPlAsPLoYKFi799kFlcMv1Yah8OS3gJBuwf8miGQtP21kDL/q9 3vsY3kXj6fk0iXPskP9plGw22GL6xt7tXWC6dS4Rgde7nEYdBAO9wIjuXLHDT7+VTFfArCFe1ftw Da2vNR7AYnlA12DUC1cVUzTK7f+Q6Wu0MQ3g+Le7waVIGobS2RA2dj+LIW5CQWGq4G9mCODfhEhQ T8uaN2iUhErW42dAQqmLfKLlLpGBXHgK41HBIOTlZCXK/BGPqJlzeHxwcMPTUBIymW3Uk4RhFxKN leFTFoFtc3J7lbfbqfjM97hn0A9Qzg5rlKPslhRT92REvOa+ucHLrf9nQMyXJake8TKbxvc75A5O 2dF3Gz5f491QpmkrbWcdX/hifR4t9L9ixSa7gL6D7LT4rORUAO98NmZ/fS8cS0LjzcohbeA+q5Q7 O3exRb64znr6pwpXMIaGEMVnNG1eAo5FuZjWOeJR6epOTNXYV/zHb8EeiJ6wyOHXxGR1WMO4UDo2 gq3somEp7BdNgfN+1O71jE8Na7XQp16pcxywj3mX9t9iHrdjvirmpDUXmckdB+48h3Ll4IgrqNhn oSV/H/Wjiqxm9WcKaBzFPsWOLJtGPnEFIWK2Ojt0p2qgJMKL0sSAN8ZiPaXvDGMX1rTy71e/JvyQ RgYDgpI719YkM7WgVYmnN6TLwpX3yXjY2SucvoNdsyWeNnDL5N2fFQGKR9EmWTAtoS8FPbAy6ijH AqpeKRZefVza1bWmdLY6xHZP6PoidAyo/eN8/4rN+9BH1rlmBPX2vOqtS2ChcA/TuD0+ULjoV3e6 By/c+C0vUFpi/1ZJkN8b+ujjonHseJn6h4fdrbJR/ieQXvjJNF9JqoR9ztD1A+lNLaHmiQ0DVNQ9 cBZmpIBS3MVMQWIqrjlryhc67oybkKTj7ibBw8TZEHm3ezbhQGxbJ1HMvzU3nVl1SxUK0z8QVpig R+jqnR1XzjjEKkMb3ah9MJ5lzWLUU1LZZahyIuFkx3my5VywaoHyFsiEgOxvN59T8wCtI3f2njlp 1WyEcdxLGcbnZrfkv7vQXk0BHxqyiRPxSQ5eAZR8cv+ECmVDccAixQSDGyuiByl2q/NXVAVvwwGE 1ogGss7WLaMYXKUFBmNdixyPrgWPEBTWMspfthjSFYt4G/b1eDK0Zvvv7V/snQrmmanNnNm+0V/O e4KILR0CEO284W0ra6Kp8HPUJ5VI9HGb4ZMpsC8YPuDM/LtPx6vhkKwZV28yqvn9d9tgs4wjjF3L 2VmPDQ8pCkhQnXisT+Xkln+l4eStu56/lwQdnEN6e/moOcxkcRJdjLi9slo5b/AOmpPo1WHqTU55 ARg90ZyrRAjtcQXYuS40yK+pxeoaUiqz6n+nT5NRKW0IQ9IpAVpHeu8Ozqlhex7/XUXxVXsWnOSp GauK/f6WgCReSPxhYvl/3B69yGklK56rZOuThyE5RMgMgAIwNavdMWsQv5lMJpqzzN86XYxBsWq2 g67G4CMtnWZk1687jGO0OnM73wETkS1k5kupQvjMCorGjlBmZsi08HxphvZ+0TYILHDDqERxWyrx IhHBMmcFpRjtR2kxap9YZIF+BILWHNduCT04nBW9lcX1dEdJciC1mi8AFDvt88oU+PyRnFn3iIdR sAAYzEfbq8oSRwRBO2hpZXqxTLfFO+EVEJea2BlP2MsUYlKpParrckkAIs0s+MTC0U+Lws4+8wxf v6JakuY3kIwN2lsi19Mgu4KYFbgp+a03neRBPCad8xc86xNKr2U/mctnWlEo3ksbp2/rOP+UrEmI 4skNQopok8FX6g9ZZpShJCLatiPkJeHblyzl0ls3R3J83UQQgKoDrcsGP2pI3/QWAuMqVQLlRhKz tlROHqRJ+rojYvc3Tb3PYnwLJ9EsHnC9gq2wzKX1IafPyblLHfKD4PkOWf4ESp5Ag4nyAbCETJUx JQ7mpTjbF8AGyVqvHu8gB7U4ZdcRYmdIwAYVVolLbPyPYYC6UEZGC9e828YWE7Hx+RbB8/fnzpYy l2j7gZafkjj1sLDoXcCN14K5KQSYIYrGhiAisVJjiOpz6nQP7QoKmL4XAYycXjI1z4XuYLeVOsKH sGq20Uk5yswOQ6K2ZRozg8MwdRs9qnJw1p6Y3/fXIfIXSWuS2xJ+doxjVPFliCnO6MKkNNQ5fUDx W8Oi0lvShQYVozsdM/OeRRITEb/t7uMmyDQHFLVtOzfxaYW18SopLH6+YcIvFdIEo/B9zX7zcDbo INISUwzAwqfzr3K0mxj9L5WfJFBx6T6FQczTYdseOmFBSQ/lJKTVFGw7DG+iC+xwFlWNoGWGLEXC JKqxk/NBdTnl3XCj9lNxqYJ+ZDtyOrgJ99tnfshIC2LOwttSzyxeY1PuDg6VyDfgjNVMWwrX0uZr 7cq8tnT973Yez0gIfv0hfeDrVNgTmMg3V+zm9qmtdQVK5lbZW4i0F/LdzrETz+6+vTNRtZFklfy/ JH5x5XvXlF6uaFMwgpkigmUi5168u7BCLYpdi4UxZ1+3JtdWHps70ZaehXwwf2OZ/OjmgUxNeDWy eYAAMl3GeNHUDlWEAO8ozJ6ctti9uvOH4Y/BKWRIEnBXwkOLIBgXVDmh5+RF0NTJXt4+kHwdBoFN oIWFxgw61pcz3JTC+MKhZCYd0W63jFnCRwusRL37oB0P2z09bINQ1akeag14lNEo6OQlvhf2UkQe b4RI63SVEq7xPCbcwoP3NAC4duBnwCVJ78dUbh2lR76l/ue0pNCl/iTpBEViG89JK+eeBtGD9CGZ sb6WF3JXeqnY2WQcsfWFnFAKuuJLRqfX0ngkOCOwwVXscVJw3zIx5m0540MqRhDdHi8BdfR3iiba 1q3csIXuRamXvd8SWKd9AIFA2k9+DoC0V5/3lVS4a2Pmez8fJkTWSJm3fG+02at7rZUO13cRDNCi QgS7spTTAVEGWC1Vjv1eqU1eKyYpoo6rDXMEp8qDRiHLrFEuuBafBlWUga0FPGSaGZaak2I/CUTb yF/nRrA9deYHp/aim5BUrNSHWm0gMJCDA15DI5lp9HKb0geGjtyHwtMyNntr3VLSOC7lEjAgy5vV Oqhjd51gsjSaL2mOvRUn0SpEjtibrWDp0m8r2WFHsh0dhNOKHJJpY76ROjPrAC8R05cBDP+BAbyR xY8mIF/VJd6+o+leivByyTKnVrMKQEp/RWNU1pUXNJs0webFBi25i4CaZkhO+ognIxEey6zeePMP N3+UlcTwcJYYZhPt+YTsrGn0aKbz6JTaovNmd1c3OgVPQmx/7HwM8YvtBlmsTshFbLEbkcLnPP+6 L7y/vHlxkIjOKHZiw4r4Wr785SRKWn5x2A4DU5EazuVA4vBHIXX8yBo1aoq00FZXoL38Oq2bVBxW OSp8Dh5c611JJg2uO+JF5UPAuxZnVyCqnmpehj6+DDUlmeKoy9YkaNEqLp2UZJZyy2SbzCz3nBbc I88UC/lxhBRgJ8sAx4CDZOTFiDg2p6d9wuxlb8nMV6Ky16GaeVSQEWgGdYTDH4xWzv9sJOdG+vYw zfsF5I7ACwZEULbOeXUXvxy62AF4tFuXhbiqq7A3MfVjdz4QJivEd3iq3qGDa/2/K985g+Etpg1/ Mts6Wrm9LkNtQ13L4RCYeQCl0pHLY+CMXb/SgwZ2Elsiy5wkzcALcpi1k1CWFtobFbjv98fHY5QL Dcpz6a5mnja711KuAbm888bnPE5ExpV7OKHj2m1hpQ+fWeIQHjum8CqimGGJLISS3+mOxa14oz4n VkqvUViKyn2zPFx+KSZu4rpNLKbBAFr13Oq/INrrWZkIL+54dZSsf9xtRub87gIvBtYaUGrrYVO+ xkaqBwO1zCYb5Q6ByVZBv5zIdRETzkJKHFq7oOcbtjfTXZnrH1VNU9D134+TNu8hFygNzIiN5xXa GvX4dsjFAB0ucCEcKjwWfcicL3KUJWj3tpVXuRHSx8oqk6qdjAbC6sCYsF5R3qA+UY3tbmBgoWYp 3L7uUO0U6vMkcvqoxJ0NGuTTdrrXrLtXAFVEMsQi+s1a1PFH0K0aH3ve9B5IJlJe6nY/hK0W7OrQ ncUnM82H0b1hKwBgo+2RF43XCTekPeNo8k92MJc9wiCttIaojNz+aeS9LJQzNcvuMmL2LHrXhfVL ICbcocgpsY1PRl+pSse2ZtqicPZW/ixV2/Cshl6E/OIdk1omFMY6idmUgi9bc6+iWvYAl5IMM3CN Qq+XapSl9J0K3J7haRlo7+SsinIojm6DyWGtEv3q+PvCOZLyE9lBOAXXXgDy8IcZdAy2KjLJ49+j MQnxGnbLi5D5BacQGnSrkFMoSZgpTkLh2iR3I7SEz/tKHUxcOtd1I1Pe+djpi/8TFVtizPbqPdff oOvy5/7GTvDYCx5lamW7zL6Vryd4KbxQMgA4lNVuj0qC9XK0DTXE1uNp4NjULKlP6wdWYvoPx1Q9 1FZO5FFOagqpoTzL7sxZpInXI1TGoSKY/CrPBjKFnH6DPCk1Khg+udp90boQcbgoOeMOmrnjlOOo 9z0+2lSlmR4UyiqdSZgC/eF4auxLiqGLVyEzTiM1oepFFvZb4LVq5tuvRu50X4FmYR7QEZTvPlsX Yt9Q4ir+ixYAB7DKoLTe0F+OMyy4wsK3VrKzM4DVwudlUtNsQcBvO+o39kwn9Z3rPNYTHF7qVqKO IAJzuj6LL0CKQElWweUtLyGhUCyOQyJP1YvPbEaiGlIMcQU4y1gKRXDNCRl6IJd3uLuvP1AdByGc FC3NmRsFKs4aMNpmC/7LnUbmjDbc5xtQMMdaUafatT/uR+/cuvrKoA7VLlL3EIC2hbShGaXp7EDM glwWipXXMm1cmdwjglkasToTmoE3cplAeCoh/6XiQmjuvNV3N+t5fKl0XOrQvwwNyhDHbeHRDNo1 pJ+LL7blqJPjgtECMX5CN/Uw1Zy7/KXJPG0k1OslDl0SdHY/wY82mNTH0/Q2eB3qtCfmUXxreJGB R+WlPrNOQoSASnH57O/m/ERn0VwwclaKKNUGzu07gaDjZUObw3W4DUM0iBR5nprkWTliMsNOUAh6 vMURPUuDkvpDnomeEngjQwHVuI3au1w5gekG97r2Ly2Lv/z2grwn26ASRSrmKIjDmR6t8P+sfMAg SUFAwsdvog5ubyYhSmJAqvBVgJf6xnG8LEQmN1gri9SEAMsFJellVPl/oQIUbjdHmdDeAPFvVvrj KAEo4EDbwgwsNsbUTEvA7+IVHV6yX7ISH4e/VJNMLCCXktrTASfJ7Fy6FcIHGEcdHP1jBLI5gLBy FO2rq7U7I47koWh3mmO3BcaLryqPYC82pyg7wimbb5Ur5KHJ+rIUQD9Tz4rEYUnH2L8ZruN3Aaad 8t+ev+t03e3F/LdUpRpWOn4VXVxWNV0Lx+nNR8RodiaJcJQgYhxPsnpFIqLUo8GmFBIzT/OhIu5f uPkWNSpTNHqv2Alxe8ido4htJWMjN5MrLeo2rtzLe2aJoWxzecxy7BXY5ABUAdUnSOE8MDQpbPa0 Z2+85KNPTgFU9y05zTXZLsZiDmpLW1fYFSKB9KhV7qkA6uhUX0Za1HEhQIB4cSyg7mN7sr05YIsw yUU4tnMQC7pj/ZxPCweAFAb48+GVfZrnJaJrv4CRrMwbhURpo6cMZLu3vbdrvb6hCXt+MyfXS68q KfApsjuo60HGDqq5sP5396BZuwK+Xy0ldOJjAnpkItTU9Z4p/wPDa2HaVBPKW6dBDTXjJJOFynAQ rUhyobWyG9JNXlgJvtxJABtTyqFKfcwBGdm3epdE6ejchJtZ1sSaDjXFcV/iY71GXVaFWzL07nWI fVnFAo2gy1dQzGI2XlSQ28yh4HhWd3Xl16ab3O/vCQlt7lEEnkwmV/FTI91Npm2R2omg95Rhg/md oH6j2U/YAPmY3lYCY+uBFoi+Gb0vizP1pbmfTW2sdQEB1CRQAv4jfAvgok39B0dID/s/336dyIPf KFckZmvQjkR+hr+rMjg+9o+AFI7mtfl/2y2IrhDW7R1l4PcG81xly+X3Zq1MgVuLO6nkI2jSUUAQ 7PklLDrsp2BK0BKe0ZiRYRRZc0XDg5Vi+eVYoLSeYQZTYbrT9rSt5e8wulRo1w2cw5+XAar/x40Y CYqj88oeH0wvqVjgyTNDaH73AwjPkmfo5hR4dEMjUpCgu/ccYRtwkRTeCpW2G7aQ7mZDup6h/CJn k4C7wNSiHm01C84WI6OwFRAShwDt6K+b8SC8F6sST5bng/QfT6X6fzkHrl+nl60f2G9GPQDJt3BL 0YLZeE0q8804sd8wwwU21RbdFqXG9Qkye3MYSjr1WkRkmxG5ueFmE4DpFC30N7cmgkQCUcTFr4y5 F3WFC4OW8k19Kh/B1tAtFQTXc6MIWGpqgEIjV4gTsOePmOPjn6LwZ+VUov5tw8ef25XaBgUpLmzH vCgrFJYuV79RzKg5ZMQyjTTznp6Vj6XcBqEkk+tE8ieQlZuw81OGI3PtS/P9UXm4cnOJrQ1zoCIC KP5F34bMtmrrfwRHK7kmeriRfeJoDM0QYNzaxsh38aGs3EpXKOGTcnKP29S3Lvyz4Uzgzq1LNK5D ll2MV3gvXJluJr/zxYfrkxvlvMIETpBM8TNCSeEuTLq1bOKnJFZbM7ENvPDQ6cn6J4xN7z1QSs7L w5twnXDp/5RMW7zK6UvuHlDtRJQgmqdNhzmq6yT3fRpN6Xpb+S7L6c3KBqq+7oB1JTpnAcvgLY/U dpljrd3NGThYumO5XS5U8W14yKd2s9yah6HXvmt7u29zxkBcUb6ew8q0yRT+aS0yfAjQj4Ss+t0L LL1LV6WcK8IFCdo2/M32yVskv0P89XsOeN99lboyofjfaqVT/iWeZJblMxPR/c81uQLaRc55NCKI vFdMYnXTw1osMQre8bAFwxEj1zDc0JDMJHpKRrI9WA7KOVxJtFgWx5m9xg/3Y65aqFU6qtnLAKXU OXidqQYvQZQ/s7VMbXu+3hWit3vM6Ikdl0k9Pf2O/a1a4Ys5SaymZ3rJePlNxYvRmWau0JZo+86m YmASopSOO8oUkP+o/9PdmdSV4OhvBBuCC134r+BGDLlAhBE7Ic07kEtBDvoKzvxvB4hGbUcnh3Jp qJ9hNohrdhzxO5ivZ8Q2ycgKXJtn+OUXAPngYksQ4lZ+4yuVrfpwWVJcT5vfvhjoqxjaWwkl4jKW OPb3RlhGykOVeS7p/RWiKXQJoKV22gpe200gKtdHARfulVgAGpLjb67iHc4tZ0KdUP3QHbYVvBzJ z2qvfOM9GSKDmANRn/zO28i4cDXlQD0k5sNud9OysnxYc6QyZEisqFlM5fjhQOEygQvFRnBLc/TH bVP5s1ye81yFne9mEqO77S84aHYT10yUezm9iFuFOS/XfyVobaIxdajt6nXmu6/jovm50lFGzyyg WdWsPXMh8tgAElTqm7q7mYggjlQYHPoppmAZqB362vFu7mcQwYTa4J5cHyNP7f3hnE3VODiWnH93 e2IPn2yIzAkcPHq8VD8qqHVD2dR6H+zzYrHuyfdFhdZJjow3bGp9mIIsRjrDd9GOAi5jA9UaFNMl vPbcWCCLsbO2hRk+EdVYy6/+BmBn4IcNyot/uxS8fMKXSvGz76yXpz+G2dy2+6/+IvrSwgZKljCy 3Z9A717FgViwJz3kNNU+1gzKj3RM+Z7A79GCMKHf6R1q9WnxPRe6A+S1FULAvbRUAVDXy7jTkvtR GE+Er8k+fMMsnHHSDFdPsKAC4B2EycWGmR/EZ91O2UO893evcdfzpj2Wp8j067QzbUUpahuhVakb FREM71kzktZO8cP2INiVVIb/XrmHnT4bkQ2NWAVnAgcM7Z6Co+tHzz9HgPSCiLQVemC0FDo+pjHd xRbrPwLKFWf2wASExyq7J0isFDowZ8YSMJK0/D91YUckZkMiK1TKQ8W6JXdSwp6QTCQbH0pwCMvq nuonsx4AtbeyOI+U8agrAuOzsjSJftPvCe4LClUSKm7ffZ4sLObWds5GP478nEnWrnqHBaTxAs7B ElGlrbbQsVIw0ldhec0sIr9wlaCOmxPsEOStmJWIWjFyfvs3kn4iUt8K/AHzR0zXn8LduZqD8R47 Z+xeeqHWOpYsH6kyWo4UAhFWvrupISMilrj07kzGNbDn2XbzpXYljnkUvhhnGzRrQuQGADPvtguu vOUsR1QUCOjnjZd9SYnvS7/6uBe6AoQBqudEHzyY9BT4KutPbyVyUY7jHxyXWsbW0qx6znC2Plik +EbWWkntn0szAv3un6kTuZE+FMZs6R8S98vplT574Y2D068ZfbfH2+oPGFu8o18mGrMw5OI5DpvA zSM8LPKKXNdJP5T1fOJmGTCpWVVeZjFX0NzrbDR7mMlqKypA5IESM7ylxabxkll0jLL3CBMbmbEn 828C/dWjZlypTNJafLF3InTuaHjZk11EKBirWGhUCYD6ABceG306J+/umK/XppA92xTyTqGbV5Dh gfZC0oNIBDhNljzi4daGKzBNaAvUSPs35Hvymf4aY0vrgpvqYYlVEPMgFUkVUr/JkwoQwbuRMB9y tKvaCeYU3y7/aFC9czDVyDqFZsGO5TDlE41X+hj6IfjwE98R9Ug5bRrg0GOBk79h/SUObko8MMj2 kgiM8I5kUE1SeArMj2Ro82UzJ8qzoJxQTyKtLJ5Anl0hRg3/vI/yE8WMr5tocVwJzVLNsI1fS6Kn b4XjYsr71eSMetPY4mPf8HTq6iNByQ/5WSHJjF76YLBXclD89ojzr+W8AC+P0jAG/VqYNO4RvfCO 2oa/e9ryacSkuQL1OARh3s+UNWXKnWp6ZCr0l8WByuzy5nIglzT0ysAmiJyj9Hp0F2MtPdC+TcnT uFXJLcZ4oSY+o1Jzq5W8I0fHBUA4T+jutdVK/cgjSBS4UZplZJn9E8oWS6L0VLaOd/Ua4/4W0Meq bhITMTel1YnDmJGVRnLpgxnRZTULOMI7oIUnSDKauQNbLDoevNa9hzpb1V4UDU82pPsv1kYbBpC+ k+wrVKuVbAQJfMzyZS+rO3poA9y5eGDixWgxOEzoZW8kZcdLow4M4dejQv64f8qfXBfgPG4dQtCz 2fI6mJY3rRJD/Tk08rqVCCmQvIzNV542S9C6wHKGFMDOOl0M23GUCdfUvMNz8p5s0toQ1FvPSIfs Fz6n+E69FSMIJoHhtTVAXZVmS3DBQgGGlVO6QVIHo02ExBS7eKAx2YY7djAqTo/r2+VSdF0KHElZ 7fgWkEX1CBVG7ePa1yyodD9zhFWC+2CSp4aGnRPymC0ji3hoVdhmRJX/pPQKzpHB8tRJCWHjUbGb XxHdpvW3Q3zL8PGvuZAEXwaszX1dVIJlj0qlSlnD449oA38SMwDGPvVPRttvfo9A9iwYD6EXiUFF h4wVeTLS0Gbu/olE9mvzOfhO8btw8bzHFMOmFicFzXlYTBFQ3L+xIq0v6dFZMcqckKJpsQTDO6BR ErfEHvNnoMXo5Eg2MlFlc+HYkf0ofiyRF77970vo2T1YRzSMR15BHe3AxHiyzQ44uA/yh1o1WG/H hzruVoPNmRnYd+wSDnxxEAu/fkUIHV19g1m/kcRqNC2l590LylE9aQlGkaF2hjysTFcGQ7JlKYCi frt7rHQAwawlY54cEe5EuypftckYhfqlrSFTvTOYYA5fFsuBQWMOYq73/72ac5kspHXAJPSOneru PqtJuhLDz7JyJsDvUr8Ols0SDbseQOCOTR6eQvTr++FrToWfF4J4Njj5Nvd0hNpN7eSngMR9O6mE 8OuAEEvWW9VVAHJfhc5bXeV/sskjEE+vkWzgMAsubD3BNfpBvWBeP1qqV8322k/xzpKmjkCU+BGa A5Uk6gojyRzGvq+a3YLy3R1R0b2CH9CUo+GosK5lDGkAcLlxds1tKTD1hyyqN8GNc6Wd1s1AQil8 m9G6cCIhrBkxuR+nfxLVyB2drupMche6LzMKLIzu51gczGkweCfhakm1coRSGDeGYN24E+RsiO2s jgGRKsmCUG5xK657O9qWEWLZqkRk6MMXGXDNzB8VsneTB/Syyq5uZoho1l9uNysB0sZyVLY/oeGH 3bPGCgG2Bfcvsv4Pl4iKbnnHbH/5GiDUewDamcUD/6Z0eN7epGqW/QWF48S8XLYCQZ21gvOYI+e6 0OQdRAclLK2SJpV5ZQ4yy4KkG5i5lFLNxUUnUmYmQLDeJP/qPZ7NSyuIExOMB/24u3Toawg+4W3A 3fCro6VCKp+Ped2NjGjXnBgeqPN3Z0TQL0Q4EGlrIJhBBcfiV45eXP6Ij10qihx9UZpofHQWdQ9L qaey93uAs99eVjBtvFJZNs+WF4lZynUMYJjCKtS1IzmEsE4IM2jZrNu0uKKAgxEonUjH/VQ/FwHH sVQcftrT3OVjISMqZRLEqhp5PtXdBpH1hIFSmoR3C58ReDMi4P3QkRpISiHG8xjG1IO0OHajGAKx nM6LlOWSNJrK16cNoKtXGSqnnsB2sD+VcynL+GBkxhoAvI9G15z+3IBXABkKVyQ6TOTtMPAzNky/ JRDOPQ3SKjrcVylvLL1Y3jzVi6PBnGdealpGbcYmxoHwfe3GRvxC8NkgqtOpRQEv4xm8E3VNJB2+ Ie9tM6RK7D3WyCC2caa/SSieFLvQZmJp09ovi+ciISuab35HAbwbyZrFMYUeCVNSoQKRaZRA4FF3 iJkfB9dXRVMq7k43UfUGGRWr7nHxpv+P0vsejPGuhfDp0xqqVhcY7+5Dy4KP/XatcvqJFVa7RnZC Ks0Skhx2pXK5810VjYEENd3bumslTraRtE+RaXtY2Q+JZO2UW3KhOnFkbRaHpL18lertOQE287kC 3xGKTjd3tMKary2y2vcBJv2J6VWYNzUDFJ3dkzrhgU98Lt8BhcSfxTa3HMP14+yXlwmRulCT6x1U Wrolhl/yaNQvNuGsZIjjad83GL3avQAmdn0xbSMnR+F5tZD9QmozUPat4ctW3bl0BQf+OIJXd90E lSDEsFsMoo8H1SfZzSltK/qI3Zoox7QxrFhEtQsClnYf2gjl6VCsqKGFd3j+L3xb/PAbNCTl+73z S9V6qAti4iGiLYmoolNfe/QJ8EyQih7dFciStxAjwLC9hvLK2AsJhG47XdzzhBBy0Hqbc5BExgyM oIp4//TFy6/8+R7SYHvi+1g12M4UfMyx6m8mg+pzNRXsFhXLQoYeM3xMt5cj+a6FYFcbiiqW0x0p BzdtTSFteiAJbDl33JCYcS293KKSh7cbCnnUZphIkVvRVZznDdvtvBLF6JlaVqVgjtlkAUH+oG2F WdXc36J6DZ4heMf6pIDtS7b0LQL4dxtcHcwY9eTfxGUE14ACC+NCefyid6AdwXzSlWyUsFf3+uPX NWvW37Y96RQFwT+M8ThtJUmJajuCh6GH5eeUnnpPKwdC12GHPMfcNm+TR/L4OOnlqE0lHDAqxE2G Syv4UygjL8NlxH/6CsVqNr49O+iITMfMteOz/7cqcysTVtSotoRi8ZWUQFGrpZawT+7BTvR3+DBu AVuy0jK5sfeIWAXuTPng6RcNFyWkoMdaCBwBuI9yXgzDoiXW1FnQkXLK98SsHefQZyNypWDz9Z2W trd2fW08seEfdB2w7+fFe+pMMXwVqtN+garyTUIrpkSddwCeqUYT3IjS7TFKjZ6bw0aIb8cVUXA+ KLcv6VM6v7SThSqKKH8RYQWXGjQndk0qatawT4vlOKRreYPLztKaYDd2TDni6BD0rL6KhD8gR0UJ 479Y7hNdqTAx8Okfg9lTcJsfgalPxeCe5Th3oXGU1uT5YJImnqzKJWo0c9CcE1DBfyaWk66LoJEZ IVebgONrhgS5d165iwsnrxcReNAMrKGdwjkyNjZKTSzRBux9fgZBFRAO0u6K7IltBvP+U6C/kzGk yZSW5lAwAXT7nzpoPXYa3KLfxLF6SK5Sy6XETlwWaTV7YS0aktyT6BB8k8ACcKDoe9VIknZUA0Jt EMSWFX2akWdSIfKPfQLZqrCIHvs6nLBfFmVoKGKP+3KaZubbsrwYdLxNfNqng2m0mXJWXsvBh1HN FWABhyf/4g9D0PAtfphguZxcp30pedhYmb1syHj+z81a3G36obsZXBom09onWhGpyADbarNmrqLd l3dNGXc1KWp+JV2cphp2CilnXMPVDCjXp9QNORHm68D1K+kMSbS6lLbYq/bM5B9Z24O2y6WBwIJ6 +OJ5CwK5nvDMer5824c64DlQVr/JwdxbSUMsnfmMRgrzNERRtzIoFDCZ7N0hO35ZnNvu5cv9scbC /HRE3A1N4FFGCNjzWX0PXeu6kOqV+YGmVNXL5iT6pm3HPocAHQ/rrC59Dq4YVr+2DNIShPai47uc BpifelssA1Ep/k1RQV40lt/7cHIHAEp3d1qP/co1UWGQRkHLhiKNjURnTGkI/NxTJC9bHsIaIK+l BHcHajK96xndxwNbf6h4bD0VVQfqofgeQ0taGAaRBc58JI7urAI98PHyvw4MKZxWGVGOBf8VLr+z Sh7C/5VGKtLt9dRw0yZlvPy3Yi7NfRGO62Rt8CBqxwzjGvMpx+nlciwUhdwE88WVNJO1darNn/4H ZPYvbO8hxu89CYv/E89dNs9k6MvU4K5zZVWvNmK5X5rwXZaCeGCFmCQJUPk+/Aw420sV4GvEQcA5 /b8aXG6wfD0hf//MSibwIubjvhR3mWo8Mfkr4qLYucF+pJNXFp2p+Brn/9Z0M8V3m5PLWMODKW3H yDcD7MM/rmyvHae3/aQbAkKe+mWzcn35zKFAlM61stpFXxjxAB5FuIDLJryJl5Z1NtJx0LJiVbby fKXBTyy3S2fZpTcWrwbAmw1OtKWlbEkPKmRDhXD+2FvA8lgl7skxLr30I6RmU9eg0DP7gbfstwZH Mk7fMvUecZuc1SoSJzuxcRXvvPglibiQqubUiAfXh8vWjU9OWUPBS919w3rKKeRRBkilDHVIrH/C XabQh2W1d2pFcLc0vEG4o4DOh+2VLWSgMRDnEsVp3hNbKyx1IAVYcGH/oUWva2uabmEpTYAHCTNk tdlt8G+h8C4ApvE7J/PuoeLmffnXaR9HTe8YZdo8aF5ZNA2GigzhaxO1GL/iI+Halga5TXux0e9O LaO/UhdCA8orDA+agWEC/45a6J265m+wcxHzC4qlpBxeG8bRtQTv+FYl122h/oFpHpxDDZOK1E9G M1O6xi2N+acaFol9e1cBaAXVmxNSY0osFQWUgkF8awjAqowMaRdPSd4Ed9+jq+z4y0l5SRG2xy1f DFdedcIrSQs09QshVq1rcm9ZNmhZTmlWUH9DNr4GDJ1nH/WdZeYmjampxOUNn4iYBu0zbowHU4k5 jiVVD4rHGi68MaWgHuSCF00VEMfjqOUpyMPR9r/SXp7gWKKsRk9Te5J+nBeowDTzjL6Rw0g5DXEk flttd35Y1LNRQSKeRNe5cAXTME8eNT073RoziLzyzYHsxAJfyVxIFLK8sdtbBKLnNf3RILrjtXIz swEQCWh+2iSQ12LmTwAw2lKMJ/cRnV9m9eF1fKCve2YA0yRTB3/n1WsklCDbTeyfJEFBbYbhP+lh V9sf75mgAuNR7CSpeN8wAUqlQ5D1DIjHUIES5w4meDm/AbtmJ/4HyGyyHg4HzIYIQoCbVvvqAgVB rQtOExPJ/6corEMZRxaJMVrNUCJjJgF5R2g2g19zCkG/xm7nZMbOs8JpHNIeAf9tCcMq8d/25Ayw jA5yjf6zsNB2AED9FiVsMHI19OAckhFK4SZSicDY7CxRigTEV4NXHtKZs6P+8gwyLhgyLlk+4wiA aPfbMHg0SN3jyLJzyiTx2UvUlyOYCWYRxDLfO32ZObOTMq/FA3T9XG/61gkXxcUUqPvm+RMK2AzK S22XSfDwWzDB9zB/q8y9r7BdM9uljVKvJjj949Lp2LqZRKMD8RIGciIyhWL8VRva8CdV/FjfeIz0 cyhO0iaIJy0txmxr9jtpJSLSDTTgM6zlD0bM1WeM8WMuiWz3YqYG3esJBNJPphD31jRC1OTCWnYY 6cFPffwKsll3VtTEytegZArOUlqJoScJjambap4i9C6+Q6zALbaiIIMMends7DWxBKqqawP+FlmH dGSCwRrBWuxeo0YA/JC15mNLkq2uM1nmouC1QnV9CL6V6tACQS6Jjl/flT41B3PQNq4CbTEUiEtJ elhD/2dlWifcN9qhxIQ8i9uxS986NHSsJKCNyNf3rnP9f42no5dhMvwWKrqHQ0F/FbwYD+oWCw9k AsgLrN6WwkZ7BOI2+6zbYqi3ecT9PS12vVw1joAQh9XlMWHPFTEFelBIa4Yfdht7+FWJHg36BN0L jnCjtLmEyS4tuZBe6wRzAesR92zWS32S089xvlwW1esKdnaLnq0YQ6lJ60aUTXeH/TCiYI3GggRK UKpWuepye5FAAm1xcPrbycbuwqGblYS5W6L/c+F5OFglmYc03ddcVsCxPOsqqikp1KUhgrkeQJuJ g+7bq1uKZ8WobiJHeBRuTMswyTSserKZ2YtZ3CrlzCv6A31UFgMDqOs5ZbLXuYRvwzEZccT0zLdp 8mhFmMd63s/SnEtXqC3MXUbVThfkhYCJvdCjV65j1FR4miwPhp7d+WOhIsjo4dzy0kzXXuMZZjCo jVbpCXeQkn3SyvbknsvdhJseH1BxAYac7Wq7G3V09ganUbwoe/GfJW6FzIA8+fRINTfCFKG8If1x TYg16gux8w0SbZFn8z4dyT+lBjBCs/gmb7jAydOy/kt3dC7TRBO+kgJkC/++JgD89iroo18DTb0v ytdEV4dCuXF3ToMpb7aDBQXHX9IcHwI2Yq17EQNz3xxohagiyHIahYAasUSd57j2hLg7rLmu7Xr7 3ISKrQ2ZnVrEzzisVEaB0eQ/Er1HNMVHiIclSacPnGIymAT+zJaRnDFrTDG/OxWV4s6uIu+y0kDG HjAAmSkTKk9oDukMficdP9RoXe/vD2KRq5LJ0Y8no72XXZl2frY4fax/s7c4TJz+188xBVOgB9lU Piph7uPzGmr+Rl5oykt+QH1vfIkKaox1eR4F/A/hu8QklXyCSR8AiVVpBYn1M0a9eOn8TtJkW1nt KtCLZxT0b8E0eHzgwGiS6lNRGiubwk1Ub1CUGytUYlqhdrc8dpHEpDVxzDB8sW+chboHXRtx9EgK THNi019OUR3KWwddZzx/wa/AJkd/RSdwFw552d8Tn5Q6KIGnr6imIweu/NrNZx9Eg6p+0xz/ty2Z 4V3D1Lt04T3hVbvWUpWdCymmFDbsSoOei9yGlZWYPvoeW/wNIk6Zi98/vOmqH4F4NQFFmkHvkqnk 0CJlBpjaMS1ojDDn/IM2k/uy9JmS1FnbBu12sEN34wYM3/LPuDKv3C2qSC+BbxZxrl8SSMzfyPnH zUlPBUW0ieydur7v9WIwMadyko7rPUF/IepZbW0DchQKo5kzIZ1PHzAwIl880hNDZAXVc8Nm1Z+q 6SRadMreJJ7HOOJfWmhfcpz2Zn4MVt3fALV8VXHPDLo6V7g39LTVTGKW7RdCdVvRpCNt00/NMi4e LsphsJV3vsOWJqWv+XlIs6+Yg4PPYtpcDD+lBTV5KOg9QmW8BP5bHxcugJn8VJzzuoz2KVkIw613 4URn14n/64e+mlZPxpwLla+bFp9Hvl0qlYpmiNHuPOsddpPgK/GO0tkgErz3ljPkuVJMGW5nhvx1 ZBRVIQJy0GHuUmZBKxmpQe0DyxA4ODyNqKBiXEo4w1u0Ht5WcnX+INhv9JM98r2zl+ZpbXK0Z1vn /5Fy6PQ/sa2WgZoR/8RAlZnihDTeiKdJAr2ohAC/bpaF3s04QVPRBvpqIwqrb19SoqifTB3aWvxw 09wZUGlspb04Ism4MwTAN+cPuT34McXJIIu4Xwau2myQS1/npt4hRE2sIezBI0ePf8eGkQq54W87 VhRUWSs2UaPjni3ORXVaKz7ItZQ4UbGTn64PSkLWwMaCzwMpp1NaKkyoMpHvQ/VZGmfSm7cQpAIE KKcg5j+02m4znQz88EkMSyio49pMenCjQ5aam3n4k/nnz53zGWeXOaik+yAp/cCKghnH+e+ZTn/w H1kYclqO/fdbgciNtTaFIBVFym3yqdl2qt2PetMVMfH4fos68lQMZQI2Af5EhdCKrySKEkbwS3fS E4avhKeXIXj1U1cuBU9g2PWpOcNi+hB3T9QoWjqohTv7EUPANbk3ggqdoUuIfzuRFROWUw5A8sLj saRx/fkf4y6cQ8yaHjLcL0rgvMnUJbJQIOX0Fe3Lg9ZSewvxXIAIY6cWUZTyP8reF7Ks8pObZvZo U65vk3ZZq4i+FxDaL9qKX4Mv2V6VuGTDGJkmzA8gRYMspUGpsz/7KznePh0L339sKR3K1VUiTtP5 lLht0ZiYF4wrX5ParogaVvOHye4aM3n28DOG2QMLIXKA9XvJTIgTgeltpzedboKshy3hM1JMpJDO Xrrxwagq56UWI8NgqQY2sZb06Cv8a4SU35BUzbsmyFD0F505z3z658cG00JV+zPXbZH7VbuY8TK5 8xcnOKnQvnhNS29Zphl17XhCFoCm+FjsVoCJUPSnSYk4IIKzruQKrovttF8VtfGDkoRCwtOsUuzO WWEKDYFkOZDxGarh7Abiaos8fSOBFJGFU6MG+6JIu+rvqU+FUSA3f1+K8JnZvGyCPdcBAsH4Bgl6 uzjdPibR0DWw/EMf0n1dYCVpEF4M5GbtK/ZiReR24U4Xn6oYx9pfYGLQ1yQme3KRcfCqrASCqnLs vSzOhMu7WIJmoK2WFfY78poBhJyhS05o3nu5zsHQCoYnkiomZx66LJD6b+78TDlwMr0H3nh28CB0 bcYlWp/xhGTyXAVNvg8BZE3wXYskjlgY2LrzlYIrxMKE4taDqCV9wsGgWO0UTtLBQFdD2hJoNbJO Mo5uCp/W7J5cDOsqeEFk4KMzI4eIn7lCxSpcQ3frbGgV5M8vbRzrpsO3sj+5iX05CcxyUFH/4O97 aML5/Pe9GdSbwQsFbrIt3UtbTgp1sWzks9Cw5WWYw7FiWZRm2OEnrdZ2i+Rn60dkaBChK9O361co enjrIg2BTVLTJimz+Y1uEi3BqytLQ47IrJynSfD2YhlWfNximQsS1a9JZyLsxhQ0aMRmR7BLvGSG sumh9JkwpiD+ggzf6AWWIDSFVi/dMDSOriNCDY4wbLo1jPsGoRt2jqGO0ndwOUDkUbuRj8JILEgO KgNXKH9S2ctjROFviiy6GT9qC5nvPxmPfEsY1LWivePH0ALyGdNWjxKwmxTzXm6e6GKquaHDnEl/ i6fhcctSYWoqbUCUVXMj1m5UvAn13THidwM26YG+9N/M3De+bAtRrE/bztnTVqFwK7QBjoq26yl8 XH0IZoLv/PMIrrnwpQVUaN6ADw5vSgoEGdMcrU8hHCoWqsrCt/cWFccuV7Oir3dWpFsh52BrMOCK QjIKdqVSaK5CPJOdK4mpeSmuuIvrN6gr8SYo03/BHtYZ5wDaOP8EF1Ry4GBBbL5yT8frKlTin54z HXL4dnS7vSc8USTmlxH6zSPNxZtg8asQsxeS/saGFfR7wc3Q98TFa9eadm51qVUFLBgn3U3yHrxg jwqm7igddiCeEONss9DrT6C4OcZsvOOE+BNqN+hcJ5vOCgcl+gS+2cD5DE7Ey6JAsV6l1a/7xl4t 6xXYnjSPt0wClxZ712vAhUrj8bD80B1jNUzJBGIXb0AlMt0ngOKzOHNyQARQz25QcvMq/9rfHrFU 5xromm+ALq8Q2Z6ZPo9dLK7zq0EBKHAAm+1iUCHSRuRRL+tzeHfeCMtx+i6y4RnIJNViJzh1JTbC DeIr0nmK6smJ5CyhmmsjMR7GHAbBqBNCzvw1/eWA7uSQ3HnJVKrT3fGUFiVFtsgsOTNBdcLSnemj +Ihm05XzzHoT10LKBKpBV+l8N2w3YTouDGwDhp73PmLeckoHlTBdD/xAsU8BqC35YKUiIO+AIx1c NfHGGeP8NqywdnjpbZFbpV8u+jVk91IQsoMPBD63xVxL8oawLz3y2Wng/anOjHUo19xp8CkqHiFM UgjPOxWygdyzDAVI/DDCoqiThDmUPCrY/7Ho63aC6yTvxbFxHa58lWHEafTc0pBQATMj8g86DsiG F++G061nzh/snL8Mh86gPVcj9a2XO1+H/SRmEk5hhuW7LevhR3meLGv9jCo/sFaRokx5Km0VcdLJ cfsAGwI3RKTZwHjIBaCafhQsq8ytqhAy/hTTTFOg7ILggOl6FRWcLJNzU98yi/dKdRNL01AKrD7e NoPo1hbaa1me6k8u5rf8tKtAcBEZJZIRVfCC9Q5Fhq3CmVzKQJQnij91DgStP2lQBcNXyCU4k/Nh b9lSqHHOAwzv2wcpKdpCB9SortjOaEoYn5hbf6lbgawI7PPwrSy0Qxl+87PRaSei6TTS2hkVswOB IK2RF90FZCmkj3aDi8fmIsZ0FxhtqlCBoaFp4tMBvAeOLox0QCI3SZLtjm1cns2A+YM4nnjgU/ve B8z5WgjCcuswTUcjZaRAO2R0YA5w50IUUcJITgx07N+/aQ318LE1HUPHEXyPt0COFHucpZpW8NK2 k2PWR5/1iwdLrecmnl18EDeegaft98GLxG/9IWnv6n1EaWOs1GEg8PzNdT1wCREahQuVfzCaVeBD T4eLd+tbcxThOR+axb4s7eiNi2xrB6lV5KlTFLyXwppwbQCl+syG/iCFdNRrGRpp9sGawW23HnB6 4j3fT7Y8NPkWJ4AOAOjFM2D6LWt6wVWJsCkFcLUcsb30pRtvV8oAElxlgUTQ4mom5OH/15MkjwsI tqZiY3h8mEyKwPpSwbC1HLqDaSnDR4TBagA9F7xq2WQyr89KR1qhgDMrCdWIAGq0PeZkbYUgtlen OW+FjtMLZR7xup0V9lq8+fkJZyndr8E7RmA5EaRT+uJZ7bla/i/zC2+g3K65YrXQ6E0dnGqRvUpA Cu2UhV6cYzZIxBVwTVTbfzdCNSWe/ODGLag6EdqVHxxdbc72OKfDyGJbDE2gCx2e2EjJSpWOxrLs +6JLjdAcHms7swR47oREVLzlEeHDMP0zq/QOVDws+nc5Fc2UBAGxuFvcQVa6xg2+9nR7nQHmeJDC CGIVkJJsL3mIlPoK0o0RL4ej/8HUX/y/MVAJah91vZeSXPOcq2xhdPQ5eMnPRCnOYaJgnhcmHbiT cKszF3afQt7ApFgjs566n7dR0A9m5m7N5qOM78vEG8aublDOuZp/YabO03PMlbYxxlOyrDmFE5eS lN5+kWPQR8roP+e98IG7Ajx9EJVKSuuEsn855OyaxCfceiiuEIzwilmNlV/HKFySF0cr1gpV8EVk U2Lxw9ldXX6Kki63agDSwxK4psTCSASCSzBkp2hWRxn53KKYFBEK9t2zVjuEFqHESLBD3q+1Z2d8 XOe6pHQS/V/By/HoCOuc3jRzljqGP5LeOBKjQa3hu7spbu0mmCpA1EI11fzXzbcUb0LNttp96EEI gpOgZafrfIJHhQzSNvJuzsZYBJ2nFCttujgZRBP8eHNamExnXGcweiXEXQoRJyXkAzQw0Sl7ymnD xWQEkllunyKqRsLPd///AppG+Q5k51zkw5kLXs3SGEeSFQgPdGXXlkZ1SI96RqI0QshN5MR/H1VT Y4xkg9NDnPj7FthbMQBsdUqoHu6JiKK5aG+246G27fEcMs3o3/HDfXR4feVEmB8M4S15US/iikuw Tk+rp71FPfZ+YmM0dyoRrlJVRx28JLpO9XanUQt8RJ6K5E/sx3hLZPnXojek11N02XMZfg5UO+Md E8V9K6t06nmBdouo9/LHmXW6pP9JonqNMJ6gK3ZsqUBcdCAbKdSUja7gX/gRDSxE+Aeo7DBhPVIr qAEQJEGIvFqTDfIZ2pQ1EnVfrM0B+iEkHSiL4ADw0ONwLablN26R6EuCYF6PliL2BKVvrADmxim5 A+45Mv3ASYzlbA0Go+Rod6j0sMXtHqY0nmwqadIJb3jbKvdUKBCSLjdyL4xc30y8XHTj5odjTy5v PGDQQsN2i4wC9gr/iFYku8twACTiukD6FnS+t65hMiZZ1zkw/Jw9zXKkEAO579xIKhVAyJ3GSrlv dilbkHiHk9spsNvInDhPahkJzGc67bc+UNFFzk33mxhF0VDsefS7spKXSkDqkxrAEAgSxPPg9ni8 ISwUH6/vV3aPZvT22+jQXmKuO7WMihiKigCa5P2aXqpprdzGLkBwi5Kf+WzatSVzGoBK0vYID8eP cAkwrrhIaX3MfIQ71o0N7BZSh+CMnP/LWI2d5zrfIXmYuUwMQ4guV8zutu3P4emJA4ekBgtoQMJU xIjOqeclGRQH5X8lkKVUfIQXMOLFUAJoYKcVXW89+KZymSGg4g64OK4bp9P/oYxXmnSCcT47om0p RkZl3tMzB5/ppdyWzPMwOR2UHeryWd4eRAtCPOL4Ff7hj8DKZeRFJm9mGJhCtW6xGPrdvQtUd/yF Sn/A47arUvB1RP7zxARWl5IYQt/JTJI4Rg6f0/8UTs9Rr5+vQu4Ob2KcbgZOQyGagZ75WlC3977x GL2dFQPv1ZD5ovXgzV1Lxl1SHQ5mSbskJxVob5fYwTTx222eZavBdmc5XqdivSWvtvhqOzP0bdrV KOvi/5n6XCwNH0VboqPzV7NEzmPvomVV1+pkGtoiLjWDEHHfNcDaFpWgKDl2KeCkR0rNMmBoXkTo 58V7h8AQQD5n4qX2N1nz2WtSX+n1o04gPugLElXfpevX9hsM/FywfjU34GMsDs+A96GrV55GHuLo 4XGvyj3RptvsbFhaXSM+etAc35yBfIYZSci+7PD4Ych9he1+e38a6uZt4GBVGT1fVxVBt35XG47W qU/W9IQsIbxHpxtTTZl2KUHuvdbbU4T1bzGrzyN9Ak8jxT0yv7+cgl371ukqvTtb+gPqLSvO4kbP hOdpUXYl6aQTfMTHEFfakuzFsz/me+S2B0vntsB+rTaSiFoof0xpqdO6OFOxH77rcVbLcZWjpRH7 BxGfD1DFVfLJsDGTv99fsIuzQkYtFRdUORiz0+hayqSACeXvkuDgkddQQL5zlxXuC9lE6wgx4BBa 7rA3P6+1f0rUXfWgO4Uqpq49Gsn2IV8nps+7/O7XEm+DIPfQwX3UxNuMMkmS0hR8bGDgTXXUi3I1 +prAVpgG2rKHZvJUfEHzAXWrDKz6J31q6mt7NC3vxJkO5yDr+Cq3IrXKh8frvmxrCKjfIs8OmpvW oezOrJGUphrxIn/+OwrnWAqxRrxyDri2vayNzabmX1M+IRwq3L0BlP/3OOkyhbf5jy8fPPacxvz+ T1OJVGQeE+44Tno1CJ/+EEubcgHTGfZJEAa6BmYMzqh1wDyILaIOZ4n5aiOhlqX07O4POShCMJ3H RSTlywtZ2nHWcdpSZtPxtCRlGRpmu5EtXlk/3/fF+cSoKVJnA2y5blRGZ1wuMRU9Nc1L/cejutJV NIYugLxLSeHbVp5mV6M9JEms4BPbUMoZdNydCjsTth6WQyyr/PE+laWKZqs8U0ZbFPFfjAKUAL04 Npc83A9dgVTS6VAYKEpLHCvx+bU6p8JvTC+4NEbgwCxogUtshbPkXwBOFcVmYpNA7RMsZfuUQgqw TJrw7Pl4PmpXuGR4MKD5ZlIsZD6QNALMlZFrfgbGP4t0dUG+Gng2Hw8W6E5AQi7+GFoYz4yyD0fs uuiL8BDbLjoZ3MyA8NsbsP3Yf6OV5ZkUG6BZvrThHhCTCh0DxcFovUDbhlWf07O/Tyl9wJ5IDjfz uN/WS6vu/IXCmywQfnoQc9n2NIZrBL9ADuDGiyAXIcN88HFiKFrYrA7VsYcW2mK9BPFps4rUu56H f4vZLx4igYJcbv94/UPaQBKqkU0iQR6hOLPNfpGRJnfBaKfATEXavyZKjuBrWkA8EPNG5ZXvxwjE Aq34l3CBliNSFZyHd0r8Qmq9yU832GcrDdZ9XF9Rrd6aiq54oplo9FvsZPgSvDFnDEvBAFlrnVK8 pY8BJLfK3AonXUErLC9i6jXF2A8qEjKD9kZS+NpgitDoW59p3w47v4noq3DQ2WSrmAu6XP9ikizl qrRNBR5S13Su72JTmqSJlrIFQ51ulS6vNbjXr2z+dLQXTQDhdznN8K65cTDcBYDC1PoZsWzXUvIs r3xnqE31gyg9CdnehNzYEY7Ssrxvo5tzNj+Jr+3XBE5U0fvtMK2ARsxjoVtxAJp+q6Dxc32hzweQ +aws2j+dU5rc7uoPK2MZDp2onOQ7RBVY0H3BSpHTfKNcUSD4ty64wkK3VGGdDALH4EEJvFzH775H D6rLqjPOCSoYHEcF/+VUA3ovu3lCXqAf2pNIzKWe6CrclkVslq/VkELH04XQ0UqUpcld/U+1mdPq A92q1QjSyccL5u7cbL2Hj/Hk4ftXuE0Dd36Tqk6+bHvY+pE8TKBzsWtOJ3aKuJrBaiNz67krrbCs x5zQeoxVEgfiWpikLPnFDALnk9yefL4H4xfiJILBs0bUNHtGBLYztPzC2YEXyGbKQL5g2wF5DYJa JTaw3INhskt2M24JdLO8xOiO8TPcP47KYqVAzqNuK3ckiK2tvd2yjZyWI9nGIi5IleCpSnbJltmY gh724NzWUYjMfWOdfAaY3mElvdNDIKQp3xI2hJF8RPOgvbindAiSh4naL3jhxOMyZAoiVAebR+K5 0foDV8mKN3XIMrAZX//PK/wb/O3A0uSDE6zFTat97aV2LK5K06HL3z3g/genf21j/nLD8WbkF0D9 /dex9HFcgQcsJmfPAiMYV3bJ+6WoVL/vkOL6tfjg61tCrTia9K2Td7LvJ6+VnO7A5ea9PO8ghvS0 fY0UbEabZL+bzQRkS6x2Km30MU8rRCTknBVemo3mkWECw65foQS8rVHH85+FLKwiRmFMt0BMIUGy mb6mZwtwgMdH38Isc0pAlBAlDEjlyGjoburszlSeqPEh4N4fQUfk7aQ3vHqcWDkroHhHsz6F5tCI XH8U3HsMdqkNNaNLhdcy1LcEg4ZsoBjjvBdFrVDIr8knwXp30fC31H7dlKSBcu39+5sSK5eUJNgf YWiN9bSOnvBAhlgpN3BIPRXjCjBe2fuT9U7P3lzomQOuickhd71IkAaqU6YymxQ/5LvwOxfEXIJ6 RGUmRczD/0j2ODurdlDuOmfs0DRxSfbHJheRsU+BV4z59KHkTLMOt9Ne5b+m2d3roh2FN78wNEbr arWBXBt6wQfjbkJYXrCKREjRZ4LRrOETGsEyQpNKSppwtHuyXVZhK9ejD5OCQ9P/Eq3Qzq2YB7ce 3tzB+rFDITBxiPGIVJywK5OzkjSnxhNO9OwxVF0oOJkJF8kCvLWHZAClYeD+in/qvzTrrFoLwM/l LoGJbwRwca8Sc7rifOVZRRdK/t4dNUiDY2CdAoDriKz3fViQARz7GSqi9QHjhWnAiESdMHjigAl+ WrRzlEkFrNvOFq6l3PvaEWVnpucsrCp+oVpO61NHiFdSLKr2f4kxOaWT3lS4aShr5+qhcgQmIrRt BdoszDMBzMmq+ZZvLW21YAfgmkxdDrs2pmbQQ0jz6QxToGNG3gKvjmBtWSVFQL8u9Lx2hsdgbJ1C 3zYy7xTY/3ifr7Rp0bdcO9xriVChsM6SOM5pm+HtTG9fRxgjjzEKTtEinRvZgH95Wj/Lrt2Qclv1 sKwbsl06ili7CNwp6yYTEkkGxetABTz1I3msA/KZxg/e++UGmhlP/hijP8RBD1rl8AqwE4kKAoiv 0rpVrHHuFX6TIzKDzgFYDzbLW+ZEkVSMZ1PhMXXKOvtSrEBF+rDVLm5PZ0+/MLLUQ+qwN5kuOvf8 HZc5WcnD1QUvia8rGz8Z9qg2Rf14sqF5b70NUv31xfUt4dWx6dKHUAQu6ybTtVUP6XF5lPBlBZE2 f2H1MmCyyl+gRM/rmtE1lKxOOV7OBfI5vuhbFNolGCi6RZOIuz5aOR+0u2QuXCICIf0k3Lajmy2o 7A+gqcY0/kbMxYQZYoJU4Gyo3L/sBFf6JnLu72A/hw4/P17rrVX8soCx8e2TuVvdPp0nQ+8Gz9g8 B6aTYrUfrqE3N++lUWVdofKcHJZmB0NcbQK/pzngbiwCBsicvxE+HInyNNI3Sk9WMB98116K1Hxu yJklWjzfu4Bu6kNtjFW2uD+GxmGn+dUidRBmh/VH7BznpvPhib3gEKK8mKBw8TnCLL86EsTU/dOl sV63X+wUjNjUIheoXOqiiQNuPIBOL6ICvgY0bf3vee+cdvNV9gNHhVE7Y8nDgVwxiiWTy+rJGPwS M/WMOrtbIs2QVpgJSUsXvdIud3G4EdSOxRGjjiQsGJfks3uwrVJMyKv1LoUT19uikb2vHP3psmzG bMJ3H4E5nVzc2y/odVa6hcM1ppiY5sZw6NAmW/BzP+YCxVf/Bh/F9aV/B3kg1csj4Xv81LzVJ7wt YGOe/3BaX9ka+DA1m1E0AL7PaWUrZXj8kFWP8WNz1zFGzVg1oPh0knhk6rFNqiFriMn2aSXbo8QN ZeAAOxpCzVUhy9iIOoAGUnye75RK7VYi3DGoFPoikDDxelF02cPTRYXyXMfDgwAxu0NxMgw3dG9o fNkAxHdIfoJaThLE0j0zcyR0hVQZEx7HvlfEac+qY4CHGqUljDWI3F/qI1+sfXeABYIUnf4l89yV X1R9Kp74CG70vIGpCUS9h/Y/OyKhi2utZkNMy2oETC7N5WhwxlvAXZS4dztq5G+4uBD1kg2LSfjk 9ChUNPTYIkb60eKLmFV/oNsu8RvE13s7mSBVDajgZtLB9DbbcG1wHOvXvShnf4Qptx9pRF14SiAD 15g3fKVV+ivRhgqzZyEaXXsqI/88UOXVg6OlF6XW/ux4EQ0ccEJml6imiG/D96cRSCE99R4Vg5zs c09twawtbCzcgIt2yGm0Dmi+FZMKYlNfmekxBjwoqGEnpI2OsVgWF42+cjD/0HaWFv9FsTgCGZ/B xV+rkw4f0ESqW4ZrS/qWJSJqndTPQWyN7j/2yS3JqxKWSfRJv4GVRJlYROqte5R3MRerykldxpVa smDjr4pxZrTIe8XCrRT1Ga1ZTso85RVUB3IjL0z/y63tv2RdgYT/UovwmmvVe9V/8kqTWf1sFLfX 3HKMiIVDJzYbG3bkQJGQ4ArWGZi+vUGkX33iOEsW4iCwrsCYhYiCSC2ECEFVgs8R5o9pJpxf5nta CCqMI6xk8I7F/w1RonqL9x5p0QxB/nHoGLWIL/5fmJrBQ3iIajM7GAyxVTyRGcHWXTy/0UiAvEY+ NYXLo+mDccxtrbHAU5QXp+Jsy2TrZWWkvTRq0LY3KdgHmwuKdRjSpX8VnYF0vkTTQWJnxUCyCM4N hLUUnbXewlxFhFGAyfelxFGfpmnoEis/AR/WfyqgG0+AFmEJBC1p1puVvgixV5eMtKsYiL0gCEpK dPxCBxUlmWXzOQqumIENCHSbxf6GXg9v+5EjQNM/xf0NVS+sb4mAZ6L+eqTTDy+klouXI0cfFhi3 10jHuPtSWwAQDgmqPk93l1QSv84buZs0kuVVar5zptm9gBOHFAPRWGOlp5L/zYXiVqm+BnYSpnHZ 9iMYj0AxhKqEiRKGdsECpHQW/+FY9EmzxH+hpehssDlkyBa9WyH2IdGyX0gSIOf1RC74gMiLu+BP BXrQw9tcnu1cnRqdUo/KrxQt14t83Tg809D4geBppRjPn/Q4j6rY7uaegKJTHBB2eVUqsPlXQ8KD CPkeXXDe0yGVG3PfAwxGYe17Tyfnp/+M/dKrblwk2ExLqq1D3Gm0x30peZHqku3kap0hGNADfeNq 6H2AirFxxMAG5RjN5qLI7kZodtu8u8tbWPtkzVL+e2EN11mUJ/tnfXB74n9S42u0aIKQuZimGdUB /h4F/Cc8glGRxgZp/yWLD74+4wlVF0UIe31hUpS0aH5eVrUhgjUss6cVLiyZME4PAPLWSPzo3W2g 5BctSf4BjJ2T/s8znZ5GgeagShFNBfIzzpqS7LNy48m+I+vuvCX+xnU/+vfW9ldJTiA9yAFHoAjA t39v+k13yWZyzE1N0dWlUxW7LpRq+6eqtXnNrr7fN56wJEwcpfSR59jmvgacmRp0zDVrojvCJZDl nm3zj08qVpT+zhSTRFdtVFvi09h4nRgQ45UYBxpTDpNyxZVTKF1VYxYdfoioWt4kkYU4D0gxBvDU yI7Qqc20tVx8e0FgsIv7aB1+qzwtP/5yFY+C4mtwrNJsNAikM5kzK+Exxiyycp5tpMinevaNopAL 9KeP+ZYCvncB5T+FoRFmtM1wQ0DWCqVs9tNhNP0gRnld7cwILAStqI2x58jdnPvTYFNxcUej7QHL /NLKBypvhNwf1rc5KWfSccNl/OiKtdemn6Q+dMMBZBO2jw4MJYDIfyYJBykNnOAZFMDmPo+AGQ0e I6MtpxWTpdfGLz0QgBQ+DN242UC3rMuapCLTE8EoXKaqOkILA98bJ4zKZvbHueHNN3DSMUUn2X0P Eu6dp19lFl+eZxNeKPMzF9NmoIwbWIYzIPh9aZe23cwNPcl5aE06derrAvD4zBri67Ze+crKkzCC wAYOp1rBnns1tR2dwUnxiKgZyi8MOI+nmFWHVANJr0ciAkyabHe1cPTVGDXnjzrOoebmBSJSRc4d i+noJ+eajPFm9jzSTm9EU81/PWsSPiE1FJ2n0zFd9G170H0fdqq2PqEsXyMl9Nh9qO5jvOoD2L5f DT3QonfoUI/3amISxEkaDlxgQJ26mifbtbFvtIP3jyTjnqoE/bKxO+CbwXt6qtWHehHk1GcKcN9t WbJNx0E0UswolgKpd/HtIFD6soHv4LnqRt8vykGJSNl1QY95nwPKOMf2/SwJGe1dr0psv7gEohWt tUgOn5khdMW5A8Ty3PLGhet6Yo/UEZ241BipMfvAWSwh4KPFf8MGw7Ixk5xFYJfWZFMPMQ9zqivw 1m9t1F9L2MdSPalsrkDJEJW3DvDBRWzwjKKQ0r1fscUSb5DeVj1/G8DfW/Uxp9fsc9aAZ5T6RuTm TfBFylycjaBH19d4fi8N4Wpdvxe6UcKEOmpSKI/GU2Z3xfdO9/g2H3Ew0USJ3a8s+a9fCGbdZ2W8 hp1faExawt9WqlmKky2GOsjiQxaiiGLUp8F49aUwRE0uFjwQu+V3VSXiXBEGnqXvoMHQsE5Sgl/c ShWoDUV4B2KjvmqPNTUkISb40DQSCyfrikmMiy+yQLyDsCjafYGbs3XtQv1ZcUb65IxcetoDY6H5 mFtfuJZnyfD614faQWMBqBQyd4+CyAJDVGw7VKoXqwLoAR2IZblaHwoVyBjlhuLRYyiu7ltIBrKU ULh0Ijro9w+dmkCF6syi4QNxi0VnfUUxbrt+s61BI5VpN01qXx19iEv0W3o14yfdBGugyq1Y9Rln jfVAZvyMIsca0x2aJYyXI06NWg1yzC4iWVsgK+4MDakvSp5WH2hOeWhiBsis/nN7HF0NPQVxw11d pLdaH8rHC25JBTXRmFYqIO+khtsxkTU2ADODUz/IgLhp+sCPPkKBFUzk73Rx/U1uzWtWPH1kZFQR knGkYG8Cr/xUC2khegqri2lLD5FlB40UxLuKiILXy+Ejjf8/XeD7WDyBeix4kSngv7qkc9RAguJ/ RRFw81VFLouwZyTHaU12lxWzQDrp5ubutZ9SFyK7WkomOrXDELTZOI6jlpqpyS0iPP38x4ubN46d FMC5K3W1f1YfsiedlfKWI7/6pnrvFSZBAZ6fHYU4v7yVaaFSC8D2y/yxay3/m3SDyLM7oRsjAqUl DnrYfXLUEEIR+Ay9beEFK01Rd7nL3LYoc7sPmZ6sy3lZ6wKNQe8l7Jm9ERugdSANarZvOAg0h0BP +JRZVyX24owWYrQuGFdkiV1ECLK3AiDhss8jmvbffbLIkvrLFI3c3FvM53x9Zs8S8gOHZbuNPzZp q64ryj4MgeQ/IW6pkt2rHd6UCrVjGM3CeK8vgTNdD/CwegIAFcy5ibRR4JyDu6G4ocACmZFG6lvt OAd9wnULiNn6/uNLS3zPthcNc71U/9exdQq1n42S6puOWDAfijIJwdWF/jH4BQDIoyLKtLqkdj62 Ht8IaorxT3CoGPrtiOJYdLN5Oz+rwky6kU5CKYmG9f7bAZzTXxUGyGE4vq6MHO832TzyBZgBBDNG O9M/o0BU2Zc2SeiUFyA8GdziZURLWHdtEf13PTFsEtRuUFdS9ta8o63wGRw8IVKZVkv1vHqaqzst mTwTKTxtS3lDESNrCsLQJt5SrmKgfloN67AxbuvvpLi9Wqyx9cRcl5OS1YEGMjwjw76Ooyqf4JFX VKcmu3V/r/gRiUIS88lCky75KfnuYCe2cm+urMMK4E55Pihwdd+aVGBziSZFqBxRuxvMZeQpiQZq F+ny+nyh4fW9DoGMaOQryM3PUWoZmEOkbodDcjNAa9sQLSzrhS3TaHG104lxPxCzYg8Aj0ffy4SZ GjwN+/9adId5AvGapjQwMz9DRafz0Tsy8qF7rFE3df8L+UFnw1zyS2Gzh2TuYKaeyBdR53dy+dcF R4nNb9jU+L3iMX4h+pa//29Ks9q7Pm+VYvRT6ewnHrIYHh0jkgX8Qp7b1Kxjg9hjhtEZxKl/pJlt bcQE4chldY0kjg7oDBMTl3eEl1xmfg93DGMKHPhUjeF7GAhzbTQPt+1odsqdUt8qPjBBMOP17rzX 9QlXCu/syhQggSWkOZYGDrdX+Zl0ikUKmW+KYNQ2CsZ1HaM/02c8mquhiF2Pxt5PcQAvuoaeK+Yw 0EuwYdRBee5dV8S0qIx7iJgL0wkNjlJGKpTdfYYQtLVVwFPqfCcG6ZXawl2d8DmwgQn6pxl9DKKP JcAh+4tKxyvkcMk1XSakfTI+pFJB/ouqJC5RBaoB+PQ4a2QxX3iduMi7bG1cA0CujuzG5SWrtxy5 h14bA1lREvjZjB1VTSjU0XH6QIZE9diffTmpvKzssADPIrrh8RE1FR+MMSOQlKlxWnvllj1PhGIz QJ0VaYnkWPGhgWjDtxUQ4jBhElQPfyPl5lHUJTyLnsGAPYwKMoOJidhtRchDSrFlrsvfLfLglyFq gXsvMN3NByP7IO9a6t16dAsLY45KVZP22DlLkUfsMT5i93kcL/ioPkI0z9K0ntA4jgP/ZwfcEjHu EFE1RWt4K0LC8URqvmGJVTHr42tKWFgrDFyJT+Ps+8++A6uKIVMNk3O/fK1x9nYyvG0QOGe96YJB RMQnidHLigfqCejIlINN3XXP+4+8rS+WmKY3av6ALgaHYSWAK+3Lfyif5TWLEUcBRs16ZdUnBIZP xL6vegfIUOSRe8P9fsLPG4ek2S5954EHMMH27JpCXR8bAlyFkZhAfMIb0FcICET5PSIOEBKQOE3u +vP4Qmend3bhbba7pNTELMNAlFitTz4XcDVI2D3lrKt3ES7hwUmokqQN4BxuW1qPFRJuWDMcl68m xLs57iSpIeCB58pgB1KpSjfku865LD1/dxa6b4LARSmH3Wmqhh1lsss6/5k+Ys5CBfiwKq9wTV57 b/wX7KBjGWv0M34tXx+FHzsrzd84Gd7gLxxCUIrOA46qhVIJ80lIPbafyNrvUPrRRaUIp78M2XSI 9xjN9Isu/fO/RiPeb2kgZ9iC/cYDRI+BeSRDUPWQob2egqk73FvKlTARJ0peG5aHUmjZyifvG53l SXltcFqFL6AvUTOl7OZMP/KPrQh/5XDByK2UhVN48KVsrpoW1/yysp5Mw5qaKo9fF3WM0WMbymhp YlPH0jltp63V6xRJeEgwufb95RsE0mVCUt7wY51IjMLDbe3GXuCzQTMq5HFlrZ0vagRHaNqR4RWw yBejDVZ5DE1droaf/zpG0Od6z6te1uLaKQfkB8rnDKogOs/sSuENrm//fKiTKIa8FQkva39LqnBF rSTllPjJhhPibp+sAK7K9AgAnJX1GflLkNhcxi9xotXxrfO71PxkF7yCiVACiFLyoa42Hdaz+BTe tQowZg3uPKes8mtb6h/x/dX+TDTnHekLk/epZL8OSpBnHOgIpLwREyKEr43Zi9CDrCo4wgDS4LvP 5w6AGzBMwzt4o6PBxbVIF3C7qpI30C85gm+m7kbXhuD8GOYLQ01EWdeBk+teh0JOcGYoLByY9nqm a2+ttXt1JWUB84P4kQEIaJ9xHMm6YxUSz45NnpDK1/yy7L2tYGuQXP2u1Mp8c3Lys7iH4rt/gZsF //R+ACJ0+24QjfO9+x/gOgg1oKbOFqbcqOBaSQbcfAsX3lMvKElYPScMza0P9yMVCvKibossdisB z0QCNaBkOrrczmiPIwn6zRXW3MZniFuA1b7em4Z8bWVsx2NT9RKYQx3rb3Cu42tWYRlKxScucyeW If/o2YSfN8l/46Ec00aIlTUPQXosmac9AeYefu0TZkZLBF1oJW8D6IvK1WEYq7GnJIld5/4kZTrZ lIsqemFYtjk2wTWJgguq4A1eoIN06g8erJcZGa+6lwPYuwCWIjGcJIAMJ/h/AHlgz3oCP9Y4lmqz Fi4jF6I131U4HbLj+zorXEqwb7dsWpcTRMlOPSnjaGSjzubHmAktzLsnQhempU66tHiDf7OXtO4H p924aoP4UxY5ZKWK5SCARP08dCUxDBc1sFSsSVI0n4ogXG55wTUtCJEnnCjjBi5wcnOXgk5Ri3G4 RyACF9jmARCYdHN9zwdAqS4XgimITWNJqKCdA93FlvU+AkVv47oNXYqXyJY/PGqF34uDTrdYWPlF MWctW1dzPfvC8/Ks6WWxvdIyltaIxodwTVPdR5UB23W7rySFucbxLojwlrzOocWykS3uiItH35Rx raEeCYNr9iG/2OKNRHPu3wVjeHIHuUdiDfX3+mwaIOCfLkj/pHjzYhcVIKgACJ43F3B5b1AuvvqQ e5s2ZvKsInHEB6d2iCa9vXelVFHYr0xtnd2+QmWySWywUD9EO2QljlSC910DUxTOTELp9m8Xdxia TbsGP9YBfDdWPkRIBwa05xTG5VZBoeHpAsrjg7UbE0GuiQlM8xtjNa0cQecp2P+zownaOfNYP2IA aezbkn2EFhB4eBDAGeO0Sw4tvVmEw2WxzutiEPjOS9Kfz9VWBPdIaWsdHeLzwenIEC9nQUTTgeak jGVwv0AO3Kco47tmO/KDomx93fduF60Ax3vndQQDv+49cN7+vpWTGUzgKFv/PNYGDg/kXU4/xO9s MudsrZyjYN+sqh4L/7UV1IMoNBPWmrsNkHtO1GEgUfEB2D3m7SuraOrHScWgWUl0nqHxJEfVeP9W DoChiN9UnmRZNrvq3c2+uVSTvMJia7fXXWpP1j9lppemEjoj4ApZr5b+xTDEDyTdJ+e7xGAxXGUe vGXWizGzwHIcJA7yhaP6ivz7IcINf1qQFVd95wBrWQXIuduwNIUm8QcsBU7Lq1g91DMfN3GEV5ZQ GNAwNkjIVyKymYIoae9Oo+8UZRJnTRfxUOopy6K65Agypt+p6EjYBTvbJRiTaMET+d3O2HgAMn4L Wnmc2JMgAWVUFlC9kjUJVKMHlkBP7DSwlSeaCrzOKloKCgE0NnBvjHfzTFfQGsWzbLdoCUTkcBKY dmBgq9k+7NY0eCee27X6eewJAKdmvHOHxcYRSd24+ra8E/wjf3AELN/FrjmzsZ1LWS0HjkjJ3Wmd pNQ2FeNfXLh7nW7btWgVQc98RA+JfVxgzeYxHclOF5/TO7lEUilQjqaIdjUq1C+GrnGTBNCrqXUr ba94E+BWzCpqTP42OcnRNCO9T1xt+J4Xfxro+LB4xjf3sp4UwmBei2Qc1hgFlaZV1DTKZzRSk7Gp rvEgVKDrihVvN1ACzYdHn+pXJpaELl0LqMXsQBP5Ra3Wvl9+BBntQo88tcKRw0Uu++B8CUGjLhgA qBJ2mGEVmDqk38RqgZeLksw97w1mRRGVzQMIZ5DYrmPR/gEg6FOb3VzkYuiyhqhfg+Y9BR36qPKO GLdc4TZl2BrsiLzbLCJNMheskUUUV7H5MhOfevIMRQEn+Cg5WZg6mokn4mNr2Y6yMYdhOZhIpKUx woe0S57sKqVG5Q9TiBWswSLGr+AD/ReRiTV9j9TPeZUf4LVuVtw30kfx9ebRGOjyweVTOw3bQPvY OCEIG9SUdWibpMMJ7Iob3EsC5iJ7WkpBQaPfXJ/aCNuMDyAcGdWH8+fdhGdbWvnTCUSNrzBBY72r s911lrYJmBXd9SqQbf/jOkpMneBpWruLGNIq4cOqFovRkg+gwiat91+ODytf4/flrYROw8FwZmlG JHM6Nqf29HpWSTCP/0DdEpU67ySjqezyfuMhFhFdSp0uPgheWp6+OPPm5BOeDoV7LAnwMryMhLoD EiGSxrO0iCBmr2zwGMclrAIVbR+CGisyYiwPCaD9AcqU4Y579H+5crc9SqAdTq0LB6LSFgCzup9X 6ZhgTbufNMV+b3NClWOqj18OoK6CNBFuKEXVnZtvJ1SuJ/WthscujPXRW5Q9bhep9FVo8JHYSKog 0xLsLU6mVSdZ/iYhG0k72GyKZEzlLQN86yhgZJF0yg1qnUoaWwOwwTH9e4+x4z+Yi9ibAiOVBLsL juN8nIqDKeDFi/bM3S5pOUXSsVugeRuIoYdmkiqBwLUHq40d6JrvJq+v3NYSQfbtIFvPO+NeXRro OPwvku2DV8rJnlWFAw5+LLJx8twfGysEJNDX3gu4o6xlZLFpDsecHsiuK7C9IVMG8RIKywXv1ZN5 YBXmZQUGea4wvKVFddz4y7GltTSP+6ihrXUdpNL6lDGwcuDeM7GV2daQWiYwhiK4IZMw30eDWnYJ 7GPcWpK98DOI606vUWm0ohA9I3Uktzn68XsH0C44NRQo1l3dYx71aEo6mPsrzhvuMVSGeJ5pF8ix lDjxyxaORGu0RT3gSmuCQI68NRoWSfPG7wGRQbWoPv9uTQLThlXwLKk+rUEvclbzKp0yUMR5Po1T PHXs5/TtxVKzOx9gnJQPp9oSco1b83mYqdgRGFSPKtN5n20F9JVpr2tHKurpRCtOW8xApL4CQ/OZ aJYlzYHhUeD+xv2Y48Q4Pfbv6mUx/9X4mtskjoBTH24Dkx9Hzh+uFpYkwPTU0OQCWue2UrZ03bFU 5bjvb430o+be/cK1YoXWhQpfpPsKIuP1EZR64leO3tCujLRu2avaLcIfKzCL27jx3FwfQMdgGTdO /B4ILsa34yfN3W0WrQAOnhQ5YQHnkLJ+wHpXx+OXOSo5lAhkzYwZ7a57G92Wu+Gq2K3c/I1vIfOM Xc+ohr95qbuWBvCBKVu9VuhVtXz8MDVQDy0jpJvakocGvqYilJ4tsuKbl30ccKYiNn67FmDiDjHA 3pkOV6BRk+chCCjDc/TCiJtBZmRlSJ89y97W6GqDg+e8uIiTfI2rZye27Hmfuaiep9TjOJ6m/wur OEpbNkW3ihdYh9XY5x+7Ppm1x5/XWnl+BhDjjEWIVO2uMr40lZcZaqoM6ekREtncOHSzkz5vjzur 63NOL6VKwPCyVtO1OSTP3yOSoOQt08cTW0e9SFRBhveJENV82lwZK9+Fc7d8dBez6oeWvcqzmoIg DwtTTTO+mhl918nwtOnI+x1u6gqHhBBLXsY0D8tuqcJ/jEcdBA8UEmnGlERBUbvYOeqE9e5Jo3gq RaO02lkVqXynhnnmhS5oQnaezYZPIwktZnPRP3RjuXgxCDE3kMISssC+BiTT4P+W7Af6rwiyHAoi qHauqT+tuRUiEBfMDgvpoqa0Fd6azTNnklGkEzfUlG1FMppMCRr0iBzrKvu2scYaNYzlsdpcUWlr dlRRtJx5F0jc4r3HHj7NezZPo6G/9m7W8+Q6I+n6hhITzz8YIDFRKH2CI8wGNI0Ba7jBXZyRRatj /z6UOVZz4nmDe2jZS0AiySnqqMcYxhA2JZyYcrnlR6NA9wbnWQpwMUQQNWIdfNHz12g0aNCao4j0 0vqMojyhWvLXMP9ItF9KZEe//jabCQHcBfIqyKfZgpi89rrwD0t+FbSc63DJtvCd/teJ/kPSbF32 X9t1V1v+hjkAO4SKTg3czlMHb2SvSKdUfU7LJJ3odkt71I4inSAzaFJNcmWLGIW+nXG5GTI4HIIN cG27CIv0fWakDMB/NYz1rY2DMPQB5HnvTAOsPFdq1J+Bd16tZQWJjhWW4PuqWFFxxPUlhp7S0P1B E4I3JmTcENJCs8YbfXdbJY84vuk8b9WfNHVAu/RggJSh+4Wb9DzxEgc0mIzWys1Rq9opI8wiL8fO Tpov/f8YVIyS0zud9rLmo3zTulHqx0ueSpBQxtIsr4eOEWQ/PiBbL/Lt8JVEWQ1HbKVVrXITvvv+ ZI47opTI0OgH1D8wLBTDMSUX2EqgnydK5Rqbc4p18CAgxw3Goz7dkMwpIY/aQsoBXfaZrbT0bjX9 qdeNryUQLfEdwLePUR1u3nJKHjsBffwk2upC6G/AThnsj5zqlhoyMkJQDn6O9tBBdhYHJWFSCgVL fwcLCkA3X3/zBBW1TWntCgI4WFxJMT48RkIxw4e1z0Ljm0yEO8pPbGqMAq/PFf49Vxi/1BxnOMQ4 0q/CtEXGhCcN/66zPmYABeQUlVtq87O3LpT0bcD8r7k/RMNj9f0VIcc/HKg41fLrjjIcrREFAqoU nCSeGPrcfwU3b2YA0Qyz8nh+QBB07kYvXpp3I3NK1Teuh7IdSDwyZ7Vqdqey8NwqUTeWDv3cP0K9 c5dYJoSP2lhnjAQLC42GH+vieoGJMJtSXdC2Jp/5Ci9FRXUBaZBfPuonZWUo1tKDYwGOgsAkcU7R gQJjwJ95Lz6bSc8MegMT/bt2VIRhr+/Lf6EWqOlvuCDPb/opanh4b8dZsguGGj73NGpkNJMfsRjT wRO9NrOmjUfPiUhCzYU7Cbccz2cphi7na9yvWp/bVR2T6u3Z3z5Y+Tgw+WjhHJpXh+h+HU0yiTka zEZTedHLrdTTQyZ9Sp/6O+1Pef6/lF1aLDvVQ80eyXq/2Bpp98hRma/PorL3GX96WmJ+OVcYPvV+ 2i1eR9x0gfX5KwQzXdSb0v9I4zS+w4Q+DGsxft2N0Remyh/s2QqctvIkrLBK7Boqa5ONe/DBFNfQ nlXsRgtLTr9luEodd/hcBX3FokUB55ES13FVD8ULnj8DpWS0eKUP/WGYIokWZwtDx5ZG4A7N5bSC zN3V4+cBqxUzJg3X51fnUXMU1xZS/4z7EF8JGZ/AJa56gq5t2xZ8yGwMnNMSAXk+Me2f7dcyqkea Nwy/1rIxXZt3EEqQ67ZPn5047RhO+uBSM9el3Btirre4s/M4yHhpB6WT2eai7+e2STcWYvl1dA3R Zbv2mw2aI+omN9lujYol8E0cz7yGfeikineghOwNkfzKfCM5/Jg+CdIZs4dY8S6ihVSjD5h5/Y5i jRd+Fy7QwKxIUf9ufxqsWLm+8kyrwwG29vdQec5EgSaqtQgUVHpE62SDbi0PCsY7/58TQUE8TjCR PYxscDCPGcKzKZMjSBnj2f2PLhqXaOZwm7vJCa1Nhvgd55Cw4EntGMCArdY5xocOBFCiOQ+zns0a 6eoCvLLnVjFrb51WZzL7dPOnJe4/hohKItLauRS5PhG4LeSDKKedt/xUAZreYFqBw4hlQ197PwH6 csbWSyKvLlAjl3MM6bF6Tl5QbmBiq9BK7j0aFo2s8cycXt6/wAahG3Ob4TxaXmy5bcK2nYknQ/rQ buRc51rSB3AziazaE12+ntmLsRMPQpx5hD1ANBHJe3x7r4gyaXdjddjH3xpvokzfBQ2d8uWpqJW/ r7iSgfwyd/48CJormCQlwWmh6lxOgK11eyCA/9cIUo5MqUWMHpW2clhFU3vlIzrKeKayTF7xs+IX uoPgZzSY+Gkp5eIcFCQK0xdVdSUW/oKqKh6pdWOXl0zdpZNNXfC9QcEzW36wJSLujg4emJWbhUEb xhYzDOSyDU9QfFCFoL3GMqErrJLHKQnxpIyOE8I82G1ZXOvsY5u1OikkOFIYjaTK+mO3pEjilsm4 yO4HeuiQpHkZmTvJU8//zvh7VmGCHpiHGzxgvAKMEv3rT1GoO9HytMOm5Hkrold0Ef510uWicSOc rVKNuRcZX00TgtLazO1P3gG+AHuswJbRkmRFW2rzvpdlZgVTBOtW/s1IFQ6FZPSi26M3ZVnOuyqz aFAUebidoYV2ZdGNDwgjtW0y4f5uX2Vdq9SnPuY9vd6wL1KvCxno3rR4PLWiRGJjYCUitFL9uGJg xb1smgxk/TQTrYpXk83G9hvw1pRZmY5XVywZKphW6KYuo+gfgkC5TopILTp7asYWVHDrykiRomGW z8mcP4ppbdREgXfFT3v8/3wBRBlxz7uElljsOmVrSh4pH2m3cUwD5Tfuw0f60gLkwAUlMa7WK/ZU WpVdeXUDVv3I3R+GWWc+5Q8Qr46m3FuPNrsXwP5WgvLD/LynF7pdwfcuwcFReIysL3Xv5KOj77aC qDvu4AFD1tnVYhel/7qMrp4TEMvUmR69tK0KSBwXHOCez9iIiKYAWLz8pVLz33jqMZc5UGkrLwsK 6RwIPAZdc8dTCyMPyl9QBi+fhvBkTKc+iH2akXqo4+tM6uTjaMgV8WVXtRI15tL4vrH2hCxPcpMw VYvD58tYj42EMWUYy6DAO2L4FZS1v7So8qpM+QhLgzmVsgX1uO0qdjDp47KR4ntZElHeQUKBZUgf MlwYtGWOqvsL8khspXF/4B8QZMVaHOIS78HU0FwtGwErHLtGkq44PMeD6TUi23194W5cG+JvI1jM vPBdDQkYjJmnW0TCoHVKc1R61Aa28TzJE9E9pUow+qWAsLf1cVPGTSoyhK/pQnrS4fxYXnpN31Ht sviburGsnn1QH9TAUK9OljS+jM+232O+tw+kVRWfCuvryk5cHQRFbDsUxhGOv04OVabna58nXZIA LjGKrKcbfsk9SuDnmkPS7XHqs/erJT383QP425W+50dSpOh5voRF8+ejtrg8drHtTMiSD8x0sg+x iI5swOEL66BYi/N61WYeKTH1qCl92vTYHzuQcgfF2XIoneyGHQH32aeia5n4Nd7nRuziSPn9Grxw ejB9cex2iwxe5HToNNpMcM0kAUMV5O7RSL0gRnDfxbJi+L4qPrKJXOe1mz9dTMqpgI3knNtD9h0+ CLLpyQOcw/EoY1iy7myNDLsSoEe7E5dlq7ECjaBkXN+VmbB2gAO638tUc41uLk61vVU177BjBYlX ckVtgULiVliHAcErwe9uoRUwppf4uHkFl7pYO9i/OgX5fB6E+8Gg5fLYnV/tC2myrSM/YkdRekbf YC87b7kzOi6RQJ5V5YlINvH2Qrv0VGWKDnC+FHLsNA7aDNoOrscE+3ODjfrBm7x5zxiuBCl5D5OE fsTu8j6Vw+Y5YrMNaVQlM4bXjovUU7fydp+4zPw3Znno869LDnqKuKzoqcnTs8uo4nYVJnOdigbA sonCow+X3N0xNIF6al2Zj+Wio4YCZDEaiYvRDsrcKFDsZOkZ9FbtQdIQf9HHCXLAaIk3rCEf92Jq nl6vUEUpPwSgY18CyFjC3yBgsB/PQNsCL+oFzKeD9DBkgVFeRCoekevFMBexrMdk40T1SgXvk4v8 QFtgssimCXSWkve64PcT5S9w+Q7cw9aPx8viatRsSiFZ/OOgeRT5r0LNjV4Bdko3PVY9nkC8Fu1M DaHDUsPQpgSy2tK+/CqrhvK9ItfFNwSnrvQ9IAaGUAaXMZUqTLeDSOOMeEFGEAunOJmHSgl2GA30 65R2e0vbEqo+MB/InVV9E+XKGdO0ynPuSSocwV03L3YtdoNJSGulYD6iRhaYvlTuA7rHSMVZUx5+ dJbdSbIJO+JMg6SGZxdzGY1OG93PwiPXWL54mDv7tQhiE9zGPr1zKCiJu72CBRWQ61wx7XIoZ0Ol BeoKQe0RU0f8AhANwms7mCPWTFchL76mwG7pllYa1GmIleRbF0Hn7PmuK7wulB8Q+k5i7Hs9vvuL Lpa1tk5ztPkT+6Gp0/eF4lc9V0zXJAECY4vmd6pQvhu6eFbEUxQlI4+F3bvB+J15+xl4rQKkB9qL 1exMp2uJeNak/wvb5T6qOrMhmRYQwfRB79yox7wWqI/YRjhmfGz6qApw0qmJ/Ajnx3m4xno93mdT l3DA7IvyMiX3nwZHlUve33hG6Yv2VH1vrugG9MpqpQZGmCkeTQrds5JGaZNB/MChnoD43hg4Vnhc 7wlxOlUAlMEmBMFSrCRgdL4WszOoK62aG/EY/sVOIQgop4pBSbptsp0TtBXW81TaaJNVXaKYYFIp QmPWC0e8Vb/UksZflMVMxbQN5PryUi8GyI3zHpF1U1AymjY5iW2aa0ec+bY2mqxVjkeDTocEAZAa 3BQ0h6YQMA5EwxTLgCvMxXZA6V0jEDRhOqzyXFt5CQ3rTA9zGZBHJgwyq7c2k0kT7X1oYsxS4x8H Q/Uh2eHiM93ewybPufLIoagQhA+rjqoOzKLBFYP8iiqriiY4tGzTdGf0GsvJbhQrkglndrlIFMfo 6dr6Bk6awvmVU/RqfeHkAl7MLaw2xKJQaCtyOZxDlm/wCfa6MEe62XjDeuZ/AaaP+rEfpZvJQtDm 8PGtMT11BYchnYreecGqNsvu9A1ziD9X3HomFZUn6Vsv8SZ2VcYq61kba0Wq9MPOr89kcovWauYe 2FqvpLKGojrSNJMSA5XejzghCY+0mKc58anspetX0lzy8lV12Q2vdOSKndWVmbj+PHLwZIIx/tIi E8ow6VwoUur5NqkiFAehTbKDeL4Ogvx+XiPqiEyZmS9GsNocsruidbqZTknqr0m5ljCaF1JzXQBa D9Pt3q7qKhSMj//dEF0fscsokTIJXqVArecuBkhLhj7AirBVU4is07Q98WESTHqGFNHuAIOkFukI jeci0uW8Z+JDPiS3dKuB0KOkkqkCWXZC3pbeA21dotOXijhaFRngGL0MJeURiXZ90/Qi3ovXY6Uv HNkEaCi3BPz+5tcQLop2HSTw6RqXuY5Z9aYADd6E80yV7NT+7Ufgsckz81XhdZpUVBY1p7PCzF2W 6IZC284zNk+WQyoxTUll9TCIrQlYqeaU+WbGHFtuPYn9ae6lxXQZtHW+TQgFKQuyKcK8aNZDzUm8 rL2t1xZfWAqu0ltGA/pYTUzwAxGsnaj91CN4Ha3RX7XhPNbIEjhlCFWOQCZPeTkDXZaObmxpZ7Hh HJ1df2CNaXmC81YmsZN/tXihoXx8/NDmIzqiCn81Z0dpTJ8DR0XqzPFaCGRwk6sDeK6syKFqz0HB l+enpW2RcRPekSu7j6nWhMlLybzZplmmwjVKGrAjjPSEzpBFpARUVt2Rf2W2DF4zGcYPbkHiKVXU 9ejreCLaNrDcRedzECSMFYiJWbGreC8ZP8fHhohmLbaTMe0JZIjKNWH1CsleiLiSQd/UQRlXM8UB 92QZw/5AFFXeBOyfHIAZ6Ut0e3JMTJCbDd89TjbrMewCi3oKtK23MGbG7kqkafiQH4364PY2vhAU mRrk1a4shb96HNSM4ZcEDnmINv7mHbtHOvr4mRlZAzD2U9hPCnAQ28QyrZ62hJEcvbjIznKM5Kk7 lXNaVkfZJc/4NmG3ks4ehlc2claWstnWSQ8YA0/iO3iYSAwaMqqa0STld2N23ijHKetEFsj+BJbc je0WWJgW8XEYODfdTeZBrWRQHPXXa+53iRPMOyW9mqxYYeXg7QDvh3B9fWMLhbyIaGYEGVmYvQHM Xeguu1uhR3KHrp/o6i5578Ag0HxesTS9nXKLSGBKscl/Hvat4Vnf+8y1MzTibCR6HdV1NYMpOLPg 3m2LtbJqlLUAYpfjxY6Xe6qg71+aNKDpU0TuDrXBgqF0Sh2aUgbXl+Msc3MaF9FuQ/34fNZ+sJj4 8HvRlnP/VY+3cpAtagGRXj+GpfP+0mlZW+vzarnAGDQtAVtuBionuWZhoJwI1THxBlp/Rx7/VhnL 0LchxmKEqE7nST9M2FHjVF2GyrKU6ubmQPV1DOGyEWADzAeepfgeX1USjdwVvQbgl3t4mbH1bEnM f0bNq0dqBmHC4lLnWO/dwakJiMbIwiM9myM+5RPTwbnYmeUuDA7Zjobc9xezqaNKyX/BGQZK7OTL 4ylmfRhFnWGsthJCjVmc1HJiUb7jEGQ+SErxTVx21sddfp6Ou4V6GrMluSWsuurKlR64t0M/sbFZ 2K+q0lzu4CpBfcGpNDUScTbM7OGDN1qkT/4ucQSVMASjyhX+NU2KyQ7S/QoH1pqJuz+7enO4Yu4K Udsx30NuVKZRWO6ZzaFDpmsDf+rXxUENDn8G15txVcKQqddbSen1+rjQqpK1afdwUDrr1e48yUzz GR0bR1NZsSDlafjsztiBLmrzETFK+ilNfKSQCshVrmlMERiO/OTpbSGPVnr8q7fpgp4P5SrNA5wu XB9kYRYQdHCUExsbMO16wuDP88UID85oKAXzSb8wuZ8Z1NcDg2vDRiKLqc4/ADss54Xsk8fPmWvc W8MeTSPprHaESlJE/qGdNl0z8cIyLZB4fblEFfeXq+erbAX2CdCCgJBJOMabc/av0Y1Z+8HU1f1+ BYcCrquxPBI6nLiJfvJ1fLRUyiXGMOgJbD15auWo/Uoj7Pr8srwGqmVoDuOtpVO0igH/+yNC/PpO FEOVGYLVTlG493kYop2+NEdb2liOVztJ3oIKyecSp6UeKCzZX2MycDf04yIpWvM//C08kKrMbktu ZrjQzHd8a0TVjPO3HrzmwOKlnF0HVLOge1u7LSsoTH3vUhr+/a+9uQVuItTJAIaSJ08BsDSgueBK qy4f1EdkzuEDa8ETulf+f/u/c6hduFx1DURNhDg1qwqTEluGhxSAgM18BtOKqe8na4iT3R8+nnNH UKlk8hOJ7ShpqelzPckXtQZw5d1F9E22DCtsmorTiTJqF17qi0+cdVCX3SazP6RpDi7kQylNpMJm V1pn+E3kNTmB/M6ISxId2VhjSa+kezKABO3IS1+5aRb1JaDtTiegg8Y+8PrPnR2LHnmj8/ueESVo zvr/h2ddcGnq+z8FXmRGeeyrL6+3wObQrVb3nkUhZPPdme/oI2vtVKzndC3OGq5yI4eB3Gzry/Gq bEOsv4axw94I3M/Z5lDxWX9NcBilnDnBytZKOckLw27uhvBoiIaFYKgvqWV0z7fGE57clmVlia6H ocNSORs2BmURA/d4tLI7Yr2nsMOIFxQkdQ5l2HQwxUvU9+qC0LTaBdoiUDd13kDRbr07jvGPegbj WIMV3kngHT6IMqck6oMph07zN+xFHm9OjBcS/FmH9lPbwuuiy2mFyaTzCosmeeEAZc6yo6H25SX3 0vCB0FhhcuU1XXNQsG8Q/1HDaZrKWFc2AO6iPLvGF92/gsNZ97lAPPhaNpMD7cbbFVDcQBSBQlK9 EkFvCk3GVTiSmkoYK9I3MOH1h1Q+D2dSHLhLlhrSlR7NsArOmdAz/GtcCyhVrSnE1nEzEaRdepBX JGptfTX2hNQ2oINoB7hMpKe6yPkxUbY7GPSXw5lRLEfiBaBLNynPbYyt7FfyfF0D9F6UuxhmRgKK IV5mgnIjR/MbPVlHW4tOA6kX5hNLp9JGMnTc8kGOtWe2zK7iaukgENGJLmGifZqkX42AunmOZCtT l7ypXLNLNRkosvWab7pJVK6LyzDTAJvAoJ9n7u8OITwFO7X0RQ0yIDPKzVxMVjBXQjIVkMR4STSi HQIoTOVOMfFmnqHwZ4hUh3H3BTffjIlHn79Z5c6wvBnRAIm90G3aVx8Z7B1q4kS3+92E1IgyUDDm ItWYzOHxiQRa0O7TT/uEmoDy3txp5xl9hOZUBirDrRMbuyJttnd/S6JD59sDsVdT6SRMbQbO+pH4 JmOpqYR63ukA0zpeaaeek2xL9N/gdRpT3GUU4st5Okk513a9slgs2x+Nvq7ZEHOw6H2ZOm8QxK9r XEyAgIXtluiT55CrO0NMfRlzgw6erwiGLkt3psemQp2WHBVUql+MgkAxQLB6WXxBt7/+c0XC9ide 60VX0n30j7ZyL4H/QqvT3Z3Cu4hYmPGRRZ/xF4qnx6K4SpuX2i1ZCjeWAkNQJ4wbghMrUnPGIgIo K3GJizqccG48D9I4XGB7gSO5UdoV6JTqCRS7Pkwi0w29fZEFzDzP7lhy/JPkGwzHnEg9RxAedj6M aBHWIvN9SxA4sC+6BaCa8ObCrLk4LlcCReJGp6pqZWMy0hpl9q5lpocBXQZjELYYmsj0V+/887pi PnAJmDGVbwNJTJuPe3qLBymJ3HJHdTYc7DHeBio51mTAr8R6D7Z0W6x9QRX0rj3a9k0dE6KsJ0Ro SQNdSFeoGMr/EIuZlmp4yfYjWaKGKtcJsy/W9U1emKXjAQ8gwGr4+4FOCts8IaRznWUvs2AeyH9Z DvLlVvlXdkUkzx7Dw4DLQhtKmJmRwmMqr9IbeUKD21JHjUlWJ0jBLkE3xoMX4NIDMsxK+xetPeeE MXDcrv9mK63bz2gCUPtnbZBY8Cfw1DrtgIKw6Nn76JcGrEbYjMUe3g3aa1l7wAOLmJYAIAsZq4IZ 2lwZcFhwtoYlhuNSz54l4D9HZQVD/cAw/NXkMEoyRKdGfKk6CRgbrtDsrTFeOcqwjNsYaXVM9uZx IkVTM+cfMviGYy8zWSYmjkgwHa2fuX1fj9JNQ+eWvSXgFV3AyLLd2lGbuUvuml/QDscCD5yGrWjs hq7O7EpgWCyonTPMBER4tDj8X5gRE2vAJStqhzHZsIB3UZkZSpDMOtk7+tQmJi6q3Gvaq21stMva ZaxpaPB+JhuvCNt5l8J0ggK19mplkInHx1t1YJwzVL1JESZiEQLN/1noiMsNGAkfrbnMD6Agev+0 Ry2bB5vX8UrzTFYkBhvWlQMKIsyKrGKC8rmlCOeTfUsJgeng89XectUf2ipVB/g/RPRw+qN00x9N 6jgiT0sOr138+cym33BwVZyF6ysvuYlwUtXO/uCxQEydABmuTG6zDQQ7nZI0/ezB/DZZ1jzQrAJ0 gKF3Jn2NJtirq9ZknZZBZaSJWMjJTtoU4SguY5UFJcOC4Af6YtSpMBT5hloIAOpq/Z1NglnTvyvA FqayDPQ0N22aa26AqwvHrWGj2cxt1Dh39myDZyRG6xiHTPp2k3b3EvOJ+A9qxiyEU89qKKb8Gscc o9hLeGxJDQsMA7KYjaFScG3ehPAbL73otw1E/Mlmb9d+yPCPwtr1Q5y4T8AALQ8s5NQLPwNpN274 oaGBnrjFmz0ZH25A7c2DdmjrwnW8Ez0P5vcnBkjBtt0VAGjFmGw7np6U2SyG1vUJ7GGfnCRR8cG1 i3IaYa0CVHkwnAsyT4AI5VGoH5Sd7nsywlmRiAVCjAOWaMOO2K22gFQwNEzyMTKQfjhdBK5L3buJ YAMQVfbgUmB+eljlxSV5DUAKcv1uOjgnd4tMKlr1c05kxWJpfrPJlAb1u6vnW7Iy9NQEptRBZFlH d3ja9Ln7Cmv6cOiO00AyGp6p4DvggDcTfWHK4Muh6LcqlAH4CxKfB+3CZLj5vcOpDzH1Z3VhAlTz L0IQvovzCHn1UDtTCFsSObDQkriH91OGA5f9LSJ68YZptvbUKAHmz2Z1IniL/FVlUSTDHDy4piQ/ 36yrOhL/QpXpejVr4x+58bMe/SMYMXrH4KeTX3KB1Sf6LQYzK/lgL/eYH8G0ybgi8Ok17OLGFOHh x6D8TLygwBNjWZhrR9YCbWZe0B05Y8l7nE3UxCf6aroXjYtcvnuNT7Y+CuZoX5cBLvwzhqno7T9j jYdRLRoY0COeCz5FffsdncJdNwWJ0Ty+aZi1bqMwf3Gi+paPy2aOI7pbR7M4rMYBCyhsiF4W2MAD VWWV1+kccXpTzxbHKIlIX+caqphFQH3v+OBDRxRSmhv5KdraefTDTI4OYaNGFA8qKrHkqPffhJ7k 1WwBmP0utKkuh+FgqQxKu9poCY5yCfm8qOEdcFzzniGAPCu0dD59KTiKiZFybgVi997zAgCFZDw2 9KLjFwgMhIMwz6NuOUemlw103Mp8dMnJPS/FaLtV7aLdgk8TMrRaDJiaPPkuFkAg8K7okvSk/fJu vQCEiL7Ap8OmZ/Apll1QSFYnWmzUeI8Ebt7KMvLLQNgDlygRRYEFQOTnM3CEqOaXSBjz/KcdVrzh MjmPRgpPqnDckRZZU0Qt17gJxYz1ds4EaFp+J9sUT60pPZWqCQoJwa1KgE0MatLRKabzAifkXKLw kteYShDmCVTgVoJ0YJkeTH0AM78XDT2YyhvOp4tW9uCPaDsINb3Ul0OxdQix9jmYYK94+HmO4Xji F2kx3M7R4mvQZ6R5i3YRsKnQRgf5HhQ3NwKu3AxD2f/w1jBh8DUgpXzD0xeQvHil+nX3d0uRXiL+ ampfySZ0S9MOCAgZsOS7QMYZOdaD40GEJzeJrf7b3fDh31Me29SDgZiCmMKtoIkTA77UsIzoRZWc Cm1BZ7gE8sE9ipSPw3eXUIGEOHooaq1+T3sYR68uUP3Dq5Ug5R7iENfxs1SzuxifTRNw/yOFhYdl Qj2s1am6La0UUdD/9TUkoiiO+0pOqyPwm8bbzc2Z29U2bGhNxiHD238Nt6PFqDOdJi5GNxeHIsbR h62cw0vthJHd+8+Gh1jReE7MNKv35cnU9T4D6K6VtrOKN0igV+AGtc10zoshXvUW1bFrRQH0qGIY 5oyWJ0/IWgaC19kB4o+4auouElc9HaD9LCsMN9hTPfeCf9FrmkAPHBiFEJE2weOzDKdaYLwiAZ7X DW2yNtE1ttpH6KDIZl1x+DQJqnwkDQl/3zvbUpfbSkSHed5p8gO4bHgtMwdiV5wAVkQBMbvVkT9x gGcGJ2bHUbCzIuzBRAKnSM7ENFZQljKVKG7sSGXxWopYVz4JPdt8QV2In1sBTh6ITcEtGlku2UfU FC5E0m4Q+tnviyfGiPtXf99XRg57vDafqL2EEtoBHezeUmJ81ft4IAj8+cT6DF7ghyQoyKFM6mlF EMYqGVKhx7HKrtRKaBjZdtz/Ann+jva+wCBkH3PYSFzRWsu/aog2vnYoVOUyRTUErWxKEXqSccEy YcOVhUYEoDNCR/zfVXefIuSVYvuz01l6UoC8u3SxJqmBEgO62f0HzKlrVBSxX3/IuvUjKf77WRLS gxdkPAu0F4krcoJwRZ9JBrv76JalnCHDdzs2CsVVZJYm8nNV1naUisF8GobxhWBwDklNW27JjnEG giPcSpy0yd9bwkILhz0zJbV4jRysrpExGLY4AMAD4qFvpLGcx5pJ3el7CEk61avp2J3aYiRwizof Qu1W6ymeTruN4RjXnAJ2aMNe2i/Ptbf5w33/W+RUhC+T15GriZ+PNPinNhqKoQkm68wzNB2uGE4z qt2toGDDtEJv1mJi1QKr9XMkWu6tlPmUIRcGFJt1t/EU+QzLcdXoPi9A+eBTrabzOtoaeVc5EWpl 0+XaGZMDQYBpGinCFVStRUM5bR8J0wyBOpU+FWA/pBcw99GeWTchcJN4dEDoUiheG2hmXW5XTD89 tHOrXkNXKNJ1ijQpz2r+eant4HS7LviRDWeHhumKnY+ivZbSoEjJqRHGFFPt1fvM13w2nGHP6YCA DvF7uhRd3BAZ8Q2s/B8JOiTaypUkRApjcj7VlMqpVO+PLnf8NAV+v9bpKqxczHhMUHrNhcyzybtu 4WCEN/0cOEnuzJp9KpkHx0dg9rg4jJhHx1ELDvZWYUahAEoT/O5N/kxogZNCz6rKF6iz0wW0/81f 3uALsFAgQdnOHp5bBlr6JPx4pA1EvAO3MoJ68S1W8Xqr/+D+YSaQV3ZVlTkoR+tqxPeTLX5cmGI8 9681LheaP6iGsQWvWQ7Xk5xcb1lbdqVj+QDr1M61sKdkaTHNihRK0NumbssReLiNDvqJX574qtSL QHdiDbNCWbOoxbTrpaUF9L04vmXXcaMdJmUN+urMNVqk0liuag30hMBbCftvXVZ2Ovl0gophVhD5 3UQwE+oJdcOjBiHYp0LAJ6Mv+AK4qiTWYwQ/FIXfBDTSQo4BORCDu2JvAkRGp0NTE7TX4Pflshly lLH8gwssiDyPnxVlTSMV4un+iM7mA+iwgin4lj2yvP+PvYR/OxIJoZPwC1y2vtzNc9QN4wdD3ORt Vb845bukCORidc3nrBuulR9DbEWX1vy4jsYIvIssWsPeX0jhn+ceDD/BLIXIeYwdtK5Hw/+MoHVv XeHYXP4MTNVxz1k3UYM7CFYDHKicdfWeV4bC7mgc3spAx/DfvNA6bOnm4UswUyN/j1/oZ3c3jrm3 V+vWlzp87+GAyndsdrcCQZbHZbkpL4e8n9bMr909FkQ542G/l53dWFxwfsXzDOng+mM9tLIVbZlB /hPQzZGuxNFMJ2iMoaj4uz/Wi6PXCamZsraWpRcRN0rz4YPX4JWu9NOt1D0t2eHzIUYM0xRQyrR4 LrjKvDGS70cWoqaBWCoDQYV6LXYr0A2AMS2/4z/LpuIsKm+JvL8ZUNWdQYLfHaHy/riF0Ce+JB++ YpIV5UJsMVcYYE5wtx3v+ONAuCsGIb+dlrcMb1/NLkew9EmcTi6gETv5et99POQpy7BOsHAZs26l oMaeynEs/n9N0JXvn2NOHn6aibGM7jLpZEH22lZrFMtZZLyVUQ1S8KOqL/rcQSmST+AfPzy8OKG2 Q2dEbZqr/uBZbbH115tLy29ZzpqUw8d2R5ae83OGKmQ9cX2lJbiE9t9K6Nr+eoCgxMNWC0Vo3z53 rmudV78BMsJkkSE63pQhFzP7Ozi/VvKV3JiTWGhyHC1i/unG6czTph1B13sdifiohJB0J5O/Vvyl pClcKUu2o41K/u8NKNqutiIwDNl/tWohzQp5ivN0FDnx+gDqM4TmoR3ZNl5WuGyTMIW4Z5oBrZsg 00QS/hqTVSyTy67VVzzZJgX/TUE6cTdDw6zkv5zRuxoXD7OXC/lPj+jkO1k4uunUHu31ammDcr1w 60c5+8Yr9FvvcEsYGvjb89YDS6k+2L3ZbZbUBq0lIf11Ik7pm84j3LsvOs6YN/DU2VRmn/0AzgpP XrZdjzsM77YrKalJFtMc6q6sZunJhTFd4Yr1ulQT8XiGIOQe6z5OMNxxygZF64FCOHWu7+7cWlH9 Q6l558jmqdiQLPIkWU3qNpa3UvtI2RlDfqxrdVZsr3qwpXFGj7gSqpXXk97y7ErNr64MbLqK1kjz RHKLxYdwc7EJ8SlbpCeqohfvSULfQNCMq0auPH4H1COydjmVCtck2eu91o3ndATAyxnlzoK4ampG crk86+P3142MdNg4yLf99c1wNnuuPbbjwVq1gyaMn9npHrvI7bbB+r4oNdA5nxjHzPqyIGQ6aJO3 p+hYC4uGFNQMjMFNr0c4og4G+0QQJS+7JdrQU+b8o6t5bhXYlTj95vvYsse+rDc8mZtIBr5blb3t FS6fxglcUiQZbxqJkHvLek1iHPZqbjsvXFH9qTPDkxlamtZPzEkFTu02Q+XZ6NJOUtqYSqvrIyg3 CVB8OC4wH7sF5+MQ7H8Q7PC9T7sRdCeY09lYE90ANpYOK/GQHCIwTp8L4ur9jtM4hzwGae2hu9HS k3ygN4w2PizVjk5PBzWiPR59fMg6oSGVQQYPJWd07hsZmdHBHytmgHnsf0Q1Z2sMo4hs6DhouFs9 ip/LLlexZQHANhnE1TnIX7wNvQLmVFXy2Gad480i189QY61URpf2YOUNtxcPEKLRKDHUJMi3Zgtl dkh/GvzHFcB8lng7CfW9H+OuutDpF7UNpiq+evAuoAXBmFIY4k3t4R0es9irIxOV058QJBAyhAUK SRVT3uMpEqlzPbETzUOhsRzXuqM5lMweTWAdCSwXbFZKu5CJjJ+abLqmH/HIrQSVFaUM+iSIZTXD pZLp5DKNUyrN1vwGN9Vn2Mj3NrFihSprEiG37UzBCjSUsk9PvxJjgCQoNim2KD5ZBJsau4BeJbpA AW3A1MQs9k1RV1TAqXuFUET2tTZq8C2RXvr4j1pSa5qTHJWqqtAjUAtmFjPqW4VfjrWYIQ1Ls+W0 bibEs0gAMVfdFSyjAXvr5UISK30OWlVubcjOdY8xqXwlIwFUTRpUqQkETYDjXKACBKmPC9wJK3XR 3xTCSqTiiOvWlL81vzgzgxGR5egfSUYl6VB8Vi9m5fpKaELRJxryh6Hf2MCJTJU+238Ljg0R+SA7 nuWyyaYf5k4afsubwMndodUwm38AawQ74T+s24W2NcltomyICbDL8z4gaAXF+bkqLk5DIs+vlPEh 1Zvm1ha2HKLAQudDZVxYrsMDo2Tp9YsZsCOJ6VZp3ariCWg84+2/Lq9b8gfPpr54aw4ix8NdoPLk 2v3KYVOzYPZFJ6lUM4+dkQIy19ufFj3e7q3t6nCKySnRFYyhvI27pIWCh5LqVjYbl2uzBm3tG4WY NaTkmOwtTFdvMgMYEfmXuc7Im6MV0xbZuHuUttwibIBQp2NdYo4UBCGy4K1kaHs+WO5LPJGB3otV M3qvWXbKO2PGD6NiqSuETYQzlhMpiy6qqZYe/IdCCJbo5kRV6RbQ03KwjWUl2FPjCkpq7kTRoE0/ W3kv9CDkinQ5lqc5tZN4igDuC+3+IZXesBtG8YPe4CSi1z2CUKgV/UBGBR3f35PSt2I46Te77N+n 4xmymY3/1nwQ+sB/Hx2gdDaTPMWD8+wyAQPsiC4a9ran8usXWtRsrj1yqA/JHT0XtUEN7SSXxAEf C849g7/y16nyRWfAEbTfp/CD7zPerUiSW94JMXgfknZhx6zF9ZWH6VpBw4LlWKc5TajDipIskJFE Yd+pF7duycag1LKJrpYUnyMaxKaNPSMYd5eD41eT3XPbTcFe8VHt22FxQnQK7v3xJdlW2qVEMyPf noQdKlaVxdxZiZklFIKF7cFvNztmiywsj/j0ePGHu8XFzcesSxxA1CbEoiA26b7CdN598fJsoVFL kkaE58craySpEHik02rB76DHtiKGSD/RXM3oIe5A9o9ChhoC2GYgn76AtaFCVTx29c+gFe1UkXeQ iNhTzGNge5emNDJXpF6VTfLaBiMQRpxWViHn5a+rioqiPC70CvKCHh4I1XnCzfNRhlO2GSuCUnR/ Zj0m0zwHIPTcfrHbIaG+W9/H28hO9ja7V+Rk10fPmxaF68AyZ/+a5Shup+RoOf+YdxYli0balhx5 DpsgbuBWGgLdjGaEvrLAOs1ND0HDylxu7hf6xOZL8CSye0C1n53X5r+Jfv5Y1nqgpvrZpn5pfnCT 9Fp9vY1QX3holw9g95LV4SIoggqN42zg+Z4EeTtFsJhTydNQKAOwoKWCdjIdKaQyVceGoHTrIXg9 Y83ikaMju4yU/BNZxmTuv/hoBje6g3/7To6KgyiM2I8qjDltRzOp8Bw6zlq+l9XJ+762+a1HqLXm 8e3rK5N6j+gXR6cUWD438l5dGdyqVJNRGU0PJq55Ns6HGoLO8m9rQ7IUG5C5oIz/+JupX+34dC5b OEBNkoPSJ7Ai51xbPindqVN3u97ceE+U2zZTi9m+ei68Kyjk+vZoSTNPLDdaOa5P/GIEouLg/bE2 MEDQtcGPfUgmVKCaYv8sRe5lbtA8y3/JuIFmi7Gs1AlAAKy4m85lWxGrBBY+BoDFp6t/n12+OY0f QzfGEWA3rfKrespY3aS59zw5ytKvd9quhQkz0lnjS/4dbsz4eWoDiiEhAFN/VBMoXP9maAIlLq87 v5QxJ2tcNPu5dOe4Et1ZxcJxWfWKHWDP8Ih0thb4hjy9dKSJvJy86t56CpdxXsjCKIzSeT/zcv+R S1uBmANw+l9WpTSwNzl8/4xkl6HB6maHfRTcQuIEbaRtVeQXSYquwQycnKuEFfi8pH5UpIdDwEf4 UqKQ3EnWcDs63R0F7VuqYANXATwJLybR3iyJcEOnuwayvrearkHwKT3jn02nHsptWZ+xmzIlxX1L 1OOOmVPu9/HDqRyH4ed+ISczZuKwG7E3GPAsl9J8gYi6KnWUwmrkNOt2VIHt7HziJQ6c4OYcwx3/ IqhK+pDc+GpqRdjoMOISUhapugxLLpEcQDCWyW1lx/S01xQF88IkkGGV6AOckynnwm/eIGa2rTRL QtBt2Ft2N/mCFJAOmssj4h4/2/9EY3p10lTYeijNMfyiDCFAXspS4RDFFpynSBd/lW3OVyWmF+zs sP6v27vldYeVKjxo6mxyOOEU18Ylt8GkK1+i5enmrS6DS9C1hzDv37zYAs5kTDJx4W82o0dedZeY 010xsNqcXJ3q34ukJ0vh3sNPgvaCDZUsnzH1obc47UM8S6wRq7E5ndBNdHHTaNiL+eifOQQgWrIu 22Jpxk2qN7IV6MYbo3YJZHzsU9HI2GwBMDbVWwzSWA3S5MFXkfGFGhkBDmObzb7aJzdqZKZnFHMg CQKXb/Rzz8C/D/7zodjt2qo2cTvYKrXxDfZoK6ePVrcz2Z8HAUoBcT5pzoAArW65dvZ5c/xhRljJ fkQoyvFUPfWREnfDwhBJPW2e311cj6WLhopPu145/i52fG+BJl6pdixL1pvO8XXO6gub8He2tGF1 8b7G+woebOP1wGYYPoz79kQPXI+ZIpMmlFVPzwL03K/8zV0b8C/VeOXTUWCpyIHPtK1lAAX0V+tU CMUrW1GL/BvlJm34t3iSf27wXet4zladUg6N8oJH0H8roTnYNZQ05XSoJDuLZqRRAD+v0SDg8DRf FgOmO0cjeeAObnqcw9m0/8qhZvu/rEasHHsU6jJP+A9gDlTMmZml2uPaTy7IkUKv4zoS/jPPKfcg HUt7Q1nUZHHAAglRFW8Cup6YcnpKN8k6QHQtGunAVT/sL3K+eQGqR85g5CJFDRvudvX+VFngKV2F zPSzvkmnYCjXaWLtI+7iI8X6asB0poejB7lhO9yYzboK2BAJ9YaYSVo4OiYeRB0STcV97Atf8S7z OoXSiRPf+s3RjJTGX9xcxDlxbgdIGbb//gGZf8nKKIhRI6LPIHFsMPwTJzMXDMquyUICggJCGli7 UUK1r19yXeWsXx++TqwGS9bLKhB711x1YE+Y241AIVyaGyZZMWw1SFCoefPs5uNNPGkvcizEjjJ9 SXkebjtuKTUwhdVLySbpP90pnEZ7xtVMamhuUtuSqzuGtefD7wePRWmhf2fNrjwFqNDFV9gLzdFh Vgw5+cmu/TkHy7DA/dTYNMKXxb+MZZVUPcroGcNLedfKagd/L4YxkdHLPOobj6YQvaxYPYMTXxkH G+SZ1CbYc7pFhHIhx7qYEsDApA38LvLMIagkHErCRtFUysoTNMbB70zx6ebEa7bFVSr0/Im2y85n Nm2+LvEtrUP3AtZki5ljVVh08H+P3XQP1f/5SaJXhSM8CuP3Iel2z2lEHly7/2JCwgzAhBM4OTY1 93jygIoMlpxyAfakgXZP1EoeMtEXbYTY3pEvX8hHnicmrs8x8kqN6jTFqll6mH0Kkbc4uyjp/ZP2 P44czrvjefL38iZUgY3fS631kOsEhmiRJtz7ynfiJimaKUUSv+vA8GHEmm+HOT1ncp+edKqUodq3 hQaYkJq3dyJD/8o3t52bLirbDgPBVxAzhgGCRosO1aBSXgTFf5i//s4L9doLmM42oxnTjmQUnWEv kRdxKTZmA0BU3Xn+CwMpc8SFvA2Z5D06tr6lVTF5ZFZDd8DBuSQeWPf3ZNSwI4riwPVxQTBuRY7+ 0uK0BTRhsb8p/wWWj/banH5rKWzRzZ0YdhuAY8ffreAdaPjCDbiQRwkf2SmOMaoHIV/YBkuKgF1i mEyq8y+2BEd4vyZWtbrNVsmuls8dLxVSeFWagVxXALzhwHtemWBEI/p+oKIqtjktluqxCrSwYWlK WIwnY0pF9ocVq4hBea1Kw7kEOF6Aa7bthe/rZOrDJ8mCkInkE9waU3ctAchx+cZpJQkIjM9T/wGi Dem2QoPVqbA7Q0SM5rEl6neamWeCIHbhcXIscWuYfHhVoSrxhj1hdBoVnHevzO2tythbeQasZdC/ NugdqfHxk8e3vkNhkzFTPT4BNkdAriSt+sE5jJxXQpQpDrOURYDI3dSYVbmdo4IGP/imw+bt1Axi 6lLVwVJ5ez3Drq1h4YUjl5cdqOxzJ3p5+KIIf6/S7dQiNpxGLK3b1qsf+DN48gVRTfo7Eiekb5Ee 8PUO5P/QrpqOCMVqneU0snA9ii5UC37XGsAc+t1EyvIbA30q4TkhIFBHCl6VldacRoqtCinNUUFG 5fUWZaO48VZmdtM8JyE1H144H8x0btPUBEFMmX6OVvJTsaR1Elu6M+z4l+BvmpgZhhUweL8Vr3Gp 5v1jbOGkVGI6uNNlKRhR8vatG9MncwD/nGkOkQ2zMho4e/C2y8DWUz3lfHTGMFDRaJC7+70riPMU aQw/h2Y2jafxzgZEN8+C4+JVu++OWx0IawD0VnAFHmTzj6GLnSNH55HzFL9Y3AvXklFJw32twqZq 7ooWQutSVyQGbv4jhuKpYOFqCAhO4CPJ+V6XQ6dkGabWs+eSniO3jCf8ZlAU0U07Zs68SdnSeQzh uNwuS1WwciillX0m/ap3znuf+rJ7lMRW8nICUaVLlPAVJPbikCFKVejMkJIlKVHyrrrqhECN7WeT BdGF1s8PGHjhPNlZoEcMTbVDpnRSH9Q3cjOjGxqZODucti8ndf8VoPINoU+jTaKWsqZLwD1DmjJk 3+tapySlZPtT6edVRsIZTUsaayBKxccApCfzA0TaoyikqNgGjjrpwlSmGWtd16yrQBzj31Ib6BWq ACkSyyKQnjPzS4bFivpC+NmDnz9Y3AT7mRN9drXPfBz++n7K171mcrCbccqiLnGgbwNxYBTXBppl citaBlVfPo/fJc7ntmyLZyo5slZJLwf8WzGLGkR7rf50sWtQ4CiarN91psAuxu/Ub0CO9tarPoOH 2EC9dQgjng8GcJVs4egIx4aWs9nBTorJCtbNS/8S82gCcVPd3WKSegPA4nIRKiW/+zbUO/2JVGfT T7WVTVkIOtlaF+oHg4bly5kbYzBjLoPlQdlTfxp4o2yY6uaOgBO7rqdTzEQdF0O8EZh06Tok7fTw +pEX0LRZgPh5BPvyYMdoU9aX4kEk2vjXnDYwUroZgSLI2En65O1jUNVqBIjFro448WzewlqTg62N 1b0vrlHJNPbGFo1GBnlV1xpNkxl5h8Q9C2B2UWKmVmzhc2++aDoEiNYtB+KJOuU64VGvdfm060+7 6h4ZssPWaRR63+Xu/s7s+iLlR/L04Prn0qgtntNT02rmVaLgdIup3kD3koPcogo78qGN7y5ddRAt 5y9hLhU5pSmcXCjtqYERyxG9etW5yv9eu8O5aMWHHjvsc8ePRFjvs7+enzlopd6bkeCJY14c6l3o s9pABN4hlWrRgVMOUBuFSRKA+pmNSl+O8JN7TVtPyB4oJzwEkAKtHR8GmYl+8YLjK4fv5MXAwgGG FPbCYrcjGHgkWGqsi/MaM88/JtzyrsVbwh808PjUKaYsali2CVMnoWUpkggFY6nD/aNXeKLmZwqF qQGcpjETB3IumABpSGzBkMMDrpeNxKOGNq48LS58KHThPRx83TKdnO0XZcnzsoWRJFMkSAa1brtM 6UyNI7KVC+Y7f0/yEryY6jccJBZ/9bVo2ow3lOWJmlW0NghybD0OszX61PBeKgZUut5oAg3acMlc MjLl4jih/xOaBEm7WAhOQB3LQOaut/n8bWGgA/IA1mTNliE3j5OBdZSYdgSId7Q006hNotP6b4cZ NK561wLAKfrGU0RjAwep8p0t+b8hJj/hnPU8xXuxiiVZuQDg6GIQ+575BpTUAFYCd5VVLbIvLOiM aA+5GCQEbduZU+58EnXTp8tcIbUU0Ile1QZM3S/i1O86EkrYSSkYZUmCxBNyFs+iWaj0Q3m9ku5R sLrou7KtrRl1TqLyB/57yLKZABmaESZqKRAQ4QYKfQ9jAC71zR2njnp1Gu0CDp4gMUqQVAJwlz7o bQttWvYOHkRWp7UhrWSlv3SByv1ZRfRZsmW0bo2dDRxbuv3Z5VFAJuTRYTzw+yi+dTxFYgD+JaIp BXBR4SEwbVstL+PqtMtjUasmBhvhgkVUODVggo7hOvqLvWedipnVM6pcwkAvDkVgC5+6aqMLzq6y EaeUgrDem4lOEgCaUKLRJw5LQ3OlQlNJHfrCrIHhV77ZjaVc2zxrw0gTa2WOIcKhay7bW4AwLfK1 W2i4GIvBqYh9fcS8VeSsnYOj/YxvzH9wGpkh5Xf6kerdctT5jmoT4dtrYf3wisS3/CLHq93XSYJ0 elCgVsQgJZFwvRXzuWrFVKCushZVCEwWmn5qEYV/ATesrYMkVgdjvA5D4BRxY091/44rJW8kpWZ2 uC1KKE07SQfryM5jHSiIUkveOgrBfDwg0kAsRDwlYdeHOmwfOly95QOK/ylsbW0jlSbmmq84G4rV nKlSELttnFbARDiPi1qCflCWPFbLdcgzmOM2/wAiWO3HGS+/9XEOc90CHJQXmFz/9dmzyHK9Ae5W YpMZz4wlhNd4C0OQshXJw3mSomzwMgtRdhKRaHaTskaCEYjBvHUreGmTIbVXaUAGQjrr8eJKuKaO MpjtPzWE6qwIKqN2of2cgDQXYygw8FTb2daCbQ5wn3OZ5n+z0D3hiZe1+AhE4pd6BejJyEPfTf3Z 8UG0tuC2yiM9bpRGmUiENHRf5LSBiTnspeqiBM2ojmTOWCLx6kBAn0LX0PIL8Yl0+UU5JbGUo86w 00LtNCz46cnl4mG9+yi9MIMvYAP2jRfskUkAzT16gqE1/0q499pCbisKJt+DAvygmYizpwtrw9r7 E75bxuFlHy7rbiZgtSrcQfU34I6lefvPfmrhliYqlUwomkySW5ZIMg8DEo4k/MN0qUbgoZBmruWP BIbPzmTQIfl8Inr8oQrAyh549/+Hzb4Zu+oid/rdc6kTOy2xdDxtdwLCfhi+BLAmRP6ZpuSgOaJm S7xVvlsUhZmZkBXFDP5by++D0sw3A+Bue0kz4ebYSfLZonWIN9A0g1ImHRGuz7CjAy6rRLAXwpoW s0LbPdlIVF/DoPRL/JzKOxrdojHVX4LIiyF+ADFQBqZyi15/w39nG66zqJF3D2Dg9BgB/diYrUn2 HLScbEf60b5lzDFG0oX1Tzcmgq63XI7zUFagA6gnzZZG1pCUeGzyoXKeUQoRfRXf2WRDMCgjhTSq FzjVRsY3MqFsdScNfRImWYWaH/zoTjg2IQ102q0zOHJphmg7QUJE+HhsbB3YZlbspd4m46MkuvUb cAgqQSyxwQfxt8wwyKjPI1PEc/YsEuvpMy5ncQELXOXrdj9Cxg4hxVEcYul0tl8Q7z7y5FErlXEz 6bszuJDTPI1zrTR7gRd1uF3ZBWPov6YBactTbo4+TPo4ISK98HaGcAHT8LEnWylKIFMMZ0nmH3vN LDXNOLTowhuiWK9UX6a87pycPH4Jx+YUKxAzBLvj2YbGPfKJ7kUtQUAKMam6GgUzn2FsLqHLWrs9 /xqisPBsfspr6GWXS0qErT+e5wLg5+SqeVoOoPr0RgR5VcRPQQsUSO/yX1RUOjmAHk4ws8Vg3LQo yknRugrVcZVPYShfOx+r1XkzFzcLThHqatH9WiD8tCCeZY7i1Brd3FbxEVAKKZOxdo7N8YQa00GV qGNylbEe4qDj4eaVzDR8NnCg74mBid0NJIXoxP1WvY2CnmPXD0Iw5Id9F16HUp3Jv6ykWRZZ+jPj EgIueHZLl3+QPSEQV0r+khXHuDRdxNxOtZMUnoF25EGVbWDYJShXQoGzIXLTlL+cQJiMV8oD6NhQ 5wy00Ms55Ud1xpLAr9Oalq/E8S0f1K9qCODcQy/W48VPbRJ+ljvAM2ugLyLfuvku8nDUZsmtvDGo CMGEfOfZ8Wu+ESzy3QlyjpdkOZbZoz7VZssqCbyVY5gyV1ZRNNGVQ57+PWJhSmtmEsiybspAOi62 m2xMYoVrUty2A9jkJPquFYGCNnNTEP5qevTyu19vngWTozcxEjbXcurVM9eNoj0b7A75QWhu6YhE NXt79g5J0RZjUviBm3rPjYqXYqLaW/6fJNqd0HHeUT7L/3zxyCFhhh/AYHhFqZsKfNj8P59LPAFh 04oEU40CGnSsXv7spmbe1DM4a30bqyVJKlfPQFLiTfftvHW5QIYSRuoRZ4SkzIiHDSruWX8VdpZn Fxdqy87sRKqW4LsC/n1Don4HBUP5B3lEmtRAWcHkAL8YIZRU4QuxcMPMR2qhpJD4LcF6oovRK/1t DkyqSeRwuHBjQcMpWcRG72gtJkcZS8OAhDRN9oaB5IIampep2xJ02toWsym4tv7Ig31c+6GUrkSA 7DyQSGq5oJFVsYd32R4b9X8sUdzrTG7878pRnE66RrOWK4ByAmQEIj7MTf0zqLkuyMs2IsEldood 3Yefu+a+DJLkuGqTQmPblqcrEdvhW1dzGvHpSv+ty6cNKRSRNefj0vlxbf0lmJlw/bvnKNa2yvQo XcXdOtHiXTvzOt9qoIknjtyosKwNushyf1GunCdo6BT5+zyZJpch6OjpzrckBeeJjLAkbj6pl3TT nrwCk7J4TMEESTJNqjDJfIvcbNLB5+S1LBWdkqCdCtn3b0OtvHFOJ5kS4TIpIJhJdiTbCSOIVVym c5ZT10UcJFK6kIZiYd9ohHL4FUWu3mQ3AsoJfqsjFVz2cE2PORepW1LSvH9EOZroaQb+o/sAhzMu BeaxcfZ0gPEi3vtDO3ZHmKiCZTaDSILO+hrESXJwrzWp64cpMFBYzp736O1tuFd3+3T9EyXeooV0 BhGQ6BGZTndy9MUz//AqZVVXkadQMt6L5H6MI8H5MANsDvqoFtYewR0XZs1vF2LrpRTb+6NLqJ6p cb4Y5PjQZYAmUvSSeiPGqHfjvfmKy+KNUf6Ld8eCimri7EnTyZoc5SXQ9zdhtu54WX7GtLhJ/nw8 qTlmsnElCQhuCOCTFsZOdyDtET9qSq2chUdQ+VdqolwbyXebaubUTJ4iSxfDGoTTZMp4zzysjBfl IVqUxP0malUUN4+NzV/J4R6MWLB6V8pgJCaI5uLLZSw+5Ovi4lYGe2WWI0om8EfFco1+8hKhYMp5 vBd2Csc5bUDk14V0KVibifo8IyWmJ2AGIyyg/VzJFct95n4SELEVt6xA8lKv0QsrPRR03npPcPId g5ZXTHHP6uAVGpO3HqA3CKZ1KVuTN9BOQ+XksD2JeVFz4PWfzVsY7XIJJDiSUUiqLgnlr7SI2e9Z 6+VeDvYwAqZ5aC8VIPyBGtyfdtCyaC7y+fefmLlKEU6dKWrIEVoEFh3xgl7nm2jVBByl2v7YCdv3 I/VFMRFZxW4fJpA7JHvfgme1VNiY3RXcp2bn47waBpv/OKOADQroL1at+lhGU8dVbm4NvqpbdJCu wUIOME3ldOM0MmGqb9HHPAlitzwHxonj2jghv1mg942cdLGs2ObbtmMHfp15VcLzlr7ZlJSZRrzB +2P2OCA7Vcw3krm4Mr5d/KSwc/kfeYjGhemOC5Q8ntW1zDjrIT92W1vXLYJNi1sIaZkjlXPOnxXR f9CWKhmwESyVYLdfyv6/eTtozPaKIkXZOzapnRR0ZxeIbJQUFx9hO8PlHSsy4wZMC1R+c0Zj+fep u0YjDlA7xAuWF6ZA/e2kLCsYjLSMxCWn3A5kMjEn1Mpg8seFvxC8GaBcmv0eVCBUH5O2dkv2pqOF /Fe6jgdlyKh0xFH3XEuQXxcFPMn0wG68l0RUCiD2hji9Xm/O0vQjw2qyDfMdeJMy/d+E81AIU2LF 5x5ejVQu+I356vtIIXJe3Lt+tdS/eCIETvXjpJ6ijp47mncbd6WhZWDWvsdoJsJ8NLwkE3Z1O040 39ozRgpy/2UpcWdUBsDQ/DrdAWe63OPoFnB4aGG/oQgbolIP9KpqAWVhTY7SQKpSl6Bn0jaquwhh qZkOUJWT5SjP5C1WSdcQ1eItds8zDjTz90zcuVKnvyegvR2YBBSsmdNgNdMrGzVr5gulErS//n6Z OSsjzLac9SO84ZXO4IVx3cg6c9m1CZnt0G4QMSpyx0/BjfDrSrckKV+6Rbxlo2sAa79ha2Mjr4PE gzbeBAlKSn/Ak/rC+VXjgQB8t4XeZ/Mf0bW/2O+HFp+W6pMss6OHSd0mV0AkPM0mIvRpipgOQArB csXgaZEhe4HCloYJta4hggM/IdgyqjC1rhmyOCrZx8xPJ8Td57gY/+f91UlBdhiRLgjp/YW1pBbM akK+OEHFkiIyO16MHwc3A1EQRtAdzM72iDKRX2JCLtq1Vl9rsuxmCVQPEZbNIBEKVYHux3zr/f2M 3NwI88xJZZpU44hvdZjIFY4S8XcXr9A6aYVyoNBI4uMdHZEPqBrUKby9MkG/w9Rrsvu0ChQvPLV/ Z8PGz3cMypIyQ9Q20KL74xP1THVUfWTPgPRPoHlfiDwiW82+cbXTWEmUzy1F/8sxvRPTYVWQbhte oS4PJTfo5mqqwpZgr1PrseWWii6KhQOOXN2BQzM2Q5qNV28X4H/4LQ26J8DhATVIMMuQKVSy1gwR Yc61YpHRez/YDb74dJZHtk0deCGPF906R7DwUK4ngcQt5C1OKBFSFKGORplUAqYed+ZE8szhKpym eqegSmqQCzgQIZ62FkVWoj5l4ZypaWh+6NOzMVp6/n7wN5wtGaP3hETBLkeYFt2V41ELgu0F8lDu IPIwcc9TcbQyvx0ioSBRWz6WOU/EKzYXC7hB7YcXkFxp+lKcfpPbsc6UMGe+URMHDOBEnJRWYSjI EUwpR6kvJr7CeXMcKglp/s50s0HznrSgQwSLbjjxZZPGT+ZEVydRiSPQf9Gqqk39R79/y91mg+zq 9CNKHLzOTXsNTQDjBFLFEXMoLUix9aICOGVd9t7fLyqZKMBVAtaypaMFHN7IsjaHffImy11vNaS0 0fCfIVS0FtJMK4HfeoxnoVraFgVK02wJA8GGHTleJ4Z43EgJ6ty4luJccUX5yTjFfR6GeAlim0Ah 1sNFsY2jlqrhAbgYzljNMH1ieHeWjwKYNWrUTJD7j3UfEMAHj/9ChzSHQh6P2B6bowXKjABRS4+r VLmrfnv065pGrl+/wQ+7XuEA8cY+fLUldEWa6X01DdBPrwNXuDPgX1Q3D8kvuC0u7iCqlbmPbWhU QurCPj6wsix9HTFiPcq/UKH81KkSrYxiENZu+cEWVY8naLcx6JGkOK+1aDWVUtVXF/sURBFskBbn 9YVMsz3kzkTP5a28TUnmHWPeCaTRCXI8id+C8c7chIN/zSK3x1ptB+JQGuaRFmt7RiN67m9EjrP2 02DcIF4RhgpKFy+wp/XegZXCWPVm9S0hztTWBheEoEP+SBlokkLkE+xPCeBNZ4KE46OIJdIiOrZM kdz/9kK/w14you2gpo4nGlyItA4hUO2Yqc9cf3Pp5GCsa24VhpvC3a4a7LP2crUjPHDF/b16enWr 4RREeRAbsnCsrj+iV82zQRUPRW1nWGWJ2/nFretjAGb5l07V2wShZyunb77PAwmloJFINaJg0T9l imuLfSlCjgJCyKPbE3Mkkr7Zm/fDNC8Y8gv5M5jp2/yy/LKK7aeaGV+fhsJo8S8TPpsvdDqUY1Xw nUxBJ+0+26Yi5en5FKK3IOh1s1+pkBUaVOgX5LwoqMMiYGdmrOI2aKyoEWxjur1KrLMyeDBvbitg sIkFdR7ha/2mpC63FH0vLUI3iM0rdK0l9+fHyzqgSs34I5vM+fIYo+V/94xjwm0pd9mYl3y8ofbI qh+jfEJAmetg/ArtVUFu9qyUaIvOGss6mp1Bi4TEvZBnX5nGx79lWucc5jWjsRYVUeWmwS4wyIWb BLJIspu9YygJl1jHSO7ELV+ZjA12v4a9F/t6XsyZh4UwjlPlnSdwzJClqKeq3+Up3zOPn+umwyHJ XLCHpfkDz+8Fw4a1ovk1ufC3dAca7/jMwTncctuu4JLn1p9ezAzZfqGqx3u6u/oLM7No8Y/t8VrG WBDx6Q/jlsoALHY9ERghWq+9VhLSg4GwITZ2q3On/SX0MkbQWcK0VYYXAbFeC2e2TQbhDdapDuzM WZ0XDqCSPKItunLguuhMKJgC1+85Ox+gpQns0MIWI3LK6bJUSwotO2Akz6SksDgkSPpOnq3vQ9NV GST3rKPr7510LJ4PCjiG+9TT0iteXU6XS+0ha+26A1kWDWoylNXZvfTbcEmI9cG5VC/VX7A5WKYr ckCbSHmrVx4lCKJs4G7S3wDswKB6c+fXJ6GpHR6Ry//DQHyRR+4Ns+7Nza0mdSR3j/aXNzZ0Y16r t2Y8zkXtArtUwndqe+lVcDqrRckzHd1sCd76iLqes8XIQh1IkriKeQRrCJpW1g29v3zNYl31OegK lZgUfUb8h+yVwWOESmmJ7xsc6JgDeIL1fjX9fySHJLKJebbf4EQBQKEwQ1N9RM2Y1+G1fztuS+hh nutIyoPZtyuZCWVpwcGAxy463n11yeCv0Ffi7x8i/JWx8JYCIzqwJEbOOQxj52RR9NCuTzUeRUZB 1IjrgCRXjr4yrhfki/yAW89Z4eXX5tHigZwuvqWTtd5ZVO7lj9SeGMPW0RUAOBzWf1RKGRMn3Fk5 UyzTZsmRoHKWEuXDAvpMlkM/3BntCihBn3EipwFxKKEDxCQQEFxFrhLWAWJiQGw3jE4Sl7gxyMkO aIGSYA7sNfpZAGy69q3OERF4h2v2+9ldSWl4iU+GGwyxcQvzQa1a/f8F4S01Dl//3tGPxNyJnb3v V6USuKkyqoeUIBb08m86lAHAS6yIoIsZkX7fkxHqw4JF1XdImAaAQKtrtGHzg3PKRUkYX8RPpw+l OR5mxccgBLCtshSsYDDNyL+bfARdhdiZAwdviVREPJhORBmWpA8oPiwj0Qbfw4HtnfGyNZn2bVLz 8XP157jC9TdsUZZn0qr3LPqnCpO9sGkIBbkfUkE57NMgbFqlXQiHKV5Un6EPIFkpF4CvR4vRQtbn 8ijxFUqGjSbo2SOPzHWIAiKhiSEVfZ49OLaxbjfwgPxY4Q00QGFFj//B5vCIxrkNau8xIDKWVXV7 m02jXMY54gihoZiopirjuqzw3q7ymcidvxlCRRBsqcT+E/zissnl0/iO0f/4AOzV8qaZ5z222aM3 /vYMRrNFf4ma8Kz7rqXguZqJPREAohKdJnxwLfhBp04xiX8llUe6K1heb5TV8nraMhUIDzRjbQ8U umdvv/FZ/37YPf9pslh54zmIEop6sLPsfo9syovj457dNKKPfcCeGoVDd1qNYSLP+vX4FsAW8J4+ wh7cim1Vq24QaMbbOYYcCin1a3GML+6fHP3/W/VYAXohmQah+Cvd8j/NYbYLg7YveDlfrZQN04UI ZEL+1/gQvlacErZMKME6HL7dRzTiTR8v6mJQHIOFLxx+WOr1q9c84ltmUxYSbdvbWKgcxT2sAK36 ih12BgA8VZkrHeLOJDLeSUbTFZQuQyrvZE2aUf4NZjdEWIiZqGi0nk0Sdz+dNCXPXCpmXaFsm/XK GP7fzlqY9i0/85c9v3w8CtAhWxgzHCGlN18RxOlQD4C7/sTc5gvmh7oKGn6jdd9D1yspVpcoFtqn egmBptnvNa5RI4wBIlerhZPWwCM3PVIbEcdNLAJoO2173p4gbedDMmSKc12Q8uS9SaGlxx5xacPQ TE+aytxhkg4jp6PWw505iPgWL/bIAW65U6bjwIX1Dmbet4rxuY1GP2YbW5R08G1srRBX7PyQ/2HM IjBuZWZQK+U6RFIoz6WfMXnFyv/6IQY+LniwY+Rh9Cbp7hPKQ2eP7owrPEl8GggiGeIra1d4aEVH CQA49XILWBL4tuw9oKBRV+XpjzdcoYvNl6oQ+OAdcLqdKzXQ3v5+YUAM2TvKAKrfgV22bGgYPaei 36jstJHe7JOvHeOjj9egPPXHvyUG17FMS+5aew4x8mQzj71xDXLFL4QBmUGNtiB3MFJ9WLtjkXAR Iqcj0g+EEGCUBDPBwIvm2u+0/LavOR8UAcRwCOL3yfCFMJhHGk1lEM1L3C/O7jpI8jfpdD1XWZ01 hDkLt57HO4GFWpZWZDV16OGaYUUVWtDxPfCedpyWlA0/zQXKFUmApYBpsEGChvx+XLoAWQ2M4TvF x6Rsxwrakj+d3414cxj8z6GSGM27wsPQ4Ay2wKGNkrD1QypnKq3fxZJgmdRzZbEsbPIn7rjhCbRc 68M06ZfCZM9Hh+Z/lVvyY5vsuDHZS4bIXiQ8/c1Ld/tFETPdBWGcH0WckR1J8dQmtluvxsE45ZMH iFvjld5kc7YJ3NWPkRr6YxUtDoPWDg055W7RycQdT3hknmstQTcW04/zaiTjadkKQ3DIhrzeSzQS KiJBZIWhtF5RBh/qHjbLXYeg8mPI8L+QecxY9ZMf6NyYLSn6pEuP3oDbrguPmJ5JR5hf61v24XoO kOp3SmfGxlDW6yUml8ewygz6+5HNH2kBER93lvH7cxELK+Yq3Z4TJMileob1gvfmi5bDUf4JyheZ NFPvs5PPPrfMrsxPYu/cjcMcU7L0SIB7pcFkEol3kIbDupj7sMZXP5c/XMaR8UrqcWd4k2P+n345 HdO5qIMwXAD3rWjCKg3NXDveLyve2Ser7iC8DA2cSXIgoK1mahhpt+XleMHZ3rbLWxi+egPJHpId xQULJAgqZLtEepTHzBv/kiDHrSXf2+4OjF6N3rcI1cNbIu+ytbviugrWzRxcPkKTaPPOFH6FLtvc qnUkavrpy5yFbu0hUcNkScwcKxw8I8tgyTN1kAgCBqOxJ0zEFv54i2XkKb9Rf2PWAOit9Z26aBkB JXMVUzutrzb+vxdgCUFUBsVgdQKmMmBcpM6XCTa60mvImMljUqQxjEUYPLojwYP6CawlVM3au03V Y9VFLwPOfMW1JYxJyw8RQPGWBaUcpZuT5M7x34dG+YFdth7gWJOhU0FRI80VKcpfaFrxUBOCNHvE O7p51fSNJ9RtY/b/VTtAxlUAPwMKKY5x/G/Xi/9rlTPOrxt9kU3wE3c6d5yNIhBv6SjSIRUnnPBy OeKv+kBaKcCIuFPQB7xtTjZOZl5T3B7KripAcnx7BtDHZFA3ym8f1nbKYJzy4VA6THmH9eYyiIbv YEZAmIhm5WyVIf+xru8w9p8Cv0Yg83R0Ibvp4a6p1X6MP2gj7roosM8DSykm8qOfoKg0Qa56mOvb QT9aFWodn2+piomiFk2Q8ji2XQdG9Xo5gaQzK6ooMEJxdlabs9BFQTQMaD2181MT4Llvx2us198r cYql/EIURnm9CI8BQUnZlCDZ4qlyh2GrzXokDf906T1JFyPG6VtxrQqFd6ASVoqnSS1pSUbqg+os 4TmGxMbFZ1sAIh/eaS5x2L7AWzJIrk3PF0mejyIkTyOGsY2xZ7YiCHpQtfKN24V0PsDv0FBITd83 uaTCH0UCXwSnb4N55djNdh5DXXCxV4yXPGUO5d78OpOE6qwtPQIiF3dStBX3SHf/GNO3gpqTHW+I /nxO0W7INbVRHlm79AKqybUuYcSgMMe/GUUuhVYB1eInBwjJSYODM2Vz3gO3444OfryI6tJor4wt /UxKHrE54ZVE3PZCl9VruTF2LigWt3ksqrHP3uKKvvN/c0oGQ2JlyM6nd8O3Pzt3nOK9ya4HcH/e WS+Ea3B0vFLU06We+jbAz8lgoiBafQJcsqIvHyqheLqE+QlLyHA3FhQwUlOZDjlF718S7qcxyCei CnWsdv2XSZhnRrjRBdlf9RDVG29ipC2jg5aB+9jAlWB3MRgYubKOOXCvbu9g52LQfhZmr34GmYQ2 RDzok78XEZSJUIlC3VAhKJgQtr5DcfBsKn9V1MyF9Kfo1/2gxHh7b4OWhtbCPhq9NgvugTccXAyG wHjZh2YAYx8Hwgxvm3qXIYzr/L1zekUZc9I+k+7fxZto7efxSMCk/i6R5FeqyvW94L3GJh4EWCOC Y/24TpwidMFkY928j3EDSlAUoPJSNG1P89GeS1jfM4C5gPX8+0FBtstoEU+wr7LwvntPYKGP7Qt4 6CRLRMziXrgaTQCFLG6jABbiVT87+tG1Nar/9kDI3AsL0T5HNewTzlatzsO34voGCt45bvrfUIF1 yv7P63BJQbAJhsXUai6aWok9bUrMgn71hzAfFdQItHTSKuh01oMlSaRoOfKqaI5w2P6TqTbUuRf9 /OjktT18cN6kTqeaz4XEQC77z7H/GqX7ppffks/dNDVdfjlVH613fGQDpgv4OqjN18Xu4IyjoRp+ SQoV7jOzaJ7cEF08MSUc+Dz15lhqpq7wBRgXaTMADUnY5CDQv/hC+fhcXEUEavMuZ3dkHObuktmV /WTEH0U3SR2ayPV/HeT689WwE9emjQucmPgKZwXIkiqtkg1CoLK2lPoR1e6tUmzd7FQaYmx+eGHD yTZA58HZQtX59X7cL+uVdsyTIAncymYDnnRUuJqT3UyEtH1ZtORqRUPozSiagFIQKYQLFsCyMyjy X4qsF/uNmJYgMZ6tWM3G/pHplaYN7fKeEhizeuGvLU8wpR2GqAmrNKIj+gitIq4F2iyHx5xbbx5d 3F+PgHNW57FtHBCYvPmy45MeVNR7DjfA9cDOp6EK95Uo7SKTtjE3OShdJh8jmaPTJ/bsIhiOxEtM qc3UHGN4t2S9b9WAt0L5sCWO9lV+XSR4Gz3t5hgegC6c0oT1E6v/mwnT5sT4mfMWlWAEMelFYa87 8AeDcAj4fY+qHfmrEZNHSsj04PwWZakEI+BN1qt01cy8eSgJkkiawtrjB7q+5sHRNQ3dcWNIvQB6 qyyArs+PyM8UpP2x8zh1H5sG/3hYERAk7aWlCGQtn5x+EOT4f4IDNo4OGK2ErR/7x4ANCtGVflZh wrFNv5ihAFDeeWlPjXv8c7agFtOrwRcLcNzKqSqgY77fI/8PRKURPdJMwei3KAsqshK1HtCBEHBH 6TNkYl7KpyBksNzI1iRDCz3iP0Q23xToxZth/48RC1lLXp4PJ7Wf6xOpPXAZBa6KvUXO+5Qo1R6p ZGo+4bXJDHb7dlLQE7xHj3uB5Py7r6ETSnytuYYhOiOeiGmmq/LW7rFZuF2EbLDFIFDJIhNYe2XE P5K6/CaFizsg3n80bpLG3z3EMp0SOK4Vi1VTzcQDqVLmqQ9kWly1FetWlqppzjTkyYl2SosrT0XC cqm2c2FqZ+eoijnHsB2BUTPYuE1IAR+/nEjfU8AQR7ZKLvcoF07kaEb1XCp+pgUVlQp3QEgRb1sH SarlJkAYDctd0d/fzSChnoxcs9wlE+xdWPNs9rUCeJPrVa0ci9rtnit094nfgFEnGvC8q/QK4Uiu m1l7M2g4xys/33c2alr9fkuIG5p3In+UrPHvaSDQ6POpMUoEVBxTgB0jUYJd/cpqjvV9aiL5pKEe mQncVU0GhMj1bqBOeODMjot1rpIeBejZ06V/NEzq++4BCiVYvRduAYXb6PrTX/gUomL3QUGc7q7d MiD/BmQwu1kTjtsGCuKItFsEk/jThT4+PEcc4PwtHPj0wpxVlSmLZb2xyNEx+4HINcedTLNLeuBZ 3dIwB7Ppv1HxcN219uMgdvmhjlQXnnmFOazaHfXvQ2BxfNYtMKT/18wlG5+gsWrZlrj7IazzJrnR 0t9S1y70W6QufNN0EiwI1hM39f58kx/7PfceZ8eHTOwHHMOdp5WwFpua9qiu/2TzmnSurwiIqniE RP6SoO/04vIuZOqn5ntWvDMlVqxbVCjCqt68fyXjURqx+PrzNO9Y/+KmUdiYMIaASCTlMtv4UanU PNEgCtMQE5Xy+tuetJxO/ygROKbyTdaWKqJhFiUGEdgdS/sv+zkcFEjAsc9u6wmDiaVSz8H6ZLMe lgl9NsY1kmC7SdGVclH1vnwFwGD1MefwZa7olrjRHyCGUVfXIHPS+XfjZngfK6+u9+e2EPiEDrc7 sTTCD1+apCmuZiccdHTA/DHV7TTrtvvjCeEtSTl57F9uzgBMbXscegV+T4CITlgpxhpkhRxbJ6lU PyezGmRGcXTYuMd0Y6lBmHVDem0USJKU9d18/nSiONC5kJW4zINPTjaWnb+i9gpqAhvsrVnd0Ytg lGVEUu6BAud4BksZoNanwENSedbVORLajgmUTpVzA/OFK2d7IVUa3NTH0tO8gjXBvUSu/R7seLbg MEWtJS1BehsLi2XmLozgS2KqQcVDvfMPU52KWiPuypuY9DRxofZidwhjLv5REHBXPZVp4Mf1/DoY 1A7UM41tsUX0LI/e8cIf/mag6ABcuCBmMH+0jpfzRFjaFtKZR9gHARET3feIeknFo3gObsSHwgeV 90bfE8WRBS28CfkU7ARejL12gCL4nQTtUaa2wIAnkxD2jUOraos/F/ptZ7PWXyLRVoZd7pTelrnj E+RxZkfV/v/7usn8a6eSRnS3ITVnrV3e2cCOJbFsSB7rXE1ZTmZ4pE1WNqlROqVbzJmW29DKWf3g L2T+FXj5pa/1niWJH/8wfESdcEaZYsdRuxBYKu4Hp2c6O5PP6VHCEFRKPrOUllXFXCTnQNOZvgWU da0Tuyw5aDCxRJzP2J96lNtXwBvCJXBeGG7zS/rUWd3RUIaNwzOvpmVISGbQBOVvIOBMA6v/7RAQ rFL0kcXFkq/MOBRFCHI50Q9sh5xmNpcRf/p8ve5sK6a2FWqxFr83IWZn9q7FngtzMOXDyWVExmku svxOpnHxJyjmlvRhjc21Y3QOYwjKWE7XVlEimVgCe4ajmwW2hawKBUFYbk7jIKso4QtwA2WIZvPd o2rbzMRx9rogB27IHuR9e+mR8VVKNTmh0Y0JS3lD4HTTqvSq9vSBmSSWSveAJswvPeQvag/9BMU5 2AJpiEry9/zthUyKJ+JWBN6QxKK3FNIYOVj8D7cNXGhdxn+6G15Ob0pZRXtl4N9odsFGP3a+MCfL uJUl5QmEdMRySHRajBcxMoyEDUYR8a/FZ8Vi8ipO3wDLd+J3nxbQjGXaVKEepXseXiztVFLgncuD IhnXVnuN5uawfKlZHKrwRGfDM0degQb0ogZZe5X/0Vc8wLuzepT9S03KmtYemF6mQgCX8uz6tMT+ MX2Y+ZKV6UxKl+G2RaV3LAA9nhmFUADv2gdiTODJRVTKogz0VRAGJHiQh06HuH1RUT3jAIf/9WJh ykmkPzQ0K+bCNtHJ/UoPFbtBISu821tbAndU6dmhAL8Jn79exPlwPF6njh+4AWaBFXP3UnYaCR/K xbw0PUeFmKWlZWeYKDpBYL8qTBarNq2pHT3s76jLhRuGCW73Jss4oPvPzcpOyZRbe/axwNrXjln0 i0sjsbcHKbPcwhfU/wyG7uIdeCziald025jqMjxQQhbAwb7roFk+k3bu+Rttmv6sgJNHGiCFs1EO 1rrcXCo0dPKRmoFPm9vXrwW06fGVM8e8mD+5gZXQHU8ko7OM74++bd3nyYs2Z7i82wJsAiInw03z dj3pMX9iCHF9KgpYvlEpghe6vXg3BTyczgMtn50EbvzOPcMTCm7v3+YDRx2u587yHK9Z5DFH8peH 3mNlu/dM7rbnuIevtAluJCiE2Q2HCAW3PyQqcxMEMbhYmPdD9LEyGdf/kJteOtJW73+VnXVrcvb3 bmjXM88KLjU12ppci7C8w1w7hK3YfRVaTFOFOpK3zRqnD3HOVPw9/Z5rospjo6gZ6t2XlhsPYKFd 3AnR66GLS+09cw2rqhBzMscW3AaMpHQUB0U9BqHNaX2COeUoC+Sr4iniE6UZmtQ26PRStcpof2ju OHM8+h1a8nTbHI5K5t0jF4QWOd4YQNfIVKYcdRTcNXIaTdCYJloVSu0AfP7pQAXHmju1iFMXGFNk S+pSsGfHnL5BfZscoKAMGoD18Rk8WZcU73l6L3S7t1u3fiuVekGdMPwtxufd5vAOIfcElM4pWXHX BjJuoka1OntSyXZFSNkimrJWu7z3kOtwRMzsBJeK2h3QI3QVaEVW9vhu3CPWzcyIhFPtNL5RI2Or iIdDy8ps5Tx4egmieDcF8OUrz3G93Mf72MNWEKe7DCzkMZPzrbSAtaDiYzIbeHWhtQ93/nNNGYNS t0a/cOSDpBI/WJpfifpzhp4uPVCrooXeUE0dl2tbjSSTkbnFswkQTNusYyHPV6EES/OnNgVIkCce ahbP3CecBQAF2bOOqNz7rKel1uaXPZsJNT3iyKc6yvNPD39qfSxBuTkYVIWaOuqchjS4ujCeHtYB 5QPIBdxWx+YctK0xSxULTFLTIh3dM3ahYoKEzvxQYYJ2SHsL5gxDsclC21YJ38FERi8pu8V0i9GW bGO2v7E8btZUn2BTqo5jdMAwiyPAJx1yPSYUj69QDF6TAkPsvTmF770fg3Qbf7gNK4gVcqBoMIJc KRD4dLAhQOPLo1uLgXevVpKnPl3ZFv7SZFHoirpDvsXmA/gf7GldTxzltGp9vVOpXqcF1SJdLSdx ie8bCkEWvswTwy9Gc6pvV10HI47cId/GdHRaWjjRq+1t9k5geEcmW8Aebbt7eJuUDOc0W0yWZBUP l2nrxR2hLR8hCq3oyorCahOonqmNN0Ig6GYxNpk3kNnPA5/TUPKal0icNnnk++fGySaMcZpJqcmy PXToE+4thyj5RPtSumQA4N4cTb5qroN1MQYRYMn+pv0Z0LYcb3YhAi9yUQscYhp5SnuoSwQkhD+n KE5m+0xkWhYSwSWGvcFA0ussyCZsv8sNvi39dxig6/j9i0zCl1RHgAXrjhsQZgomFIczzg7sKeJn HlLg5y0I5WL7U5RXD6bssgPlgZzjcRjlCxOaafnl4nRhxYQk8FPk/DNNt8JmZp7hFdp2qr2DYpTf UPmWBEly9RSUbPMNOrTPpRxvbMzseYVeVzGcKkcBrjq4AVc/EO4CiLMUHMNvbNLz76fZPUPERoyR ux/nuOlg3NfE8V/Mg0IXlqaWrLXHZYSJY6ipcq9B4usV2nIATPQzFCuCxZaeztJss8pLmddimswa G6x1fDkx4YtdIpJ+n4aVYuq25iPvzA1aph6ySwMFD/UTGsImeJPw2781L8EswAf+khtZOxQmXnW3 EfkuLGHVdUXZ9m+NQ/5D6wGni7K1MGFDoccO7M3wkmwxCaa2fDE1Q7J09M+dfL0L/ymO/TUgS51f mzq2Z8OGKyTgYxB/PoSj3NYLo8txLijGuvjRsJCyQBxKsStRyUWClm7HM1WiUUDo/nuuIS9zlFwN sHYqGU7F2GePMBa1Aizvegp1s4TKkO6g7I0cMrG+SAzOjpwUKiDE6uX8uJsmKjHvmkSLn9iaCUfq se/3l+R/nlgqK33dGrizoOtJqHeilr+KusaInaRxbGTlsSuJumxzDPWw+3tnRycRMaWCej2rlcjP ABfuNlxMEyoHG2Bd9+hf3zy+Cp7GBOLJGyOPDQT/ouxW90g++YtXfIMgkz34mJVEx8HgW+Vaqpgh ekeO/d/hxGn0g5gTcWxrBKObqDqk1ip2azTljgeBMFTugITh8iiMtMmWGITrojJgV7ungxYMkoPr lXFkY454X97bzVMDRayedk/AomYlBHQQfpJ126mqj5Pai+cnixt51Y1S+nKJeeVjfl0la40rW7R2 LsKP5QUmLTk/Yd8Vu9JUwOkAWxYCBxN/G4ZoGIoitikp8EvgK0iCtBwPl1OjEBq6nglNjinZ/nkM /wr8EiOcJfSnIx3170VJxYtanhfxeicHRdUFCb8JWnIuiUQFBHWnSvT9UkN0LzKmLDr57ZsR5gCu WRWQk+sMq9C3yhCIQWJH3mdxUBbasWdO40TkzKUozZlPOwXDOPg3kXbjPTqT/+LMCgonRimh7vCG SdfvlrAYyWcz9v/kVLhZyFJQC9+6whHwd0x6lrOy9aa0CKJvs957n8SDW0pQb+JnhpW0I+ttKrZP 4I6mmMkoos6DQF+OpOXeAnabOtIdEexTkDtAHPbcj7/KmhTBuCOYFTeT0jZxH1YB6yznG6/U+NGB Y1jcfSflBngEO+//5VulCVBoRVNbzNeXf3vtDT0fYDDLdCE4vBAkZHNgJHjKUxzdhUwX/iDk+1Sw kODcFIm6ISz5slopfLvlEj9eP1OVDTtCYIelcBBCRKGeOA3eZDrygoUJdsYJ5Ono/hB97qbLpaQA id3Exi9XkWSH41C9Xop7/IYX/ZUq8W3OyxEI5+kdin+T/gEigQoDAMfz/POACsznX9kotokH2jWG X0JvjjC1WB2fC4o7y4e/AhyIx5gubvdbkfh5QmVbL3KXxkarNhLrNyzQ3GKp48o4qC341PPAVEHA iSn3Sb4EUoEjiPGe4o6aV0WCqS29w/pzeGh/PxRXP+AExPhUr55CoNykIKyD6eg85aiutlcsaLT4 QX0uxUmKxECn2tWgyNStE4aKbn4uGH/7mYQSk/7puVC9t1wNZ0GvwEHMYwco4RO9AP1YTtoRfhdM bfeOD36RVTpBg9Mm4x5yblSyBY10FhTivFODz8yfSRPzlld6Sn+GoctXquHICTcqs3IVy8aYzFGZ KRf6RKz2mWkGsgsTQU1seMaxnHgplEaqLjHG6ief01nVbDJS0qThyrTMAcMh1n9FJAC2cswFRvHR dN0XSlkljzyIn031mXNEjZfzQ8GUOIOT8kJfk1gi2QimAns293DrlCR6/Z7GsZHxuzGw5Rdw7D+L zLaFyBxy7lL/5oTCQ1SqFkje8gOuMy0iZY3pdqgvu73nsuxr/pqqze8XGRR0JBwkZJelL9VwBdd9 JyKVBtTyS1iUb0AfqFlCqf8eDjNm7wPBeD3Kc9ne/V2VQVC9B6J7auc2bU5ELYxVYDOnVQufFZ/1 eGHOnY85DgJi2srxnyL7PEWvCZibTLhoLeU1XB3fxKdcgi9l0rcyhqzY5EY4cmJhCFWyohfQlnod 9GaxtTRbJZLRFOgLy+O2RBnLbCw4y3JKANZiv5gu6oTrVPJgHmFyN3BXzOOVWXmDV8rZJsmfCxTt dO6eXpw23OZsCRg/ht7kJzIw7k06rvGYrDVM4GDj9kIOCsM6WUaYQ6k+sBy17u/GJg3xxe6PY/aW F4xoVNPb0RW64KEPXhn0/FrQM6M+UiM3JolUJkrd0kbxSQmYUFtToLTFRTcg+y/1alrFVnxiTRmQ kJXQmiSXIh2YheiUpt7gZHjRlQjV4Gjvfc/0COjXRXPGqkJ/gn3JQPz//CQLEF458P8K5pDY9ljR Mxbf0P/b5FfbTTsmryvF6Gr0MpYk8HuyGaS+Z2MC7nZ1h3NsO+KQ7blf6FlRZO96NtlzLY264EyL ZntPL2P77E0n2OxWq+Wd8l6Ze+/qFMBlWh0d3HMkICIN0SZgjc/D4WzeBeHFG//QbiG5uQfxWqTg Wu3oUrWMtyTMYHk55pKwg0UYNglTMcjvVjV0YoRy6z0Slm/q4TeP15TSfMLZvtX7LiqFA8Jnl8V3 yFIabzI4lBXcJ3G1o3/GVz0o6zBCPLiY83C6aECSJ3Qm7QluXVxHrWY3wC6bL3J+LVHImBjgZppb ThVIreAqxsjPnoI8dqo/6xuOaztymTk7Fhd4e+4b9eguWvLWsDMbPUYgO5TDtlrdE03QqMkTdvw2 RRQDYKNmN88AZcZgowANBGNuvwtKbaGOxTUDCIQq7wEdC4NycPE+xndob4nAVvjr9+ak05Dv5/ZP my/HybhERj1wJTO6uP0TEycF4kpd4RYNr4WTIogV0fJGbf7VImZe+RG7u5sSICV0z9caSb1QV7+1 9yLuCB3pRn7+qEHdMzN6u30UR5gWvp+RPntQlttF/f8ALmp3cm5qSOdUKdoRxjVA3obArtvU0ZDP YHruaS2jsT2N376pxLHRKQbTdwhC00e/UNfMjYfAC5xtL0U8nOjOr1PFg8ERVDsIV11A5drC85TQ yrlo6LQFPn7Gk/EvY0IHe98Z37KG0SJQXBig+dTQ3unD+Pr7Zxo6fPNvPqbLFr+eJNKoi0kisb3o DWS8+bawtFF/OJi6Zw7Ow2il70vU8CDgup0n6OAbMslaNqk4Qe1i12OerOlmpnRyMjHPknYrmI8C YgCtX7JZuVpvTglOoaGcPBAV+jaC4t680d38ZKfhHWaCsEK55BwJ5tOiyIXPC+0G55e88Rvq76ar PO0VBJCY0rCRuh/Kxe2IPNheRJ2mIpWsE1SauaMH4pUadzPLA/HTsJAuiwZxcCD9iswxDFU/2UR6 nHyZxveqL8EheTTlFRVevZnWYykUNx9j4uI3XIw/FCwidu6MoH2Xci+uP5TSWd/ON4uqS2aSqEiY pm1015w8kKiba1dMhZZmffquvhAbtm3ccsFoIvPyQcRI9HBLkZp7fawjInYAj6G1WecgU7N4/OAb 7tAdl5DrHrGrZRWjN+PxLZ6GiRmXTTW+mYNLcad9D/m6/PONmP9XKCL61v3YSjJA2aGbADCZclWq 2SdtGQhcF+D6L4T/UzJHE2ihjkRq2d0Bnp+BhzOjLEKzfuRGgwdOrS3J0sAs0+mOc7uS97SUILPS 0aIy0gB3n9rcVJU+0HWnt3G/0xPtKmh0SAO9dtY1pZkFJuvxxpWKhEZH3pEpZOQs1rl4/F6mkgcp x1RvQPIw93HicaNDruLSxCb91GMvpoWy1nJJh9tkgFBEMMIVUvIo3O3Zgonal8KS/12S0A8kdUKu tbsDUAnnp2+bq8JY5nTcfVowyWlDyC8UWG27hQkQXcbR/YWivUwPHeE2PVf/a1M8NQBvsAGb1nM4 DqdRB4Vd2YrfbiEH5qfCr4wTdxQRVuA82w38fPZk++scqsQy9mUcc4yebT85xg09+d7iyt5hCCzx vh+xML1VOp/A61+oQkqsxaO8jwLGGFMeLq5HaLauOPs+a1s7q1I5PCoh0jINDSeK5SrzKltunpdU e/rvp0hUnJqRWeKQq25j4jvXEr/ZXQLdYw9V+bnpfCZJBw3SBWhoVaa3Vzo2opJC/ONf+MgvhmUT FnBQlEzcLRHXE4I6Agg4TC6HuvpetHryumbeInVn/di3pN7QwlER0azppagXM8UeRNtWOWEa4T6g X7ff3YIVFPYx9ryehEatMNA0r1JUxl1MMX5f4OWVVBe+AIwzXc+BBKFXiyRLVgAVSKgyoZyO2rEk +tvm0w+PCIBhLcuQfLnRlWyvgg2J1N5/iO2GsgXNJfytLcOv4l8mN+bFBMRxTN2OWtS6ePLlglxe +ZCZfjzbYZxx1mClekEKpf8D7uY9G3u1zt01wfQcC2nK7xMuXFsq88PlFPjexZDG6mWhbYUjVGbK wkZdFhmoN4ZEHcyLRYBW5x2mGAcYIUkz03SrerygfkTxX3aGh9DRQ5cN7EnMek3dB3gHVfQwRfer ae2xiyDppweQ9DMMUdr0WZhvv1EAYA4p8MSUZ8ZidMMPQf80iNQsVxDaDtcPUoQd14QQoNM1S8OT 8rRIq/zMrZRkieBb2sNuwOy1V9Ps+zRq1ZgMcU2DEZ9URzdjlIs4HVR3JAWXzoktOCjNjFLNcOGK ZzDBn8prethw1dyB90iESaUw7V4qLL/cRGS609FJrseJdxgg8dee8pGPM4DO488Sqh1/dSZ0Jvqq HuGs4uPs0IjMvFN2wYOsMO+37ZsP6SzcD9RlCvVUkP/QZEWL3yfZzDFBcP3YJ+NA0RcQfADr85ya YOSetvMdC89lVY5SVOIHX8PtrS4mgSV6RuiCx+TwBFcscmRTmh4/jOHJIrdwtOyBpN5JJYhMIe9s uCpROtOBxnIbJeBKYOk6Ot00nkTR57bJL9WoQXpfiIFpYGI4HiPDDQ5JSoZadja4L3uSwa00z3hx t3Z0Satf+O/umt8lYXPQtqOsWC2Poshh+MkL16PmamnCA0S/TipLRPwzWzVlrhkhcsjVxzH71Kdl U14AiUKWat4gTj5eBcKaRJQLuAScKmevXUXw4IuxMJJqpnuji6oKlnsBuF/nQT+D9JrCnS/jJKln cw73ira6ZK1764c33tVdOyRyjq5W+c5az7MKKf9NBoBQxBGFRz+Qyzql7Of5t3LzqPMHSPgCRYRe WLdouOm4ZND3fZ59dB6x9bxQ4k24zW1FgokQCpD2G7gE4GV756yCx/aKmUX8Eb2NpA8672z5iAQn pSXCO3BRhs9VmCH4lISgnqZuJwt40ekpk2CE0UtWQuMYWgy6H8LME+aUbXmfcGwgGRzcZAniip/M BHu3Fv5HmdqvMRrgF72awlvf0HVYiHT4zD2hvXKiyKO0hHI4+yY1MhIT7IM2L00fAuuzdAu4dslU YEZ3KnOXDIC6AK4n5hDKunAM1ijAVH22LSoUDcWXaWzTEZ3x2xa90CRfuha2l7TUs2dU09qxLL6Q bWBPd+WycJ5VnGpuh2Vhbmd1TITZuUcSK/zd24o7k2WQeVV7Xj+OeyXkM55Ckd8MFYrc9WTZCKSD OoA74JJiPYjJCLyEA7b4FavBvHvIP+gtxk/A2ss3mnHA0fkydMCCLnXeEFjwNFoZOb/ERTPXYXGM WSGQVP0rFBtlcJaxvfZB6I3JiwdCmiz+3nFKY5rwH7P2xGd++cUTt6FWcNWaby7CEMZQTOknc3Xs 5rl76Vvt1SyyRGqaQSzuc/QUBJUaV86VnpTsfUtzLxT7rmv6ySXzh673dRYR+/t1+fOP9jt9MZ47 2ykW6qUwyUoaalJQFUpsd24h6fv4Dc5gp6R95KwhSMvGF59BwwAy3EbM3YBmGQP+jc+ZObeTu/su zqpdI9qdcJVpag8T5sJ97jLBsQsA4RlmpnIjWnNgO9nS8q2Ykog3rtOKrH1B441I0bFxYtdHFLh+ U9r8LCVtulc/Pbp/p15TrXqDwysz4gq0QRZNaVr4/QEYr2XMfkK4uiJcWnY65IsfMGTC5femX81U BrIHJbiOI7b78Y4OlMC0KMKQKCLL6lNVZ6iGLMPpXsYO0i5hWmUqQPRFzSZhmgchu/cUBzjYz4K6 VZEfVHAkTfJjXaX/TXVxwsl8cgD8kzcSXdbPNFZTaLDVOJM3mUR8lKygHZ/E8/+kdMjF03IHMw6Y toZpPdkxJGbR2mtjvWO3YWeM8w5wp8KXwjCnUAqfEBZxBOJ8Hep7erwJjUTcLHRjlCMND7HQvnhV R6lgoUZMdS2rcqWQ6NY+NvzM45NfgdTduu5uaXrtTSdncSU9p1ZG42DkUtNLEkPhS9Cy+98+7l/G Sane2BIiccPKEFLFZYi+T1RNGBUsRnTXexsz8fVnIYqpCP6JiHI/77kns6U28l9rfcSevuyF/qv1 dD21KGWyCSf3PBzWpj6fVNhYA0ghb87ONcAY67/gDSfnk9joPpA8etZaLe1qlnsX5G2N6WIu2R+E ks1gqxRTkZoWEoEh0ITkqllj5JT6prkkuimogQr76g+ci43DuDH93jV1+0YU/hhCjv3C9MzXShT9 u2h852NsdTdR+SMC2Jjs/mLdznfG59zJ+5tWz1t9MCzPZfLLYIviEynsnN3g5rC+pL1YRj5MQkSS OxzHQAOjYcu1GGUcaMQ/3hOMXJbpsmWZih4CFDztM7jXKIQDxlAIpyxTINKFGkC8XMLL8SDTHprw YDY3vn7xs67ag5Vm72nGPovZl+U22bVDHNonDqgHXPcucsH5bpUVJzZAv9vsIIxYBDN8wGl2QhlH t2U2FPLv/b/ozw93Yo8D0pWi95s8FvXalFmcZoS+up/MAhlOlXsk8cWLRRzR5+Lj9KnFD3BjdJUV X8N4kpI0YuMO7qz6X8Usq9ZHRqXHnD7DEhIFTOYhChQDdcwEXDdbjxwiHHwFdLRH+U9kGoA7UZeH G81ZOiAfayCyxaHx5+fE4rCN5yT09c44YfsMnteD3NCYuCRR/Q4dvFaKnUH4gJGW4pCapbzTybs1 VqpG8LRTtTXVPqXQnE2OeAfo805ZuTVKdQvXnWEUaAWP5j87t7nQP5Y1DjDDJYLSk6gMNPy8OrBL qFf4IdrhEWuGjhXlRuBZRihJ8EDkC/q+UtIvkIZvRdk8LoBWLFlgqPryRtmgUHq+XnOwEgE37adI AAA9qbTwrZsWdXXW0GGY6yB7iHIZAxvSV7LNmCpw7jXBCFiC7MvE10lLFtW9WI04O+2afiyeI9mZ S3LF8+9foTvm2BXjM3Z8IhwDzDzaWv5D4clcrELhNoF8UXQkX41C55HvroV+X4fymFMirSdRAKfb R47PNk0NNfdMi5O3SaWv5wMC7SfrepbWTwOPPEZpkydSn2Ui8DozyfLPpAZrMKNSIH23BxqeVGA3 lqHnsceXgfhhFY09QjOYd28OwfBV/Q+MuAy5U0vdaZJPTnI8T04BFzx296Nc77Z6RG0OENYqz14h EiMD8+9OXNZ+WURV2DE4rhQrE85F18ID0XcBvTl6PM1A8uLz/jOgqrIKii5JfdfzJ+oRQaTFmqTQ VS/ooSfzfjYvqYC8np2mfxrNcwFkSJGYAwMighZEhLiMQSYnd2Io8LOa3QL9I/FURHJgg5bf2Ye2 MKdqCG3q9EaQ4tavCWK/dZ9CKrr2EC9Ym0SZKcJAG1ti2yzFbrCuNd3D8SLQ8nd2PvWAqs4umWXX X5ECJ9/rBIlb2jo5yogYjxFz4/uOKa3QiRFlwFVZhCn/j/HO/5zxup26VBSGnT8BU5zu+l0EP/v/ PFO+dKZiiDDYWSP9ZQtFDh/7qX7/yPgUEHK2z0hsQoWwMF8zDZscDzU85p2JMepU/9/M9XkH16mM WXtE/iWOAH3DNW/sEU4WtGSdleALBFTf44pUMbUA87w2I+WnEAbWbzVawB2dHLZTePy2xxZyreao S5VsDHsMkdUY7fKRD3Q7rXOe1FffMFkXjKmdKCIVlicTJZaZGPGTNP3/7bzbpJsTRNrCL+W8NLC6 77bubNQPKo0KQY2hhvCENdErTNgdqGlcLSnYmHMNaRW5c9FDpHBIjpqgcu997uq7BNsj4jrBF1vB SuOWxntke5mMMqt6nYd6V4RPh+92bofpHyT3FZUh87oC3v8H1I3lggEL017weiGey5kNrT+G4D7D RBkD17sfFN9sE2o960iy3IwfHx2Vqw406SAUz8aUWR+kaP4zZo9cvg/Qgfa9fySkkJvwVmiNQaKC 7K6Ug0p75eGu/CbYXbN2orupm3eTa0OxwNubTsfnGkV6ooUFMa2PgN2L/ClguV+/jwOX9VyH7aZ3 mhHAsiVOhXUNEthzQvYCjBHQWYwixIOZixPKXuRK7dCnrjG2wMLwnP1hdKBYgj59m2+Vo5+os/8H RmtaQIWfCleByiKm4qTP8i7yHv/RemHXy3DuNIbADR38m1Vsp6UW/cBkH8cWTzfds3yoSTqhvihk 1wIy1RAkVqIaSj1NqpeH1shjWn4R0DyS32XjDjlNTNpRvCCKrW5EzYH0Ki82OfN7ZC7nuhNbm3He jbWXgtRgkHzlvWx2I8d/c9PLk10FqxFXvKyGUh6KoNQL8Tf7h/81H7aKDhvuyHEpcYmk1AiU8TTd J9HdBxY6fszPQT7TpFZJwKW4YEAOqF8KwfXjWlGPzI7t58OqE8m63dlaE8+i5Ikt9irkJg2zuJEk EJAzzGCfytpKCSzb5ibV7TAFDUP0SnfpEsY1BQnreCaA7ds/7MHhv0ekV1Bz0csbru5rh3m2f1ww LGCFKhkG/qZp+dMi1bJUq6JwBAza/s1YlAz2MDloSEYIqcvx3K2j2HV6M2++DAclsUZF6RapONQG LNdJCuXFKOOC06GBqEqM2/nWY+1fjemXNXSTvkA7iyotq1PP+jK423Vnja1aPTssVNW0JIWH5yaL KysbvqTpwj7eYzy8kN/eyBDVDRzxV3wL0Jbof1dAczPqP585/DSxjcjlNibCwVMG9//QCpdjU0is JnoMGaIi0y+EO48K9fK11pstLhuLSZ65NQ5wtHTzC9/S0Qq+qPHoE7P6BegZppLr+8Kosqrq3fpI RNp8PL/OZW/PgaKg41rOSZuj0YTcLJMNRmsCMp9gjtRGE0y0nPEfsu2C0N3EXj0VAs5YFsHmvlJM ckdYQzRqll6E+PBh3p+CV18TWxbBc3rItx20U6ba7vDfgo9hENCXXy8nJXo93MdtJCXhvPLRNR31 Uhy5X8O7dIwqOs4auM59PPOivFyf02N7P2w4qwUMIIoR6RS14uZDFoFr30Y+fEX62HyXmKDSYnn1 +5hYdU7BUAjFLVWRHLfDnd22dZgz+DlrIRC5heAITECG5dc8Xrq/IKxQOC0auiZvvmBBSm4dLdSE omhsCGCh2Ja0VjppnOfRyiE+XWkaKeuObTaHtDWoh+qC3cO+7J3qQVO+6VXGmd17RxbB2XNhdIGU ebSfU9xqUJ6iO7ld8O9CcSkl50qvhnEH0xNh7fgWv3DBzC3pSOdrM38WxswklloZMeK/zWWVqO6E 58bLUe6sgwzMtrgerLegxbnztYSr5hOFCcENGOhj3kBvAEfXdNAmPB22Y1VgACeWsq3CQYgmHcuw yLJSA+TxWagao75SFVhlRns7BF/0MM/wyKNMG/fwc7Mqb3B12PjIO91dJ+3gCgqtDLAam3JMXcNx D2M23I49AIezSwalf52+eB/TQf890RIPhE/aOTBoWE0WWfdAGvCeuAeeIXJXLbHKKYOh22cMm0om HVZ0nUfwXzXcfe8rtiCBrXVH/stlAX5MYxgnnhat/z77MnvywLtTxOfAja/DdTlj2QYBstBI4F6c FcH/Dd3Nu+kbMEotiQTeHuA/77xT9sgfu+w0nE+bLnl6Zfg5ORJMQsoRRSrQ4si+wpw4Hc2WTwK4 w2ZJiXHhJrm/fj5QGCw/IXCszsPzCetBeQLyAAsThRxlGW8j3rKyVmTkyOZAnRS9SyPn7uUjC/s1 mqQzDaCykCm3qBx549OTh+5fTMpaRz6zLK0ENpNKrPNVzPgaE9mU027PwWYtnTb49rwC0lV71GMV ZYhgtFatGEcxKgVm5h8G6wI5xd0CNTxjMdFHAx454OGu6lyfrtJntaRACuW2vCuK2quVt/0rY1hG 3+StrYGizK5CBuZfI/PyRfTnmalEgmF1c6/st05PjWn2oC3ZBXn+RbqYjz/VDSGEVLY6vSjm5nCW p56Q63meBn6bbsKJg7v9ITfHeePiB1RySEZU5HDA9kBlrz3l18GZQUL8eAvhLRZjeVejuB78CUXP Fto4yFKilVRRVNYMY0gkELtr5i9sm8m9BQnbBHnBgkBDLAStJJK46k5z0/Gxf/HhwDwfWe8C/ceW rJ6DA2Vtj211hds/E6FExFLZj6uM+aDflN9bi2nwWeZrm85VYOC6C2OCpECaEQfby23Xz+Q7P/t0 sW4x3PVE4lVFJySXcuHyLga3PToLSfW6aLK7DmtJOclTbwpOk/wX0hKgozdShDRP7aLyEqlbr8lA HRInbGdajI1OXn89MKpnFESKntYqMp6yR0HeRdtuPrLHFlB7tcJzuMsEvzzNMX15XXXnoBkx69PD RNrvV3pl2ojXnjXk32cdvz9lYO2Suc/2mWR9iOGdwZRgeKUCqk9Kw6jol42khc+TXYSZpGsVjF7p 7hlA2mKDQbXLdGP9me6xpFneFi5pGXx6U5g7zwkhKDS1tkJaPfqiuIN3F+HFa0jQrOFGcpwaPZD3 yqxrtIKb24kOZInh656cRGUr97aw69Zj6XSWHhAROHJW+4oNy4vq5i3Xd/YFbeXd48Qw5aZ/n3+b S4LRn3H89Ihf5ngBi4DIkonmGU7LN0FmpiR0O+++3rwXGGm5d6MwOgQCa7eSW7f08DSaeCVnK9Jv 45GJ5H8osDZ2CbrJLpk7XqR7YisKbbj1NYBjZg1BjJvDjwTQ2yjzIvuD5JvNwFw8O9sYmxOdT31l hfj2F2wGqJtrehsD6nQKaAnT8rcC/2tYATIFXwUiTg/fz+f2IiErPHAGwnWB9CqLMI72R47VtUjd zY9I4YUv4owi4nnX99XKMfjyB/8pK7BXw1ufi4B8uXieRx0Ry0OxX2yamIL149nmnCVqoTJJLaRO djYKno2QveRf0geNm1iezp+cTh/6+EEycoZcKVuf5fTRLwayudRhN/bjq95I38XyzGaw7SwIwoUi hSQ0YkV7pus0dITNCoC9AFw1jHrRwQU2A/32QKKxoYvUD+sPh4tMJtUUG1QxNoee+Rj4fw5XSmgk zMRftCbzq+BaaCw9BM6ti25AYH4FXHUvZehNmhFqvI60/j9JlNAhx7dXRhXXaiXm0cEiZGELTtSU Ld2MYQx+LhXph9HGsAXnOeHDKxgs9s0XYTWgyuFcichVhR946cy+qOdaH0nKqYDW+TtWHldpjXCW uoEARsbDbYwWdUaQ125VyXynm875ITuN8OQczL5G1rfPXbrgvq3b+rIod16shcUNE09roywRpqE1 nAX98eXCyUrp+08i2rNU3S/78NCRbAOk6wnrtD4viHHPOMios+XpttpLwKwQIUAddoxb9wKZasmt /FMwPa107P0PGgVcFA4CHnvdXGmRWFj5yew892thvmkHo26OFgu6/FUnmFCfCxcrwBJS3Y+oQES4 2k4+bcnA78J8mLJVJKPp/s2XcZjZSHfAjguY6fF70zLeD56W0muNQFA7dw5w15hVGECHDO8phmg+ izcIyXmUkKhPMUH6y5LwthNU0Fm2F9ydd2n3LsdZSZO8q6Z/r+ZJPzzyQdydSaU0PpnvKGIVOQn5 mdFhuSK/7lWyZloUEhGOvxVZQkjdZOZ+rZtTeJ3fk1K/IO619xqgkTvHi67IY+rHmeQbgxfjD/BB ZKXUXQm+5QxJxzPiVc9BaeuR5xx+ioMjEyKTZCGE4mh5z/Idvi6m95j4ERFJvIiBD0zQwK+VEH03 wdWtFlWk1dbz/R6VVvr4d0Ke6BYiYGDkQ3yiSVoL3KOWb3//uiqCizy1CYiE8jSSXagToNmrz7/8 cFFOAUwr/3PabEqpGQjYAtqisgstHm/U4mDJP0jQ30LIgY5aNND2Wi4g8Cvutc/EadCLF3DhuVGg JOhe4poZkNJHlAmu21pY7LImC5/7wHORIspznsukYN9iW1po/lEMpybaHfURTNIzLzChvQI3eObe NkBtoBU9UXAkSfY2v6CskPkvsbrqx2ZC5/5lxz6sS6+NkSsV+d/8TfVQa6Kr74qy/um9zR2FPDeP 5iQyc1SsXyQ4yBlvcQtAIB/m4v3A5PqsTzfVu8aFA7gIaIxSm91jgjbIUZHKMi+ALJtop76ntsNr csnlF9qu1QfcwPcRuUNzPjlRYzCYhEb5wamyEgSkAhaxL2Kf+BXdfwvQE96aj8OF+ZQVNcQSBe85 v0CnEN+V0myKCBCQkVlYdbAMzv5apc2xzQRwjpqpiMpEaQllxebe4yFrqgmFpXce2WUDxJGF2FfR SYN46FW/2kjpfqJrC3o5m6mGiArJaaKY+kqV29FV2vCkO+dkrao0Hkp2A+fVR/9DMsI0dWFG/92o QQJmEVyCnOw0O3eBYTAVT0+fm+I8arTLCdYQQj4q6EZSv7cT7MlScAkh2W8nY0dews7RYjdJ72ZA CgG7FPCaj9oo7+594ee7qGWw7JB27MdKefsQJZNswGaKBe3CumOc3c1cFZGUQI3Uuy37HBLnu58s yXb0PVGuS2a/01Op1FCyQYtn3YEzHUbibWnPQRqUfRYJ0772PEZKhCc303G/Za6u01BQN4b1Acaj ZY4YJtWbcQ6yqxmzCMfEc2OSYtkIAV5A7dptm8Lb4SF7ORMXg6qLQnrBr2u4Af0Xmf83fUuDB37a 4O+2hqW8gkwTb2V7BxMzZ7xbNg2hkc7j3mGxZly6qD3J4mZCFZvs0QOOwCQJrcC85mAh0q7rl1fX iWjFiFh5RBbgrcqxm24xvSrOlmUSnFDi77GFbQKdiJ56neQ1tpB68RxhPgA6JcchBWajHaN5OnHs sP9kHJaUYRJdb4/DvNnEGTGK3/IXnUzZTPIxBkg20XXG4WfI1UssbeYeaoIdYrFMlzxjTwaGRdAk cwHVzIlNJK8w/1qkmIIY2oilITIOWZTjtSRWBTXSAl6g7CkC/eJfSV0PeUInfMoIMaYWlz1TCSSQ yfCUSAKISgMAEm4pqneMSf6ZzSMXEp+6SpM/U8g3mah9cr6fvjM2VGx/Anhm+I+7XXnVpShs3Gkk v2afu1ldcw4b+Ny1KFvrc4N519bf5YKvBvjBC9R8EzCAA1X8hHo8N6I7OqJbUU9LwPJLX8kuMRy9 yOd1I6mV7/n3cw8lq8Z0asuXc9Z9G/VXck0thZ7Kjuk9g6eg7UdPipz0srOX8/TUvvlWEuT6w7n4 rQlPl4y7cCHpBbuwoCuR6Kweh4w0PJhSerQrDZZizTs0RjcYFAr0fPc7ymJy/CJNfK91+L8XoAT5 MZCHdJT8AiAtEFwcwpbUyH3DV2J2lWfdrUKV3wD62BO4nfQoNC4XeQYEz+ZMKTHWckKHjDqlfa3U TLP4hZZ6Kj2CIwJl+cIhWm+rb1N+PFolqFiLoiy9E7vqznEl4ke9lVJ64qMMPiCSb0GSILJkR8hy eGJwceCJ+IIE+b33xLsE+tLlUpDn13A0kvCHuu2+bUA7e53yng0j8so8Vxi92GPuymMlD7eE1jqO 1dwzSqYIYnQsJnOY1xkVyQEWwC4X1+qiKcrSepu8uFnCfniklRGQAupKyXD/MmWSMUgUPdw5cFoD 84wYLJZ5ptFU4D6NtRYL2E6VGk93Axb3PleFaRHFjhOnxoBnPtSi10njWB5b6F74iz5AKq1jfLrU OzYrST7rpEjIR9s58XCRtohKACHC/xH+ybiQxlvegGqlyrZtyNaL5Mfz+BLingu0B97XWpu2Ctpr IBNSsMxUvBA3XXlNKiPEPDWDl6zDvMzEGQNQexzqMzauQ/+sOfLeZHHZxgk6C7j1etaDS00fhycc Oi8ZPj3V78jOhNvi1LqXv3xafpsFM4ozezeK8yAAYplwZi0EVwrKtEt9jj8dSZn++2nahIX2o59K B9Iw9g16dJass0zrgnjVjuImPFUUpGIlO7vCxfDUEoNF37pHurQDK0AyvczkGH5ofPVUgDujbrWL LVhb8uHlXVdATNljZeGstMrvHLqQry/bUXsD2wr4Rn4ZZwXpQyt6/t1a/TZ6EEOp472zjdWGMB0S 2GfGs975ixddyBsCkgYgc83TzV34NuRhywdiyL52Geq3jaZblQx8X3IloahpRK1klRYfGsGfwUR8 MIeCe50j6SZPJ7PF8y5zo/lRMWKhjb4EPQfb7kniZf9Cm8F7n8ilLhtWqtUzUNdTCBZvrt47xyzp 0vzkkJIhYfLtF82Qw7z3xOEcDDlxdkcbreVxHHH+8olWrHKvF0yjzg8Qshp3LGVyt4rOJQb7kD9K xBkmniGKE1Ks4T9d5zKrdEIgdpJNjeqkKHcwd9v/Vgh8UakugjV9AevRSpUMjILCneYEepEe3NU5 dNlyCEovtJeB6i3xg5qasAnoCR/XtJdIt8e4ZjkK1qu6Vh9MKRQUA9u9n+jcjNxqC5ZZjdV3MPeF CSUhTAU/6Y1DV+nXfZ5UderP96ySzrB1nWAb9h7ZYybbGGb3l1GG9CmqyZ9TqTa54vtBpxYzO1d1 t+KoZUq3FN2NcyhCbs9OH7LDBmLLfuNdzaKEsjThn9I7I1DWqjHtlPfA2jENtiP1GIEQ7TwLynS1 tcEkNASQgrWbT40grUzRSPYccxE1CpakclGHAUeQqpiaZiT3tseORMKNyXGsVs7goTX4Jyuy18an ZIl3HeesUdifaJIr66X2r792gZY/wZviSz6ljCunh9o6b+nlprzxk2Qby9Uds8atgIQYs56WvhfR 3c/Pzk3vODSYzSMHAlK1hkdvWtKMJl1stAWErCzAstLlO3qKuNKoMXXuIMbscWFLID6QcWDES9iU EWdjOlfMVUkuz9TS7fcTQs8xkabVeztG7Sw+eVU49yFK45B/CtnJAHelf5GENPAraq6yNRtMKcxI 7xIZNaWD+JjOMU4LE+za68ue7L/4aempaDZYjlf+Zj9rvZRXgqZ1EKF6iOm427M7DIXc+UJWC7IH TowzGeUFvPd+xOLJVy47Ex3M0bCTEjmbwGwZp/J5f2/5aklq6t91yRvoWJ+7iqI/6HbN+nE8hYwH fa9E5dxtsBQJGu6q7bv9jMhjjTLctGbhiVxfDHR7gz1o1piZPleoecDq76TBmdsSP2dnRRdJ/nPJ pPFKJCX9Y6hmlru0gQ8pT1/ew39tVSRe2cAtugMALPPjxuzqRcqJzVVj87BdH/6r0/jsKtBXweDu t8d8cv1CdxOqIxxv+by6G+DOpR7t6rRDTqM8RroNnx9NbXxXRJyv+2tEPV+YDJ8NXMBAF+caMZGS +lADFIASvrRLE9dDOXJ4FSlDsPccKVEEB+hn/HID0T/ynRbLJNI2CCAxnXfDdlLqnjWvjzwRb12V epL6lUlsQ8aoM6b85X8RwPsiAHBrZcpl8efOhHKtz1/wyr21iEhpRAQnIynz/eFkQHBAiNkAUqfi SqAdD7q6PouHwKL/zEknxmZF1vOWn2kTdlFsYa+W6+51f1uQLOlB6LgCmSOXcXhEeZ3HpA1w/lHY 2jQkOXt1qsqM2eql/uMpe0A2By2+NqzNQl9bbIyxvYV1NfDhE8dkY8VNLSg/drnmXr6c3bQYcYNB 3h9zCPm0tYtfdv4C5D5FKlvlKykYGX87AyYZd21TLDur4PruH/kjaupDuU8V1x14wB3PJYQni8Nu qMxSN8YOnOrqMpqCylPsNzh/UxaHMuk1RaNylTYG21GWAgeVfwELUvhua4Ipf0nLM96rs66v9lmy m1cWMPg40vclYz2tNGa35f0Z/5VFNwO+WDJkeg8w0jPsNptCmbQRbaG4K3Az4q09Ym/FhxVpL4d5 mlU7i7604Bg/zPidhoS4Al+smPpmBHOtHfCCageECQT8fH7gDCqndfSQTTjIaNVE9lJZ5fcWgcwL aTeZEKh0i3WX0xBeK9ODOrgZLC0Myrbb87TiPeixjb8nrXbT1yX1Seh2LOpmUJXz+f3FEndUNPQm jLAtujQg1JnjARI0kkuQQ9fCMSeWa6LBxgmyfnjdP928NhwKHYDKIsXGMf91uUKqzWHF4SlzQdqz o4YE/vb4oPRVZH7Qm0jEV1Nnxx0aVCJIDjI1jCq0Hqd8xS2cVaxxLI78mT4IGrcQ5G/I/Qd88iwo vxgqqB4GKADIqPaSzrRcK1GVT7Q8CTlsAmKIzqkjgHgGpecKGH+08VmprqgteuIMxmKfPbskJBfG xcJHgSWZSBqzUp0WdDF/lEomNPGOSrxHwCXXcY3FglaUK25kaxjC189+p/m+rntoL6RH/CFX4kMD Wyv8h0a0dP8wPVS3TwFJYm7hIa2cRFfPzExD2228C9CEZu3kFxS2zxYC0GOEqMNLDhACzlYrzKUF DButXOhqbZfqtmdKEx2J4EPBAw6NrPwdJfXPczQTeBV2iSkQ/RGhAOaAMNGgmt2wNzCoiFHQV1Kw BDyzLbMYZdCTWSIVZDunOPIbBWURudVPHpHJXxFOY8i0xayPbsOBAi4OuO7v/ZUp3uPD74zWfIYx gDpxvse2bXNk0bYgi37TW8SY01viKV6MUA7VRCFqPX4413UxDsGGYmCFJBPGBAelS5NwhDnNWlnR m8Hz2ofZ8BXqbBv1rJh0J5dXlHTMru45NgRLPfb2yeyjUoBeCXXl28eboxpJSTs79PenHJfOt5h1 D6akUeceo4lD0l+tVp+O8fv3+5cB3FW5MC+xuSd0XbcSfPhx6CcBxp6gIFMxc37CIVls5aSbaJhF D9HguW7OrB6dRwC5SA1jjj8cr+L6gKen75CbTX7KvyyVpMBFEBUOZvpAczgw42oqEQRrq+748nCy +K3jODT3+o7IL8U+qbp2mG7DwCvTx8padKwwIwjmS5nUngHFCsOVv8LOgBj1JISZS0t/A/7RB1Ff 6aEVwtucwFjojCS/vMkXy0jd/WOKPjwB5CZYolAU5O3XlUkb/7WxbDS3fVQDGBVz/qr478oa6HpK crlnWG82NdAVC15zv5xwvvRgXKBtRuWj2bftMLp96ZdncTVzVO+GWXkQwTacbcxo79UZhLs0StHO NOlFMdT2i/fsAjuNZZIE8yGCf77nN1STeUq0J61SQqJBiEAML6BjhnuA1Pd5KAVWnpVqV9k0KFLE i8uRSL0aAFk/8gbSmIJzBEMmamqf8IiGxIhPBau4QvsN0TWlJ9mfLgfBR4AX4YwFvCy1epvZUTpP sLY3RcSZ6zTsZh1yYfhk32Ds+oRRdi8gwhzbQzzY66sHMu/Vx+DjkMzF4cdd600tgKXq4isBkpi6 mWFC40SPEV63mQnM3X+Sxv9NDOhT1Svbe2koCsiJfFGNf11eSqmXngAFlze4tk1uFi1hiBEaQWs7 j7pgEsu2my6BTlkzI2Q2r3G0FB7L4zjzBOAFTbeAPm3lp2QW4eGWAcnUWL8PhUl2DxANaTd4wk9q XdSngMZKeYwtU8TVT3Lux1hM2daOURY8r5y/fjkzTuxADFO/OI44cmBqTi4enbZ3sKzRu3z7/RsM GZ12AMzfjbdosEPbZAHoenYoyp6y6raYbCqvaVkFiqlSe0YkpHcGrID5q8HpCdc4JCYlmO/HUtRn vxmV28l6+EUDpt0smCyRiC7HUi3hLuc4Fsg7IgQa7wFlcQ4cwm7IJQouVMlZ0VrBpaxb3wTjG5jN aHuchYbdqkk+p35iC2K7B8E63um8SY3lonSIWbfUSx+mAD7BkWXr/xHYnQjAQmBTajdkQf4S8QzG x2tDZ8bqZhloAY8lG1Dc1xmkDknGF4DYniqF8TsoFfaTrlrqVSyqjK9qjhPIk420LPJB6btYL5Y1 S7LjGU8mfepq37n2AtAnEu7myJTFMHEpJahqr/MWVnUMSzEIsPvswAYeQLGdmHh9sZ0icuhJjmy7 AEyh3vfyX1jvA5z3LEFTPHCJVdmwOKI8QohOMhYNE8q8890rvJKOojsJy5NiVsBi2w/+H62AfmbH rz2xfvVS/3i+CJHmNTOXYwdQxC4R25ec/1bkw7nrwminWXlIEF9orh4cGteE2z8wMqg5f6o804Jv lJknjl5J1oAX+N6aK8dZdgHJwMbxca4BWAbniE79t6tcWfhkSQhXFYEMz/bNjQ8r8S53Wi9juNWW 5/UAf/GkYVD4AcrQ7ISzlDYCGtr8kkTzEMyMtytz1UjUJ6lvl/aESB+urlkA27b/MCNMHuO4u7uR mBRJqGtog3KjsHXZhOf9imbaM5VicBgTFXpju9mVw4IwFgcsDW3dSBcgV5IVAWf0fWm1rlwn8MQx Z/M9jo9cIM6tIlAkicPLX7tXSPa9I6Vngfw32cCg3g3aVVHmGBBpzZwhTFx/hXw2bADSKzLU3YB9 PKFjXVK8uGCsS7r1kCfp1YaDann88k3DBhL9PtXVZ+o4p25/UOpH4UD/J89vp8NpupQQNFOz1iAR eRzPweSRbHoOolq3LJkZ8xITU775yoPoAxZAVc6EG2Ga+veJiDldVY8NvH7a3cBYgrToP0a9XySL MgNCiAsj8yW9aQlLSHgyElUCR9N1s3xotoaeiQqTmOZHsRvcXrumcxpz/dsiU48GpT/JL36frYNE Illfk15nPmWtQc41eAI1xRC8w5mQC82yVzRVgsQWV4veTpYKN9+kjLm8QQ4MjuHe2K9gr5ovyn8b rLueLUw/Is2vENdCj3mcfTlievwl07bx4SYWGefKtxzy6id6z/GqXVVRIaXEcCkonDNQVxRspT16 VzjrboJAECMGEziHxCEHnMdgVjxKq2HOyVppBZZL4DluxvOchH8VRBvbxNTfKAlHmPPnhkqZaGvE GEvXN+FlCMTin6N2icGx+SC6OIRYvnWYcXnUFdYhxaA2kNqptN4tzlTWJJN5CTqazzAqHwY1g9r1 HnpvIOHUQ+r3uTDpZIZUcG+XNMHSxM6j26DYjfvI7VAmWPelrdHFDwsUUGqkxRIB4ftsGjmrCh9y 9qVu45aExcneNNNNxQUOAJ8WkOAnMEY2Uds9ONz2ki9AVcJZ+cQkKsqdProcjmFVomPbhP3yEeBa naw2Ym3VN7wujVwTPKryj6vRrj9SUhvyvnz5DzCwVYgsYi6nx7EDVzQ9La2fmqbP/s6kvY5Yja91 M13A0VuOGnyR5ih5GPIBFQahNrJvN/MUMsh8k+KXQVqT8ufoelwCac7AN9gOaodhkKD14PPg7Vc4 FDO3IvGhvHcubcD6W+AE2k82WV5JmQRTQyoScPX+ju7JpisBv8IyS4Oz7qdu1ZdqBAZ6kecL8EtH z59F4nH/hx08Omvhv+stuUy2oboawfYAMIK7lu4lnS6xbpAXnuiyAAJLfQldPfl9UWfipMLf5nZ1 gs+oKJRhGO7ymQeVe3CE7pn7JSCiIpAEgQStzPTcDaMbYFnYHDpkm8n7OD9lK2CYcHvolzgSauaR zCSjZv8T9clP2VbnY2f1rXy36xoqF3BWgnjD28opxYjcn3hi+YVj7O1ziKiC8OAkNXJdwSx6SxI9 3SechHbT1bMI8lBDKN+RPLHg/lBuhVFsSOeqPqrJ5DRix4K2SHTpM7Ba5JA6k+QFOZiaXBFL1pIM vQF/AiOLxH19SdSgXJz6sBHddlqI33Gr6NFQUhn5OizHUr19sR+kn7mDdBui0FEQ6gtfEbozvZ5y vNSnhblD+bLVZPcNWBjeiHYaO/CG6f9LBAE34IC/ZwUGKtV9+28uDiyIl/4wSOBRpZPwiD7cmWFR fI84enm6Wk42yOBn39jl+ddHtaLL6/fcA4/nnBp7rWMZnrF8VkYjgBjiEn0QHOFQR9eAB4oWhjgu Xc98whXYek0hyZ8FoJ8Qqme+ucNZFlXf0nroLSkndK8bQYmgwQiUj6AfhA5cbbXFk0XfLQg9cJrM 8q7WWWSRJ9YDTZ/U3pUOZ6qfAW2UuzpL2ldtZ43NxmPR5dW1NnjC2KkbbgfRKtbYczlaYC73tXTt 0LO1SCpSdJ0TOlk2GYqzhWQGrhl+adBJXgaDgJSqN8tq/ZiZdn8KUr64RQENvoK47acadbdIrzzE sRK5MZfxh4UlPk0CHf6dRHGQ5T2OPz9xvb/ZxJy6EmFBpaYi6VezEfYXZwAxg/d4JKlzIrrbHd+B oCpzWMLSwoJKfrAbSVAAbOPiaFZnEvBiSBgDdBIxhki6ZzFLJ3rR7iNJvX/TvQN4477Bh3u/PjTt 7YiulHo5m4d0QclepSxW2jgWI7fNe9BGfM4PPZV1vbEzwJbWVH5z+wIAy3E6A2I3iWa3DICmf95p ACJWV916vcdruwZ5vUUExuQS7XaXzFdV3wu1sh5RtJipHlY8a3oKtQSrnM89xQFbA/Zxv3wmjXcI zsCKINH01YoU2dwuROutBfPgZA44Oi6ZCz3ube1yG+sw+cq3nw7DObai4EWra4TAXH6laGWIVqjl kmfQXCjPtRQ+BQQi1ENWGRBQ99otfsXOfdA+chrHAqtkuJL+AsL9HIDsuCHR3bbmt87lGeh7I4xL z8ib/lsPvBNiJ6aOZQggvPCkq2A/ly8ABMiTdY+rxR0kCnCOP780vpyvfMyluBDmhrpLqInHEAuO /M/lsnMQsQTAucNcbKw7PJdkNJH2SLWw+J41/IsoDMg7uoxqO5lOp/V4HMcV+RHJO3qSmvRluKRn xkE4vkVx+7bQmS5ON+/Wx59sij/myZTcxaZegUR2UmdrCKt9YFhtH8sw2D3bncSUCR5K39kC1LXl v6fDtcN9bQ9RFbjF3NDoDkz3i96hJedzxibTqCudA538VE1XA/71FabbSbwudrA8ZFmmrjD3MjKM fg3l66bhWaDEC8iYIzn0NDMCx8PhMcT1pLx3RnSLkB2Wf0tusPN/opzTOc03p7qekw7JeeQF3iey UxSYupFXHO0glHu234PCy7fDfBwBexksNmJK1X25om/+FN5pqI0FZoZzDO94MFpKSJu2cca5FDox YWRjOOfG9Veh+7pDbnTjqJCna/Kz3mQCV+Z7VdQhJLXUQ+ZlIgMiRTlZgNaj9qcYQH1OgKM5RONB FJPiBkVPFz8TYsLhcJ02DF4vrYzhHDqfooGqCp0NQ0SJ6x2vgqXUQnrf07WHxX521EhP32ERSvrV 9z6Sq7B1nLXN4PduHgPTSjX0PLL+R1NEv92FFcQ1u8B6OYbxpkJld+e9KhasBIwl8VrEzhkEk7+F cmYWMutS/6EFO2p32XHEQoDbgk2N7m6njGLc3VTrvKFZzd6GbcyuhDsRhw2P17fI/FxTknC4t/OH ubWpg2SMM3bWApwplygsxr/LYCPIPm53XKgTRkDLkzqKeKXdeaxCO7yuXKrrgFE42Q1covhfDNSx a2rGmu7POcIr7xeBnVEZSGpMc1aCpNugwwc/01PkwzxAf8JkaGm0p3mn52tkjdfz2auNwEXEEm7z BiNGFeWDeKaRR67zVo8Un1dxBm3yuwmYWAy0auXrjqQVocgpke9sO+FJNKzX6TnTjQZNUjQJh9Ou s90ul4NLT5dWniTP9c1PAj+DsHKyP4cBzR/kOsaN8TvD8/kH+jYAy4FnR+AZ45c1KNygWlNAVN7A 6tPJ0nGs9K31uorG4JA7wOYFGQ2zLkLia4B/pL7EnQ1uh75cKZ85zoIiDHc5dVOMuiNMPdpaCjHK QfUwsVJtKMxpqdeNmJggYKHvvnszjRvYq3Iy2xIg3SosvsTWpkerw0aoVpQZqEwr5qlkM40BF66j V8jg0L2MJCMMFbOHdQFSGSf7Qbk1lGj4KE7GG4KC/1hhw5ysIrceprRrPBYR21+YcUXPBuBb8P31 AUAFGqSaYUnB5ZtyTGKHUHr+7HK85TkioykwmfEnozJ4EfvtaOGJbVdq+D8Gb/FYiScZ5QlfdguU yQpN42Fv8SHkx4OyOSF1uOp+nRiYZoiCBOuk5zLFATHusEmqxtnClo52wLMZOXgHPz6bWKorfs9E DR3KgSSDt6Z6i5rtXnfvMnU5Zry+a2/pnEnU0e741Cms6BBIj2D7RIXjo9SHSqyWMbzuUq9slVNj 9kJJrL6eihLuIUZqJii4zck8iG5DBnmlSs5anFVbG9hQhOWswRkXJWKwqyoBRqbZPFqi7uVR0MFU nwMsOd/4Ef0fAJfquoYqkP8+Mg2N8misjEs7gE0i06QeqLOOCCduQG33ca+0wAyzh3Ql3UpIff04 Vo7T29kvjtx1ZCd3S3s5hukY+od9lgzjELih9xI2QOTa6eH5iwLfAE4ueu++6WmZsOpPkL6P3eiO KPQaErGwvXOSF69vWg7S2Mx3hRKp0RwXJw1d5EpyOwsaA0ElTtrln/tQXASRyBFrZoUiNmNIaE2w 5GGr4aAiY2ZxOth6lrwb4HCRRg46LvfztSTdW2mCaYCE2OLERvK8aU7SaPbeqJvZPCaUXjfcMAme uY89CxoayieNWzWQSm1KaVWs7xGZ/mKMKWwjAj18PV+EwdvclkpQk3uYw5y+J+XIBe+1rCDd/Qvu PhxkQDj1CqVukjKVAlSBqb6iOVFRIw39sU6Bp2VgKmrAfebhRy6PvAOvImvuTqjahWSLJld8IZ0W Hlk5MAS4wEGZsx46/tJ3DtEfg75FIdyMunvYZgq4Lu4Tl7jbSDq2wspgfNEWYvWM7kqoLgNuEmHt KSIUIha5S8Kdah6/NoDKovr9MT3fkkEAcBDCDNISeEdnSBdpbp3Zily5JoqRIrOgQ3YzEMa42JWk UHt1IGXt3eksYQaI55UhV8IuXZEx3q2TXU/esKFFcEnuajB+8oWioXHqbZHvfgX7SAwyaPtKGgJN x0Es5geaj7yz5kCjqfNOXLLP7rM32K71T2XP6szfbnbRhb6ItsOVwka6JueMfOzA4VK4cE90oJyj vpCqJYdCn3DHCwJuvMTPFSTKKKfj00IWkJLwSHS5dpibp2aN1iTx7pP9I7zLhvH36ysRyYXxqZyy P2EEu+blRj78t2BxoeVH5vy/NDMdHPXcu1s+e31Bo40NZo05Qb44qkUQhWtESX4QEjHQGEKzpxlw gSb1+Qt3f1CFtb/cPEhJJ56zYnzO/at7BzB0stTqlrifv44eBEUB1X3vPFqyHON21IDrsGNsydMH xakhewhCF42Oi8siwHdh1JbJljWXOnQc0Ns3qsMq5ppfgh4n8ElzgELQDoqbNMndKTyFxEhoG7k7 G9x/Zoj9THWhSsamOED+2uxUOkNEzAOaEi9ACiwwuaK5lZ/EMKg63f3Le69BaKHlE3hTfc722aTF 15JkXM1gQFJv6poiHFvSOV03IbJsUhmCu/d3x8XG1ztCKRrhdgdSMMbhdtU5HDbrYnjY8iITbAFS c9vXfH+tncRhsNLWS5ncl/Kyas6aTQTb/qys79b1FBY9fhTJaCITMLFDu8nkOg8i0G7B26k7qoCE lZ2sdu/jAZYitLOW9meveDX0Oo3zABszpD8Q7YkfJNe/E/52K2zDmKeVih/qc1p1LNJNNkrzmMB3 yUrAx8/LZcMTUdXFZsyoCETH4/uleWIxqCfwId3fIjOj2jHIPzCRtf1TJjf6IXNGQnIGjZblh1IM dgOOh1HGLihBL7f8LTsZ/9XnEf6Sh52/8KNctxW9H+uwwXShp5ArXrStnT86v1p093/XnifoTx9t SwT2SDLFI1jZbb9WFn1vANABo9wo48Zt+OiAbLlFRzO6amLERvYyUXNVeMIhxZZhtnM/ex7zoQlr vQ15adUEta2rTXPYQ6pn6CydAdZaBqzrjzlPq6oqw8JCCjGTsJICiFQQTgIaSRumxO+ORkZ4u0b+ CN3kbZcIniV/wnjRVxzxODyXUU2W0sprN19embAEHKql8oEmmTMgXI61Mp0VbvVrpc62XKxtGuGY BqWoZfAzjCHbEMFQ2t05Gjrd6hV04NPNhaj08C7Z3SslsAUL7xmxwIZjstJOg35pDncioBN201nK sJDG+tFah8gZXLwg0AeEiz0JuTvpCCaxFMbhnDxibOM9H4S3flIj5fNXJlyxsvIBn5CZn7+74+rs iUVjPF+zemgU9KlHi4k2TFnket0LvfKPanVYKgQ90QuZLAiqNZ2Nw8qLhw9xMiGjpK+5EOsppC/R qfwOpfx4mn2/OGVEnwMeJp0jeTIrUNzEUfigr6R7aFvfm2kHoVHJyQ7wPS+jR92V8Vx4gSvayryN I/mKd7W96Wh2P+s2bmZ0XzybWbQ5xHEtnm8EqcqVNtxL9O1Hp1+Z/4Y152gC5amHo+7XQ8LNLDoo CRbyPU2WaCiToIq3uGG4MVtQNkV2pP/MJN1AkJ5t6zpk2lucUMgDVf7LZ3nHrLGVnmEz/1q1u9oB c0h3AvTIlE7RiGzKXKGI6D+V75ZLxUZRS18/9V0OVStyvA/JPU+uEGENDi0/5Bw7vlf27U5IzKuj fh9TvoxP/fCITk6LZ0gml7QXXtabbamsmr+DA+l5XWUZC7fCitKMgOQ1mCtyqdQKwWMqCRDDkNc5 lE1udawXtZzE2BebSstbOj0k5nTbH29Oi/aM8B74R0/btufXYlDX8LoI2LgrHP7lAqbMyl4nUmqd 8k4ljjwk10PL4R6RpmyZxazoefDdrzu4fTU7tnhgzMJZf6zarmhUafl09b7yD0a7xQpz1oRO/EMB 6j6SKiyZRayE0VDUmlJe4BCcABFh79dtogTSNcHsq6WSqQhuRPoBj7cCjxV2uE+K9J3+cdqWCF/q Zc62VlFhp3393EWYG6kXPzawdgeU605+ByV0nV157ny/UaGKsEkKor8Yo5RpunU/SkXSjssFx/fu DGTRXtyiZnI38GpKEJ/8PaDFZ9imME5G+W8KE9vDJfQQ1ognw4q0w6aBbbOtC5Ow9iDD7RBN7sYm VWU9SaVv7KWUMUiluh1eI+66IP9zdQ+EwR+ZHs+M7KvxAXk6zjURLs9MNlb8CYiJwx4B4/SMH6+i XI6k+VQ+NB2QifMFsffg8GEtWjQsqptbhz+CoVbBieNcx1Up+aaifGCeHW3adlxJmQOMyJ7itKkA bFLUe31MzQPSjxtfv4apOirOW2y2TT+M4OyMEjaElBfpZBvg118QWB2BBmZDBRvCedBcSewm47Fn QkpC7yY6RCGK+2Hrg65HZ7OweKSpikEMtp1/dusgcL5gY/n57AzIB50T1SFpNeCaTiTVDefUtt02 Ns/4VM+kjkpYLEqcYYfKu8ZWgA9+fnvwAq+lJtDuHYykcIpGuzzWTnHUwUkfFdJPxaxEm5nEctqx JFiFcLI8P9lxWXwZmrhcPnmVRtFudMOLJgt7qtDmpCcBDCLkRWXkdvid6AOvape1Mp4LnNOFGXIp ZMMKeYjfzTPXwq17C8/T08nXieUhIiAdFoCiphJxwXWUMgcyki5m9DhOtnTwSaVWNdJTd+p4p5zC UyU7eY2iuXbatxsgNkEecJIQHnlM4PIMIwyHDgAsj/QValuhH0R+DK0kk3VdvQo4PzTqyjKDxUS4 FfRzCLCaEK8LNCzbUnFA4j3mOC5tic6BcqtIx9zd1pSx0/UmXYYuShbL7gDQDgumnleH6CezTSYk L+r0VvXInUs676sGmDQtQkWG7nYVLeCAFZJtqE/AYMPL7mwWsXTFojmfhQTulQHn3gJyi74UkNtp KQabkAOA/x5nZSlGYCb6HvPbQEk/7fD0ZyCaLFPkN3m7tF8fRx3LxedimG8HbaSXjTItWNPEQe8u es322ZGiqNb8QXfHfF+nTS6pQSDv+wnN/dvhSdEY/h2R6nmI31+2PB97Yuu6fbhNUoiCVOQtkZAz 4cKLBYK/K1RA3jObblW8XSUpTLg4JyABM7l5dNWpnydKgGXahaCa3O6m8TTkzInKW+70uTXz3Oq9 M+HhrkBtNiIZ9PJd2VbZaHgswFWY87d4zOtHrfboW2SZB9tz+MaebRoVrDAk7uEe4XMLjWR5GI4y JAEhU24jhg0QY052Qn4Gefit86pyRA2zdHUTb1DNVq54w6ZLShCl8s1v1VTUAPh5UIHRPNn0t4Gt WtYRHy0qIHPnELkrnpT3W3lvONBxGmS/g6aopNm8ayUanVdNf1nkswm9CQysoD4ILaM7fYzl4u7X ihbI17nlnmaWylkS4BMtIKTdoAkvdN73fe4e8UIt10UdR5edN9VYG9Nj6u7gzjnE9jRH9Yf18GOP XaOZiQOIWevDPLuD26Fvrq1oBPal7Ay3svNFh+WUgu/QO7DZS2wLDxIK/4MG0UQKOox83YCCgCqU jCn0O8e9rmBUDu4FlUShnFYtNfBQ811tcYYlvJSBpjjCwr5F2Al5hhDoGC1TRC4YvtkL88odW04y xEtij2Yh2PplMCAGNhxPBw/Ut7EvJWRPVswo9Eff879FyPe79YatrFLGdMWUGkJzvcggKDqVv1tQ ce9yd5KUO9LWAO8O7D5MRsv4VXAuDuNalI1szHao2MSCVFIQcfvE7Scxj/V3hOBSIpPJOnn9SLjd raR+DPUB0bAee+po7kzV+7HL0vn8J28O0WiL1eGXWEiUPTG0tfPUoGCN2IlpcUZNfgbSc3rppNAS YeU/hdyvZ0o07Nxc1w5kVxDE/ach8WDwXjcMeYZais8xRx+0rN4ZeYuH5aglGL2O80tY1saVg3yO 0ItpmOsuhwCWNQWJSv4WdmZuLIWmwCDSUwIxlb2guzMSv1gHeEL8aaDwYD+31+GAn7RDkkv2KN6j ariduMcjhvzt3kNS2FFhT82AhH/SWtkelEbWA83zx5bvqkqrm1tTsy7q7Jr0TP984+6sEoyI5ZaF 031CSlKe/5L4KOe8OomqAtN5HxRXfmF97nFiwfbuQ7TLe0oo+vx+WSu5ts0RDLcPeYLU/xUQMbFn fjyLxXH0fvr/D5SpSG8oAmx81kcTePHokCKB0gra5rDMzhgQzAoU7u9Ch3A9EsSSPYcPXX8v+DaT uNpNLRRINJbY0n9+xYi1tQyrdvgdsOgJvMHR0KlnFqtgFyUGMOJwCGc4ic+/gVM378/hD502hQa0 NA6RQxZUieFKT05vlGHxfK7ksUtFL9dWA1wvgZ7UgIEka96E+22RPslGgBJt4KmAe/4bcPgNC977 57MG2pPKUiisQsjJEPuTrzqW3+XZ+M4uaX0TFih2uQsBiN0tJrbQ9be4jZeis7RPobtPbKv/z6Y7 q8AwB4+SEvp8q/fYSEDvN2vnTKZ1v8en6QpaXzxdLL8P14TepDP71d7QponEx+F8AQydGo/a9SJ+ 7hrV4i5JVwCmtxkrWj2ln82TegdCggkdV0arIl9Mihv+aM4qu1Mutt0i5Pz2tAwTEXxyxsr0opq/ amTSmbzRrkJ2s/Uef0ZDvfNXNLZ8Y4FZgLTc5voGtwnYiEOtL4ztVpCBmsFlbQLA+KVPG8a2D6o2 C3RlongVBzyJedrpRTJaMQt/M0Px+t0/sgtl+m3c7pxFHGYEwbSsYWAl/8VtiJARgT+zYVOCucJT beeLYwA/rzqMuZ4Mj/lJsRsPGYK1KeUuzb/e+tg9gODwEURfZm8xfjr9NbspeI8/aAmMvcLofYP2 2EEs3+QghrynR01vwtxYNpp7r0o3KWFcWNQMvvozPiDaB7G4ejxvhmjhEvfWF5oUGhzFJuvopIrQ TjPmAG9dAzYkelOy1lBBzGoRHWAcRXOdE7Dfac327lKqzwouLK8tkS24kzbhsOT4bTGSL5TvTJn7 9Cqir9PnDZaYSteHOtQ+crwlDzTPLfzUiSKH2ETaYT6qAVk+RLWJQaB+4fZKZAOL7pQynmgrrOD7 yutI/nSqVFz3/Ohr9qsQIa8OyS2/CQpE+FmWOxUAcAGK/xtvRRSRPuouzSs/eR0M9bkntTXejeOG yEmlcszAQesYoHbE36rMlAfAvosV2Zhdqh+PIs2CeF4YFUlnP7RZ9IVs8Ww529tazu4meHFMpuu/ j7uFiL2HoKuJadyPFxtCG1mHF4xXzIcSBMC052xsnDYQD5VAqn0Q6qnWvMCMbdNI34fseoaP/mxH u1nNIIrvNx97Q9AUNbX2NWrBaVfAuFJrgo34Uz9OXdGa5BoCNELmcy8mv2aYJrO0+o+VKzjm7PBi PfMNFy9LCEzRjCnvNN9xAZuuW/K0Y4rqHUv/FF0vqOokx76MnE7QzluL+hyVSiw1yFmBWGkICqH1 JuHAKbbrefwd+OBbuDiAdbIyzti3Dyb/YNioylbD2BnMxT8RUnVan1sDxXaI7iN82qnfBUYudHiz kuvyXBezNYO9hO0QTcZtKKE3vILa7fNQ2TH5OlXOKmJIIbVRoMzHihc6QQ68yDY4Fg4g1RFX8bkh CQaPfqEvKnhq9RX6PaCno7zRZQfUB1ZxTWzYl/KFT5t5KBUzniLlhaduNC9zckiWPvBl/d+c3gan ReOAkrKe2OJFMWYRu6RD2qa4UL45DlvUE21Z+FHaZF8FMnVOwdqcAEafOIGeuXORl7THkk04EbQv 91LVxmIWii2JGrWM0nlVk9GzD/1cfdW/d+itUeixJFjWewlrjaDVTiv7+gD+5kC3JUSG+VPedK4X qMNfYNo2w4NhG5vIjxt18dmE35BUxsDeeQ6HPISjvb4hA6Y6TlnE8xAcXKf16bwuA+XN0WdDQnUP kXNlrlkIdYhRQLw8AY+PtCXV+6ThH135KJ3OV+lA+HwX98zGQR6CzClh7IpDsa0SZS3uuIHKPGOe c+K1Ms2nr3BWlZmjikfq7U1yNf4yq3vgeLDSUjDVnbsMJbtMjlrVCH8Dnqk3FuV9KHbbNAzcGaq4 0dB0kUt98SPWfrf9jBYhpIZnijLVQLViPvCfKp76cGXmr9e52FQmWy9ksn34WOw4gixh/RlwErRq 7LJMRXOgLuO+AVgnK9CIMArUIwQ1SuBEQIC8qel1wnkXT+5jZX0S1hpRy7wpsxJT6c8jgHwwsbiP e4lQ+uV7tiglCK+MH48F/XyBnJ+0VjU1MFzI8MvM8YtMTUo7G1JDXZOl/GQ3J0KSxpTFTs8OI2ZG PZXiTY1akfn+s+ZlUR6lR9izEAcIn1BeSKffR8oXP8SlCrZx0HyZ8delaZnzcuuUJre0ji9lLRro WcrsogzDZ8dqbEFVuHf2fAsv+OuYNy3WL9OTQpdEqyRd6RwN31kGWnTEQVP3RjqzfgbAjvFWGk5Y fslikOFS52YqIPGL2nS8qbdCI6Hjr+Qapbd1paLbvdGZ4ATvGZ1TCVgFn/FcR3N3x+vcq+QRQmiF Yuxh2MKPzR78LbGjznTQ+arkiIADBTWh2xeUjb2g2RvQGcdAcUc7IBOeC5DfsrVcOg3Pzz0IXtx6 eYX0i6zl10gwoQazCQvFCgEkfkbAPU/CBzbYu/L+7bDTU62nGFv3Eh82LefZ+aPq9A+CtqOpy2Jf ygqpzI0mgYXv9CB/REssTNpZKtDpFV53P4hjJcT3WkbVVlMHT6HU8ayZldSk219BNoijoMNhGLcZ BOrVXmms4FI3u/qnqRK7Rm6WPshG9IrVYAnUIsTob5BMyRS9byoNXHakQho43Sj31JBvxtvVABwk HFyvrr8dVK8R1J1f1GFR4c22HWFMbxealnaq8Z3ZrdCs8P77939nxOubum5rhJfiiR1O/w0fpyjY X9wgriFF2SaERvzN8g219l6lxi5nDLr8/rLtOmvmkGV9Zsx/MdqVTxLs6xdDgXQ27drZcNWNAJcM yLxI9Qo/4JsPvM7sb9oTd+XWaNeBQaaUFzqB7qVHmC4Nd1pgAA6mdjxbKZLE1Z6nRC0aFUTXIaW2 xrzHDWq9US/Py904Or86hjMzOc7uSxp1lEFDk/78p6pkzLsbu9RfgTKz/eOckdMAYKRjOb+UQi2v Y2Eaa7GU8Hvm2PUhihtYz+3FFyHLdGJZyjfRdPiNSQAk0pYZhAihDbyWq93UvhRFOKBVW+gHPg7I Oyv8fFSd5z7A2MW3Lcl06USmBLLqdS4YyMzrPaExJxngCAlS3dp2FBqXopLPZv5sDec48NNhwWZt C06NRO7OvWPnXOzOdzRA/gzokokbzr3JzQwpQIFI3L65AHhKgA6ULIJa/bxAk04IAbwKqtCf+4XG twGW9XKzJZkUnoRoxplevusvDKCnYQfWTX0lXGaEqdPo5QrtBGRAyYlKiy6yLJq/Jy7BX2cdzKP8 ZT+PvlK9JCspX/wAKNYJSDz45OiEXeODI28HLA4QZ9WqxGeBgCnWBc0tz0wXy2k9S7xrwYJdh+Se IjBKS/JbC2pcGvH4BeTUfv2qx24c3vxgYIjc7gj10TYCjs24OZkVBC3hxKD/avWJMwi3BAwO4vcr DJml6KTpfHrc24026H/i3m0zpG5mXIOmA+f5rGTERJqoxaPE9yepZQLCbOS9sAuMqBSJpnwyaoWk k+ayDKKGbV421/6dw/3rl7DBXYlF9csGCMFp0bhjjOezh/FcF7zrSY4GjabCTMu64GqTlCQ8Y6r4 LIuCNgl6fwgg4pSImtZq6Wr/5qZmwBs37sz5bs0aZb5KCPRBHxb3ZxcqawHjMJlOQO14ehZW/men 2oLJoaIiXfS6b/tcp4xQFZMEwKZoZUnHCFR6L4TvvNny7ZwithYquG/PVPDwxQ/FfcprC0OXBLlO wSDcpT/ysCX+aTz8+/WNGDul9dnumCr6+sgBKqhG+91atRru/GwoATBPoXzkLGBp2JJH9674M1Dz LNApK2sqxyQL8jJJ2q1BYdHwzbhDjyurM6mn3+m12axrj4jIZ+UN7VTNmhWmUZpa7+yks72uKIbO xgSfWALXGlnSwtEdlcYSKgTOiHLYWmyJM6eIwZd45i4ZjhcTBlB5qeBBsVGw7ZeR8T3CidRrt8Ym q0R/r9tHwjZC2urwn4IW53qKVzdgc2KNnyqT8ZBz8foNOVJoD/MpbFQuD/afqv73xtNZYcATneLw IZngbQr6atBqIqtCAhrlZwTfao49zPLnB3DFzMZCIfV1izpY+/ZLyEUr6MuztjSaTJG06pvnlmoF xo+UGpmS48YjjoJCVAlGC2uTf6+rPLFVQMK9Ou/uK1UkS4/L6nFPmZ8RpSUGmfTucoiEHHXpiJ6j 9KrF23T14/AWBbWYvtQXEAmbkHUbkNzbYyw1ov63uvymVFBfhKoGzymXpzIlle7HpU+xf/suf4Yh bfjIeMECfhXv/q6A/3A+Ysy02Z5OmCatPLnt8kb2xGJth0grLkCPzuo//Vs8iopf7+zcYGlb3O8u Dk/3OGMGWYO3oQucsqJ4ikoNOgqfV6YDZ8ga8rmOeJQ4MjOOK/4QskeUyYcENowlSOtJCf/7yM9m Bs1SkZr5PSWz//ZH4btFLUaRfk3kccvBqWkWP7ku1ymTmcSlHVIZ5kYBxSl5AF7y//Hcj5kMZqSO Kf+rkytXjP4SHNF9RdoL20GiNcPChBsQmDQ1FudIXMVJYVw75xa+YGztYVTAVv046hfToJlItZoc cJrDw879GHrmRyTtvnhYpl4dEclAn0xtmFZ3Uj+Tl5khHkx0+3b4oxQIkFnp3+Yk0uDmEQYLPTQ4 YbScFmZSMv6HItpLD2yT8dFzL3alwUU4E8zd+9FjoZwFWjI58fvcOy4nV2m0Ww+vPE/4Igd+Y37T GQzD+wXxeuotpDWOUFQEi+qKh+I8UMGdvKJE4l9EJ3CsyijEyNJ4RBx6RCZiL2T0eL7OtxTc2eM5 hzC7TRLuepjzIOYfoXE4sRZ57xcJrEaauaQLU2cy6c0qa1PvSDr4lbddrloUi9MWzXGIKiNrXEGf 7j2GWRu14XgzAWeGdVQKXeEzInc97TOYyft1HtV8iQ3oVntL7c9jvh+Rt0SAglSY/PKFVzIskf1v 6tByQ+TQGDKR1nzv48EERxEDttH661fEOMA/B3DytwJPycGm1MNtGmbhcvogjWqPC3RKpRo04iSf y56MPk5P8G1L+ur1X8jMMMwvcB0S2P7EYTs/upBEOi84Nw6dbeEgUScmtuxYmkmZPm1WzBY0BJyO 9OagPFFUdgeZ6S+yH0an7bz1NSjGZCKzgn/PwLOamwouC8ecT8zZRWtum2lBVzTiOOGc4KMahg8l QLKaPMadE3pgllAHxHbZM07aEbe6bSM63Qevoivq7W856RnEs65wlxGaeaQqXNIr2zqVAKtSlHmv E89Yrx1mGxKRvQOa3jzpF4o4xgy+L/ZK81/o6v3FJCyXaT+qeK8mQEcmVv048ItV0/bEf7fmWqJD SGz7kIgvS2eNkxCVGY5nkIQs3rUYMvZBD3Xd6KzKCq4TsvTbH7x0VrAl81onJthKSwY/HRxsTzJD DyDyIYScDkCphBDZBL3Strl7eO+N2+yEua8XypkaXlNs37+Z3sUG79LvoQWEt8JibhgmFpxvTP5r nsThku7MeyVVZTzDlIf5xhYbh/4Xz6iY2iw1zf5ARLUQEIuiQ1E9ypUXzxRw/kpzB9XQawvzP+2H k9zzFI4Puc/qNl+pGkuEfBAu3LqdclOhnyIDs4vU0ZuAXznMJ+7kwhi9I+zZOfnZfGjS+5SvbPbX C30EH9EpAhrfkJLOx7dJyWdDnBVcjIuPcJWnVv5moaq7seXACjM1z4/JZpp5E1cyzfIwupwtQrLH y3oDw7xMVUSDaGHOm+P2NdWcaI0a8cFtV+AUt8MvID2tbdqknrlgE+p2wJZpxvTYCLktdHhhYUwG ToX3CpelLd2s6+fSlR3UTX026vEjfatpjjHbnbXHnDbrtOHI1Sl8mK4xJ53Gf1rAwAqaSewQVg9k PQ0hAzuTkyTydzF4dpkBZTgPSH9iSrz+NAtL2lQ/tAL6tNb8p/t9iUa7dsLkTHd/IHjtHktgaceG 0jYrCg2I6jTPm4jK66vrhLT7wG5l7PKameCPXsWUNgwl6h5Rgj6IqrHUoIKXYqaba+urFvKbt6QN ANY0UB0W1bW8E7nXdOhwl3DojrzfNOypJ0xzaEskxOglXE/2hGhWQukXXLSKqRaxQxvl37KLLEnp DU2Q5xea/984kvJR99EgM5o0FF0jn6QDDk6qP+XGWhKbDbqFFqX5fqXex4EkHKnQb3wFrtzuD36A k6leAzgnz8o6mJqEVphag8oLXiYY7+p7FV6p8eQDbNcNzkXeA5HV2bCw5L6aqYBUuj9U1RwgTyeT yklRE6Vl3zCP0qT1Yw+u20qFxrHTeH35esoCXLKGyAzVt6lXNU16CjnwM62Xv5RGjGHHboo0Qctn ZUqLUWpcmHrhvQXQNRTZheYkxAxnisgrzZ6CfRcmcWcVWIPhmgBNGrrqIRPdnlOB6n4i6+RvIlPb tCp+tBA2zTFttpxxNMEcUYNRxllUGWHAsqUNmoRp1rCka1SUSpTm5HSd204ns2lVfp8JoRHxdLR3 M0GcLqydvOmPIfDSUtDO2vEe97kbIgwpibDtwvK9Dh+fsN9GV3MGtxvd80GLZm2cHHQB3XyXtQ5K HHhpFYHXwyAvSEmjXDj9JeNSvpH4lOTyeU7HslB9p5Qxm2fiuhhWNGbwVr7m27jNtd9c1WgUbltu xI87zvB2krDEWOTshJsFygErYkgD+Un4NdPe93ezVmaYn8a67YVNp+3VaoREqwITNugFaYBl2kMk Cok7r842tm4C3T1WbnAkT2lmlMJGH7d0lT3o9oaqXL0dfYr/OQACHWhVgyYw7g9VwOZdsGLRHKyq JlqNGgJUoCva9DSgyMOrDBUZg7KguwscLZ5Mu4UB8KWIkV4qgtx6iFw5kgeYXfaumETcIZ/qnvv0 F1KEtWjtQ5a34jeTWBKkYCzv2wiKO09YBItBR0GOd+gZ6CZBRa8QcLckSj/6ZelaQ8Fz3V+9o+7O vrgKQhLw/Iz8tRi/oA//Xu2BdnmAUgVHRu1/hz78n7ISlDmWPL8BrgcN7tJUBPV2aaV/Y1/XlT6P 9HPNqItbkUj6l9AyjdONfTWdb77+4DR7C+It72OQQ918YsKKiJgiuYavxmv3EmnJN6qxERg+ut18 g7x3Kb480p8R9mstqpWO3Ix9eqlxs6cbUBX9orfe6pX+veGLwbBPBfHKFIMmbfaEaN6X+lh7tcOm CZKJW5bivefne6PO+YyP6X/mK43AZMgo/d9Z6G52PzXDkQPztzYRTIJjD2aehQsnPIkIYgh7arJR yVhSaRQN9YPVn/GvnpzZZwx0QLfowz1w2slhO3jxCqbsgZSXHbgQotjQP73+qkhfMWrf9WOxH+7x oPQcY4HUqAMG8tnEedaWCyePTHGX9W8zPzHKuv83Up0YL/bwg/cPUV8j/Osvyr59HIwMMGj5UIPH G2G9FZFANPliHPAhW2t3U7bxcGOL1J4mPqzXMulbUmFI+D682g4POOdLhbxtzzT0oVKNJm3CFjfa CekPt/80ocG71M5gNpDgsXfXuSVSxcI74V38M2gjSi41U/yItAVUb9/h7b5X1mMSmr2oEhGS/GJe AdJI8tdVfEQxW9nVhZz4vAo6Q0scUqFon00CM/4kWXHdSS26vi5NxCR4W4EAdLm6p6eWumZOluqx R/l4TELfJ1HoNy2YP3iLF58p6hHJpllVRznqYTc2Mv1Q93YdWAzCNwnQNMgaRCjfnWCY7cNWofP/ NT8JOXekYBJzhxqFBZ0moK0H0x4I9jP74qotoBrhHZS4Jrxpt/ff/TWUA6ShSSp3+PKg6VwUuEd1 LAs46WWKeRz5+HoZsys3l5+WUF20rVTQUhb1IvMWRdARyk6kCiazwlZB4E6QDoY0TXDbnDnnqsFB X7r7RrqxpVvVNU6yGr8UZ1+9KNsv3l+DCtJYXJg2V1CP43kWFN93VWzSouP9ujpHkuCruqO86mFy jdBYLqxA29Ti+GF5KMvVlpGA8g+1IOM7fuASRtvrGkRa3qh9qPOXezAerHyqLk4e6rbuwcRZKwTM Aohk4pcMj/GvY7HL0wIgsBRUYoPrtxe7NzhuopCnAn6+imlJTgJeegwM4XUe6htcApYpbedvBxmP U7YwvKa+BdqhEdFMAF6DfE4hw3lYVXaYHsi5ZGyfGUp44AmhE/hZDaOdC4M+s1AMGIuCPSGEryhB Be08wkNPUtuRaqLDykqufTM5IErtYweQuqONbF9P8CnnD7u+wLkeACJwLtSy0DG5PHhiRqIpNv+f bkBY9OTffsAupKQfYvq42+9zIxKr6bDeLXsxRre7mT5MVLB+RBtytCCt6KVUBttbhdBsrsKwVl2o cXW+LqbtsMVoc+R1KY8JQJmPwdR5YIgax8ApVthNw4sDnxCT/oHCBHQFCp1YFY7CVFMK214WNLVy lGliAKxNGYHvJYyDqlMxOT52PzJxouwvIgb1rlyZjl9eGPsOzEyrUqJaDWMDvJoRlRAG+SSLKOxr K13FxXrszD40AA+Fn6UqqFNaBeQHVu515umJQLiC4lp1Xc3K/fdZExouCxndAoziBB7onuXxpY90 CUqe7OV5imPMIXK+5iu3dpvlt9rYcIpUDxv6zQwyDlvvWkrwLddqo8JGEe/x5pUHIZITsKkz8EsL AuqvXTwCWg8fXA5Is3MeatPiVclqm6hbarhsxnGBsgk9QwmZPn4vtyohyAFasUitM7KVbgzPMQai kDge7oSUzxJ+z7OZnTCWZ+iQotWgEBFky7XgGOPp/JZdlxrekX3wpyD2h2310lc4obtvWH4rHYDW 9SXkc0tWH82H2KVkux6cBGvNkjj5NAHr48Xf2nJgIi4F5Xxd+IKxRZ66v5RzkvphFVCpOcW2qzO1 A6M0gBiU1O2T2xARrHPslSI1qQfLuuTRptOwxX3ozKFKfCrwzRf5DzxD4AdpOHtA7hTgxfvHIEQD QHrcrmwcPySiXYoGlVGBvDu1GXouVJ/0yS6GypRIT92qa3nEFBCpxtUIII/fSmBkzDfgt8tnvGDZ NWmT6fYI07vR56aYY0hACDsneWqHxgmjPxIV9Y0NN26vcL/0dKx+rGWEw7/lvoVZQYDeHKQtMtti I7wfBch6mzH1jwLlf/FJ9Pwn0+NCwZky0K0T3g9PrP/RewG2zjs3gN85+uc1kagwUTq22L7iCnJx Npx+LV3RU6rUaJTezengFAPzmWsOS5y8rvAaQe6PAfx9cXNXFPwOvHAIWATHhd7/q/+POzn6iuDN X9L6XANT2AXUYHWx5DQDZfr+lRveLBK6T3gl/jfIZO+qnnbSTPvrkg2tIrd2UL2z/+usFRDhYh7f 1BAs7Ywo6otcFlYpCUMKEyPWYFWx9YzlqBTkO5r1vyWjYfXPXAwLihA3ehUq6LrqnIe3xeSSTtfX dyQoC/gdOSkDiwNDglAXNAkFpiaZADuRVoO46ezc3dx7d9SZFV/MDPDckrCrVyPSFxWNUWd6LALP 0L22vGsTK+ANhHEDkMd1LUfTZrQNGxZJ3dqNPVDH1pzoQfqTJZMTUt9bf5xQ3wDnuH0seKw6vezy OhKQnbq3sHCBIH5KMajnj33JNW0Stk+YkBeS0mXwzyoQlptZjporYc9GTGJq1+Ok79l7aDjHF84Z iMFyLIaajLm0V6Z8qpGclSHu82HrfeoduEcscrSxVCXnYj3mR6+eVRGmzEDlGxNxsS026uvfN2jG I3gxqKi1v+pF8PYd1ejRJ9YQVWMGucpR4d7qpf7DWyc8gk+pbrNOvDDKeYPKLANDCCUXigLl3OJL 8gDoE5eMcpGJvMrfiyk/I6l/ROsdacbHhQEXmdKprWXiZgRhbe2g7Bbm20dgir1/4zM8owSYXa5K h7Nq4sDanqfyDUfglaheolyYB28tq3g1IFyjliyz4h+iKm9jI63xVeC5bb15spdxJoLfjsIo4u+T zr7DCitCFIdEkwUNZEHNAjeyMGTlt81TAem1SN1aRc4ycToEcHp8pL3vadfD+qTkShcOzDVQVhwX 3L3lFEWVY5x19jqmLT/1guOd0Tv5eJPwP93hbNzJzTm1TDljuwcLxO9207jbBBop/Bbz/1G3EtxD hpLSdow5YbsgY+Hsn7fZ5wnG11Ak1Lv1GC73WGWZx8IQ1om4Eq94deHbeeHjGC6KasKBEER2A9Eq bZOie7dAI47vbshCepIobNWMEuZqRHqLNTtv0jGV0LA0q1WsXmNIM3mIx3lBEg7iTeQk/M5w3K4e zWSZEcl3Rx+MrLp4p8s90F/R5bfAVnN9Gdi855puSrt7kBxDqlKX/5VGzt2AuWyeHUyhn3oMRaNs KNsnAqFmsC3gZUfa0MZjqj7o4VV7KthpcyEsjkOYMNjpFIxDijCINEo0Lfem3NiI3kJ6wzzIhdkJ tLkSNPSrN5BNqLiOJTyf3GUK3/5FqNhGwtG7mIHbg2iFoyDWu03mGbeWCazOEoOdHBhPEEzvfqVL 3A9xmArEqmkFYjOLb3r5VA6tzn1uG8wGeoo7q2p+EeChLOnoRwASGifPQGpk4V7/Zw83tLLogU/q FoHYXK3+YWjqcrxQhKTi+9PEMHDcVuQN3FBTHRg61em4ho+ojUUdo37qcyKebeG5JbD3sL0br3gN IPSVP7ZnWYaj27bi5GDemp/8NwjGgoDMj7+FfAIxUQNfIkFdqakuqNSlWr1EIA9IpkodTnCln4H5 CVVTYr22/yYxTLMbsx8rMmyKoI+Z73DXQf3f/5zEi8p626VhuC6W8q9BlHDIl7F/8pwmrOVVHsQG eTceczY7Sv1zL3/7HC7nC0rDOcLyLKIVIShocvP2XSFYXA2T467TzEgLaT6Jjz1u/N5hCgqqCtGA Afgy6XeUfUA/LpTzGuteeY+Hvem61qCh+/0aEOiEKNxWGj8gz/mQWjpkU4U6QCsERioAa8j/0x1x Be67Bx682MOLJUrBKn0zsoKZUDOkVJ9+Pcymr27P8mZOYO24bilddlM2aIM1RKC4yOKHO1JlA5aD AFDfiLHGfz21otsxrPwjBVEUbv+vAVPBSzF1y3AxocRFANAKvVIfIRkM6WEhwiihY6rpsVXi8oQr V4oLakT4mTdQKSKNIfrxqAZppeHXo1MbNKyau7vb5TsLOrujwGVZ3nzwHKYu1vjXFHQ5z61NcI4j ACImLZFvJwKvHPjCLMIwKJn4jBoNWGfwkYhZthaTgY4j1LnjDBK2BKh29GPckAtgZOkqguqW3TII 1EYUxdWLuNm9ZFpZ66rFbqmdSslvDbPYlLphdj0uE3dooPxEC7ipBo483rr8FdS6XP+y2K6qGypN MgStyTCkt1s60fk6TRpGBkBbepcVXcGtth2tF+c+jjTK3GSVSsTk8jMWvuIlpKoIPftA4OWfIO2q ZV0Q9q3nUEgBVtp3pTCxdDY3LjHxjbpO7EjmDNa9b9an53fLGgkj4Bh3boZu7HijXcNSlnBWdzLr 4E0NC4/0OR/MjL0tU30TkThtsS8jt2OW5rdvv1xbkmyZRd1t4XQlyCSAwvl0dV2orOxwhj9dsyrv 6ft2YeS9q6uqmS0y8/ECEKZ8L5auIyZibC9SS4t7bWXPkGzN93n67xTJOLDeyQO8WMWAxTp7XJtp BL3H5qtBNbUgyEAhYFL3VidSv7jkpUBnVzfRKFxDJoSfQlpoG3xfWqLveQxbdZV93lb8Zb+40RvX lgzWV27UxBfOW5ZfIxMtqrp80hOIE6YSfMt0waGUv65Y6oYYbD3LopWwVAk+srrCVn1fO4DZQVYN 0TyNZ7i/kZuyh3yoI3JsRNzE4qbVAM9vlmfMMKipJeibOAr5f7eAr0Ovu9TnZ6CG+xLehmBT624m Azd7P0mgRFTzjmA1VGu7Ga6sYflc9Ek3C8PADEDtRMWqHih7/5STjgsqj4ZaUifglRXxQ8BUQQGU /bu9n71ywOchsCqfpZMMyMkSwJ4nAypt0kYtSih9+CKBU5i97Kcf9N23BKEg8BMi6nAZOS+5ls7S I8n1UKEz0z2amUlqL93j5m29uu6+27Rw7FKMdMsFop5edaQAgXKUIsvkWYVZwb4Jj0CI+/7bv80d k6uCoozeDty2PhlahlMsUaNxNpDIHccslpBxXOFXF7w9mUid//NVSANGWbYxjS6lJ1+PYmMVwqnF fln7bmXp04FNbEqhTRU6Ejh18qVGLZJb1AXkJhZO50CwrbZX3+DvUJgN2p3y5nT61fn22ZXXSoOM drBuRfF83L7wHf/rwV5nkykDew6x9MXk1lmxvVirRPpG3fZ5wmbZQ69gQQLwZ8vLpm3YDxfGGS8P Sk4h+ft5Rx0oVFz/Bo7OHaR2BtDQUaNIlzNPDzhQVptl1e725MUmN7LvYIvAZeo5qlQHq9/0ZFoR UwJ9myMVFn6TsJ+0lT7QOPSMcNSpMrYZByUgUhPQ14FuWm0zFX6vMRtRUQa0Vx1Y2xOkBWNtaLkK bGnMzp/Ne8e8M+JeaZTqBpsx807Mt5rlWb6oucbSgYMkrIcWaKKEA3VU7Ml5xLfa/XQz/JhXp62U BJBgC8d16yAkBQH0Rz6VoxLveE/+XxntBOZedrGaIVf17c48uUUXXuf7Tu6gh8JxKA1w4szGLbm0 f1HRbslsyh5agoYJTQ280YsJNaIc9+zYNjwBdqUa4n/b9qsvmyPpew55CNjbaWhQ1DVWUXl70pBJ 3s12yaT/P9/nlm3TCxSyBgYrigfIXso0X5Pksi1AnNb2ju0BjIx26dqdFA8KADznylESxMrjDQbP 5PMp5xiEx6K54xHgQet6vo0BrEjAY67yPDda8S6xwLQdlhpY0rukMnAhX0RBoSIKrNjFfmoLUXzP XnQq2L3SZFyfmoEMt1Ey2BPMY0+TPmI8UoPbxBxQbkJSslruvOalEca8Q9z5dj/dv4HCkW8QSICM YEW0TXKQ/hkif9SvDz8TKpNMhjwzn3OHALqbVRwYzKRn2BczWwj/8UBK1+ufVJX9WHDrqZyVQDyI vaXVV9eIwFQw3rtAScfvW9iTzKqqL/Pd4mXpIzOkW7H5RJgGHqikaETYeY5zdOSXMuej0UgD0PEB zk59MUKG6+jS52jHfwtoZfXGU4+o7/A2W4PO6ouatemFfx++nDwQTR6Kyx9XONq/J4o1Jaj02wae 8yDpPWKc1SMcy+R5GWc9AXltAtbefhuwQHSN719KCVSi+dUQ0+7xGTbtg3ejLclyru8EvHBAOobV 7RfndgO6n9NbEL79SIgAi0cgWDIwTHxSiHX/KoIhsNVCbtw60zj5OWA+k71iDCQ7Jsn1roMW3lUI 8qmkE9fHEBkZcIuUM1LC2Wmiit3/KS7NjOQFRqUdd/yAXQMrY6l6duiZIrYzGjs7zYZP4dLaxT8N M4UrBsqUycDSO2o4OVXQfhGCtX6gg57OYvcodsWxy5jb9QQzNHND1VP3KFGraJ/SR+jwm2bCaMIg ORY4ejg2R95mw9w6EkPJkBoGttozU0kxJKRMlUrA//uIHie4nMBWIr8eCy2Xf4OomLhIY6/Env/7 eI6gz6ovjKG6siqNEudCaOLehJ3yG6hmzrXzH2V/pzfyTm/nyArOgHx+iyrd8lRC+jsQusKPZ5KS IFHzq0WzxoCUqzOk3RWZQc5KIAorOgCafCC9UObShehPKC+5z3+l6slEZODR2qalti5ojpMs1HGH LZ/NwNfMqkKUs0E4CujkAOgCmdLwH6FYlTcc61fJDPGwKCV0CC6xZkxFhBx0VaZIYUIMxfHjvTfj 8fcks4HUv1DV1U0U33LaI9xQt2xMzPTosdv8LByzd0z4PbYwtJVUln4P4hWEKJeGBV8tajAqBghN ea6yAb9F1BWJ9oTSolwRH9eck8uImObzvbB+SJv6PZkoW+TNxvjHaHQgk2fLx+aDNaE5TgNEn6sC +CVXNA8ZjimxTumPjsd49wfds/ZFjuYYSxcUTNPmg/9EE3K1FqxS8WhWW2I76iP2tctyJrmkYqwI 0lTj5HwdBcsmgI6tLSY2Xd6UxGMreGoelHT6Kz0y/iwKje/lphSS5cNhIhwBB7J2PyDOY9lHq/KL nKSODFpTv+ma6wC1ZOCZ5g80ynOmVuHs9zcNEWGzSTYG7fkGm2KWwlwsgej/wWUl0L8p6MuXpcag ZwGZbvkcpgTOE/Yqj7bhJ4pMdC3G8dtP5A15iw/hQGRZQge1V+YHZembbT//rdPnH4G6Hnt7VOND CmXKWrka6ZiQD5RNTwiiqhZtbKcLZXDWtoEI1QBnFlh5twf1gb9tVhFtRdGL/jbs4h/A6dY3e8GZ 2pYt17btOaihCR4tpK7Ui/tGl3ztEYCpzYLsaxwRsFij3c8KJDYp9KI9EON/AsFTRFk7tN2HB09j y80uuNtCUWhXeoHzj+g6cwwNkonMN6N0+hjFd+gOgyvK72QkU5ChCIFOucJjbTk3JD0lJkaCd1cm gl7p3wzMmWhT4wJyHUwD73tPRjWEX9eOAC9j8uPT1L85laKwAJ/g+WHSZBdBzbEumxXOxiqAn5Ed V41UTQ0ybqcYzCYoHWcYGcga+g0DkLvc6UqS6AXeZWqAnEQrtv82hm0izyiqhXAaEZtyR7b9h5r/ UsEHcj7i9WHtKWTO/7LqBdJH9pl9OewU0D9ad7E2QH4EBDae/Ctxf2sR/3WwaMNBt3qp0aPvrwxf L4M2iKGep9qKrnoYY7jd2YvoSwqGFWfdITx2Ttc3+vb9p9wY6NniN5L2jf4Lk01IAH2PiIVl0+NC za7aE88mKdX6l/s+uTLA21aCILoljuFav5Hy/3HxKsaVbx8NIH0yzi4lvc1CAnikrx9nbhgJUIM9 68BELK06d/Ksj2IphDDXmf/Gku2JJrvVWM8aSpLpS1eH0UihNpAUR3cg3MtUL7wJUVJSynZDY62r mXCfTMB4ME+WDReKvwarJhTK0+a4cYZ0bFX7T6P53CvnTz7jSd0h7Dng5urE1SetiaI0S3Apvul7 ZM9h81u3kayVjPZ9WXa/U2hn4+NS9pLm7eSiGDHA9iv4sQsz9WbgSJyjVOTp7KJ7B9iZzdV5mq+m PI2I4vV1JBjvM40ETH0dAGRcQn6dzNVkyJpIVE5K7pZ+wL81GZes2NyejLkT6A3CS74BiCrLE+lY kfvVvNawp3VtfdUopolP/4K26JOS/9tWkFkh5GgwczUsCOIpSSqgV62CEqFewRG8VaNDfb6l5vL8 l5wyFVKZwvGu4m9i+AAiywKnzxdwOZQmFAGiMqatFNj+AvkNmkwW6uILHkuzSM6Dr2mDK3gFjcjD R9bEyrXzY7EuNCHaWC2o7fMFQf9aTAfwg8ik/LpQcHKiHG70eyTbN4Ofq/vrlF7ZaeFRkBUne/Wp ZU6UOVcABfQnzht+4TFT1T06FNh2BA14yVf5A+s5MnqKn5Hdfk4VlmXGsRZCsjVUEU1H0ALkt4P5 cLXKkuHxKnZg88t5DLjAzUWIAzdE2M9aAGYDJYD5ugGuajrPEPzvcb9NV2kVP7K79PvC4D4oKdBi whx2DMVzpiLXxpQ4hnJd+96JTEmPV8D3ZSB/7xum9cb9KGgm8EsCym9D4F8FdWFaEsfGRNtdLflK tbpSNxqQXNFQy1PZ1lLZB7ACtlPiFt+vtTpUmYRJ0b3wmI49dugdo/y5JX+f4fR1FsgThVFrdw+8 jKNAGPRP7cTiTUoTWxU685KgRMqGTh+aHxinASaR15xjh7fD9aSWWI5ELqMTFPJC1jTgNohipns2 E+VLVPDg41nUBCDhetu+RTX1xT/HJwithrNhU2mnZT3nXwO2UZ8giF+m8g3kY4Sq1B8gpYmLyqk8 x1DrFV+dj+3DETUaBy88RuGrORyyQBAXjdVuGUWdwRtcWxgKckY+FjgXgj9jof5gMYgFeRxWe+n2 rXgP5ypktYvbIXK9RRsBWTg84gT6zrNZnLxJ1KyM2z9hpuX3StCpcFk+3vEzxjhXfEzIlGwyHBB1 Nf+oyd5a9cHTI48t4uty3W6JJEFSpN7GiJ/s3CLUUrEIF6v5DBMYq9LfdgznCxyhCGhcG9A2oP70 MieMOgHn9xwOZUVwrSy9b1YN4XG6SD7YuZWtkkYZU+hczx83eeQLr5UkU/8ZXLuZxlq1TY1GNCBM Ofrn23PtqE/vh9TNg8UrQTv2DG6Dq4gtsw5XWeDJlrRFVW2XgFcHScR756aln8AY0D8Dem+XO+/W G1X2Fuaxg0IhlcoHWqQH/X/0KeZtjVwTzH6vVD/VguxasZEDgdlXXXFezAlB9CZr7/NjFmMoCD0l GEi322Foww/rmRa9NBIZbwrpga3zKy/OeafEXVjQpXePFxTD7LehEmKeJesZCt/haqJJGkPruu+p 2xK6RTiJLJPnTCBV0KtUj3KX8dFFDor+YGbtx/pzF+Av4aDhFUI5mFHmwylMCTUGAVT5wfjkWV74 rDF2w9sEud5dKtwiutUCI4TiiybqtG5tEgorNnA/SwQiVRtNS6oGGCCDWsU15710T0PQHsyYp8x4 UV9k0nKmlFtCJ1TQliGkX8pGQ5MLmqd1S/VOe1fffuFuhb4803153d/jfe9gMOrgtRRU2hF908mp ZwwzJRK3MpvjbFrnFPw7V4puEGrTQ/tIxc1PcygGMK1JJLASNFvoxVhGEtmLKZ/BbssHYefeiz7o i/MD909zeLa5X9HjlrysfCogVoeQODMZieFWBpkMhDSaidHK/rppUTTDdMnki80Fh3dMzbx3g/zf qN9VeteAic+TKYBaehjI30/wfnmIHQjA1E5LFOeRewNB2W3LJssGhBtaCSqXiwmbBJ8OHrB4yLDb +Gv3ieeOh8WFIjpbQ2yABYIBrwd6KKTOrLivwNDSh+ueSxTqLGbDtzaHki601eyxL6rrQbpIXj5g AJ8W/jolvUMhEbPEd4Q4kN5DTrpPoRdiWv3RhQW85t0BmCKNL+AkSSzHWpso4Mq6VMCBFLybHKCp ShvkMcYuPDAwq97P56o5IxBdWRxBBzBnVC9Hc9hcTZI5Tvb3DXWgLBTfcsC2AOpe0jBCDTyJp/5Z Fg3WQ6Cd0cY9UhJ/8MBSARSgz0XKqv+SaoURWuS6xyEucBRTwe0p7fT6toUccdSDG5zPLQM0+lvL zJffLjihCcNqogqr/UQmaVvNz7BZlO12DgpcsLAxLEOgr6udcjAfPJeVQRls1Lac2Gz1jCVqQOYQ AUE/ESHMi8UBZ9H6oftzHvYZsYtlsIKk0xBMim3N368e+6dldHKIgDiBMczsg3slCjTW3b7WNZUC pckDqke7G9P97YyqyiZhgJCXvYNX3NzZ4HuyIbsquatYT95tZtqPri9fUkO/12zvZFd7n59P5So0 3NK24uJN7ZpZcnRDIA0fftvZUxi7cZeTFuqNGEUrfR9BThJDpBlex/DmcKahYJzinGAG6s08ijwV r2psAM02s+u6eWca1zRAN1DGPEXGRhUIY865gdQbTnu2yokwdBZ+XvIUqenZ9hB6LTLAboMUJdRN WIp+yX5j78MK0djaGd9jmaLMJYAgVXw1Qx0TG0T9f9Ik9tILEiFnlAgvlumey2ejuSjQOSEU5kYT zHE1qSbHcdIKXJaN2EeYhvK5d4O0FfcyezNs0JxNB1oleq1hKqDCtZkQGB6QKhmStNXNypP3Xatf 34CB6ZxXm0f9psH/NkA22d9qzjeIi6DIp6xi23hRfDNjfHoLzOrHPgAbogSJJHLW4Zu8Se8yg5Qm xN/GSGcfJvDAlp1R5nNa/aK90XsiSmjiD1hs2fw3trHunAdMw+4Pzd2TnzxZMet3GG7iBphZpxBh FJjnSCE0yOBW5Y+UujwnNY7hkIEYU1PlA+AIvDR7ZMSGAx/SYJyWPmQkUAtS+eq4iLvg48NdpXj1 G9xNHfG/hHzLDT5J6Aw2uazT+0l03Qrem0WweE8vKqs1tgrSX6ggtIzXoBfXFTMaQkODBSM4l5KB XSrqkIXFWXbQI/BvN7cqxPEVERiwuYvk0/GG3T9OW094Sb+U/sUToDJVkir5eZdH9y0Z/orGfs7x gyPNGSwMslvMFWUrk0B/xw0In2Tn6zpTQBGeniKmUSknFUKWucU2Fgv22AsPbwxjo+2L8bkuvC1+ x6HxVXe98ZDTuMvx1z0cfS9NKEsCg9TAsyRu0wN68+ysQVR6vM9tRVo02sm516hBMNIGbfm7GN5/ EjvwO2tGKap4Dan9RHaYQR5eppCDKxBoP+XMBywYbqJZiUgmCwqm8Z1ggSv7biZESECUry0c4kfm WU/hAhmpE/OBXirLS90thdeZaUy7NKMrhkKLkHuNtjYUgSFgLpZMWufBcHfZcELX4eyEhC8vpKjc 1j+m+bayMG6+AiIR2SYIq894p5uddzeBOPEUX7GO3LIxwQQCAxeG5WSmGapamf+ofmEJKx4/gwyq Us3m1OwZmVWTog5hLxLMtUycAofSZtiUMd/bXazDdS7lxxIfl2pbPAZk2Sf8pcAP8rh/GGH3C6TN 8PDo2Qk/+vvO2uwD4q4fbDd95E1hNcTk8SqNgQwFd9zOygtWffhMOAXK0SVA6yC8Lno6nA8HdXsy gQIVWKGmQdKGYIUHxDbqRzxYYYxQ16tOoEV8gAlosSkzlrDJyJh7yza48YQ5jWXUfoNPbMFJ4nGC PkjWnOdI5jZyQNV0+A4pqY6vDf5Ib271uyWWWqeduPU+cI7N+cZ+wyJ5ojTgWn/ZU3KLfZ3VzYuo 6S3FjqVWiWWugqShaVI3oHkVLhY3UmuctwAqIeqZVw3NuG6v/9f44AgoGcAx6XNGIE/pFba0blBO IdK8XirVe1c83oEMNKS5GTXGUL1INeM2yp15J22pgtd4al5xHC9gktVUBXsnuYfAsfltU2JuVo9c OXd8iBbcmM3OhNB0Z+KHFbUxi4Yy3P1TXwunzuQfhiqXbGzKBEpkSIyATZ/hLypkecaEK8AKPmm/ RgNdaM3uXGh1yIjniZsG+i+2aJMf+yW26YvmYgAGyiDy4oL42M/hi89qJQ5SrRHPNMwX/Wg+emYz mfJNVYDl/T39NFsnhAYL0zNkYtm1fiYexXE/Q83bhGNK13s3Y9lt8QJphxaXG0XFu0LiOYASJS72 hWXOxY8AwM4p2so4OEwOjILQfpVQDiXYa7cxcuNAdKkR7pymKyi2rfsEwPYp/1ojn523WzaF6KX0 omPZLXNBNA+xF+Xzj/IuFYhtswZmqCVsI7LKvhN7ztf6BeHLUoOSQgpA4l0YFuteLKm03nPVG1Zu A+71K/4Z4TztKcQkJHBk56kWqRXjUCWATvsaD2jrIy7MJ1EE+QVbpznPBkJDW8kQ4E3DkbmOGZAh CWwSQbUQYrEdmrufV2s1Y3QVRWLjI/IZPOnfc2oD5ebVGhkOc6yuiiKMO6ji7FTZyBPIwp83ogaC jkuAezifIicooTOUbbTh4cb0SjC26YM8v3eMtb6e8ZNCzyh36pTqzLS1HXox/5Xs9rQN/sDCXLqJ MCkiUgQiXTb7/JJaULe8+OCknJjlKe0s98CC5Hxc2zjlmRFWln7TUE2d8MuRU04IAJmnbRTS+hde 6c06fBxQjz2dthWoL50P8gtXc7wNyfgQ4RW6K/g4sdV4tA3ZwrswwICvC5D46Kf3vnUFkBDaoY2G oeCxN62oyXCbORm1SyroANQM7RssujbahEhOef9irTcAjwkiyuq5FLH/wrucx7xuuI6uUbmDE5Wf 2vSpM7KyuIo3oe3Nsg0jkezBn3vijBipvpX8oEIK/8h1ifMwF455Vt5iCrnXp/kn6fEaxI8o4Lx4 EELC2oiiho2aDILyaQFJDpd7DZRP2zY1ZeFBsIXZfmRGH2AlYCEN77k7ZHKNGrpdEeTGmrCyFxpc CyGvsQ5qWe+gTNdCM6NxbqlH0OIrbp+g42aJzCT12yMjC7W67BLegNmQxvyzeMf1oftuSDzGfdNL MtqUBH6z92uhULi/tiIxoCOm1dZNG5GddMAlJiP8jYS+Pi4RtvQbkyaLI1j2eMvAK7g27SnQ6rFH R29R30h42396MSf420fjjhfRqAnd71O3wYRhWZLVQXzOx5Oo7KAxIG4jUTjdjEHI7cOpGECWFKIU G2L6EJK4KVvmicXQ7DgtgXx9MtM+N8y/c/IDmmUgci6JlF18oKfDJNv1zKSBzCQUfL43JvLnBLkJ mZu8NnCDmajS3CUIDNJLg5cIF/MP5fq3Nycp55+F/0twKO1saLvvqIoLol/CVg4TnIPejJQFNUNA 2N5fxiNC8EGsRrME2B5VbRGddhd17gcofMYDotXpod3sdf3PtcNbhCU9doxDfCFtw4+aJ2WhA4Pw em800ffmnVqAUbW6m9AqALVw0vr31Vz4AQtst500s2DfIlL5Vk4U9Zylk8VxopV3r7+a198MephH sNPIy3a42ROKBMvE3cmWNRKaai97dVtxI4IWYFnAMinRhXLRVFHzBphSyGpJIKnjuXHPKEs+8bDT x8GRrSN/0OyjNXEeqq3lC0UgMbvsiUmIJ1Ha1duBt4ycp8mG0x9ERpP0mXp2soh8yttV0sYcE/93 wBvip8WtoGSfy8xb9wLC3sCpuBt3mmCx7IG0J0r6GHz9lLRS1DawBJ3ZkumqyTy4ZdBrWA3p5rCG qH5gfbBoyslPqA6Z4u81OkCitrivoQpEmEC/JcWQ7DzsKgk5Mh1AuD9qTrZDp6ealtFWb8es+SjK 2fR0mfL+c93m6gCXPHh4Rm4a/mbcGJ5thMcJxki0JEPUhPLjLvwFTG/6XwbKLxnLVS/6dZiYsbtZ rDeVZPf64p815yUFuE6DNwIk7DIESSnT0588owW8WqnkiOn+sww5xMJBv0jLK2t4r8KTdTHLd0kk LJc0nIENUUZ06OzD9Sg1mCrWKjRGabSpv+nu9ybzI7YZwZ1yqn13CJS4vPSmM8TQbR5IGvdTc0Up xgx4UlVNwDImqgHP8o9QgJ7oJgwFjQ71ZmjObWjOL7hYo+0uIFDBCLzYFWS0XFjIoBcimWSf9Usx 6reDfTtEzHUczpUTNVmBcHqcXg+3fd0F4nxHcXJZrAxirv45RuSTx/DNhF+qHWcvq34TTwvLMJ/k yLSHRc6xRkFM8CBB6yp0ISyIOBF4XPGVG+bb0A/zdzcCAYj73BiQ7SVJ8s5Z76t5na6MHI6uCCQs 0+xE1Kn8yF6HRwGE+aiuSCqMu70jIP+Y4yDP7u2USKPr9hlM3HVAXqB6on8empmcF22jTYnnf1QH xm4HLsgwn+vg8c2dEBdYvM06e3ffpIcIjXPOjXgj9B20VGTY8bLDAfvOwNdSAhJEgxsunjUqGUca X76Bpl5lu+UXoCS98JAJiNpY7ITulj68vz+H9IpJHMhpVUp3j9wCcs8ChACyu+6AmuyLQ994Y8GD xCrZDFWZ4ToPwXH4Ko3OTKA8vPgjq00Rtzz70KEQRS3j2hOEASQUAUC64P17yNtWWjRq2Vw0rdfJ hBMs9LkyIUZ954AMqN8B71Gt86qiug7qCvM3qGUuUh52w4W/vrRyHbOev8a/7k/iFOAnrjHW1MjY lQX27qD+bpvbe+x+jicvu3KvMA7EYwWdTYNyQ1zGykHaLTgzX84i4sIAGDr8+n8lQ19wXOB+PIxC xFR4wHASnEYIHLKdkpTmuaz/ZqJVqBW8U3/eIxmc9ya3NI0CKyWg8pFwrFXEUCZ04AT07U9WiIG/ JTvrYMjNkvEnnqCVQgJnF98yogCImjrHvVW6n65U1wJ6T7tAIYFBHmiVQ5WM0IB9zlJQdXqv2rW1 MPa++oshV2LkWsiRVTKSgIbKG5b9fUrz+HDJ+7pJ/4v1HYPcAw/azOG1Y54steXVujc4i4oHPQNk Gm9TKloA6LbMj37nsJtpXyFkUSgfrWnJkg+iwkllsUiQC/m9a8TSRCfzQzIKgLxaE8DUH7+t4gtZ NVDWMA/+sR7wmaektJC4t1lVMngRnxutdwoNS93lyqCwxGPUKGLHcPUty9LYElgzr/sdNFFQGQlc MA+7Ra8RUSIaW7N3YJhn8Av65V3IMDp8uMS7hSHW+k2xZoYSPeRdVTIm6XIzN7kj9tnooXnA/dTC O7NJmzXPi7oVe34VkzykVSJeLLP86z+GpjASroUO+Jj1bazv7A62/eYkxL4Z2Bqnh+d4qvaU7iWX eiNBWO7r1LsEpjxToQUT/zhIFELgzhCz8in0m7J8O4PX3y7CPJLIiH+b3HZmc4eDlfsp4OKnUkCM 2DTsWzWe4uBK0/xVXu+bhJ8/uaasPl/+wwPA+pfxrYBfoOLyPWZi8TtfQ2wRb1Fd+hopVHx/eXti 3WWax5DVa+U2R6SEPjZe9QydRjIv8gQUwirA58ZeJOgqATbg/+RESKcWxK92YDE/qLmbxCSP/PHR Yi8rpkxWDM4d6bbL9UZENoCZf7N2ZTgrxC33YQLcL2w1HoMRYrl3TvASXZhmQdXM8KF47S+5qan7 lDEvSXXeOaTS0Ufhuyw+VXeWM73C6NDQt9oD7/yLuEpBdVpCO53r5oKS/JkkAk0rvihKs7svZf3z SqIDYNWyczXlKK/01hjLgEy7/uUSmI/uSYcT0SpBzS++VP9mbbMM6r+N6TA6XINWpEB/G23wOI8R Q7YrpsySjHuhBXknghThTfpOjje8TGgT5D6PkRiKqBhJSERnxMog3dzMzgs4xbSHTtAwqPRg75Cj Danw8A6RgNleHpSoN5w6kOHZtlA5Gj2uKZFXHygLzvve8gvEPirWIStvUsSnZ7S/dWj5gl3gcmCT Dgb6Eo5lLwIbX9LwuCXN8yk/xSpD+VhCt38k8YDboP+fhG7xZ4gO9vtjRkf0/UQ90ROVMJeooPVT YoGuDm5XRPMbfMT+nFrSUKCT48/xBRYWbjaoaQAOy82khKgjhUuqELi7H6IMYJgLqXl/10hwRz8t 4apF7RjBMJtvgG7JS/rlSXl6qA9wqhbLTdOwfFCgZSqn4O8LLMcMY5BYnUnKF1E6E7uzQpQRy+/o HYepTv2zbpTNUS4GDCo9PX/KJ8Z0DaADTsCSltxXussgoa34s4XZJBIzjSX+cvHicXpL1GST6Wz4 PFZVz/bHFi+o/8mphr6qUF+JaeHXs90S+HAq2nhDlCFA+8uqwaTCAs84Q7YWQy24pzVUuF5f6j5W ktEyQo2WH230TsgxTlINf/ZvzomD1JaeA3w9vPxiDo14vWTcEqJrdOahVqTRk8mwZ8GNFNtehSpu S+SNxH3OsmdNH5Jbjk+H4GqNchYPv3QzR6t/owF1qEXVMLbDtwvf4mLD2O0ilYxR8lFIFRdQJWSt MweQjy/gT2wkGTjkrsuVTpx1b5UDszqRMLAXmfl2Frbv1GD+Zc+LoGoOdwq6UB8ssT/P+tD8DR8X TQkSNuizQFzqCdR+cX5mzZvmUlcwYrE/4Skp/8h+0NzGwMb6/kyaYd7R9GLLT18/04kIOfwX+2BU AhpBakgVPutK+ZAAvFD6dqNPu+i6dJ1wW68mX08WrvQrAVFdVN/9v9GAeeeHXkluHCj/pLngveZG BwL2fK4dN+HSnhI27wSfu2a3m6lOfDpHE090dPfkrb6iIUH55xlWlbeJDMfBBsYJKr2vo8b3rzcr Xhy5SCS4ScnCMgip5oS9sL5Fj3eo86tNfNf//ADFikaQ3K8ZYf4gOLchLaZVxK1uykdtHw74yu9x gVM0lb+kW8dxlxwi22eqnT5Y8lZoixYCzcT8BOLoBCAr+st+PzcqlZDoVWVG7guf1sC3kLZqrTlR hzbZ4HgU/pLyKZQWf4i1iPH6EEVSWkfLS4Rj9kMzkXaxBgmntnKEAte9vYWJiYZLGyqaZWtZM+K6 Dl6ODbPrjNXshpTyNDlMwQPXTyKMCqXz8ZkwqVjiZXTPvnTfdSBjis5BQHn7po5bl/9JKbHiz2xP R9zf1aBBhgynL1nCj0esHSarnDFiuPqjU2Lr3bq4nA0sKCdMFv4yufQ19z71++UtsWcOkpfC9nUm FoSxf5C3gg8YHKdomjnPdoZZqgHcUjAfVmq7RHSh+H4C5qmuhnKM+sGzAqgyFNb3XT6mRsGrTdN5 KuGugVxm3fyZ/njc/0p2ZJ/wVekFD2wyCGFZwkMoIJdZKifAdQpr0Hj6LdrpdlCATW/DfIxvYM0b HNwg31sj/nw3SJtLOafdC8Cs+Riode3fVgG4kVuPasdKhFc4JXc9nbDNuS8SFLLDxpelq2Q1ZHir ERRScFipbIILN+qN8MYAU2ndxGDJvm7e6kFrElJ13d3m/r8DlrBwAZeY9eQ/D43RUEkrG6xLer0p Ty+O1tZAXbGR4vB/JM68glkkHC91OMNg+UT3XH5zSEbnKefaQYyPvxKmLsaBlFE5MxJP7hoh9knt KQf87RCBipsyfTh88DUZcYPpU4l+Z/Y5Jyj3dXPlsc+7yy585vZntgdRQ/1d8EDR2BGp6CQ8SGir uV8Emo0A/UKNhhYMbn91j72hMT0Mz+wgAmAjKHrJzeBWBubQ1nkbv3FGjAEtPlL/KekO6sMB83xs mUdnc2CORrGXKOoHTwRGtcqZVVRFSo+ERF2Mg2fOvFahfRAZU0kg+iWmZ/XZpZYW4xwh5mOfXKxU c81jFraNdbJ96vIQdxK75sFheFAH3ibWlQwJDqhADnFIrZTOCanO4frA6Peynmv2fkPi3ln3owkP FcggAeoruKDnjHPv8dF72t8MdGLXUELFhn9t7uCE/aZvJVUgdJAL52pN2vX3y4TfOb2eUikirKjE tGNsFRdKbB26zlPckqyK9ZQnfVRmx2JaZPoOIkY1gJvtiP4yCucSBPVkkLLXiq0VfHB8v1RCy0G8 O7b5kyEaAziZv6RWmUJLvdGqPT60aadfzb40jVAO+WM/rBkafbiKu4U/ssWgkbqwF7PpVTA84GXY bvfQxbTGmWh3fyey6ttcUD46+3VIRMzl5SB0+f+JuYtd1XFHG3cTV499xaFtcLE1ErgvYrGUrurz NeXXONO93qQRyqya3glr7KPXKQTzVrLIcrBQxrjTnrKF1vXrPkljyrK6Pqce1SdYq6+n2cyLeOnP R6Kdy/Z5VgPZoSqzVpXBSpQnZLR0fZJTNraZhs6WWT28VleXNb48nttnMBDCHjXD4boo/ZMPQX+A UFFukbRL0O2FH4ZMuWzlLJ8m3fxWeA2iuJmTEoA0DDOe1bU+eT1HT0rHo3bYNELW8ssWSUMlU6V0 eazwHHAuzYdI51Mb8tO93hHeGLghQFVrb3RXzBkFbecTIiQoaD1CDnQvgLy2w3ep+ZHNrurp2SB6 6doagCBdCLHYFaIzc63/zYkHtOTH1fq2aj8rPFa9jLUIsbq4FpthRVXVPfkMGAaUtCz/X40kNAmg X3d2rMmr/cX5VPcvoVwgK2Fn4wJRbHMvGfpkIrzPIxKD3oPKmZMIVAUMzD9k+axADxUXV8lGrBl6 TX7EffAsXKmo4olqIqxvEihUD4B2iIQOMs6rtebkv+maDwPQSJD7/NoJbOSPpLlLRV8ewghh3uot sy29MN/gWm0yt0EkFwyoEtVBfJZgjo4BTZb8Cza3B0he/baFYQUTRNIb3eUtoW3EOPtnHjm7C+PE HpBEY7T/GogcfhJP90m/e2ffyWe2bDlkaIzR884rOn5Tcd5ooUhnbQpiNd6DXyYmZAZqe5MQzoGo HnDz/YLNA04rtRBMAD+ujb48EyWcPSSPPI3OCgfctE/RUKgXFxPJgGPvC+l6bYzLR8VqBs4nkoEc 76v00hua90Z77vmRllHMb93f8VICPOjlwcLOHsSE5PEMdIskX6rDTLgjvCQEJFPW25Op9LjVWwEB FUhbz50MsfXxJeAAcYg5Og2E8AE6l30y8soFvFzgUeZQNHD57/BzcaGYsrDQ71HT7UspSCZZOxRH W2r+72N6ghbr8Q+75bzD70h/7FIaWftt2o7cQ9YxZmBZhxso75Yu2uY+Acpc39XeIJ4Lb0zic2of x9y3/H9lBU2qSSt2LIkjhus6bV6eg0tWarEO8S6y2S9t/5wuPOZ9Iao7ZjDbYA596ObJVagXNwNu UtmT/lCB/XhSZ90Cy1+epF8ANuute+6Bq78ji1EJj20uSe9B8hnK01O7IQWslApIS769k3jk4G9Y p3pzEU/0u8h0lxFC59ujAYHo19Yo4wZsXm3bisrsnXNA12RtwbOJhiKE1WR8Ww506TK71fIj3mDi Q/wWS0L/O5wErLITIUxgux6vzrGXa2QooyMVulQk9tqJBsRYXcbIRzSLnJalRqAV9mGKeTbAmcFj TknGIOrtq2y+lUCBOG1itji1cbQNLd09TdtzbvuWxWh0CvWSAZibf/3Ps49v2jsTgiPsmyMtNTkj oQz+iU5tRG+oy0ijJEofkAovfQoX9on5ohdDF91h7KW2IXDD7wllwDm0yQlZnOhCFMqujFBaUco2 pLZ40hX9bRHNQoTG7GiPUpOUooq8CbeqJ+8k0WIKiFsRnhEM0+I/7p76tJOvuskhSICxS/5qdzMa xIwQoakNhZdCKrKJCTA5wxyFrdn9GfDEca/AHcmQ7txF64cV4u17cdjzy4s5ePbeMIaCSw0kZQzN BFDtmDH45me+dc0NXIQH4gay78Rh8K9yT91aXVuZzhALnQoJTJmnmJ0wOyU+k5R2eXityWpF4rs6 Z4DbOoWEMh/41q3cBTL3uU44oqkEfxa2aisXvyGuIqjuff4mNP+7L6ZwDV77kWXcOKDSz5bMRvUM 4c98HX9CA4l/ptXNHzdZgrtcMVO+vM64APzJgXHyO1TNJhqAYLctRiQxjzmk7y6/hfRJDOfCMOcO M2OaRe91UKTg0CIo+IIi930WPK/7KPKQvlowV04ekx7wdShreGnklyV7pVo7RHxmKMMS2XSsj9yT rhEGfVCcuspaJ7CC3JVbf/+7bh6ma1fKsu02utjQviEjxr+EFRpIguL6aWC97+wlnND88UOhHKlo 4o86DA0h2A7JZVYXy1BZ3MU8MuwkCU/OT3i87mCe8+o+BZEwgmTKgCaoIt/vkSyKm2GhVNRPuJx6 xQ0+ZdghvKEz3L2ZEWRTbJOTGHV4rKEDZ4mKRjumePuBRVJQMwazmNPMFcet+zaqpVbZQazdZ60J w6InQOIearRbMNq+ezZag4THvRj7iCiReX6ISPaLDGlrKFc5DgD7NwZRSroWdzXoWI4ejsn1nSBk mCl6KI2rFJkRyg8Jg1dL888czcJQXtw7aXqkZUiwqpjiiWq4ys1G+AEEEP6suD3n5k2ODPJcs09P mF3rUSlmwc4fE7kH6zO5nRgxXDCKduDAAPyVPHEH9mhrIDIFbiVrWtBejy0CWVJ9rqtykUlmZun0 LfdhJWnocaUUbA6xqjd8P6XK2HLWXVoc6Ku1z8CcqVf+IonQKzlbVLzAG4ZRbXJstfmlEaXGq7Uj IDTa6dkMlFwZmpOyF4Sgz2Rq8vEPq4mpQHZ+zv2r8IbEcHy1BLbuu92AvaNe8Zhvz+R6HX4ihhM+ gI/eLi3oyDiRDauxIOb1GZ8BYUaCOBV+HcDhHVy4w9rlonzbi2OjBbNkOeuZUaWtD1ggzUXW7EV1 qLEBge9rdNyssV+CxfZUxGnrRqGIXDfMP9twaxPU9KHUjikSNab37Lgu6+K+Vt7Hs1RUPxNtSY6W llNqCZMgVCMfsVCy9S4PoRYNZs79TgnzEb9phwWOveOnXXhRn78pdixOCCTr+VpMGKIs/07nhPya d0SyhPCikuNgj4AzxGPABtE2vKiZX3mw0wHS/GYjA/ZOZaJ2BknNITg44N/60V+4JaRdBl0HdjYh q17xGZDNYo9748PSwPTaQ8Z1Y1XY9je/jZ5a4TPE3tFtnryoL4vGuE4uZdri6pWYTLgMaqlqZKYD fN90o+kYQd1vhykAqWdQntMGB1X3AzxrRJxu8SJQQk6WpqxEdEsHAYs6L72uDKT4ePK7J3J/6mip KTV/cY1/EeVVEQVwRCBmsWfTc2KWwwevUT9VhfajnZVKk3OxPkFBCuevftIniDY5pHapSJuMKFZz yvuYV02Ci/gbVTtqMKfBaisAIYGbGU1ZW0jv7fCFU44Ti5l0cRNKRrgws+1ecMFG+d6K70TK40Q+ cjqE8Oi7EXlR4RFdK3gGP7/akY35jqe1Xus1RPC3+ZklhvWrr6XrKZdKx9q9el+DYncVxX5kmMsO Czsx3oyjmCcdQ8pDEWVV3yXpxgonrxz3XFDf0gqONxk+n9mNjvND6APtQFN8BvJk4FjOET/M7FxP dUpnTqRM1upoM6GEaIMpZEdOQbuEgoS+NZVLCFYaMN++Oh7MrIqWNTFb6ahJMvHcxknrdzrKMRXJ KTc00hNcWYP9UpDO3C0UwXqMSqvAngiduBunlVh+vtfldDVYQUvBZDyXAj/0T0r83TvYcyZuyhVw 51Y1JedqqBvAKodIsc4z8shm8VdvBmnRq9mfvPazUB3Dj9Zbei/CDDpnazLNa5qe1+JlD2MguHFK cNud4gWvD61faCMtrC9LnnMNX27KNgZ9XcGSUutyrkHoowHGY77A5CKbqURGQIEoXRZkJMieTglk Z8SJb1sQu4BG/7hqCVANlCq6hSV1TFJ37bDHTDSbH4K+VX9RFM5s+417eIL1utGtRJLBoJJEOvQZ 8I8dsE1HjQuBuGZA9LgtoVxKISBJLSJFDWjMMAectbNoNFiLvJ5tjxLlJMT14iDpNiv6Z2SAiAEM K6FzOdZGZuIMkVUMumIUuqKaqa15N6KnEYgHaa6wkgO9xpHx62K8+UNH9q5Ys5c8swCfjcDT2Gj/ JlIR+Vhfm7VBIqQbesZI+tePlaMNUMJ3YDNFgvam9fsbT+xGqU2Iyn15Y4MXB4LdUHs3mWlhML8Y oqltJz8Vcj4Rg+4dnHGE2UavBtnBQLcC3D2yAm517o3AED6WRBxWnDH5Jq3/JHYuHLdljiJLwXf3 rtdVAkvxqSF6+2O5ZKm62jthUusZuh2nGPUpYGwV/zRibajgcSZlqhsqA/vZJCcdw8DRAfHy021w t02HRIyW2ULBgTt/kLOjRmnC8N4piI6g2/NDmNdnZw6u4TeMalT8spLcD8dByZoEE5ljljPO+A/X OkpGJJAwMNwSHv2V/lvyK7mGpfiYQccBDlRp5SHWCt0RX28W6VZd6kMEu9F3KgIh0Vgsnm9bfxQv 0KM4NYxiWayXL2jifvMN8pCgVJ3KN7T76BdDPTb43OvOAXDqxsL4/8Xedhjcfidf3eHrEE2PO1i1 uaHMLWDnTCiiciiztVRq8mY+ydbMVm0TSnZZbo1Wt3H0K3OSJOzF4cCn4dJdCNI4sONpxX1ttkg+ DhVukq2hTxqJ9YDtTZIGkMj4Q9N1HcQJO6ToqbZyzqHNjfQGD2Gx+7A40R7UmrBqkMlXDI8oNid8 JJ8jlFoiSyeYIc7QYYbzJZ8SRlqXWDrF9Ph5h+Vr7ejGJ6ZwbiaQv+r63YnmqcvIxKgqb7TH4Yxy cq3RR9YOxgMLgnj/4xrwr3czN9U4V2fFYcND9MP2GRC3p9Yv7xAU25VrdClyLSE47ui3y0ipkkL4 3IFOxVqEEJ1+l5U8ULRiCEDSvzwKUHOHAhIS8a8MlkrKBnEcLWEgqupxnWrg7zOpItLTzLPgBG5B KZ2mW199Uy0snz560xSGO3bMYLwW9lI49oRhN2KvaPmeYKfW0XtrVkCYxV7XXCzpNn5j/hwPK9Qa fUZe9khJULnwWgp7f2yjkwjztpDljNWjPtEoVbaJv15vnFyjPLP7mQXRq10tgfplh5R4b5nILnMH feKuJnmALprx6e4fu8rfSZ7yRDnrXS3ox70jQTVZbTFXzj7NrY/VKTXLmpDeeNchkiK3KyWWInEy 5hXYrYIcGxuLkcNHvPfdgfZesZE88H5MP364AZQnq1SHJzcVwEXl8sZBPiVq71Qhl1MU2tiaxzGE 9J5U6Rk7KsQofHqlHfmJ09NMbisKafuVSpd6mmTlWQttJ8iVfA6yjycDaXczfZAoFNAK1U0K3gR+ a1QfiDFgXBDiBHXdQ5laI7cHecdnRxKr57Ej+E3SNBikMqeVmwePPI+P4MYArbu+zl+XBN0IokmV 4iZcJPL7/qSmIuwY3bMliLb50UfaWf+Q5mGGvVC1/oaSRJSw7SZ3CY7tGn89J6pyGJ1DiBF+DTAh VZdynkJWgWxSzkV3IEwmjh4FL/c0ANHJNMy7nqHkHQYmc/NA9Htuav5db1oDcs8J1N3Av8boU7/S idcC0fSiOXpbkD9+O5ov5PX/+FnudXhY8/yOoUd7njepuedY0ucHidflfssQNhTiX5tCVXURtSEv RjWzEmF4UHLyLpuWho7ldTr3ysiEA0QFVSCtzkBwh/JKxk+H7E4lMtDhuoSr+u7XwzSUtsAuBfyB zVzoG6yu2F/KRehtPmveWHipHIu00IeatKj7udzMBcNaYnRsmDcqeFY4ngJhEhql/EdSCD+j1SVi mmarXP8NxGKF5hViLegJkL7otHPO9kuA/Rlm1tZ1U4Qx9AkDxg+hJrxxiBHhQLP22XeTdjbKt4vI n3VnqyLi1XqC6YDtvpJQ8p6ZDSy+s7QV4WTHqdQFjFL1P/DnaHbMyI7m9xat4kYS/Q8mISA3URX/ DH30IBWkOF895dckLze19lx5h9E6BMe+WLJkwSjVImbPJP9I4v+LWixjw7pR5AWbJnl12+MRR9X1 Wha8ZQV05c8t5D7Ki5/mRGli/+aWOsHVTsq+5y6rKUnm1guGPIiFuLnUciBTbJ2Jm8/48dReFDSr INWLyW9K7r6xAQm/Va7ZK59ozkRO+LhrLHVilwz7wL8Aqa27CwTd8iChHI0v5/wd/UnwZkmPcb25 ttQZL/bm4OfDRAEMYEQDXmJnKOMAu1d9vYjRUb0MTCO/OuyrBY13gYtDEmHDZKPlp0O4B2sN0O/J X1yzLdACi9DTtL35bKYqC+dFv3An6hKg8ooz7Lg7yBAOQxHD/7fUwDiWws1NREhmGSXd+huzoQ/g 7PMpQoIf0LzzzuCQNhXA2ehv+snLDyBJtjM2Ybh7F60VHutR1Hi4ksW8R0KR8OWThEOIFtp0JzDR aufPokbDNocXlINalcxVRqPlKkcbYSd9b+ANLFdpff2zSfWm90jy6IzH0nYMMclV21CBKK05Pe26 mFlQ2UoYHv6rkgxy2BBrSKCVa7caUqvrL5Cp845VWtyvqY6uCftcGjyq9jSbn6FjHvlka4rXgPYl 81b5OIJK8zbs9/DnFVnmVEAcfbFn2su6FyJzCzK357W98VgNvzQEJILT4rqx+DPl0kAC3vV4gqsU u5WsPgn+KWsNDpSeDwVktTSZcMiCLyMsf3WOsi48d9F3ZNm2Z4Zk9YMdMApD/goUIqg4+EttP/Rd 2qUuEWXV8NJZUvK6rmLIBsok7BIutyW9P4/HA4hyQiEqRviYMGtxE/fNAtwDNqPyNYW+So/V2wGF I/daZXENu0JjeG2OnLmXlcEHc2+MRXqXof6bkYrnqCOlAkEZE9Jdhxvaox9V25w3mbFc+bv4ZO9+ lCXXbpZKOPv44cV3HMnfaSKXt0fDaMuAcZwjaPKtkMc2Sl3+VzRrGEacRdmjgYXLnkHLO6HcGfx0 S+W3YOa6PEEFcQNAjeh3YctaLpfc4seyCt7cJNm88sm2K0k9wK8zxpprap67KuybJxklo+/g9ThL N2mY64PMLI4kP0RUDMZof3vzA2OYQGol0JtYZNurHuMEO8uIVmh3haDcDOyfle6YumqvoXQa/bly nKyjITsdSfju9vdsZ05CRbCHONV2/2fsAgdfJd6aMqfmKARdP+lNGuLmqA+s9rLn0+3M1Celt2EP IJgbjOZI/+zgH+8Yg4ZVSquTFQOiduS06O8bU6xsL4OsW9jpcMcgT9jnZ0czRfionJLjY5Hwk1GW qkDr9gs1JFrxI5gIt3p/D/4kmBuxZ0M/pUSlT0RUoZlclznXYbAc35TmEZdYgt9ra0t9cUeA2vsa 9kRJCvlrNl/zI4EBOb5cS52x7zOzSiGdodHcGlhtNw7BqCZpmI3Dt8a6NJv4q06fhU6+XZkgM7JT TqRj3XyyMeRPYXFBoJKt/TQNvjB93zaCzuLIWSla5ZttFbMiPNgmyDlSJrfe60rtDM350fS/ZXX0 hKjpGaZ+4YpM/jWtXsexit6AXxiZzH/Z1xXpTkvlZ/IZXylqB9o28/c10CRUsQUFx4cVto6EMJsj cWU5erlA3MMFVBNuZVx17dvoaRtuSZc06oN5NjNOdPgomcjLXY6J54WoqQM+pItcfNR1m5k60z4Z uPdODiaMlw8E2UyMedjLoOoCtNvWRVfQIXYU0/sLJPB70x1YR3AGvwNO5XuwgcbTNb8UmZZ0L3wy zi1gUacERWSKjfoXD6BfWV7WMCzhocFjJXfcJWmNS0HZYQxxI4r/WLLQVuvcek9qqINR2X5UxO/M IcWUD8ugMjIYm31SmuqRD5Ox207ZF052salrvzry0KlbhWpL/unBk+qeaCXULPH82Gyvg3Bl6FaN YFa9TxHNRvyV1Y22S/PHpdA/Tw1H48W+SI6UKOxvp9OT+8miQJXTt0Uhc9NcClH/xDAchyfFxPDy s1kW5ZKKkXJ0i5Z99FHa9CK6M2ta7n2cjRwiKcnnelWcquOBI+dGfmEP+2/cFrtsWD/sP+8dy8E4 gZvp9wJ316G7N4EunpYE9Z3BPdKwDZq77wK3v56OZdFZBLIWdmKERmit/F7wuK3l8sPK63HYQ562 TE9PE+zQwjFSfavdOqMwd47BFCvbX4tvOALVUTdNOR2lWC0fZZF9hQXUyUYmicIgEib53f+y0MFE 810nRIRhoYF3SK4Beb3YASCUBu8l7Lz1vwJC4ufk/jpGG9CxbcJ06UJ9C9JgMxbRtA0C8X80joCt v5NI2FQpEOLH4pgVc6hi8QE35Kv987AbUQ9E5Q/1HpCk8M9XguJMTHIIyPfhqcm9U9Q8VbU+DWNh BhylsaQYE3SKOP9oWwO/pSQPUJRnTV0XcGrSEQCO9jGiBGNvlPu7x3isvBYG29xdafk5Lkv/Kvph K4TFOm6TcO5xS6HACxOA6fY+OgOqXvYVagY0Yb4lF5yPs1mTvsi2Dksim/TJPmscThYISXtMuzM+ t46yCy1Duj/YJwsj3oi2kSQghOV8ztsj+1MhSLiFHFym1FHiK0p7iOW6sX/EKoRR3bqLuy+3IZ7s yWItPsi4q5moEe6qu5OkA0SLJ+/9ULpEsVvXIH5EenrvNWjtdmh+zltqGJMCenFI/3geuDILWH6t 16XeZzEj9zbHnbxPw0F6AuZY1e3QwQsEvLGMWpAazfhm2xxuX3LqJBRn+EE2GRwlJrjxJBhjSROX 7jhqaAGmgXXa6aCOhaJqh7llow2NEfK8d6W5dMkeMJW1zurgXQLfgZp1+plRWtdx0YwXjumiaW8K CEjMu3v0lw6NOHK8prfWeEjcI/qFOOB/Y9W53M/40tOelJc2ELkfrebpIPy3lGYlTuEs1HgOHxae s+mW6rNrk9GPbDNn88/iAUlC9XZTvKcpwNKToKR10rROH/chml7kMvh79WhyDm10/QbTqRXtgdlO j0I+FyqmZk8FlKyYijIdgita/aN1v2bNfBv+fOU2jB0cSS2z6+mxa7cVHhaXQODR2cr9osCOJPDv CCO/kwM7bm7lN+i1RUY8AnO1rO6T2H2/P7NgGnwhf/GiiBSxd4lYdJu0F5lhMnXXRIUQ+yLb63qX ohrhhm+Hl+picEhcmKhQ6frAgBEaGOzynGqo0UNlDDV2X1wg1yCj/8Di4UG+n4AxL3YSH455sxhR pY+aA4d5BrTq7gaQ/43/vG/bz23Qvmqi4GkegaYUbcinN8N9pbrKm39fBgcwZ2nYwACg/M07OYx4 ira1x7RJUmH+WfZOTcRLnqyRcb+ZMkjKpkm8lvs5iVbT6OFOdTjVoWbz6CMWHaYEdTX6Z6Alp+He heXQ3KH/1EaYmLu1JK3nx80ebqVrv9gsJsti8NBNI7/lD6vRUofPEaj9OZEhd2rbtzEnQAQvsdY0 e177YZ2U2od31iOwOE5d9fUyTUAYcwIWhloBF0Hv6h4r7+QlUfXrfF5zNO4ynzwyFFpCMLuTOF56 pciMXaG7xeIB6h0MiqLweS/8rQLSUEKqlv47IUZ7vDkpmGEhAcuW+JLruEcRAmmGY4K4qf57DlkT GUJIfhKAgDWbgyXItEE574+6Ejv+g0ItNe/g5zUYRh69S6LuDEmC32T4/u1S9HG32PcQ/pvwwxUD gVG8hgjCm30zRmWGsNSTl2qaUwsSnyDmwOhYmmuBUu4GvQWCFfb2xTMrVkFlV6FbiWOcoNgxj37N JwaiUveyi6DXbTJohv69Ha3JvWRTw6tUvo5e7If4TX5ykG14FY0SI22kl6UKtnYG1UIkS0jL7B/U iyRIM406Pp41eB1WmtyBGNPhW5Cvat83zB0EFQ+BwR0Lk3awtA4ID8D8gm6ct+p1zvuM1uCvlBtK aMwEJMSATcVs/IWr+XbeqUHsWQW6FhW+yGwxrKTB2MWmrWTeoXVDpyKM6owQ9nQqQ5FlIKCZLAvO YeKKygclEdxZTpDpHh0XNHenflkPH9T/l3IEWohbAiZgzqnrNRlpzu20XnQsbM9bwp9Ga2bn/iGr Z9ubdezTm4nDV/bkFoBJJUMEXDjGLyfL617meMwSs07jgo1xXZ2q3Q0Jg7aq46d9bLbs2En5x+cZ i6BaorLQJ8Zv6dmkHO+BtrbFxfebqe8zZyirWTWOxUQJyQBAzecjU1kgHrrufy4OYjOURnGHH0NQ lm9dsN02AInbnSoGcKiQ0XulAuSy4BegsvP3U1J39GjLhvFQXUxuXU0PrfLcGhEklP0Mqr6rAe6O gHAL8ctoKzJVT5O7SW4tzrsIsIYnvR7NZd7vFN8IrGS3guG9VdrPL/gmIPcZFQ/fYk9aMPzh1xew UMMOL6Rcxo3PviSzjK0jXrgfK2JSr3p1N1C6VXrM5M4yEYW5KQBo/g2DOifBFY2I4+RhtDPDYTIS Doc1zRUXeWQ70ayrdkz2mrOVvz6EczSBi5wCnBWGhRSJwCeOz/s64EU6fEzhKA8penJeIqiohMLh EqAotd19MLrJb4Q/kHItX5cvicobzxoi6bYrcTZSvNo2W4SoifVUy55XQQZagfQbsfj6K/lucrHf 8VlBKWB74jmpbBjGltSWiWJTrT2k+hfZxn56qNUc6qELkgzVESobFtFh7DF094hKRxgSO8NwwJ5M 8sLY3PfOFpVRXYNdzcGwAEyEe95t0fqNmoG1mnQaKRM3YDG4sTGY5xZCnXs10Bk2tymvrY9PUNW9 igEw4EaA/GfKM8r/GbssTFADcMsnocLV1wzeFrT9HRIddkHn4SHpV2C67MgPjQCuUIE+B4htXFTV jICijX8gHOOVZkNymFSJMpX7lc0p9VkWn8dgN4D01lC5wOsv0b0hAWfk8Y34OzY7pqf/GEyA4xAO 5/VmgRsFy+ilLH176EdSPdiS+mUm+nfLHRC+HIdwJTxuGivBtqogYt8oVM83cfDRAtpb3vmfq8Nq jp1Uj8wElMzOAX6lJUN1yr/9vgx1Bigk5ogpEJweQQOVZ3YAsMoDhmt/q8HidVQZijzDmlfzesl/ cHckQXAgebAQjziuD8AyX5seGsypYw9ADoyVuetvice65KXUQ9WIjIius4wz/P//KflRCnJfQzx4 WXWYjWYd8m5OOwBIBSsMI9GlxKElaPjBlEeC6//S8Z9Hz2TZq2k8RYF1Q9kdtoB6VWcCXGpJRe4P 5blileltn92rpFd0Imnx66lA/Q4D7hxKHo3NMMnt3h1cyU5zIG/tD7DZVYkTVQ7tXgZbckMN3YGf DiLIbgZy3c2BtNc9KvrNTWm+m0WwZrqao2lQjkAgxOVH/NeuAk2Bshty0XKpDMe9RHhlh360SL03 /doy7hw6Wh9LMz1cEgMZamQ3LnNGr0AheKYqYOi3XnNvq5Tt92LOURqPB4d7AGNf/JrVr6jPJSHe E7dzS2u9pPlZM45/8CQ1dLLNEhHDBfciUCPEnDlaKjmuWoZC9MJqEk42E+e8sF/zVF0dP3Dw1/ug MO1d+iso2l0CgLbE97f1veVuubvmH4CVBLxiC06ND8mR6pv1EkMS/625cLtx+R/BtoY2B+4VLutX BdXcKDLMWADGoTSbJqoJrG06FQxeshsn3coNHGQygzKbE8s1ZBb68H9pmbGQVuzEJbMSEAC+ldKx MznGiqrontuBBzhet9BIKdDu+v7QSQwzXDgEg55xjx4e+0/NM2ab997427XxxdAK3SqlGN6tN0V0 dIDSRapXFp5fsn6BM7pCPUKfNq49pTnFEaxlzo3r+fXLOaN5kgMU/Qo6NDrjAynxa2ddEkTeyKgk M0B+2eYwdqhZZ5y7E9bPjwo5l3i4SczMoW6kl0SDAeaPw0qNJg9IXvsimpgrx4mb/H30fKNQSRHr zPW4eXOU2EvpnSkJKzkLRkxfV2sRzItOPgHyae30gsC+/CoEinNv+TehprTDiU129henQl0rl1ba 5TRij06kk+09KZ+HXmmGUu9LHCWr16w3wrrIu24yHmpeJkQNHuW9LrwQU+miEtGAAbS2O/XF/Wto HMwrA04KZripuevmqq7Z5MqV64IrzsRFKU/aV0vqRcG7rIDsjSn5sYaYiMNA9U2rPNlOxnwmh3KB nRW/GILCbX3qsHlnmhf36KIHOdInAWwoijtjXvPvALIUN0N3rl1G6gl5hWwk1qHbLihLsDq6Q3Oo aLHCTmptY7EVeNzm9lHQhs6hrw3CLyNZVbtoweZ1tUu3LzQjIHhXu2tqXhxjpOI29krJs4YwHwHY YHZE1YJIT81P79ptVwTokcFk5IQQLNnygjQ5Bxq9sAo3DT1Yfx0DGkuOxV82a3moraZmdmy2q7gj 5syh3uw6z1Q+IXisUwgUfUWeyuSvxcigIptjXAmldQHXg713wLpT8Vd6ZRLiyf9yVVS3vEqUImfW PlXYk1WZQOQ9sqOC/nNwBb02pESkOBH/4HEtrLf6RqiJtdvLOuJwLy1PLCpCbjiyBEmzPwC5h52U z6DZFQnCSaophFZNJ7ridcH/Smaiz0JhzLid6Yt3kblwWRb8gfuHGB2asRqsaa0q+iW/qY7Mwa/A COz+jSXb0ZztU0YQQHxp8RVmvBQXvaze1zv/KpAmgIGoIkG8n/VYTjTSCJcIud5GIhBeVbw9ckNK JTFywLfqHGMP9bBS4KNfg5Evu8FVE7mtFAcAsn8VlQh46aLE8V1INS8qed7hDyX2sxq4LQDmwVBJ h1U5hUEQm9XqSgN8QiKDc3724rZRG5GM1o6BP2nz6S59Oi8jab9r9cvmGjQEn50UNklYpGKH9dKE gfuFrvrTL1mMOxNDz8CzzB1TASTGpJS61tOf4PI42F888Ht2MgZu6Lj9uVWnAZm6jfZ50je/ta4O rzVBbwDwSB8MwTEsUBfX3UbQ0ple6UJYf9tx0LTj7zKzlRYjrJkWTjIZBbP6oP88znpVbThs1/8f DxVwaukGb2OGnV2s3+2EC7h6dThPSrDjeMZgsn62qZPGewxyBqQzz4S2oVfgh5F3wDvagcLq5MwJ s5p2TGezOej/si1SVyq/GeooWxVVHnivrHOB6IrinpC7DOCIdTcEBYerCi3FBd7DyaN4YqRMIu15 eJwmjTIHZupaUoVPccStsOAA1Lw7ipbjgKMcGlJLNvqWzN1ihcQCJx9pKWeFdVowc3ZSvdlF76su vNWTJAUZNQfv3PURszz+EW05VkahXScJmx3nCAZ5C/lmvM4X4zYg/aGnFtyq3HPFy/59plVhKgvF LM885RB08ZUksyDJfGK9nWc6xsO95MBsOH+HpZlbI47hexSayPvbdOrMEI3vdxUP9GSrXJ1Ug9zD W1E3tgnz6HcGsufKmBjLEQdlIFTfuDOhy0MEsphHjB4mvq+7bapw1pT4tBnUsxLpBsxDdupwkp6+ QZcVKb64uZEOMhUBxGQ4+Qz1IFxxYqdYiWYew7RyqlKy+6CxPnyBgwzZnPtOz/d/l3FZyVb9fZyN zii0lpr1mNpkUuNsxiKUTpfW0i0orgEPyYfRHRP+mgo/sp9HOYvCNmVjU9dKv0MY4DnMCI0p+89v d2vlEfkxa/Xz/oa6JiMRGO8ETgF/X8hc+Jh6eyWSmrBHSSK72oj/ouh+c+6sn1qjR6j1BHrr8ezr j+fbfdx7psY3HjuAMRsz/WWRj56xWDcoXxVBCc4tD3KHibGnQJ/aGZ8oWY/YM8cRIXTOJn1kKGvm zoY1PxA9M/QQ3pFAIqZfPbq/1ENzSY6yfZu4ZxJCB/mgrSe6x00vNjdwsJNC8hIviYcywLYaJUV3 ADb2PsYWBFlrcV8ylpv8AVdPQI9Tq+HrO4jrjGdcLh+qP4KC0COb3wkjWMoRTCBfjsFUEGRm4CTK SyULiM8TQcIwSx2d0arh+G9FHTi3trlIAEFmBl5XhF6nAGIcsG0U8hXd8pW2LMzPqPd2AEMpfAeK UuhaAKmvMC42dv92gxT9QMI6jFvnW+MuracLmSJbRJMO98ItO2DaUoogh80/yyAwxLxZknrJmkUm 6O6QMrenbgtlzUHwTDpeOVbjT3CPqFnMpF9ouj6O4zGTQVrldDTLqxbyLvxf20RMPAFEQRwAnFnM mJJ/+nAGQZf68p3E+/qL0cvzNrUBrc5/369hMHlM8+7wvGp6tvjqP9e4tPykmeSL+ffyV9TwmsEv jWOW187b4Dtp72UZifIoap9tyn+cVYnJvJ1T/Cc+sZVWfSjtRCOwNk5BLrtfCEqXRNcReolZyscW jFE2aZpBk5glqKaZ3ipHWa96g/8ClsgVE25Hh2jZ+n8xlJN4H4AxF1DdRGpO2Ir9BJX/CiekDiAL h1nYYjR81IEy4fs1caU1kYZCvxBOPUK+/H2Jfkm59zOmRMRkJ2aUvlFYG1rjVWqhmqlAQbbX4IKK sOolZsKE7Nt6IX9C6HNQFjiD73lKoGcANoAT7f3EkMXiKKyjFXkXYFDjIM2xRrd1bIycFDKRlGXX TC0T7+O9LqFwDJIUOUC4RGbuGqkadB6+eOd0z4Drmje6QP1Clh3P4tZH8n2CYQutzwT7ebSuc+MJ lE3RxJSokP69/RZB5XWJThJYiQrwvNBFI/YEzbt/mLakaAocU+ssazRDzg81MWm+QNIKU68cPLEx tcsIFA2hKVLdpOEQeClDhSJfha4+7tN9BsU55Ly85gGEC3PYFtO+0QuJXOiUYH7e4KrYdXSFq7YN QeqAm9nJ4A6XH8rS+h9Yco4rNoKKOy3FeFYAZwbNDb8tbotwxdp4eBTSUodOuJMbdd0e3fJxBqj2 3F8kGw+G9+PuYKGIx65ze7D98SjRjTSTvs8FTe9jwwWY/xLrvdcotw6YVRs9oh6HjgOHaWViuC3a Hankd+7g+vujkIAY2ZWmld9vQ1qghVQDGaYBOF9pz//vV2I1Sa3gsmED57Ny3TRz7bwBXA+G64nN JclYNBqhkuWfpcfyxVoqzboeTp7AzTd/DUEvYTFWIJlzwAe7ZhOAToi6Orul9BNT9Q8z/6o9zET0 l5pB7hcbEaiO2nMqQkmY+Nxi24Xkuo/MEpeWsggcrXNsecNDETvvZHdFQa/Xj7BqbBCmiaV71sjo w5UisvZgxjo8IlnkOTokFwyeHyjd63rrgIRgiBGXSMxPdoRUAQhZz3dFE4blxVHdpcbD9y9EgmaZ Qlxac9cjl9XM2Fl8VC1Er3lpf9cokphvOtEiS85mwR4NgJnU9pNF5Cn888YqibSqsnGo+eH6EnB0 TV+f/TZNXDQTb8zLUOM+SWXhTXUmbgvpvV1NdIvW6Zq9F6F067YOA3zqjqnY753FscFKq/0GcyYp RQ7igDsjNsrB7vBk0RGRfUZ17vgETNEDK7Wy3+v8i8HCno5Ezje1ed+HxsT95+eSRbokRw+plog1 3BHQLNWGhogfTLStNPelMwS05e1WdgFMQd7t5QwfaoK3IMqr0CZbsVi7RoPV1WDqM0D3aPFeBPIM nxPo3wfnrDmHpOI9lM/QiX/ga1U8dtq2mSkodGsGVAAOA7iv+PUU7F2Oj7Py1Zo6fv/yvKDOTLG8 l+gukiRMo9FgGJK4OP8wNfJIfBtt2O6PGdUtCZMGcGxzSoJPXtu5AIwzixQHilwdgQqPrqcvg8hJ IKzGTj4YzCjpZ5khpBWLNaP5NiRuBc6Q7THRM1yDcd/pp2dnMr6qAFstUtzN+YhnSFKzpMHXqBtS MCNHZoTmC3vIeoYVaNADJzzouAHjhdy7LRaV5v9dVb7Bky+6XvHmmHIOfGyIZv/m11co299tprTz ceOOwHPkhIxh3m6Po8Kgwp8vpf1ng5grhXE2x+CAQr45LvrZhbnyXnuV4MyV5OVPZZIvB/Gf+k2A pFYXy4wHSrzuurx4wVdQx7nCPsYhpO+Iy6yinhXIW6hR02w30+1AwdlrWgm2IuSrZkGS5F4e3wSJ jfSMFDv+r2f7pz1RaoET1X/W6rqY/LO6qmHj9Ju0uTvxpWmyJoCKyoqB+ta3zJeDBgV6HA1/w2US TKKeuWkCozoorib8SFXWGas3NNKMA2A+7KpbCAZWkOxCO7GF9RhVeq4uD2FyU38N/Pwaz06RFCI2 e3JyGoAdcP2bzFr212eZVT4eWvcaORocYlJwPmjuSiRIvxhvbjV5AaRHMrbw8SDbY5eN+iFXnQX9 NDORNVkPcaoGdKBjm5kPLoiMDW5H9+dmugPZdpsylIywQwtN8rwsJSToWLiZ3i4ejW2RmDLkwXai L2iWlxK/hTYmkk7n5pb8xBavbXPms5Fj46Lv2iEpfwFctxo2TwGx9csBPT8q8EEoJMmlEoSVsAtm LCxSyWmCcHIcXi2738Q3b54CuXOhmY2Tx78OEL6JdDsvkl/q8XTlvHubazMhuL40fCMibkr0DJjV /SgDlhheHqxQ27R1pBsbOBo4WdkyKeGQaGFyUnM0xVyy8BhphAl5S0HD9HPz0j/KuUhrRC+yU99c VAuK1dMTy2Gc6ZPb2upu3FJtHlFgZx6zGhnH1+3kyAZkkX5LKI/yTkRrtiOzuSa8K5HQuwEXYQI5 OOJgV+3uuXRzlFewRdAXY7ZI6y43RwLnvOmBrn5Oi9TicBPw691YeWPcYsNR57WZJE+uyOOdYrAA hqjC4Ti1yvvxWcfgyMZe32QMwQYk3JgsOOYv94j3djgpkxj7utXZI35zktTRO8ARVwM6srw33/5G Pvw9KzkfiyNXMIPk4Qz24q6g00i37TCDCZLVGMVvHah07lutQvglLmhyKWQ8XiPufhGan2BH3p6o MWDZ5Y3/2j/G3hg1iLlE6n6cFXCqqVuMtf3DwMzVDSi8CRE7OQifzto/BV2Afuo3S7W38B5A7Ifk Z5wR7p4s5j3E93OPHR2LK92laFPggHJuFFEoR7LFff0yuXDjYA7OrxavVKgdHBYxGt1TBZlNi5dc Ma4IuDD3W2T3PS0LbKLlYUIULGVItO4NLPDJ9lFsUPBH8yg2SJFqs0UX/26KFqTmZqL/dcl83Knw MvhoNSWJGqZMNbI7xrxiA7302YcAli30wxi6vbuT/3Hf5/RMgYw+vPZt8UrtCxc6U86c0nU1Vt14 O7qRkGjH2AYBszQXc/T1y5ZLN3TiYYcLBgPSZAdw6hV7gnHCCBGpG8Zghs1kyhZHAqQUDw0Rw6/E kPHrrEVp6bGRiEGD+aOVel/nXtp2ieuY/0pNEk7QSxEeWns3+Ut03t+xHJ9/nEvF2ScWIQVjIhDt S3X2GtFzEl85e2XvBt2ByeW4BFpqlv+Df7ooelCahbsk98Cdj/n7iCJ2euzx5iU+XWEHShl1X4pS lCHjczm3F2PirIt+TtRuQkjEz1bQA8TZAFhPSEHgAC5RwugaxxkfyU/Dvm0plDa3A9G0AVrxBEY9 xHQBg9S0E9xhFnSkZ134ltX/Y1JI/vFldehbQlCQV1oHCHbaLg3mArhWk/k1G7Z/zn7jB4Pwo2zd 5iktpDCd+SowJ5r5ccrPCDDqu/eDNQM90PLV6Dw29ezenMLHDJi2qFrVg3h0AKWaEIrdpgkGUQZH EDPriMKzBQwlz2UnsDd2h6/SE+AzO7/nvk1SPvQfLSFlP1jyzWXuTl1b4XhXnBkbqCLmZaxZcDT1 MyaRxg4ObCfD+OyEo8hReDE7+4AIlke4A7LgIYBkA2jjXWXrNlSkJePUFIIirB0LFpYDgAF1upuB VxcpfYSOYelcnAUf3Ne9qiBzUnRPT+tt679lEC/dQ8N56FWJ3yw1+K8knwA+ym0g1V5xxpj1NtBV SYChK/VJnCBFU2rrE9pDJWmYWq6zaex73x1U5/bHmi70M1e/WGyrdYeU+8SOsrM13ple+LxQmYMM N2bqJXLVZJobFoWToVOmGGDbmoGFJYkQPSDZdkwzC/y5PRcq9uGGD/N1sXR/DB/OBFILGuhYZAvf BiG9DiwuV5AyCS9Cy+/UdEok3n1zxbYRK0pTsPXA21GA4jKw4/GgAs18I/7TU2mPuotbJRrg9aiL iiM0GGEcz2s2DJc66kPmADb/thyopG6sAl3aiTZk8C9HCdGZJzGHHM00/WOTuqh+3QriLs7EYq6R FF0BvYPvRU5oAqwTkv1qhzYzKXhmA78TaGEz9YDJXoEBUJBTWWWdyXuiFtoCHJ87dsI4w0tC5PQj bMUTPvxzqrg5w0JjUsmZOuc7B0cwL0EQ4pjYmUFx3CTeYcX1rG6j5pSKhExD6uzQbLAeyV49SzyE vOrBkTTBMpYhkt8PokHuuCTOkSZC+dzgBh5GqGWvDQ+b7Tejgi+QlpSI/ke+vMBlPtFBe9OLvnLM KoDzlCaop8rh1ZBGZ/S1cgsmsfQD+sYH1hwV8wZGDZJrQFIFwGcSGZ40jC8B2pyqHs18xE2bk24u rSnOobLGzkAFbVstObIqIYtcqM/G+Xphd4q4zMlW/RSmXgVSy9pIjpx1MCZ/7HRxDiMJ9Nrkob0m uFMiAiKrRs/vhnLIoCLU0g8FC6HpYkeFCHMbOuemaXry4trGtODY+qmKmjdsHM9DoT4PU5jE4yAu 5ZpqamRCkKIfKcSuLe6ym37ED2N573MiDVjJfphPsDE0RU1zPHkKujrFa1xS7eD9UFP4wctz+iKP 8OTulOG8YGh1YB1m6CxkbcdoRV04GrVf8WoZzygXUia9l7Snw+gkRXg9+RsVczO+dvf2SiRRGxcs YaqvW2DQ/BD3T38ZVCUaqc+It2TPWaqTDiP9luQP5kaIoBnIxi8P94XR01IZMvmMIPoP9WptCPxD lZNiD1oWQjjjBN0jR0ThdTbdH49GxZQr+7i+6aPLPbludEYR2gYfYzw8r1zG75DVqQnQyM1OZIcb scfl9/lInG+yWz+qJHUvo3otiPDLXMNwdhlZvZl5Lex1uSJA4O4kny0++ssykp6/XWA2zDO4jYAK 7p9NGHrUgh4dhyr7Eu2QXkoyj8gt3JxtZtx1/nIOOhUTtNkX/DLfpgYi/GyokcK4gzdDBAZR3PdQ 7/rFsLAsJ/Jjz2WLwV9+hplzzCCB3tK8CV8qWhSOSKyrtUNTYiy2L+Nf5naGyr9WYleM82CXac0Y 3XSuLZOVY/7wkR0qoEyDDfQSW8FbUeQZUnuTigUrwJHEFu6dmNAv6YAJMD6hQBSzjXXGxrDQ/Dkh eYVvq52dKI2L310at8yLizU+j5waYepdussHdQBPnWCQwNgsAZ6QuS4ndz7AlQxHwjpxnIW0/NF9 sIYTL6D94N7EdIKcfKK70Bl1XrHsNI736jItRZ/NXYi/xml0/zTR09uTy6LzNIxxVnAUZ95EAY9c kCbaYib7b7n6LzBrRQo5MzkJOyJlvWjcNTBKoebWL/uCMrvwjQKAhXnD+Xsj3j+Y82mF4x/F94Sj X564idIyJVDI57pHTw9pnDMwUL+eTRyrmo3BWVUsr6N7Y2TnZuSQM9OPIIQqc4TxdhWoJii+1/UB CuerG8BthXsDnvfCFQQbWgOlmZF2XvLRtFma5NWpTaMwiy/04uHWAi2dq5tFjV6xuYR8XiKdIStO oOQI6CunOG2dwYhLiNY+QSuWb3qsaAtfn7QelNmNQrlxg82aJax963aK/jYuQ1wQFNyAMjW1baJe H2OgQiURl9TMroQblyIZ7E+mMIPqb2yXPidOMV7leeffYVRm4sdSWPxy2Ah5JNkEAQypFuqg3X6X 8437FiiEb/Ca0Sag6F+8JcOh13RQGWNVSXIzwFSBDTGlLNlDDMRiDhm1MkXRhuANoB37jxigwELa iVC71YtsPaGo8MgFudkQpozbBvFfkyLNqMTRZnpN6xhALLifti5C37tTyONKYYrA97l6JA97/Fwb xAIlnEWlcxV/3IB3OWuBYqTok1hXMPkLymZMzbol8gGzhoftrJKJ09Tr8NEqkUegveX7AQLJct+A pOWvqvJ6P8G21I7s4k0Z36XCJDD1I9z5913fUB3/h20+h98P0nx18B3UgPXt4COxuQqDUaXlEVjU mUrXSp3AeH6D92HCpzWMQmqEza+VEMldZpa5aYkLvX36nHWsGz3JhJTojrULIy3w9/zJ4/xTcpt/ EAxVxni82BWIm4hR8lWoI48XRRgNm5qKA+q6/fBEUDSYxNQR+v3z9Sjva+dxB3amBNpImkOPIBbf rw+s7iRRvfJmDOtqMcHQCDFvddA2aspp8MyqfeXa3QFx5ajswNsy5x6uG8oXWGMOnp8m1K578WOS 5s9kJqxFSwZ0QOSQNeFlakEHL5ey2B9MlCvi8PY2kvBpqkNgGkB0XKEImYMPLdBx1UDM/OHBFDCW 6DHRVy81Io6vP1MbtIn4ofNXh7kkCnPnsGCazA04BydA70rfQsXZFb3FVqeKW3UD8alMvCUS7Lpj wrItIq7TQyDx6H76/b7urINGUB0V5cOp2JZO3IONIlpCeVNgMvXii0ehG1hdWfVt3My4fuUuWmwm 5i2s/quoJAeVHDh85+Pe7KZkm8b/Ct7Qh3X2CA37/zFMGR7j5pNfiOiZrpqHj1DTbf+NYUurbJ0e P56C3/1OrP7G8X68H4AaL9QRNzvdyRyJAOnjdZ7H2a1yAINCCKanpbYXzaMRBl9dCoC11LwSn4te IGXfr1Uih5JxuZte29jk0lwVaKHLRztY1MeRntI4YkcWEFEs3ipr4F+Ad5otNFp6DeAiGLfMyC3A 6HahnDmbAimDrV7xVDn8PVfLGTeNLxaZf4A7xdVsa7AGrqHNq+hnudoZsPgcIOo3Ir1rDquUOl5L opvFlycR/uVbmTRRso1PVhUvkXi/6BTLthKj2GXwZ+4eSnFxcVRwMXlEz13JFz193ytB1CfOJ/wN FW5Z2AdjD8DDnGl9+iKWyzsb7tojQR3KgwvoXLLpqJvEr9N7fa4905cIKqYeYzjPNyeE2WbKJMFT hK9HZvr+kvGEjBCkBvZUc6WDSY5BRrnU6O+YtDR6gyC2t8nE1xt+gOmEC/TjZCwao8FdUmf/AB6b fSVVd6mnMFSMkKKNfxkqfj8T1z1voosfEU9OpgN28ZA/eE6iMtMRwzsBINfJSc/4BP7iXUg774iz c69QacNVDt33bImfXyKWIiilDAehB3nI+1q27iXpzCjU3qLFn84JfYurID1rIcoAFgLBD4bR8hiH lcGZNe9khGKkbV+dTKnZ7B/+f9zVYgFidOwORR+J7A+afxcG3dyXotZAZROMYyj0EAZ1TAejQ4fP 61Ph/n8RWdqzSVjNUQd0sesMuUU0vc6Kv9pxaTurqH6AMsZ2Fxah6DiqLwDLTFfnCyzrN+nB08y2 fuHXrJKh7KB3xnSlGMiaPSaNXloKydmLd0gcUCSf/whWfSaQY267YT1uMJ3elAlq1AC9ky1G8sEL R5zXqSNSOhu70joVYqE7EhMU3eL07Bt8D1/UxIz6G5IhIijj//BB+YuhLh+YAKTS7lIVDBoSjGJi ADirSlgUuVyvMe0bs1L6O8l+2PKoNYnLjSvrSQg883yo8WYZv6OiEcCWdH3eTuZYavWThiQOxJ7n /ImKNqW/SP8MFWoR5ms7mudXmnarDOVM6OqvuZMh0TNWrjxOMRMgsWFcXK2arIyuJJHxjx4v92Sv dj4mpExIq6WeGrEFz4RZDQ0JDVqxb6kzaJZYuzMRZlBKM2LZxm2YAPXZlwWzgRN5hDojrdTAMmrR mx7hr07BB/tGWTaEqc6M3UIeRr42cVc3aZsTYbDab8ToPY8uwqHYsy06Myk82MrQph/fbVnSBHFS A5hfi9T43tJnZWRw7F3Jrky9Jv58+rzc9YwcgCO/6QKbZX1B6Z5bfsah4bgIWfV2jPCmcvUccfEC wm0IQ97vjtWpKdKaomv+6Ob52iDifLDEWL0i/zM0GjWc77XsIojXqSkuDgYTLnQ1vkwDbLxSgEM/ MQuyoNVMWee9Jp69d1wW9QCblZkzCLNwYGETBOW9MVEEEdNvw9ggNc6IcZJdFFtosei2ulXx0oWH eyH4ivcKzoF5EzH7OS3LEHFip0Ogmq0r5TD71m+QP5Zn7WBtjYFXD65wIgrSNwtpqcPIgLWxcIYY W5K1Q+13n+RG/ak1jOPj3vlcSzOz5dLRGfMPxj9tPYwPPgIEl7yC/hPqhLuGgrkuHODkhQ+fYeKY KKGVfSGY/8Hrbo2JWPCdUJjz4CASybXcOJHWKWTaa8nBeV9hcaGl7X7HXw4W8DQPwvrOO0LzrIXe luz32NH/afV4a9UV7y4x1OzJaPp/2D2CaXXWK87bYb4ITP3oasBvGOMybaungWyjAhjT1JyqfG93 Key3uAH1FbOcuggtof+f3XdJXB7+TwoPG8w+zT81GloJbW/MiKkjwt4mRgVln0QCG5YqKYbkaUuo fSy1EyCxoxoHMsWhj2pL7XJghODFLSXp8ORrCKHkSgLemfzGFyFq5eTUWJy5PFvT+Yzm5opjxImS +GmB5ZBEmUqfLmxgLniTLUEQLxhZAxtEf0OGfrZAc+sEuolJegT/QE2UALAKSHcV+lcQ8BMwPiYg 27Gju+x96pV5djJ4um4Pu4sclDwM00aIsuTkPpK8G+Pkc6AHNW+JDKQQ94Z0TA3HPGbUZFtyZQVU AzmkrMykfFeaOpRHsmIolFJCd51ZgrV1ChfBYZsmsJgaGvekzwTJ9mvjO2eDDhbeSd5ocP3cPf7B erN6NUgIjHVtfA+kU5RfD74KWVcp4Us+RnX58Vq0AKwAx4Q+htcstbdf88a/SXKnGUDDfe1EYD41 dCziw1lrGJnFCoTMwuSVytnRs7fIFkhOCP9q+MvIsyK1aqA1owpOxQZhabOE1b6mTa315kVjl5qR ypgUOw3f5I1z7OYsePS5ES26qXj0J79XAFgsKqKw/GQ2WcBwmxJVqtt6+ypY+IdPkboCuO0COyRF gbm2uFyHBEaQD3dWzx6o8GH7AKE1H1bwsKFGa6XatGSj2TrXBm50munats+msKU0f8hPWVMcfJoW gU3esGvhLoc2S/tjUPkgl4GXc8xuvePKvJtDYY/HtujFnym0ZurtsnpWzi5VwUDdB+ZhzAHAY+Ba akBexDVYWTODRTrrC+7f5TrLh9PqiLCZhBAyVfxteuhmT5ceEbMaaNPEi+ooaXi1b9AY/pRwsJfc 8t4J9X1j/ixVqedYqohtgl13edBq5/eZMGQXSpApj00pqD1Rl8qQCXMIbVitAEE2+SXXuyZn1S+y IVqgO50nSUxY1R31FOjHQbSH8qOz1IbFTkkjhKnJ+sSOwexdZsaHX6XPYqAD60PExeM1zhM/9cZq JKCvVMcYxOrwMOlubub7bjMiDXUDRdQvxMkjA+VKzK75P0duLygTaXbhfeMO/+rIElepOHUQCsqP LYZ2nPsugnTS1OJpGwNvQi8D4XQWSXijTMjMU1P3GVEl0/KS71nWfKUXGevMlqxtHii66QmD9rJQ HpCypvm5wLHoKLv73Q8kYHsSTp6HC3ZGrXYQX3A0qHAbo98RI2C+S9qb647+4IoRr11BxXgYjYT2 7vjyZ9l+h9AhiqhFgAy6eELt3hynwzLCouS15r2AFgdNSM8fkudf77USakjTRuCiBtYprAIt46XX KbjCvg3gVxBhTT4QQ8fuOHbBQ+hmloWQMBhitXggyC3lFX6y4SVB4RX5Mgar6bHV5Mhre0uvClrY TtK0I9TgI/Yyk0pBgXzeRpxN74u0J1TY+VfQrbd1Sm4QgoeHP+BuEcA9Tk5pXBCcYZXvO+nnaQ7Q Uy2kniVqvJdGTNFeC4DB7PzbE1o83OjTRPMdHNoDGvWcGaqsZhumHTB84ut9q7ZjIOlg6Ha+HJZA e4eGAkYQKXfKE/++PAl/vcCmDEwJ/oOUc53Thh5uDvQxENVZ5fUp9YDCoLLNPUxa7+XudejzniIN Q7hUkkJyVAlGLoD//KXf78wf6jiZZiVIqi0IkJcUL/iCSxkmXU2Z6mFroRY8WMZLCGF76hvsZFgZ gejaZMTVikmLhD1QNH9rhajK13AE7Vhr5OdKUGJKvjhvC+GbztIvNIeKe9ZW56J0T0Ye9hwQNGfG 4sFNfwKgqJguvzYvpL6lHbPhmMY4PIgliXiY/r4mspxYCR9SfUnM0QL+QqevxtdLnE5PK2o8MNd+ DJiJyiS0US8KihINuaFlxUed2pYkU1Recj2YAG8DHHzO777DmBCHuY/ZxQeKrAYZ/K0sbHYGWJdq 8BhXGGzQVdJHMsbMdB5hmSJvTOXIEwM3POLsrqw9JCwvZovY98kB8HVEXP8nanlt8JUscGUIUfes Cxa1zbvHH3RwTPqWYfTK3S4MkfFUEHoPNWGb9ow16RV3w5T+RMKOWLuvfU7c7OaivDsH4B7x20uq Odf32z9OgeRlw9ZNtgwbhIXi2h12n2ki1WNri00k5dou3Y9eeDVrh9L21J5r7T5hhG+7nYCJkm8B HSfWdK7h4iwOQNOMIicpDfINHGxDhK8RH0GUaNeowsnNNA2mu4syBLLrpYCD4OZTcgBNWm15cn52 8UvNPVKVsNEDzi5mUd3nf5sgez1nerF/7/w/m368hptcBUnQr6e/F2pXLMAV1J8oWvtL/L+feliA yVhd7t/l7yw1Dyfc5ALg/xDVw7QGqMJwcsVmRcOePSxk3rk2KKBKrdA3Dv5qyukpvZFw9EJw5+m6 Sl5kS9loC/+HlKG9AVj1MiEIboYeA06RHRJ5egQYo6eCA1Z/YUEfU5FOP9UiVxRrg6TKtiYsOd6H RkLW2rHlfSbQ7U5RH8AZkE5p9PyJNES2j/o3KjlDOTAEielyd/Kv21MrGRJVt5TmgnGw0wNJwO+9 KjG7AKBIwPlszjOHd+FskkvlpNyz3TqXNW8kERC6jjZNpnnB0Osy36X3NHSl9HdtqUZbmBRKGvNO alb7uGKbB4VTtXbft7Jbi5YHuD3o+uidBvQSdp2wONGZtKC0lANclO35GRawxO4qAa8MTRZY21Ww +8Gg6MerPzqqcVuc9l49inV3INDc3hDwlGvzXFbc/65S5+vXW39jwMnr/Dm2gqgcxhk0VfQR+fGS Lxo4axEFiIhDTWyeIGiRjGYLcG/fy04lideaEmmyzT5RPftkGiGKv4uZftbYPYtkCFNP/Ii5MKaa nt33v7elm0yWwMJgNzLpou8mLSlBi3pAXaTCtv/svpNqWDmshC36Y3t+ZRAnmfhuLJ6To1xnzS3x 1j8Xio0NOY4x4A/28/++9Qqt5VoILbuokx1wIe8qMHqGg2Biz5U5re2SsZAlnrQo8cKrVaMYyxEG WvWEfbqVUUUgq9M34EmH6MtabyGFx5rYhL0tsTxw409lFPt8CVKNhnJkYJVV2yoAxy6oRurm6dPV IAjcZxuJZnEplZzOtVKb/HVTM/M/ADcHaTQGDL4xzp3A/Yp+m0EoN5y2PjreuIQzzMkpdkvFGBOZ GU1Rv3CGD9tD1fIffLZvUz77kh777PvAUnxUF2o9NMajaiwASE2jeuID5kaWqjvIde3Zxqw7Hq8M hxnxpXUiVw0tYIgtEQ8Zz8SaDPy7V0LMfkhBP6DsamB4Q9gKiONehaZz0Oy4ArO7EaHsZlswy8B9 gV3jZF33Oa/FF0HfArxbMJsXOzsCvoL1xatPLfrja7DsINT41PTotpvPqASYZpi+75YORDcL6rtq y/c3ol0En8wAIl2LqSa6ivJmrnNHgmM/2L/WzJ+sRXVozPmmGeG9nxyMp4kbPpPEaD/z86fQmkuK 7vvRnW34COYGPMAHukpNvdWFa4/REQplzrO5uZa7bxyS7U9Yn8d/75dGZvAJR89Edi1ZiDFAwahm /KSPvGiopta5QvI1kRmi7RKnoczR20EHIXjUk8AIS84t0lw41cl4aMmmdeqGY8LNO82MU1V/Ueuo rR+k5rtV6YBKCYXkTzQj2Dyz5oD/eAqWPJZSvheCHwFr0CrH4JaYdQU+vQsJ+C4eQmksUeEk7G3E znQc9Im923JmytJau5K40+6AhYNUCpqBA5yGTeqDMkoVJxKFnCYAP7kQc7jbXvCUoAKcf+WeMayC OUOB27gtsx7iLv8MJAR3ZzZ9xm5FApSlGpoIIK88HTFbhUTrg14nxjnIaZ+58tNPhf95Nt9EMlDF HaWSOAZTRl4YOfr6Kt+++i3IKb69Yd+tyFNRXcjh7egLz5PA5mUG8W9a/Y373eoD8x9Hh3YKKfSW 4EQuYHrhw0QWpSED9ppj4+QVn1aqOpZu4UVjM4KGvjVdbUllG3ytsMPR9tuwNg/HDwe3Ybpul2C/ 551LiYgYA2QTXYojkuyrMmik0F3sBPki0PEN7rGhBnMA5B3dRLnaujx57rq887USB11JglIp8M/W kW2v7nwv4830158mQlyIwdnsWS6CMecTPEeTJdLUA6zbpeLE3fptcp1xKk3Pu5emqEt0YHcALk4y brye316MfFqSJLMVlCu/ldtUkUy756aWAb0ZhZz0XyyLR24LhjX5d7CBvmj/ikKAQIJsOGFZDj+G 69Ae74qh1FfDs0xRlBdWa3/+6H26juMZT7JvX0aEcxxyRg0d76DGtUQyjpOM6akA9TJ8SOO9NECl AkDZCOEiGPvf1EWo96Tn/+OtccvlsVTDNzRw42WWQ57/RUXAcXRDDj+MKN4EBHtPOZ/mny49sL+I rowBFQNy1q8lwIinRMJf35Om+HZ0dyIwKGI9KTgU80JC3sSNq7w7KPzUF6+MI+0YP0k+tnTQBpzz T/Ni6xL2XMPcoV7Xej8MLJxi0tqapJ6vvlrRLP7/K8iLAjFKRgTSbl2ROCbhrxUH/RZoxO+DLDWf fP2XxKEF1AsryjKkhBS8X1yDEGSMdturT1cqYzOPNJCZmHfJi0S3Rjq3RcKaDiHAozK+FFDWQYe0 kQ9GeRTUsuk/iAEcX/uZCXqiYspgz87tSH6wVDAohACmsyawNguGbciaITzCyAfoAPsosCbmfCdi dX8ArRMEBbtr8QCqO/V9ShyDGiSRcxQRN7i5t3G1kf3UtWA77lu3JEECOt81dfFNEHVJehUinjoD nB5BdB8f3sInaRhHU8ID2NT1zLRHFGGcRPmLWGIb5YMzU9esHUgV3IkIl35c3zf/KYCJ2+wGsdvz uW93bfNbvLQ7u+7aAP8mAuY0HQq1eAQTmDiJ+yCjSBUKq8H5aRdCEpBLb+Vpu8kJWsV2irzcuVtL 1ayWwPzsHmU18O3+rLZMi6Dh3Q0otXU3mJEhdkRWu9z4IP40pH70RPbsOy3Fn/+aXtaIKp7g2YFb 6gp2k3PQJ75R57hSO7rMICjeIBD9LZEDc3HGnX842WdGmaI6rJiuBMISZntMPKG8kvSowFKvYuDj 8WBXcBpsbyqmn4cU83Xpk8qfAy55cJ4/IFErymKFDuoegR8i7jjRcl6mav9iK38NoOm1i6GTfCZR 4miN/PewQgTfGauOwifcScl07OdgnHt8YR363vqVQ29MogsfOUsMpzCoQkmJhV3+VSr35gik4u9g cxaenNmiwPTJ81OHzqRb+ZMbvA4KjErAwT8fT1nEGVqEw5Sz3MJOQNwLWYi5lHTq7SR4VNOlVuw9 F9i2067Z91gnR89LzClXFd++nBuBcCJ1dlb/QKi19bGtJaMgr8BtxUWnVCDgbLEma4KA0xgYEAmG hbuRnjj7Bqxqx3aMHlTf1qhkoV3UIU6qE1kgpXGa3Z6ZoPLqKF44HzH8AYCyjd7itl8P14ZRjMtd EmQnK6AD+cXdnsWa1+k79Ms/OEAQFJVHp4wnxBqW9rdDAXfhtNlEs9VPse/iMgj7sPsMCLYIP5i2 TozoMU54J9/q9MI5rxLFUz03cQ81vff+xe53Lu0Y24sVMqZOpkexhvcubfzR3lo4LH7dwrJCa2Ne mXBR3mdBLPM7Gv8XIEBJADLmSiPOVKyoQXvgdUxdJKw9Q8F7vfJVwkBSycOsrCsTgxa8EmUg1iAy e3qGdmJqu9V8BwM1gjqPRDLJCbd8k9NppdB8pzAq3OecTown0ajmHfHX+oBVtVIqrQBPBXAkm2M1 +Y7woZUysrd9f4i234pG/1MzFyrj/qFtiA69gERhVEES7ZiLk+jHcSzW0MYMRHRUJVlGwUTCqLuu WzmcSNZiIUjtMZ3dlMN66KqtmmcTzFlVVQwuqSuqf+RffCPr7sE2ga/xf/1H85ZhGLLkgNYJy4rs tUrQgnTpaosId4HRzoWmqd10fsN0A3Ym3UUXDu/VFzeI4neftpLw1n2nXGkRY+rUFeIoS5lezIpg U7Qv9G0Y3G3vIKa0fjBcJmIoAbqPHmUL2rq+vBEKGLjHcjbkV3Xzv3fOiHwdhov5U0BtXPH8xlO0 gIEsDnK2ILz/P35U8evltXvPMLQA4yz5+QgbAaxSYVi22t07VNI7jaT2N51gI4CFDRaq2HKBb8+2 otP+BHH8SDBIeovn18G+pOvMIYUoW3d5PzJdI91qhYwJ98mYKfSCWT6HAVx/1x0A8C+25zfr3dOe GRZHWeq+QiMWS5z4o2TcpZMqGGedru+xOC9S2XSXMImSOALLY57Ek1I6guACxaHj6YygZnRYMBRK 4k1yO1IxZRtv85+90SjIVFFF4mOlCDclsUwsLI2TARgCy80d4uI/ZAfxHOfcyLII3lDPZw4Qtt6S 3/RI5FNlxZNYyFD+gz/qbjgElS/uvbrhEFrmg87xvIc9a8bTl6elT0Ww0goi9zUkMIVJP7CgT0vz x2cTUlrAgJ1OVBT0KZi4Y8BE9rNWQEhwhv6/2wmBuXAwfCo9bDxVidg5AJQlrum8MD+Rvs2ahnte K9uV9zQMKfoNgmkrCe7D1dPJ9ZyJ/EuzyIzvAc8r3JFkx+RCTG8+ik02adM7pR1aCjQerznAdAL+ 50Rkz+Aeqyb0RbKwwTzCHm4JmNekWeHP6486dwfliKpXDuxfTFfuOFAg1AIrTHAhtmtbIrfA+lNA gQ4Bhcrt0W05ENNZfBMJRewPRNNqZ0ytaRhP6t8dkdcTGE5oB7Uf3u8LFHViog2ShrF2GFV31LiX XEXJU3bvumuv1Ni7btZKRzi6SfAqXU6Zc3XB/HiwPnhAm4gq/JSu8LkNJXkKrXk8cF2k1J+p9kEc 06YpGVxpHWf0Gqt+Y+Mcw7AkF68PAbBl8ET0JAO/Daje3IsjKx+YncuM+d4qDNxRBlo7LXWdoavZ 1hWN48X//L98Iv7eY+AHco1p9wZMdFkdLzHcplyyitKWoZw4i2Gb32UpB0B0JCTDrlAPCQ4vuVIH rUIBGem2EhjOlJrkrYCLtYI8+ywSPUYKUw+GkyTLY8p08IHzV4Bot6ah67FGeAvw6qGkGQN1qLJQ j+vy0GNkeWzh4SQPITNeZiTBORqtDAehjlnN/AJfvIa97yOZzih2FJf7m2Tt65lVZ8PrRiFIXE8o H3XtVqJp7XWjb+cAk+dCOGSM//CdF4oCFN6za90f6Mscs7LHrNhekIT2gBwHXlkPc3xVkC2qkxr7 0Jt0jTjlEXSitd4ZDuiS9xMN/wk0QFHk6XkEPVWpeZ9rSGtCj1pUwAxLAJmlTVdxMfK6WgVW/MNz ToXhdZ/5oBbGS6K9pGnpJ/PJU9vg3pAwVOIrHuHJwkIxqmvTZtTiVAmjkWPgohHGY6jJCdyrfXuK FEPEjz5i4rgwe6yVlMas4Pvw3UwZUCpGvjXDnWPsuVvR35D2O9apC1e12RfS5KiKStLOqRpKkTXf Zcr3iQtRWAkvuSaS7awEVjzjbeEC7/ePFnyVvxmIHN5uOrp65A2wlzaLr2eiGWYmKchAP6SKEJLx u1cNbBHfth6WsQCpAGxyrWNV6Ux5MR21HISFHxIE7Aax5O65KO7WdT6//EUwlBn3aZsjCTv2wN5K 8j2bAc19P3G2PtYHZ++61JY2Psqmj7MKQQ6HXXLbkVZZWPudMHNY45wgpjB3/vj4x5L088S/zDxW Rpmx9HT+ltofoZ3NwTG2q/X8komAPp2kIIfx6VujDudorSfLMfpi1E5IXwEDM+nlkBMq6h4dGOhi 5ZXRkjNemZfsKWnwOCTe9V+pkGf/Szcu/zisdf71sjWdHlSnOIjbyBCBaPuskOBUU7aAiw6KW3V1 B4uLvhS9N2KzY4/Bujm0k8zdurV46Km7DOSckpjX7K4Hv8SxK9mIBj4w2hoguzEYJDSCRUlPu19y DTpjqd5yJTsnSQ/GCp1U7EI1oetTaa7PP4Mi6yNElRIyffvtIrnWLmX5z2ChX3aqj69g1IU1gOrW 65axE3pLJZo2x1tMg6TPLGGbk+SZu1rva4XZk6qdFrCw9gZ+CHNMziXhjYsCf9R4ifhwyIpFti2J k15qIn0PCci3X0UKSL1IgQfM73flTXqpU9d14vxvZTQkkK4ODqIxUx0cNooS7tzW4QbdS4hnni3Q E2C5eiQAfHOrEiEeR5e553lpuaoIA3NNtY0vQIUFl4STN8FN+aB3ELGwWrio/WIRbjFDhH04bMSI KAQBioixpcwBwaaniK+NYZK6iy320k45VZKnK9CsSlIGAfRXizZNzFIhGgtvOYpFCmgfUMxkacxU twQWUoMtmmBbuBRtfjRw2WCfrPS3V12LNzOdU8n7i129p+rQHWbk69pamL5eur2VFfUZQyZDLpjn TxU2zJn0A6xwIfvxle6pCssWsucf1UkEFvMoZymGw92yhwSHMq8ZZPaNSaQILDLnoFpNxSklKojo h2QWp6icjCQ6Ovxfgp+kT3diYYAIu03XGvjacZsVbb3rrVpMLq+2HC3Fh+MK0sZVqkDaI2J5RH37 os+jty94XViF+IHOrHXP/KkmwVu3RNlYtivTuEFtOjHuL8NVE/um3Ie/sINiZ9zs48lzSy4CcBee wAQEUCJALmkeAlSg2cUAwPN6f+ML6PmOFeFFLmEeXdSxGoku3964QjZjwmlcdBvrcdMy8CTvgW4e s3C618NN4XbKiROv77G5Uuxqeup2L9AYKyMrV+O3HBx6SahpErRaNawIHwZRUa4JZw4eRZLGYO5w DfHdwCzvaTuMBME4eDXZkB62cxHk0Re2Ys6s5W3b/BYmtN4IHqFz6pRSO5AVrpyYoHaYtxOUfOCM BsC+e54luM+U5dx86oxD0r7E9/ScBtfntSjuApVpIUsGrG0NWNjKuGl8g+b9y1qPIxFd5O3pMXyx lFDSEreBCHXRcZxxwH8WHdIAykC6VcxqjTVAiulyeVCKr9lkHc0NYK00hTz3JjMi2O5j7kKLgV9u 5jcaTWP04IEBUUG1JAhBZRvXIPcjI6lGFXFd+viAXnJ/5cOwr4+iReKLbWq+zly9W4Bv1W50lazg ry149YCqy+3Y+plGJ4bIN09fRYsNwWZnOAbDckvbvsAlW2ZC0RXqg471mrflNpTvghF49b24Ay1F M1dL8/HN5k/JtB5/ubE1J6nGYSeqXmafaDp5Vx6H0VZrL0BH82C4f/22Cuddxnt1iqk9tg/xh44K 2rDtEpQNzn3Qaka32FwRpwA5S6kQXxNF9mbxYkP0dMcatN0bTn4EbEU6/wWY/AL1VH71yBsGChxB BgU+DbX9cZDMMXRZmnNgNfecqMqlOyT140wLDQ9dSKDLcZVzU77Tw4Rjl4oFlkEPBvbrjJiW1S72 Lr2PGqcZcv0IJ/Jrxp2qwv955eBpWbYmFlCihcB0YFyDydshgx+YPzfBpzXT9SOts+bo5yZEhiS9 R0vdyX66sjNsluMZFb9SuvFM8q1jfJj0NDcxq2IdF4mNG7a4xd6ZxfrxmF7rQdmWZF5g/zpuypqO OQv+mokNICUWhu0SE7WDGuR47Pv6OfIH1SAfcIlhdhwg9DBpIqFRpOWNDh/HhOlOeaeBiYMvsHlt Ib5t8j3MzdiEU/f7QVTr2YsXndykwNkR/9LAPRE7Thecw2WR0cp9U0K+AdkuRPchNsMXaevzWIdZ RSxCBRZliL9u36aGO3kHWU9ApaK5G0vCRjJbTR43szBtp+dY4+PRkGhwi9O5kua/m2sGxR/3VntV /x++8gulrQIqmuInawsuQ+92u8pboe5YGwI6nAJK2OoPF1Sl7xLdH386xuxmxvL/x1LOeVXVAvS3 f71i6zASJhbzGd7ofVR7CfryZ15s4noruXAJRmwAvxjAK+SBVyu+9zwhJ98PD0rcveC+8ILFbnpq bv1hw02aPejf8iAHN9uIoTyQhs6DdJwzUyfdG2lvz4ftFtk/EBtUDmHk7oxJYN1UlNwemZgVqa7c Xnd5FNh0JCJMEpXN1Pq01P525lwOFK0fyUunmfk1M+2luAQca9onZoQTVGvW5VSJrIzGDF7wrEGR 1A9nwUxUL8lqIP0c3sYMCP7neaK8nA9+OK724b3Hkk72JV4Z3/bSwDTJd0QbcI6LOykJVdg68R9M TavTE1TegsowU8zqWtNXpmcZfp5iVS8wTAZCx/A7r1V8G+K++Ist2lCt/frhCn7XH410ZqlGADaW xbVGZxOTbtpllT4A6qhPN34aVIOsQroyTilgE52CBWE3EIr2NQhwELhHTZjFpWWV8c/bwuT9TrWc zGs1jbaV8HEfdE9qQZNWmSt3Odlfw+zWtaI95KwOudtHs3xq7U8v4gNlNtsuRGj9WOgoDeORJ4Uo pW8XqYy6i1tgFKmCJ8Lv8WAl1B0/T7sOgbncD2n4p+nVAzYMm7KsVzdbweE6VVXuIVnRsak6QbTH T7GdY4OxIqK40GaWVeU4QdYHka2XIt7i/JZcH3P11hTMVb+lGk80wX5VBiEN9xDMnPetgzSvhQaq XHqilvGFxEfdhyJOaCutkZ8pWcda0VI/Xa0SE1Duh+Yq6R9V1E54HW/+KrcIZ4nOWHhG8ECyhICQ mvlE5MednIOCoRv16GfL+Rl6JgS9BPuHU/h6L/J+42PBWpZJRO86kfQGCX6v+M0ZbQyuGi+Opmow fGo1HZgfaNfFNJqECHGLfTVQYQ2yrkq+EZugeYNF3q34UiDNXrTiRxZN/hrmSdKbI5BrFyfKklEX e4QXGR39ZlqaRTh05oYROEXgmrGCUA3zZaEhjNke1TWB9lyNFmDEsJdwOGjYwzr1YaNYeqSD6YOx f4KqoOQazZM0olwZcuodSS5a387yJYo6hyZgjyb+qEa3tLgxA6sQdSU/R5O84FCN3pswAbvvkcO1 8HoWBMT3V0oD8smAojz1pGUTRsLI8BOXOP/Je/wo24EklI78Picd30A2Kcr4nrmgBe1ioRONd9ep gC9EI++OACSNPmzW+jV5l0H1i6oxrlAMAG05/Y8avwQVG3vst5xpl3JTsaygPqQ0UAG7/JJTkWGi d0XMx3knAi6k2tLb4s09WPd7iu2RaV1cAJnuCoVYtM79czHHweAj7RRMYlb+c8GYwZOSJd+5q99g 23POMzndrrkRD9TziB9zk85VPuqP8qo1Pe32LqAUSnY7bD8O0kPF4PmTzXZA9bU6sjvr5kYz4z5o ma9WOQCuQYcg9khAC08N0wX4LKpvmbcuQrVGhr0R1og9GSBQ57z2xuXXenpV4y0bAzdqKRG6wOeh 1xqJkQ3MzMGA5yXdc26pc2F1SP2iUTIliD4V4DiUO6GjXxnfqZLAhjTDOq4nZoT4u1TPKxE8tyBB PfZNhOnc4DqsMRkRkep6GXpXU1zHna/3JhHVOvrw/T0pqNXI7B8k9orRTiJJ2C5wgIrD8kVTvXQz 7gpJoz4wcla/ckG6B0M3ofiir8OtRQRcHe7Ob3tL1B8SkwXtwJzd1Hkwegap0NNyfEADQWU6Tr+i UIJBIHTCe36IZaglbSabIBzpzqBC/b2FbWiXNBIjbZcKTht5ACtqavvUhDsM9G7oBaO0BDJLbIeO tNl7S+4GZRf7JwLOOWhZDklCJktfs6YA6d8lia3dMd4juzNbfm+wyiBSpeZtCEtW4K5NmyIH8A9q 9enyRLX5RtnsOqaTuddA+exIB3PbP8lnRJ/MAujDV8qktkjZpDfrtsqlC3/yeCp/XclYcHDAJQ00 f7RZuugHXySFArbXAJ35WYtuqA72iAqCljZK1PxrsY0tevxsv3jkV1qLs0HBTdeD66FFnvNLSrOu q9yoNb5el/Vv/W6v1nsm7fz/4kp5ocmYWzyktC6ecSJv6Ue8KoHnyXsXg7iuIllg3VW5fwKLp8La v9XQYmW7QUus6lxySpaVJ3u1bw8Vk3qY9L1kXbaZtHlfnKlQSw0BowS/kv2/JjzMCvIqGjW80TzH MBPhx8MuBZYNr0aLGktvllX//fWdZAPnms4O0iSiWQ7pZ7V66yuyDGAgnfZQNnUo+G/1CO+yO063 iz7wHu0l/QITnnmcEB9rszOlYK0o9q8FfGAtSbEfcxmJjYVMTEorZg6Ta589CPL10m2nn8BfSqrr CJIlxVXEg4YLRuafxuBYqek/7mqjoDfuASLuuUvC9fhUrTZsjgHDNxPEt8oJEZoJazNU3igjRZzZ 5mrg1Ucaz6p8v1gwwWU31twXrWOlLg2X5ZKOvDJsYMFW7kHRNMTg3jS0b5+LhQEV0R2ClZwilvSO Az4fl2+wG5V5P4HsXNs8DMSdEz3XtuBrZ+0ac5MFryKh58N8dBxGes0YrV+kIaNgvfLk8/yEacrb 9Jgexv1nq0w55Z8oz5g95hxdoHy/oQ4zQISPBhyRQ8ptRQ1vlMThHwam9WQHm7H4c/oblBHMcoUw H5QdnFDs2saa9INvgdEj3xhkZJb3oWiPlcZ96/U48GW2blxcGfyB0ICc12NK9ZfqqyRVtcmVYtV/ 7THJoIBZgLw2fJHAe6R4XFHvDEbbbVT0ceHaLayCO8UdWIyjo53Te7Y1h0j7j2obNm4oW+S6GCID k0IE5bQxJAiJgHFhBxKeSbVkblQxdA9MPg9EA+07zvyrTVkcwAXVQcvVm5BxK9WFhrsdk2VPS1T6 PZmmZaS+HgwKTDe4jrRYp3UQJEgbcvl5RYCAvsq3ojBPcbP0pKhc5c/LDBC8T9rDyjW2nUCIdlQC J0xY3wWERWUBtFi2BJRzBkzXuy40tG0y64svPNnrZYCJbczc/WGdKgldK6DPSqgHWNG62PF7Ne8a sY6Flar7M1wle8Y/gUIbmh7wZgqcG21pElMtJil3Me+ftISLsn47FF3eEz7FRPbSuafuO41dukPn xXyR4X5Jg7QKzHnnz97PFw5f9bGguzvT1QaCKZIuwUI0OJn5RW4+iJVU+ZpecrksqoyQ0mh+4UJT c/HtEtZTGQw8H5yj1Psb0+tOuKFXAk/TPjWcJ7MJDGm43Y3FA3MgFGPdxH7aWAoOLfUvKo7DrVUe /FW8xG91cxV/n3BLHHbDMpNrnJHULth8nZ4H/8UApxuR/dSNNgCGAdcafs75GGk8ZvV9ITKLVPnO +uQgr/bBrWPuW0WtDZirjNEL/UvdaV1LVuVvbzLrUAat83IzSySckrLfpBhy+MWgh/0bNoc27Gxk ZBb6buvqa9ZQZVr6JtYnlxpq1vmLm2f8ZpftW8cpS8nl0zIdIBqDiituAqd3k43KuHXXqhs06l3p PJphlPQeUrNJEmjq3IvmhoDHZYxjJev1gSynk311Xj45ebIjqIp8uPusJSlDPwYLeJOhoxTtT1MA eKFHoyZwzAcDxurQKa+W/CkZDEhW5x4bgCfNxyAH5/MfNLbXAF8KOWnYDWEdGcsVKDsHV2QY8BvU m9s4kQQsQd043raLFXywt22LyJummYRFMzrCARwNHnWfDz1gk4CJxlAiSjVTZ8kif4ZcT9tsJ7e8 ZctiKjsCPY9JXge0imMnhv5MJs5e0nvqJgI0YNuZZSs0m3wTXN9G52tR4xfDn7gOC26t0aaFCdom 6hvQ5ROjss0ugdxgVI40QxKGcTvKbTAug0/JMYLMAjVQ/M6EZ5hNFl3rmUqOXDTCVMz7HauO1LRD fOkuHUVmwgM06VoyNWlQ+FqY9UfFFbh8dK9InL9eweSs6Pz+NflKEPgV2d4HDtjlaV3nXwQLQeQn lPhxQU9Wb8lh1A/n4+ktVUOfChfOsIP0QmeK4AvyD2JbXWnAT6zEf9UB9Fl9wDQiHhyULvBDrTMn LjVAgQKlyGG/SCTzJ/PzC5v6+O1xwzJNK8Vby5G+TmgKnsoxHOrdKlZvUCo7fQpgy9ORTX4PfseV 6jq4Ly8lD5wNZ1GKKlvLejfgE0wp1oHYGU9DCoXdOMi837s4fLJXBcOwxw+HoifohXZEHPhuxLYx 6tKOit4XqvEc92NKum2HGy5h5H6Xw6fABr446lRB5ZC24SdmRISEsxdxxzgGM0Yp0uBq3hQihEJe jr5qLzQm9hUvRRa5wB06eFycdQ7CqTAjhvQxRdvFlT67zJJmpE0ehzrGnYcnsXY0/kUM83QYk/RD BPnv8QJunjmoltmOQHP9QNl+Bn1oBrXMNHjsfEwFIdSGxoEmKqskljmRf6syykHxb2g7L4BLryUK 0INMD4vuUfkcfrc104YfnNEfFXEVTpMCTEpXQ0ni3cnB8YBuuKskfb6GnLvYaKUdWbWJM9uUUqk6 dMToJuGGBjhpRQdy3xzASJiFLsC1L7tdd0TujVvpA2HdCLx8+NfxzilywaVkPstd0YB6qVObKXPL jgT66wKvQYrZHduNiSvrQ+VjVgMn9ooE+Fx6VEhvhCdYIyQFH4P7I9IKRYBrsN6GuikIxXjuJtKH qCwMIHC36Yp++IJAEU3fQQoGLGQGyVAaFDdXHMTO4d5rsSKme3HrAEGzDMeRv7XmbTrKrz2W63mG qJiZYLPAeUZXHs/HJB5+zan7A35uxK/P397OEH8ZTZnxDef2F84YHI32MoYtLk2EzubE5zDZRvaq 2CCjutQAxqA+YZlNhIRGRwwtFW9TGgfb7bsHbkMcwoCc4rNR39GmwtGKhJKFIQZzNaPyLyWq6Jf3 ibLuGhdxcT7WzclKZv0muB7PLFDgh12LSrwembdQzmPkyXLC/4SzPAMtk02ctXW28J2oOzM0Bbto H6XgWmFuoLb4e6bOix/NxGg/b0xgduA0gbx73Tc1toOH3ppv4WaQXxNfMp6GFC6rlQPuc7MRPEsy HLol4DS0qYckNvSFSTggacTp9j4VognxNb/oTZ0jNBuKi2fu1XVMJ5BGBv/jSuYEczYisBcEu4BM 7QzcrhWP5BFPzXwpauyUpnzSfRjxBURMH5A+8sGZpziprBcAXWLRADnBS00Vazn40TAKlBjVKGp4 ApbX5uMsOvh2Y0sbkRzCEAoEle+NGShEiHv5rhL223mcgeJwdlg5EjY+wSIrYCLl64teebGHnDBc MQm4b3sFktIXB5NXqOTkUciNDGgdiKxpmJjQFUFbThlpwPdAOu7L4rwD43KwoELO8CwI7NmlU/PR oa1CyCJRNUrloVRTuhAuL5RMx3eL9j/jc/EGxd6HBybuDceV4/HnUo/w5LHnbSUM9CnBGI5qpJVi u0QptZTO6pm42zPDcTANb9xIfNFqEmU4FS0y+ie/wSUI/ymlAcxJ5jDGZ5GCCUGUwq4vKO06l2Cd sPVWlifY93ZbpYirNsw0jvEbvzUQttlKaT9ua+WmoONfw3TVI8xX3IxXuARSYsv2teQNclcWec6h OIm1Ve+0XgZfAHM1W91YBR9zQ4FTXP1rcIN2TgNSma58yMhI3nZL4Se7miol612W+GLAKn3+Gqga khZs86MTGvshTaNlXEmV1EaZ75PmkBag88tWCMM2Ri9IR/oaG6uhkbsFHjvG3x83aTCuXFbO/ZJH iWBBB+kkQD23POCYAhBqoVcgIpqrZG1p1nGPZ1M6ipvpuA6HfkDZCOHozzRv0MFhMZUbqlV+R+5j uCtSYAassG3XVu/GKesGIj+kW6mmdnjxpFP6kYV012yAqsCZzeOZRoVGHiTbyPpbaEcZX8LK8T6Z 0KKCHStUCPmQrOJThTlNs98tU90xL0QJFf5tO6gpMD5XIJ4HT3WG6V52J/qpL21x1YqpQ+QPemBs mCMNMS6/eJxR2WOB7ASAtprNU4rbyJe6dHpFeKnQJyBKp98MiomFrKFplVhmecrvEO3e4lccVtzj OdDx927z/tTeWzAe1fVBi3tGruGaE+I+6OKxLo+/FsbwJAsYunZuCzcvqnVfgagr7y41wtVhg19o JH88CRGTrN/mCAnduiufiJC2wuXUzli10KX1zb46QKUoTcNW7LyjFbfKPjhH5OM5e/u4C7jsNSd4 OSYOilqpOjQZMqcyiJ/pP51LcxTGs8gwbvY3T0+JJul9pEsLLKsTpFCZ1rjt0TlQSciJv/xaGKa7 lV4OromYIp8uQaOYl/NeiMFMKvJ620lvhchC50o6huLiAa/NtUGUFCa4rYIhD6gzWEH6BE0znD1n r2PN+pYg96LwGInvMCoinnyMdiLlkT5w968A3L7EtsH4NlHS7y7ZwmWrfwJK2hFyMcY31lEY0xum oeshGGZ5W9pOCS7rWgr/NL39yrDpkpiUlCtgg+mcIPJiQgn/T/8DAeQ+HHnNvPlSlb9oX0b3kIQ2 RZdlcAOpRtKsv87dbBsGg9AjuZhWe/CjIH+dmT2YoG1a/i7lEjwmsexmjL0nvN5EQAfmlIuI0S1S kRtWUPGk+13AqLl3DATiIq2DvNQ7rHvTyA6TUxyD+V1qSQyXEdYi2eGAX2JuxSVhX+Bj3+eKfxj3 jncG8ew9KumHd3NJ9dqP9lXuAaHjat+OvmLPWxtJCoFvk+b3vK0MxzrfnDiyNGXbsp2qEkcsWUD+ almmDvBAvT0PXtVGyBtGbZezpJa5rlj5y4pGL5T4aRlKx0iXLeyfKSo2hkucwDgjaAe6OV8NVxHB iKHdaxWDmhRdjlKmlZ/MD6ahe44+belT41oMb2cv2ZzdUOWXf0cgH/COVvZh//2yOHk2mySeegmM HFE7lOqYMLdPdFNd1remREejuQid2QjrTOZ87swjVnkU/3QlNPjl/Q0Lf4QzWM2jrcfxd/DCO8pS J0YHSBivSDDzEr4CIFXGX8D10orJWshG51WnNEdZx+qWErloA9UZQ8RQFNpBsKU7mQstQSmURjSH U17roEQKlPqjoYdL3n23TrsMPrQVXMsNwbqkWfaMm65hHZYDydGIwObKBMFd2iC1NDSN5zLeyERa 2y9oh8YmczvTSc1W7y8nl3zwfti+BQBN+yYTUUkCDheaDAvUUkNHKXb0UYcLySXI9X+CAlHvpy+m KMggAgN4aNTFLUxgxp3u5pAToWCPTAF2438DIp+IBhjfRSbvXYTpk7qHy3dNmk/BPvBavfxpOMo7 v4mne6fqOTUQvPvwg8pLpkGxr0QbikiXAoqltNHRjQ38k0dmIsBTW4kUqSLgqxfOLuNXDCT/TwZk zISzXAjW2iOCc83e1YKpB459v4VcddEYxbm3gDCzICeten/gjDZbL5rW83hR3fG2dBsK/JNVN5kt 46JpJ5LUIv1GTOcpAeMPTId7jXzujbD45SwHdNFLx2+2ENOG/93dbGPxWwHDgY19lMw7R68c1fxq TSvS1HusGHjKjCUOUVP7EOcAuxPfLjkcLMVyaL7bwh9WdsnxkpwQCsNRIRbzuMiClCotYh0GFcp4 vPXrPYYtTguixlOcnwuKZa+rmWnJxgRabjuFo5TKe1iJjjwu58Mbrqb7CFE7H/RF41ObXQwxB39U jXo84hwrq04LUdYx0Thpy1MOW9/xUEqx82yMN885hp/aVuE6cNgOoPPrhqqaGDOSMHVJ8yUkm8Bh YqJMdNsrk5J3guNOi0qqJlEqDgW6gb4/IPZx3Lu/bbmQtKZ9iC4ONXUBRJynMSLyNXi3gdp/3GfP 4RD36palPxBEhWrKkursSzNfUHLB4s8oHcQUm8AEIeC+WiXab0gZ6dXb9PKLtjd0IpSkkRgL2xE+ stdgqBseEz6uw1OS9RmI5kys9vXDpIvUpM+CJVBb7XwPEm/Tl70au8HRBbxlSpI4x5Dr9yxbgFqB LzALmDOP8zxnYFi70nzUumUwyoaPSCuI+9GCAiOOrb9EzDZgwKS9hrfz7kBU5ugqyq/N57pI50yr g2JTXovZzhtkNK65lucFzw/ikOq7hX8Xcxse95+KvhY81BxNY6N8KYSuBEzQJLgGFpbXKxh18oA4 rLPFpeuvJMgIkI3DcaL6ji71fIQdVUZU9Zo1vUSgAY4b0qs5qkTVbgtY3M4NoRMnNFW4lxxkJa4B wohm49yoWHQKHvgj1qU+MaI44fIz7n8wwcUualh/S/0nPe8uXRkPFesyIQ7KCvzn2mJpOZ/pwpov KATiHmJUhODuPv0wHxMKbLHo7axbStBdCsJk33Oz8OdLy1CISVvGFcrnIqOyt3KI0PmfpyLskZK5 x1OYYlb/zFOlI21ANpMBexIG2aN4lNB6RW47rh/VNQoSE0dkyUj0uM+3zzjpGyQTYP09lgNthNBR GJ2BqpNq8VaS5c8IHJskmqaSNQao5fm07l97XQYV9JXsgmrap4itYyHV1VpSSeTstm8JjFgQaD7p l5luaYz9SZUdaOgD1PykgIU7nBVAFHNzmz2+w0qUBUeCPL5uZSsUrHpUR8ojghNNM1lAZkZJTfLK jur0ei/TMyDhGsbCrS5Mf1B+sdwiAo24t8X4DbetkR+IvokkTSjuSORP61b0U8XpxxzRR7UciSmP CF4v6I9cnQRj9fSsMi5tYPWA5kYoT1jABoWpM7IvRyShwFwNEQJabejXHGPcR76rP0XBECj4Bje1 vbk7QrG2d6mSyDETZlQvqTku/GWloDOEuzwAkQMS1RKSUkGH0rzNsoSR4dtx7JqppbJEODjsOi5V AeEnn+MTzSq1MxKTTnsJ6CSdq4sUfal5Rn5E/yGzortTVWlPOD9PJ7b+ki2scLxxGpU0yqfpE8S2 m5sttPpJRf5iu3AwjqWEbsZjQNuv1g3wQbMM2ZFIwoJCJRVMLsCIRlfjuQaLKIjPV1D0HsptbNwB 2hnHfQUkTYdgViImLqFuNxej5PY90jAkRHrjjr3sY3O7EL2rtA7kKitZe3Yi8O/MWLtxUTHZ0Ce5 7YRsQN6jVsIRwVvrq78RdB7BPNxQhPegqbBB+JBUyrNMPmeaBEdapuWtGsawOLgXm3qgrgpKI3hD xOLC6vny0q+wQRrHGLlgfSrf/SJap8xLrztrVmhrRduuib6kxwZ5xl/tR61gIG9//5F0t3xlbQv/ Cm+L1/u6zqACD5nqq9cViiuyHSlizdlAYDbbgK4hLRc/OJY3iPbNYvTSEi7zknJSrtBp/Moo6tYW aPXfQLYVIWvshqx/sA2LeO+s4pmswbWN2EmhuUiR7Xkf+ok98XhVhhMmXZ/mzsBohdAU1Xfo+gGK 7BR0RF5Yf82ynq/ALY3qon/TyhkVt9ZIIulvjk3xRDuxDshpWionj0l5hmchRrC1O4AFUF3aC4OW HdZP7+WByRfbm7GxVativUMZfwJmPzQqnD7dvYHoP7jalwOCvYN/M6J2NXQ0M8NCBteW+0guPcaz HoTirjLCd6d/e5UpcR+gqS37qGP3JkMP8iJy7PUjJ919VNYhkIjwz/c9Y3rXuzbnqA49EA8BqmCX AKt3SZEYc2V3TByjX1ZF/2FAmUXaOP2uqWLtS1dCna/ZaQyoKoCmforeHoe1NCmG3g29z++uwo/y ko8gL3Y7iMQwN0cWpfDYudrk50TF10MfdL/O5Tdyqov3DbebBTUDMmqrs8Lu6unZSbIP2PFlLPJ6 ln4a/O8aOQR4HjGSM+MHsTYrMm/ZxXjh9b21afR5h5AOvfOBd6yjJL75o/GVPfWyRBxuK36jFJvd XnZ9bZ0hqC2XZ9EajZG4at3/gVjDibqnctc/gOKxRKPLyi5A0E4AO2sCovP3Ss+lB5lJRzBfi+di 4RwP5KQn/8h1O75E9MXGkaXB6yHGQOpt+74BcMwEsuHk+sWP2vjZbKF9zSJnQNd6gS4DHDiCK/HM 9kS0vqqwhjA1jMkSliIEgNlpVh4wwEpqWMBOROuMiz3bEZiJuP1qZUXxSYgQT7ym0OeWiQKItNpN iKDlMpMyvokz3cg+NmBpuw5A+Q/sAF/3B2qJYHpUCU2NaFzbJoyww2VmcYwiNmH56Cvaf1OR+nwZ KV9btDfxRy6yTxA8H6nuCrf1bXR+r1iQ+4yEkf8XMAMy/XkIZKYOYMQtPkRUXUp3xHndEKkhF7Hp MSnu6RYizb7+YN3EXTMPYQdeN+9LkB6w5Ev9KmTsT2AviRbtc3Cp6ubNGgZtUXrTRbQXYGpM1xY2 5ZO4xHdT2GA3w1heEbVGHfMP5DkC9PpfMaYVzHOzlWFIBEDJa3kn8hj5ej8Nly/8d+jgQ244wUvX 1UD4tXZCyoQtgzfqSXXFP2xFLztxobbt8OCXf/9bwJoJoMHJSzilOiwggZidtXWB7sFQHp1n7QIs FRbjD41J2o/4hY5thaUg8X1mLPtyNJ7QQ+qBwbcz3/bAA2UPFI6Ej/XVrZOIaK5XqkMQLhd72qZv cMY6khmA1h+FjvjPrlYXtpAoJQ+0PQCFsVwvPE5TmICDp36BlpMyqFqJ1JI/wwOz2OjEsFK2/xE+ pasmSIMpX3GVBD6j39By/qUsK6xZ3OCT80iZOUDSkyjdGFnhgKVvKiEotiVupGC49v6Tmo0ZBTWJ V+Cyl7cOJC4npZwiC4uop4ACJa+idSBS4NpQdeuimoVpq5s8a9SmxxEUWLQbV6XbEVV99DngzY06 v/DUnzY9t3+13Ojl4TDDRhRCBn8EssYL9rGSEBij0xJ9d/VrSE3w0c2f1neTVLsvN+Y8hHsL2B8W Z2Ylx6+Ju9kdEJuhURxFFOVGjQD7Y4NN2kQFLALAYSOmddc2y8QnbWqL3c72lRVK3VQI9WaV7u7d BLpL0Ex4zdLRMkosvK/mKJZumdY+GH25JTuUvktGYKXmkP073jiGlkVp59PdY+soW6pYXZJAUfR1 fvjQgBx/xVN7sqt09YrWm7ayIVORL5UWlp2BQ5REvPcCHMTcNyvzvHwqWoohso5o8AyetNKJ7sq7 uJrMI7NsGAy7ZkPxd6J0AXMiUZcrFhXj8LHYCBir5bls3e0cPkNHU47Zvle9aqRjgrwAeZK85Vcr 2bIJoxr4x4ZxZ6a2ey7WAOiQywyctH/C9/aJLzuUnwncT8+otpIHJHE36Km1gNuiIRBUzjkG7j92 G6m1j0ZQtdFEMo3YJNmjWB6l/e84v3JaI1EXE+4swP+YYC2SAntks3BuIyWz6OdT3lmjt2nX1QcX FwiwmeMm6/PFjKbY5x0NZefJm/ZF4TKnFqUMeq+ZXFa+K8nL2a+Qt867MqpqX1elglJvBT2ZMpPc zn7dhR1kZuFqwCu2vo6nYv3ynbRsI1OADrFrr8VK+n+TmHa5jgIanZhBjdgYDfRpwfAwChTBU4h7 JXD+9K/IhVuf9lxqg+675J3Fmjymllw979NFHk2YJTNvpalxjWc/5oVQe1j+tQY7+7CDB/X7dUJm WseAjiGdGkw56+ILPJvgBZ5s6hae5AqTpKpVfyO3IYjvU3uSeZcb0z5w+hhSfii5HbJjkQxBI8LX NCHWp6fbXoDsOPXB9TNrPh/+z0heMwOvM1610mQ0P+UCvie91hRjFL44io/z8EymDXEmk78K2Iug 3kdvlNgyXd2ZIF0y0IYkm5o+yGQjEXDjxEsjClDm+QViyr7Q4qCpCRCm19QSBULc48sXdCgZAozF rSjJJNqc6r05MOe7dIcdaidI6aqUmqr9kn5i+DGpx/SytbYZkva0u1H7zc6GPxSh0xB+GtzPpj+L Dew3KjVBo4xzzM2Cdo2G/yW4QUD3qv/w36T2zjrq3J2SPzu/fmfDczGc9XKPBkfgqwxl/T/OcZ3T VGsRA/bsx01cmc04TLs1fm+YIZIiD+hAnpz4Y0sslF3alk/0XtEg0JKrRsPvGA3dTSX82YSjIMYT rrYKKdI8kf/cJZ7DUFG7G6hDG33mc5f2+kiq2vBKydlxIal4l5FQwGMxB0HJ9ULy3ObsbcXb1rL+ AbrwJOrWdXGYqOSqzX9HXvkR4Aa6qpHPohItjSy8Yl0BNQa5X5/i+gdB16zX+/Q/rJTGi66k9nyC sby+iDvN3ASxPU5oxuW0NDgUbwVpmfcq9EQhEpa3pf6l35/HMdOiDe3shRhda2KuoBKkrlhv4bks y8qtSweYd20Fq7ZdquhPtioJdGeVMlnKXq2TfUrYPYDbeQ3bJhKG5Eahfi1CFTsdtifOrXNEd81U r2B3VltGRvuM34UW+qy9SM6zGWnwEyucDZZtRa4egeOLRCGoyZr+M7CDPaySBe1TKkkTq1JEnj9s KDmIvdM15GT6p9vLwCuhFbUPnxMnq6c1895s7GmuGTYq7javVmAz1+tqjM/HRuKU5acZm00J9Iyu 6T2ezpUHILqbJ9ttKPPEnCtcVCnRUH85b5SP19ucX5znZwGFcTgYrhh/DJk+Rg5IgW9vnRSeKMvW OLbzqG0bwpM9qfaITPIg1qIAVI3Sc02+tc3FWKor9oTOdC7Z49UGOW1tONnlNHh5AuB8abJYH3xG hBlvKFANDNoGz2q9S5N+Dp++SavrPtHl9HnWw5i368M2OlRZv2aCtY5kJEhYAg5zS9p23w9yNLQ0 tVZ9Z4KNxZODRA1HkZfIjPmQm33jniRaMHnjD4ws2UZXs05qg07Cl7GBPteBCz+4VVlVXtYjPTnD 1tm4KQqHtx1B2tfmcrACXkYH8l8x0/1Qc6A67PcMZr6OasximPZNdp6r1AocAfXVuZO82moCYWF7 8x4ZBOw1RCbI95cE6RahygEY9xA3lUbG4TiyKt3mghj3ZZI0k/bSas78B91U7Jru10796JKjHxD3 r5Y0KTI1ujw1ArY0z/aj7WbzoSA+vxi/HXQdLvOUfUIXVWKLmvPLMROJEKCujf9NK2GpJJryzcXM xbd3F5SswbPezdjcBjxgkCLBgAIb45rg/3fSR1ghCzrcU7Lk6mQxF9BK0Trohw2ozhIYJ17hutCV Z5LnfM3fWsUuDRl+Vo49Wec5xz/kYIvDkCxJ5oXq0HcaMjJNVGeLp9JZ+iQeaE5krzoJVTp0wjDP ilGH1Vr4RUdn3p37moPfH/xeKfxqKflD5I972qRlGhCAIHNdq2KfTw9ZNdn/5768pfGgXLQe7Qp6 DeOhpX2M2ye7iwd054wXIDWZEnq8UVOcybpW7WNtvPuZ+eaf+sK7Hp5uVMv1OYgVIgrQwb3tYKKW GkFdzFf3gy3m7l8ajKNLu24JrgtIo/cUDWA8pwvGutX7c5TpmPFQXtj+wmmv9HUjUO0LJVziCRmG qg9IZ2si/cML9UQ/VcdUWmX5hzHYesw2hBYWQcGDPuQ5fe34iJToevgGCTa6YQoGnfDCvpXE9Q6+ /bTTrHOeFK3FBKttXida/j/UxWOftS9kkvWU4KzUCNJRfuq90b0cCXp8OX+x1oyazFmyu031/Z2Y S3e+jGpuJ3TTwdl3ghLaly+Zy1NuG7CGKaPKJTgwBJckjz8UOf5EBUPPxN6AH906CfNfBIgOhwMm l61mty5TtoIfAUE+ztn0U2OjHypv/Q0HVHgsawd/EFYmDy1kB9udTLXpzHAUkKboUQGg9QFWq5YJ X/eOV51sCJQAwWef3VDXw32cECJh7wEYh40CiJWKru69yb5FMHrkuIerGll0HyZfn69kU7cL06pu hZgTEEREoOUg+l7aYmZgeb+cwQPpJzM5WYeYZ70XdLoy26rT2hwvrD3z2hjWyJqiDRX0avFFPMNW EXvBluax5ltPIZN5KsAySnolCkgOj90PmEkTigUjLeIZhEO6pXnLIyNXILx80+uGECXoVaScUtEw +XF7RJlHSQ+HrRnhW3GoE48BcNkQazefYjF4LXqe6B2H6QeuS0fi51C1iW8VHCO2SQWXrn8fNhgq xQsD8v2L8SANOSm1XnPHo3JieydzA/xiyglX6Ag6k+z06TxYZAsZ5Hh4CyZT/ZXioneJ/nW2ddph wgchvrK62mpHimbIEMM5n7SxMcZO3APaGZ7fRsF68TjCFhs/5C5Boaq73Txg/WP7M0zPIsBUFGhM E9ClBdcGmet2ev4SuMZg18M2jhxjj0zi5uZHb36bp5lXFmvJlPPCADMQ3bNkQmdGUHaEBDp5UquQ l5cmfxiBubqWDNLLogB6G/DkrYibloWIqB15xYB6Z9kqdGGtU0aMdpwypiMmUeig95jeAbaXmvjL n+JdFs79Z8g1c3R38Qqprx9ZM3LLawclHWUsWtH35OBFnWMY11leP0Sw/53xdMRF06JlXIXJiJIT KHoqXWDgacoJNRRZYq1LmNarYOS0mrQ05IPpuMY4ec0AOn9c+EnKJOTuAUOi8iUT4ye9vziFjxTz 1OpnlldupLrL/PnfUJgnzEt39q4Tq2vOQuDgmuAUVe22NgsmhfG1xl+ET0bc17HLZ6mXwR9nmfUE tN2sAZ/rOEiMYCI9+OEG1KmCDAxgb2mp9umjXaFCIcRpT5nqBWthDKkuv5QAaBLx0/jRgkBDFHIf hFNKnDRph7e7WXyWCacs/7GWwv3hTJUk/z//oDSZ2M8I92JflFEpXQWhhRW+r/Hh6MO418eyGBML EvHWAoVXLp+89BEEscNitqBp5EoT9Wpk+0WzIZvUQVe3TUdHJaGHoln8AzLFm5cPTkYGs8f9PbQX nvNMzCAQKtEz+n1qTrw4VvH8+KG/P9Ycxi5k+6T2Rh8tkCnaEN5v3bgxqF8SlsM8vS36LvWzZAs0 YKoOAfLGTkkCgmy+RissPQ0PiktoBH526Pb+lqDqlJJrub5E9gMWD5zy6KXWQxy1dO4s5SpMKfLZ 726vYYSTLf7m+0LLCLFKSUiJjsgiy6w6AMQLT/hsIPo6y2DR7Pz8jiENzs+fQSDGTszV+zPPdF8M DYXorPcZo62rmAi+OA7dw2Wo3WkLe0eEaeaYvzXvxBnmoFNz5sGkfhUySSBpyqdwq/rqU61D+12T dnptltCQbjdblWZVeDoSu74lv79L7BeJ4Nxa6CJj+U0Fo7XzCn3NBZ0O3tW0m+kRe2JYa5u48Wrn VhalE+zOcJCf8RlKkGM1xZPfVVMbM6/eqd+CwktCy+ifnXWJt20ARQ/yYmrL4l6bB7+EkNrPUKRP qPN4UOLUl1iWjFYpQP8eph/3YPH72XbgOgtMeDVNm9LfUGZQkbugAQRIco91xeObajMjnMMrNFzn BMoZwrC5BPQU4HhR8A+yUPX5z3Gha5kuHkUtizR15xknTZdDKkRfAO/AxJ4JH0QNoiV7klRskEpt zmcRZ9okscvtcISBxmYhctk3pD88mrvcMqK9854tHfIYUGvKrtPZ7yzQNAjYrrwIR1dXpn00tZVy cqLmHOrluxjGf5AxyBqcTDmg+CFcEspeDL0xdLG1Ot7/mU1xT8sL6DVH7stw3lAZdZe1/AUV/k+1 6q5f8rN8RJNe9/6aGIoQbomuKtALfU/gUWb9UXbNuy4EIUCrJDnw24qcYVuSrPpNcXRvmESIe+Dn fxqKr9/oQ1+/r75a0ZPw4n/4fB3t4YvziSM9fWC6Ao4IAel0BVX14wsg1UCJ06bCAfez8PiYBqu3 DD15SUd01Siwmdcyiuj8to5lU47PIVVI2dIE/CpsocOgDFXQTgD65QN4NsRJgrTPiRACXOB9hNRy 7wT2iwR1XFxtEqldoV7/Uak43egHu4NYOBGMozEL7x4Xa7FD3lxlzLf1xEA8d+Azubc/pRVcldIQ k5hPrFKAaPA3P3Io6+o5Xxcb/NoFB58Bu7hx4gpvPL3ym1tiFskeAhzjwofoFM3PQAcD9Gwn1NCf ob+j3ojIAmnt5cnc9LBjcLFiwv+z+0/AI6+LfDBq0wABN4iSK2pm38juYTnh8eQHlFeHHHRmZ1zw o0MpAsCgK8ICOOy1Ix6cP04MPtba84cHE4JgQSMWADjWaNmHC6Rpvgh5CtLkt/PNcHsmhVAxRxNt a9a46zZaU1PBjUczfh0HiBfj2Dlk7hGtEHxhKKM/c5109nGrnDLEP/a2hIeUomzgS2PTZ2+R43I/ 6lVDoST5hfFqG/fK3UQbU+cc5QAd4vRLvXcBrevtIiie6mr7XS6uArsfIDF1Rl1Lao25zJcDFhWl wOZBlMKgGXeaECL2NJRzGjmBIRCI8zqv7Sv9YZax4KcwsieVJNXF06wDxj7R01jNofsC4jSPTeY1 r94AXtKhXz9BMWlOHN8ME6sSvUmcXm2buNEH2hGB7g7+jOlXpxRv4TfFOJafzXoEBaSmsVRUZ2Ro Z63wwiypWpJ5a2pVN7wCzlAQU7slKAUZjFyyWlhLR8UV+hRl06vt1RzDslaFbtwi1zYqs4HYEuFl B07zfrptGfxSkoRgizcEBj/cRfkBwzDS3N8aRls/+Lp8lgWy9OdwEqpoGFUFn2BORGdHGaOueoTZ ZC9/aZlM915QtnYjX2+I/g4s6UIy6ZHUcNG34X0iDsEbGr1GjVRIcRnAH/T/9mEBlx/i0BkhWnYo okbzQNb9x9MGvVwkP86iWaz4hXce6wJp4kaZ6MSAokMNmNzVjoMJdrudgiaIorBTmk+JQVvS7sW0 vmck6lvMEM1Lzbd1FkVQpxVOFsYzY/oXEQQa+m35+x9j4SdunlwKNAPb8BVAY0YYCDe/oDsaVWA0 Ar0ZMTtgb3g5zeFJDSTxoOTSNW9DEZO6oCj01GxQcRJlCz/id/pBAEDrgATnhYLnYd8flLI6QABp kVO7qSGbaRwOoLRJTsnG9biCNs1PVIClh7tZa+Yc6NYwzwaljTcy/S58ll1zH7z2vI8SucKauBGb v4Nzf6bowvqL8PTwPeCF3p+BCt6fKsPMMkg6O6fHW7wKCsCLf3E8Q+dTLNqbzttGQPuh6STDnBZE 8xdGNnPuecZCDDnTgkYjvYfvz7DEym7YKz/EgP/jcMdQnBL5mz3GA7EELUiwR3mVBb+Npha0VMbZ VnmBt9lOsg06kd6+4q0tDLF97XfeqCBxK/+PDuNI5LI0Vn49lcd33mCQLjz5sr8f/5lfb5GJg/MD J6spRa/seG+9cKuaQBeDI+GIbLdpzLwibnceGe6Of9gnJ/7kVFLiRfjFqV4vnLFBs45Ffvv8CNfc JkHdmg+c16m8rA3WnDqx1rKkTKx+mlCkzgpWgK2+zH0Y8iULAf//Qy3xkouesHkloutG5IVa607Z X3UYoPFTbBchdWWbKsi6AywrvuernwfuTGaVTm2Po96YfzxcGFtthKDbalTdpalZHl+tgpNglTVS mfnOHB5E0qgxVK+mgY+W2pxaMW7rGhg07Ttf+m0l7w0thiT/BVFJWeTMN01fPrNkfq7uVtqnk3ro ljLBBqObltITOP4tGZVrAOKqiXfkAPJYn/O+kuXamlqROnjfdg86cJf2kZDgHkmOmFJTiK8AVf4l SnECxpL5B9bLLfgdkRUlOj55V0uY7ZutcT1tWnmyrYcOEN1TM/sxeuqQ1i/FgsjHW/z0p15cOLMn YPJAFUmvQqqHyb8nRx7UGR8VXVFsmO3vgVAHLDuZ0E0IiGkYaXcYAkKaIbFZcjh2DSEVo5Om9tfW WUWl0kwqty+PYAZ7ozNe/e2oHrqt0H2Ha+ryeaZhpaCzxEhpFvj5M53Pv8Sw3KrU0aYe0f6wilCK hwVyZxjsHy3AWvCkGAifByAc8ulOshqYjwEUAFVHw1Z/5j5UUNLcR8CQCRK/gojSjMybydEt8et3 okwA4CcMlsdwNNOcD8C4urRrLybAuq9k10N/0j+jp1zYRyrV9Y7Amx/QtxZTvKOdOWMieu1XIzTR kYVX9qc6vWXzrhcvAeO6zKw1Zx97ONBn1WbDcMxCP6OfjL+y0JcPqbsMRfQXxTb4XJzJlrB9E/MB 09KZszFvKbBbDnfcZX5CGXD0iPSTb7CvxzubshwG0wEkHh/buTWBtiuczV9YsVWsGyvZydS5rkoC MXCUeSPprjHJMHXTChjcoFJgORsOEVVUBU2vgRZT9J1ukUAWgJvxJnVZQ2/4zrAbQ5MmMfUtCNGV +SilMAaNt7UW9rdaVx23460gNa+gOEZsMPexQXmo4D8bPJBnKU3fuiJWpD8SqgaIK3nNiRzbK6Ux kSCYFlzQr1vNK9K0Ty0AWVxD0ZI6MjyRFS7osM3Y2u90Ql4Oy1G4NFUD2ObJV6Cwtf107TU3qrVu eWXZAYYro1Qa0UpFXN86UVGhDOz3T/Ze9m3ndiVnLrlWDuhj2sgKa4aFHSC5nrIm+IeW9LexSDpd PyL4ZTRmGVhzm9ylztTNGXAClXu7M5WowRUH7PR4d70PngdpwCWlK/9p/+SYATVX1fGmxLcFgAm+ rXcKW1NpgZ9LR0HLZ+s3rmoKy0X9hqByJ/GPc+Rt8gLfMjagn5aZww5yqKD+if9hM8ekl7gt5zE7 /UCRQk/XkA/dq4KYKgajLfujQoScUL17/qGjUUlwzpZExxKuSS5mJSSrGGtumu/V7O7AJNxzC8Yu Zwqo5RT+iLE1tQ13k2rEaw+FMR42rlCOXPlolaU5985YyOL/J6jXGXM9za3S5B1iT0+vxItDjQnm xcwexKUSNpn42APZtkusm/fnbjy0fNrYpQncBV+X/zyXwJHesxCJI6mK78kWYttlnb6ovRdnzXae bOUJvCgGRhKqRAEySsfVKXZkVNxNsY4BiZKcUpb/jE42N/btfrmNuUAELspyNyyl2Ak162qNJQG8 lmJclFcZOxzQO8U8Q8h9atwEMrLidmxXSkya4TrCie7ugEuYd28yGLu1C7vHgJ/dpYzSoZHOrMhK fOJ4qIiY8e5909TrsxJ3GzHyXtHv/jPWu4O2sMAqLmcdirlWkNLSFSS/zhlHjs0ukIwWcocDLxYF PwvE+7X/ydbWIYLrhfuV+wDBmGhy0iC0mtcBsaViSdVeNgMY1FSsDO/wGwsXuF+X1IDmM5OmYBMy AHB0q4n5XRK/mhULvb0FXBppNgc5SIH4BmEfhM9uLQLrhHcNzqefKfdaRf1LOFFUzcoHZ+zZvcJv QAiEssQsBuTlZ8SQw9qbeV5XgiO/Yo+OnJG4nOPpyKZs1DClOY3pyM0d/vzX4bGRkMro70oZT/zY ADWcC+pNXFqgQIxNYzxea2Alb+shXEtBUPgUtCqUB5Ae+lvJjFnS9oa88XbvuN8OmEeI17DmyApk QxZWUaDSOatJFdMBRS11hXGOFsrq87y+N6FWNseFopnvb3GgvKlC9DUbfqiPOAACZSPALCAzLc5Y xoJ+XtKcA+GlVg/XOa3ba86g4sxr+yOPKw91E5A9lUGSstsrUcwLBAkcBZc4RKA1gTLj4FHJnvDj 2kYbOnMkUr63ioI76KbAE0jPqnXl4pgJT8BNj/KeYoAAyzC/oGvKpZ8AvF+sZroxIkuIbw42BVf2 iiFKiIVIR52NH3hp7E3RhMUOYu1yOI3pE3I2s8uh3whO5w1tPCd+6fU5UHFaIxkU9Rnp1SQ7MPA/ pGmUxJ0hy9XpD0VZM6voNvL/D4GbCWi7V1fIlap+bYOfR9/bdsixQCXw5q/rUmAuMDzBYwOJmP11 dXLINbnUChQX3hy8JeyzuBoUSEDgM25QwE0lE9JAlccDAC9yoHrUbWH77OVpmyfOD5QimG5hVDHF 7LvTpvW3NvvmOheCk3/V82Nl5LW1TTl0a34U11vIM4FoxBl7iD2hXrBKyySgEAqyt0/0mdzHWgUY VTrkBdB2Ab6cC2HYzQkBT81m2B7gx9aAtKKHsZzuhSXqfPW6v76w4O3QUqQ1b8MErOlQ6rmS/YMR f4rgLwPwHiGdAl9y5gyximA3rTa7f0lS6TTDJk2DFRACVb1FEfp6mCSc6Ix9KMa3wJL8eqfnrZre aGT9UULOdEnqo/1EfuMIRGge0lewD/hEA7acd6Ho9QAI+Gw0kBY7rPZ9pBpVmHkpNbCb4kuJIW+U bmUIJJIpUlN6xgILn4kUhSxsSIvkmE8x8VdN3VtOw89RmrxW6UvYA8KUat4KzP/Y9R2TWPX1Rkud Yt2DhZmJsVmRer5LnvzVucNPwaWNgtLSKuezrj6bRksxJAYewA8COcGN2wFC1mKs//Q1yqLTea7S OwuyNK902GB35TSH81BjJC14RT7x7/YwfDD0xrrT46iR/YRVZNc0IhZsX5N5OVkcFc9I4EBhhdSW yZ7WQ3w4R7nghtKeh5IiZ2FptoimgTjTQ5/5iUlK3W5EEQ6J98hkOvRdJZ4vk9zPcLs0ioqOexJy NIqZDB4mBzqofdVyLGAMlqHujHUuE/2zYZzFpuZXVYdnef+xjwFU5Vvxd70tcLlGD1ZhMxbje7Fj TRftBpAHBZVCiVF3EbY//IBSmfo93AEGXSmJbBSVQGL5MVmL1ci5ezvzIKM3kEHMQ7tqZrm7Pc9f UQDN3UGtqwM1NYsPaSJYuV1pAwG5AXzq+22TDqIub/t6kEN6Nb39RYfXNi/QsR2ys2/ElXssEup8 ZpZQG3FZ3yYA5Z4ASIjgnzbCI+sOTQu+9No+uFyVJ79JrCST0BHD/dgjZ8N2k6PNrqMION/wKmM/ 72I6SknHEGR3DBh1rLEr1YOdMuvs6ybVKSgdg4ud1wZmh1wlPVw1DF78r+y0IHNxovLMMp021R6+ NEI6QywW9IhubJGM5o/ZIVD6X0rUgCpbT/ybuSzehsA7W0m3SFHFb57XW7L077TqZ+9+GNo5vuMS ZRI4Vmx/hEzC88VlbgvHgvqqPlSa4VoztjwogCm4+nVHSrVWt5myZJ3W8p9eGC5Jk3wf+kanuWnC 3Ijm+7zEcxcXIcZOeHrcDvRCegRZ/Qa8Jjkqg2ibYa/PeBQsFnTimxymSjd2unToFCYHcwn0pgwq jPlYE9STzX/ER1cOxS+3zy7601xb3k3oYtdSfPhmf9nz7QInbKthyZiO61EWv+Z0TUxHXO1Mnisc 3MRrt5MAeMKKml0hF9qoq9W3JYM94WiKb6MpNmTyPVIe38vDxX8fWwNm/dDkvNT2NBs/hWLucXem UrhWrbhQFNnoLVktxJh2zng9ChynBSj+2QZFIuKncnz9aDmHX8O4dji+ZQEUdEmfHQBGr3n0CBi1 6osM1RfrKQz9AGOIA9IvJRRQpfl6G0Z0/TWGbl0JvdFlbqehN/CuVdMqnAK3Ah7e1pVzADSMvO8C 9m6jvqljWoQi1JKypUMqT+bS6xMU01AeVChCtc52w65RZ6z+AYcIfqTZz6IJxgBt/y2I6gVDV6Ec P6bDCqZ1DY/qkES/rhpKQqfzskx7toK5gYLMAqXCVpRhP2lCPHCzL/YYJ3qyKWPD/ZmjhFtUv7Rm t+o6Wtnf95b/y4vdbRigrrBwKLLHdigdEGnp2z6UA8OFwMCwqBbE5yOs0pj+QoaUDkbAGxFV/4nz nOOkGohe6J6BaGxoDIO+r83edNImEs64SjU0TV+oKuZSYCss8i5cZ/lStrhvWF+N2PUNZkYAVGH9 88yQln9R2e+6a2xQED+5+myA9JFLV+30pW+79yoD/t4WGKWYso9wSjQWNDNkxsd6MIdCgRkY5qjS CZi087Px1R+R7pWI9H/nl/GyopoySym8BwuTLI6WAVmoVolcgzZ78nPPtjESOWQ1lPseaPD7urye uBaYE6dJRebG+E9qv3IO5bB1TPPr58XzR1OHmd2UGZftfz2XzZqupBpAJYU1houuZDRNCwWP0rH7 NN7PTi3tUgJIE1otIN8BSHu3iOjYaILFuyHhAhXH6VvR7QU4oXeOSLeVQ9Xb7uZWOJn3pa4rpIEx HP80E6866bJwnxRfPXZHF93WxH8DVlMJXdckDme/CBuA+nQAYnCWa9fZRFCov5YqkdXBt7wJctP2 zktisOP4r8tcWgIAqRdbnpAF56ZJKMjOwJa60glTk6jEUL6jBuH+y5fzl9i3edJ+UPvfhnTUPY4W l4r9qovNpaDHZ4A/h9WDQNNDdFfi9QNgMxN8sgdfw0I65lux2fCavlaNDEbyUu2g8q5cn3JhYgC/ D7NhwAOpxfBhWYxIESUVxuCrXJExum0L+AKzLLH7UnV0DH7sucLLnqw82hPYhQW7SqF4y1p2wRAn wd4eVlsNwmlSllwmqGrOgO9v/y7yVFtNVNdvARH4aZOsjpZtkAx8VZfFBthUrGOCW50Ad0G8VqhQ qatnZ6eJVMuXIEP4cnlsW7fI5hEl3485lOVPmU9o4zfiay209lYcRLm5qWksicIseqKRx0KbfEzp mlMwjv8H1EUmIh6Nci0Y4buud4Uy94Gatoy2ZkSymVutwYL5tCEQ0O8bsQTQ02O4r4hUTAXwg1SI X+8bJpYJ0qeGzBaL2eocVAd20X4OBE1obPjUKxDdzHybLNA5/QAKNWbNUTR51eoQBr/rojMjlKA1 t/RCCWLENuRXEBEkK6cDYDOBJTOGlsNlqK+euPo/6EP0BcTmOjtHCYwoKNRYnj4rkfzbAWYlCw++ ri5uqTvvhvDQTFu6fKecHAUYuQJHTSGpGqVfpWfi0dfDNaFE4SAy1A4KzCYFZmTFdJVaLbj6p+QL y6jTtCDW8NECS6KlDKMK3JLGQfljfmcSFjRsuRMwlsbyknXvvy/1nAhFbBgqGRSF6VoW7qtnJtuH F9vOICy3WJOPXMFoB2gj4yyL+L4yRkQ1I4D7H2m9bm53FOhw6Bk8dCY/Ou4wXADxlXhfnZF1FbGk sG+Xga6uLgIEmtMDWkgafWot/LcBp05fPgSBmxwlcxbNjowTqsfMbu190LVjMrrAD+crGgFrcu6v Fygw4ozaQpnBCrFDthXg9ih4OKkWEguJReSHFGIz+5RYrGBpgn7rVtIB/hbSHp1mOsy4zAN3dW9Q JYzTJZMF5ikHQmSKtms28poidaIBdGr/06mzKTgQP7z85MbqbLA8YLm6c2fIdL7BEXMWP5jpkNs7 4AkUBHINKKV+A4jZMz5Pq1LkcIRMGillzLNgCbwxsk3DdLuFN016QSKXhlf0bx+9avwj4uzk+eOy 1N1HjrAn1TFgTlC8CeQGA6WuCuPP+p8jMyC9tFifWk8BZCQzyvhGrG5uVFbBcPXVgaNuG1z/bNHX d/NRlASrXijsJoYvotNUqoNXFrqtneadQA/W3m4gHqLW8xuyucHttG3WS9Vh8DJQJ1aXoMsS/scy BYbCAweObliqCfA0QVMWjhFxPpp4ze6cmKih2XYkDGs+4L+xGS7fTNBEds6/kI6WtIz2SqdFqlNF QMl/YiykB3y0wNzungIyZBawidZJdMkfhggU4tVlwVTAKB4RoAaXY254hjWwE7xSwv0rJlrA0xi3 32bS+JYyIienaYi6RURQbl6Z8OySHEazWEpGWbRwg1n2AKT2GcTM2TasVmavN6UGqAiDxyyBrgPA 86oTannTauBctmZvlQTf8dF0obv3O1TJlPRmbW9QWF3znw4gUNwHuC/vAwExQz8LCC+RX50Ip3Su Bh9y0yoIfWlPYaoRRO5RHgYt60gIt6trM722fNh5OJQjtNV3prQrfg/wp9aMKwkRcaz9GuS/vxOL N5qVUREWm0G0Tuma6yVyQa3mtxjY326CB+zoIN/gVv7RJlUejVCz+R7EJwfulEMBfcF+hT47rrbB US7QuhWKy1Oah4Ufx3f/td9VwxJoJmd8kBPsy0KOgH6npABFOaLt86pMwV7ZIFCNCRnGlTL5lvjw 3rdmq3K5F/a4JxMiG/26LCMy1EBlhYc2+G/yB0nXVATbEKNyk977iLHk98JuRbJbxMuCnp4MiWaR jpKkfLwhGQ3V/iGJS6PZrIvykl2dLGR3UOZ/1hV14gku+F0hUwVAeaRfnCIo3BhXvVXlaaEFmTJg sompNrtuwN0XvrdEdbXa5dyga4Qy1jbPjoZiXmJfNFFCLdTmZooisL/FmIJsnuluzioWYq64lUk2 lMncRYcl8uEpGvvbnNv1zhUlRiD2lv46LQJla26LbbBAnqEhD29AgSB6g91Nzpa/oysBnl7ImOm1 Tx8K4IsHHj+VuHL4fJ/nkZIrLcZ73UeoEKHwEhqBZUR8jxo5+1E0S0CxBdtLctJoRfM6A1CyemHG SHFvIDflACiMwJvZWBrEUS9K5RsW0NhsGNhjSOa9hyLB/WCWstraI15sofB8YtvsENJTt9qibCrK P5lZhqTt8Bl1pAkigzs+2tvwvwplCqWnGAvPD9grYCJ7XAGixGHbWzRqB/gyet0plBMOu/wJ32T8 ij61pz8miihLZdcUvq6n1l5GN5h39OK7rkGg6pyavVoN3sbHmOxsaboKfBi632EMR/gpTNI60yFM GqQsEppP9UORyKmEPGxHHpt8bnCITb6Bj46WsGMGFPZiYZzQk6rVQVBm2hlHuBgxhsLSw96sFbAA SVg47snKrymOBsickopL7g+GmbRcEUN68YGGkWfXTcPok4wrUGYIsgFLax+Vhjjnbbtdhk/XhQMd Tv+KpiDCEryOZl77uR+jA3Ohw2hRQTIfTmqKH+2MtK3YErlpFWL7iywRCHqKw8sLaMwHCVpwKSsR H2Q3lU8vmWacaxwQTwsFbqGh15BTA9aG6S5rVQy8VasG6wzLjfvaWU5nDx4f7jHrFTWBwcm8GDpq PzJZE60Wop8jANpGpFp0AyC7mIsg8zl/ls0hly64oLLb5huyyc+FpTcmr3ByXTXDLXErVGklvJHV PgUkjfCyOs+M+tr+JdWNoEwg6+wLnXmX/R6Aw8bpFAz3ujspUTNq5VTAoPDMvSVPVei+FcrWFLuv +UaPJbMK/yAQmfFMlJ4V3miH3pcFRztRN49a33RG9eSn88FmKJLOCidZ7SyzU4MJXNfNS4M2coOH BZZ2mIcpntFfqFe5d+jGUA//dhY6lWChbVbXmc2iTU8VopgyQ5EHhQzLTTbhRUh/O4v5xyzxjDWz h1anU+B0Imwf3J/9o98V/M3v7eu+X++7ImVKDA5OizJ6ntYC5ZI9corcZ0OcQNoulPv67afk26ar ShDQ1+FA7RVfouf4OHzvY9k+geJ43NlDAWJG43924qwMEbI4SkLaFn4dYkqQyesABH55PwMxAqdp XqhVYFbLCfzVxPMQoAaLgGk5iU6sT5X7VoDP6XitZtw1rOPoS511LMaU8o6GrKywFPPbJpuMOJHh Fu0cQPXf7BLtspbkqC31iKaYc05EhQE2IK0BWqZ55SRdvbCe1FhYHcj06k8617kh70isJ+O2xptw G2uQ2YjBNnOpTPD7MTkIlsiXqP3Um8G5D+eO/l4LQ3HWldWyuBXIGSAHZ07rh1u6808MQlcpgejt W/ponJQCb10/EKtQv97hmBc+kBTYalIjkVy+RxKb9v03Yajhf+DJ0+NMHapIlNODjABIC9EJhC/K EtOCKSnamn968ZU/FchxRmvGlbWR3HkaTfzcJEHqVxZijMO4CkOszfSec0b+Jym/KU1MYHNz0C6u vcm30E53HtrsagDx8Hvf6MUU6eXYsxaSBFdxQ5HXxyOEtVIGQYZPkAo+jmudHZBSHUtjWymHn7tV VRvSRip5pudbvA4UmIfB4eeT2KMSwvyXPH5ql4a/U5lSzyY0j/Sd7/P9S9wiRp1llW9wzBA8eC1+ 6hxZX95VQ2rUMdWjVjyIFcjo8OL4BdFItgkGnsjztrgL6QlaVZhPSU2ywciVTlMOqtWERfS7lDVO mDUiK1DPE1HGyzsRrGmg3138KUeiS/qJsEHkSK8c4bR+x0It3ofuGSlaqhiElfTagC9EVZVMWd3V u3flQVklg4yj0x5QKOraPtjqJXi3mEJbf/BRme6ZLBX8oPEkfVTaWDZahEGTXvrYqm3v3h4PoxBy l6HVDBWrPRZmHJ0/K+pj7ufMY4voRfgYEQ2KJzVwlK3IfBZC1KhFpa3sb+hwDdmwnGPDiPD/tBzJ TSv6kY2G5r/zHRBX8M0imAEwlrV00QHqfLZc60rMA9YFGpx76jzAhmK7q0M+YZCA2EjDfWgeiG4O dp7MkUEGRRYYde3IB8qNLoU1WJ33eB1x0/033p4kYoaFLxEk8eWxOOJ1VajHHEuD+2ihjCaP2BQU zUTYgf1e9PFuw1iyyLZJs3Otc7LL2VIgb+n8hm2Xc9ttwu913fDcc7p/+p4+jwx3NVAF6o70cHD+ u922xLtpxmtjKIKgsUKhZkRAigG6646oVDZG5L1m76czydMQOAQyM55+YiHz/xej2GO7H9Co8GVt jUG7vj2Zavtm+OZcDuzD7aarbDpsdXS8boJtul4a1Gi9OXb9a3dSNWp2nMyHo48Q8JR7WgSlm1jD 1I4+tX4NkyGNwmxnOEM0A5E10kttiMLhB1HQ+aDVeOmX8LvVZeHxfkizPTYxua5GD1uvb6Q6p3J5 u61yjcBtiJbLFWwfCxYx7Mw27nd6F1Akod/ZjPk9cwpPxaWwcXjcRdjdztA8mwg3jSRDn6Mfnkgf J3O8WQPBq/jSLJhjRpkl4h2lgsrTGo6myg4JkikW/KlVtQHi+ResorfjxXqt4xoMwzNilGfRcY9t MIpqWzBZumyYcHEA6h72QEFxYD/l38aPgti1RjfKepCwi5XBWCJcU/E91lGMJQ3jX8+bz9WFLe/+ 6XM6sWjm82S/9x3MwLGk+vSm8+JYQ37lINRT9IZ9bDHXWBNTVpa0PaNt56Kt1Jh7Hbc+3pXmPzBd P78gYpZUUOLbNDO/P8L7I9mBcJ2S+ACcO/huJies9ylBoP9p3bJUcdp4d+Od9MzL9z9vWmw9UpNz IzuBwLIsZKYRZ8Pl7lEH0t54Po1dK3ecYu/afMmpI4BknCvOKOFL0VZkgSM1SKwIeOM4Jkn7CB81 xDt4ktK652YA4z9jLs9EqBtTj/K3ECsSsE7qUCQd0Sn/Dfr/ia/IXbDWNPeiWcQNHfxEqY+pUF+V FWdj9LEQlGHXbLeuGsKSZXo5HSqLnl3q65AA1Ah7dASd1oUe9rlz3vSfPPVIB1rYJ55nL5ZbJRwj lnkmhrK3TE/ip0obBjHdlemwU+MeZWrgPRj+YXXx8N0smt/PpBFYxPQhswHWHXp6gDGmDuJvBKqt lSEtDOG8maWFKHOPpX3OOIXwt/OHZFZWwoGtElb7KVExZaCUCW95GGRdjd5Tv+1wXrfoF1eE1oF2 ZEwR+bciCZYQtaV+squeD15KzyR0adoQQnLYhUW/FPc9ra3Bwq7tjTeC1UH/u8qBwbwQk7NyqYKJ 8zaDSHHjcsAt4ANSI3+KP+aiRrRIbkLwS0I47jv3bzzgOKKcqm7T0sl/wCMi73wa4j5ds4sZcmQ2 x6kPAicg/ATReSc5uLwyskxLbs+ovNohC28+qA3f/+Gcx3KxDy+WakrDWzL02vicXTJGgFJUOZXs dsOBklRiTLGX1qioP/yljuPUsfAuuI/3YSdLYbngMdHv8k5Zn/sipFrNKKPYH4Q4urFmFWOaHwPy W8UPxuutn4VxkQCBjAy9IdHuX++D4QWPz8kox6Kik12eebXJ3DdSvdF754X2vcq+YcaWjz2A0BxU TzovMVu5VPJTa8/yihe0vczjLDMVNHvwNSnFv9bgJRxjDmJZpCCw+JpnOmZ83brZk48LKq45le8c UZ8UU4tDV7cmAZZVDejg7SF4+wOAw6DgGO1zWvZBaMmPDSTXzQ9Dxv1pV1ZzFMHBu63CTB3qRn22 N9MNgz6Wjnr+fYtBzxB2znl5XneOu7/yAF1n9eAYFlq4GDMPW4QBVGY3GQF3qJTfeVPEAMKnkB5V TKotS5QuHOM/Lcpd6oWLO59L8rX5pJ8dIMRJNGnVit0/zjzNG4+qxIjD9J0d1icD9NgeCzaAZ0rp L5xCuhrLrNlyNxckV2D19YFiCmsB0WMmNKjhVrITn6L9L8rGc90q71dSycsyE9asAqBU8flVYDQL 4++Fedb37vWz4en/aTyEDznV1bXi763k1CpoRGa8RFdgpV7sGDEJsMF8bx+rY6sFObRVSyaoTN9Y jYhJtpNOYt7iijbHIO/SZe7lYeGy+g4dUynkEi6rMr8nI8iiFaC83EE/Uk5WEhUU4Xd5AblJzudj MDHd0RsvY4mj7Z+rocGI9iXTGlCaDAyrpJd5g3yQ36EOqQjeV5KNbb+79FbkTZgViA5dFUfeSy9x Hi4oXUa2auZX0DNiHfEJPpJRx4NCTPIJPeQLa26TwiJ4gaChI4k4gY0AUR8NvYQd9cS3UizMMv8Y 2in985+gUUwq5lkD9242Rxq9E6RPAADSq80+g8r7xzWYGisHzxWOkwrIO1xI+4QZz4sOwyEHf47Z q+eo7odXr3IwILI4lZHFYKQl0ULKvb4RDBrI0sJJOoWhrOGIMS0/RuUTUJt7kXO//iWdnF2nGcVx 1u9zXUW1vfuiZJMXUP86dMWwOlzNwyy6L3X40XDxTZLQTnlqxc7uGToVh1gqB89v/40YWseO2D/6 0vluVtLQVEQmWNk5pQzcf9zUtVNA58YwaGCyPCkHUziwDi4fYrKb+8/qIoaSJpd3HSP/BTyuYs93 cuh0N/tVhcGziAvFo77l5iJW5CZpjvZvibp/lXM4+3YQrzrapstyhibOkoeouVC/7sYnLA5HddR6 M6MXMelKp/tNktvrxzGorAqf9whgnhPmjeGSpqjDqY5FDd+EbWgGEpvUYlDwM8gVB2fbbAG9WugU ouXid8+PBZuspvqyJfGKrw8u8ycIhke6c7RlSqL0YX4NQjrzUVRHXdrs213s0fj3C+nJWcPTRPwE VPL8SN/VX2U8tz8CQBVNlT/cuJWXjKVCER2eMCPX3yWAjrn1/V6FlYv+46ZVIYMkf4G//1agXDID +mSAdsuJjdNdVMW6aEMGIyUyZWDh8MlpyisIi9CdDNmWHH+fxCK4ogyrBgs5JOPP70b12zyjLWrY 9R5Pv4w7mkAz4RoHBX7MEtB9TqAPbOSntZKemjcctu4kfBeRivmCBpym6s1+uMsjjI2FSeKX4ot7 ivXS/wirga3KhZIEx/1NcYJi195cOYJphN5BkTFU3iUyijsWYUg5maLiUS2hNnKS74M6QcE1PKk7 94AoFr+V1H34iOwhbA4Vrl16BkKWhKACFOkJ6duqGVm1SrCbbYSfqz33OarYfDfhDdNsRWNEqH3o jlkSQkBQfjgY+WoJA1dsM6WdPieQd7sKI3eZpgeiI87pKVONtdTtXGiwcD4UXiVwxQ7qMUxvJRan EUr0lw9y090NvuxTzRCb7O60eIV5HnxB83ACvguy5fGmW8KVBBpfkSNK3bFw2Gu6nBVlcu8PQyIP AKIRCHYu6gtVJmCZjdeTkalpuWwJ8o8Kl9MLPpDBU5MC5zkUSgHYHruHzhkmZvAFGrIqq810cZrr qmgPfw3lUSds7Vp2ANHviSzfZ9Qkbgofdey1X/aHNGViFIebV2u36pDUAfG/OXJY94BqC1YhAzPX aJai8PfFIajRK7Sshff1d2wD+I4ahxq+B5pX+fetCxLdwc9bD+QbkkLmN90XvKCkIEX1MBuw0Ag5 u7LscroLP9vJtn+KEEmspUuVjkY0C+cU14xGpgBnau69IrrBvwmJ69LVb8qW6ly98O/yJqCnqojU 3sIdzwXneDvrrVfSaZmhXOBqsb0EH48HwqKrwU5xLmbWyw58rGfSTedsGolYXY+k6bQmNc9FVlD0 HiYbKgOPNIPkMpXzQPm/1sme13uhtUPIomMcbxi8x5IPYEJDqUbJGRc9FzJiQM27P1FM7M31+v1C llWRC9o3ls87ijRahv8r98qvxUPEUWQ7T7nuFjMC8+ZViGq6RRjuGiu9cvKTyneVwkm+nihvEXST 1gnxH2xbB//7N1QLoFHgKxLu+LSEgy9sWZYyH9Ti2mgTfS4p7esYg9yTs5byX8PRQXIHVnvnBqIY ONNK1p1JQM2LpHKmzIykfG5pufXqjW+mycw6PnCaOjK6DB1/oPQ0WyaJFnUZ866IDLFKTtsiwytb 8o46of6jg6ogKbVYi3zH9rO86+OI1/Rc72OnXJLjsUVmYKGa2GzU1b+lEL39x+0RJamrAuufCz7p tMnA6jXKqckKDjB2UdQxnJJ+qVDcz9SWUXKUXp5205xLXmHpCfhvaNXYbuL0B6w9HgiDuDTmAgYO B6gTpImwSX20hrqLMR7IAQ1KxTymNDu2ee055ESke8W/Y1vn7xDXpx4/uHIT5/IPlyMBVSP4P5Az PooUFZfctw8YsXvJQwUUL857hGy51UaHxwYYxyjYIzyKygxC7EoUgfEz987wMBIlWa3qfaj+Ykw3 GrQ2wKqQgz330U6vmnMYcUYxu0+SNKlK6akzAEfInQQefkL3rWt3O6jwrhi6BhplCjoz0HTpURnO dzyIzAOZQPkc1ACojXNp8/mV98D0oKZidalhfTvr02M4dZQke4rXaVf1tLp44a7HkD72xsODke9m mrK3EEUINaCxkad/bqsMIXoWi6MAWfxPCRZYdQDxwfCnCrcDXf8gIFXnTl3ARQoZNs2cjQzhum9C T5kCERiiQIWk9TkREzy7hQiRNMXCgNaXzq+4PKmxuJtYf0ImMyLQ23+r735oSuZ2EENTVK/hjI7W STqZmxf1KAp7RPbxF7Xemz0kMVX0AId2cFCl7idaLx+kVMrQINH8wHjhPGo2JH9cbSA0e2sE43tE 7iwuJUgN/PT2iVOHQ5tv8YpZCWS8J5jXnYgmRJ2y+wOqKW7iXNdbddUXEYqnvmg4bvLudSsSuXG0 9HsQ5Sxlhpc/5V0vrWbwYpZ+HrJ8/52heGGLBraW5lk+VSrwP2lWpHJDQL2IFkmakTVR47/6MGUc jgW4WoUON6ys1gxyXjghBuESUepB8trsy8fjeFMiZ8JEqym6FxaZEuJEPiqt5Nqbp0PBEbaapzF7 LSwH4rVuFK5S57LoUJlS/T+nREmjHa2jYaKT3O9vadE1o7Uqd66c/ZbH5nFbyPmgA2uJ/PQoaVbf F0testi2lThCgAviahQtRmzVzKeLUm6TAN8G7WBMLTxK5vzRf+shudctMlmN80YqVXFEnXjwNnf+ je8G9/LIxNO6QXVs36rPE1vFznIMh2stzp6KDZKq1l205bCl+VAl6IPKLA+fEgacFLe149PPTEKd bSqE4h1F8/F4EMVXm0E9xvQM/CSBbduuX134yZKnT9xiHIGptC2M5aSWlXqY21sZgpI+5tZa8Tkf Hc7SqTZN6e5dT4W8Mni5lHas+kC0l1rDb8IcFpfFZJDxPRc5hvkIztuaHO7YbqDDwhhpWX3RH5fb LR0+aXeXp+iABFWQDYmNWNKo5PoCDSvV3twqv7c6Lo+yw3nEeuTr2c0C4BK8q8aTgJF4pQI6ZfR5 iH+zq2l2EgvOnc9yac5uG3ThzdOmb4vG5qULNiLxb6ku4KyV+61tSScbZxPSMR49Lr52PlnpECkM wlxROB0zaeR3q6Z7iWtVaGRGQzqwSgR0tfJ9jI4IO2fBvlZOR/DAx1Yt94VxbDvLQe0nhi+KC5UT 0YfmnTpvTmfvw4ianSQzTxlNR7iB+xPdxm80qiKXNZQ/c/RAHqS8a3uxCyEzaounsbMd6zoJBpAY +ESv8qUkqg1SMop64KEGyXnK+O7M5Xna+x2gem3RWajcPmbaIQLS7MP2Asnxx3INpT+78yIERHHB EbLLzQgZK69XiNSWYiGpqjOiqo6CKw6gJ3PCSqzCnnvZYgDQgKTf8ErvUTUgLoDb01+EpWSDUOse tQo2RePSVQWLCnrVyuUUuH1vFxBzo17blL6CZrKrwwBimJcSPfyIjlRiJD7nRNaLzlbCCI4rqM4I a2P/VMzfWkNuUHjftG0I8CUEQh9N1B3yFiHvu0q44mAQuL532v3FOHo2fw9CuhEDA+HLszl0UA8a Xb3FgvdVBRv5xrUZXBxhRjtWIdCDrq6GcKsQ/BiSvoxIWYhUhMegx7VUlUpCOwwofQyYcoM9izBd H/hZIccaSHNIyttpkdN9RNAICAjF1WDlKxXLdy5jLR+1lLFmF8Oy0m5rb71AU5fIIoR+G38bM9hl CrqFCjrQUYt6dR4uioeUduuFb2zeqMc44WIJ4UmZcnKORcxEN7poBSszMYD3Eoe8EX203c0bcs5g bn6dlZ8pA1HR0LvzxEFPUdLBCwCyMCUho0R8ngwOYeuoPHNpWd4GG8p18WNhszSTenssrPgO+xUj 2v+SwyzHTLNtRk27rEcIbroQgJ3CGkEmklQp+lrbOL59maBwK+G6bX4WtjtdVtU1nVxzVnqpZR6Y 0pv0RXjyMgiVKz5NLDttvl2o9Hi76kyL8PytbfksyEz37jEagLXg2n+sifJHyCdVtQ1ryD9zinmQ BEFWIXl0JTVRJOwfS9cqcyx0YkS52V0QVIlkRrJ+nqso1zpXJcBI3a2TPxsjMTjERUUXQbnt57c7 Vh4A7X+xxc0hp4NJkNaQYUh2pntuT3oNYVMWQvUCg7kt9KAv2Fygh21cm5FXYJmdpxyMhTM3+DFU XzORMTNR71p5vmhW16Y1tCcd340+ZvZkEBLoII8CV6/qmqxWggE2oX3Cgv8fmbiDpJ2NXdUYZT9g ZkGJkdOmSAU0fTSDYxxxMnJGM/cirqbr87PgNBkAYmMrnUw6gfj2Rq1Mhh9Tusjycv9RmDCnMd7j mIeKWrC4gQtmXA9eOVhpSz/EPyxuw5XutiUFg3XyOdr4+ewnxoS1rw+3wZMNE9u1Wj0sv82SLvzu gcXQweTFnS8riHItGlYWtunvQ2eopsJYJRFIlGD6NyMUxc7vbO5zsJzwB1DP4PWse/9tyEkCKatU ZJ/FkDNxtpFcaRdBomNc32ZIkv2bZzimve7FbmauQeQQFFGtUrL8SB2XsfDQ2aGO7Rvlt70bQ9EI zQyPpDlBXfniwZC50wvp+yjBHB/fQFq3/G5uNqwZR34J8xe4qXudijQJPVCr97T34x40kxPUqx8e PZbni33mWk0T3G8jQBKUqwUSk5oZjFmq0qzmQF7DCs/1CuAOp7BZ2aLtnd+9qUhRlKPwO8jZ769e yR/MlNwLhnv4l0iuz6ehBoGPWx9f4pEqe19XZBpYidvJ5XkTgsitJc9uTQ0Dlco5g85m9524FBWv zLvuSjW6kW85xCMmDiD5/HJGDtIY5W4ya0LnMWGaFs/1oqi27adTpAb8vV4Oar7eZbdVxL0tmD5E f7WurpGB+Yxa4sIiupTyDw74M6LgmEZa/yefboLiZc25fvVBDEq1v3voJHeFzuikzbhNxiZEDlFq AjUF8B7Q0mRunQ9or0PyZMQPJwPTMFBuTNPmKuMQlyrRO1WnxdpcLL06+PZHgMUvGCYONYYbkEYF R1C1zEmdYdclqElHzXd9XMYIktSitGkkXtFf4xG5dSJMsp6A/wIwL7DOrENtE30YH4bJ2E5/oIFb s7AW9k0PatYteZZwshcGtMcIYrNmChSjn6nkjwFlUiH/ntcQznpnJsNg4irFj3Uj/zhGJblzv1S1 hM0E4NkLA2eM6sbDSSPeeY69o1hOHs38evz4l81Rt+vvzIFcJFYmGa9+ymendNxbtsF57505gHwZ XOA9QoUASNt+o7Z9pTTVVu0ayWcMt7yVynLOic9M5bgUKEWwpspxkUkNKEnTUJ+j4V6h9npWs2kZ RYG6WuqA3WQjWdl/DrFYqdGksKoUweEd26v6Vyleu/avDEccDV9b9mzbdrwbZjZ987Kgb+F3OQgB Vz/hKQjQsau1YvqvlyMnGaF4zGRoSOrF/3DzS7Kw1IYUH5aL8NjOmTH3Utw4rmTfmnx5eqbZwHQf TcwhX/+e6vFhBzshu1c2/5j77YzDgBFHhZ4gvvJzq7vuM6Flq4BHH6nF+dQvrWEevEELFRSUWXoC 8OvzFPXmG44X2ZRmCNmZ8n2D89YLKQgG3L74qne/+k56M2v90Nt2U9DFnMcM6Ntpl0qjTIbqdvlm fTUFkGYvRpyaeksvKcaDGptHnMjFJ8fibaGqapa5AYhq7M0sXlXqlF1bpaW9t9caucvS9whdzBG5 eFz2kC1rbo4Hds6SZIWXX2BY4vOI5jwLDv+WPpI9BhGhRrYaIDj8wy5BbcZhqvWKpQYMIJ4+nwoB DmVlubfDY2TndidQK10bnhLF1CLTHr00fvFLgti0lUSp0if+FvpHjmMskeqva9e2jYfwcrKK70Do sNFgqACF8UU+1P9+zqRgShbcA3mEZDSlGA65HJQE59bpYcf/iT/ojGfMzrVH7Jwp1dW35FKoFZNX ZWw345HYwqE6d4vHzZdwerAI/38EgrZVNVFgteLu7kkXlNTEOLIW3VHXSihXdB02xOyapIWm00bO dx/L7Lqvw3l9/DtQBt0xMPF+pYa3lFE0eDHDZQSmsQWVo7LhaozckvYF3ttM8VlS+Et4s77MsjBM wD6LqyiispP/J45Z2J+vmCvqQtL2FojbEYdjAnmV6HxDlim8KfLTlQEj4CzUupBar6YY4sx7pkrw mBpDNsvrGCQ3zQmzMk5cZbjQeeVxcv4RWRg+fZvNqLnwFdD+44V9xSH41XO922pT0vChNKm6JqJj vBBfTJr+r6SUqjFEPAHqlUo0FwVr321J8P15LzMTsBB0n2wwPyoMHFFeQuL4F76oAoZGvmVtwEfC eoaqrgvTvu0jtloHHCm1UeoS31UwLet+MEr3YzgQ3r+HcYuEU/mk4qROBVKxLCOxFKFyf7ZjM/LF T2iHTBPi+qbpyam3WMiTG5qeb7XkViDwlbIVRZuz3ccnGLrMFmqLjFKEyMgeySikdRbmTlFjGTv5 2gzhN2KSOiWbkKOAyf+HCT54+IaF9s9MHmngAhMMytaFEZancUqN5qEI04rilQhPNsvg/0DGAqVJ hDKUIw24AUkXVo2NmJ9JmY5L9nCBy1qYVa4qhLheSVOJlZII0+Fvkxf2rDG23jhVFAMtTJ8Tl8U8 NbIYrvlFuY9aWV3w7xaz5i8BBfuLUrQ3AcE59FOYf5XLVxahC3/hVld113obA5e94E3Es8NJrIGC aD/wuCbHf4PExO1znmP6YfrWGNUZYmyLoo4ZJ8VhWTcgva0GO5ZnXbIyeJHcpQ0/CLdBbHwUPRyw 4C4knWTjtAAzNOM1eUglMCY2kqXdf3dvZbj/DI8rzAqLASW9Ns1hfoGuyqNHLLCrk98Q3RjWXNq8 OpGGnYJ8+neCiIhRaZg8PHyfnXF7AoOBUu5P0tUOt3mv0QuSFocsyCoOvWkw8Tdg+xONsRFsUi8U Y1IsFvcxwDzQe1rCHxNUrcFbvXyNkEl1WtFipH7S07QCUCGwZ81cfe3ePxNsmY+q1CDVar5OYjjz xRveaCegQfHI0crKtHvPYNJOc5+NdeyNHAfdrL4mscWk62VTPzVaEpSxICnEd/bDxu/iZDqNtbLz 0attqsFnfy4kDXGzGJKjAGMOTwJvIH+nb2hTWzdk2XXU7h8fDTNFTJrt+TxOL0hnQ4hcEsrj08Xa 8LGcCBGKRVnou5vP5RxO5XAbC5AlBr7uQ6JNP4h9sQfQXtKz8PKfVPUSF0/5ME/yX9N77xU7CBBc gIfu6ZnoVyTXkGxghztlFvmSexOodpe2ibX/t0d0XJ6frcbu0zj+YZNxk+W/H+aUnR0z7AzRob66 ZABehebe36Be4t68fYEsE8X7s+0jMYUT+n5OhNexdXh2saU1WDQai5N7qcyMOh7VVlvRll4cF6+i 6l3IrzeabQ/ctkF8oqVcn9ZCKV/SyUKpvcteRncMYusbwU1nYaBKGewcoyQ4DMb2OTiGnyX0eVhF y/XFblulIHwyTdehTRbva+jgP+PN/fE/NMal/ocptVSHwEy5RKKe0uRZbofaZVanBZ8ixykjEjRL v/aUQx0YiGCDTc4f9m5wiQkvo/jEjjdNcPe/iYPkloYXPytopKj+lOKLFTuRMIUjvtwbNlUmhrtF 43EIcey86mEnlcM5sfBP/aq7rn/dJ02AoGsLbSj1x9+SNo4mt3NuJ+GtVBxbu9PnDEqJxXxIWB2+ 1IMce2gXCR9WOjwv1748HZrYUumPbe0oc6X0HXTlZikPuWEJSfwD/ddxoz6OHYl4iWbTZCQhtSko 8vjNR2mQhJf9qD2ll7iBZtmWwBmxy6gwIfJJoCVD0x0E8gmBFfaZLkRk6qxHJpiIuIC3+vCjdPQ0 cRqlHvBZdGVmLqIewXJCVH4+1mrdmhBjZqPzrLbKKinpxXNiLzfYbzPDMRMgvBw2g5lqHqYqaRii o2W7qiI6iq4VbU9hzkqmCNfiLfvYnvZoTPJNqolqNMxgjlP1o4/tW6EPYz4VY+MjxzMqDz6chP9A mh82dn5k0/LX4gwMHirsImKNbPGXP0b0asJd0YhS0o1Xj5fYupebJWSFPmNY7lzpS6StICBEvbxO thz4jaka0pFLEtdIpOaXHOURYOQrA+fa/yHf3bmmh3i81Dgurlq19uho/7n/q3BPFw+BOKtxDQnN El77hangMFReVDW/ZgQyT4SVC/oV8YfixCs532ENFPp/bJglT5MsItJkT5leH1YbulGSKkg5tzXq w2hVybutuzhhLaYLnW81tWUV+67bhn0QU1DY2kLZpk4Ll+TXHvOv8x1nwlnW4cZklyfJ4QLJYlXF Amq0dL+H3ONoPLGYomUO7V61n8uP1wnap6mgq3DffKS2vNinJP0qHGzHyB1OnCMHLLDybf3Oq/ef dq39KJX4YAUP4vhhYY1zjqH1EXSDoiKt4sjCZScqi0ZJWPRGOjmgHjrZmarbj3P/J9UbTo7e73wP 1VHT+VxodcZ7jz8nEkQ569afr88eI1scqSA+aPDMn+iznHAAhR78F/tCVZlKumtBU2fpdyt2MMy8 m4VBGAITS+hwBYdeAGkI5dicKWHmGpCM+GdF5WchOgwLFO/wALgs5duqKVJBjhZIYAP4iKybxHm5 L171fVrAyil3dwlA2UBfgMBl8s3uWPE7twsI3kOPAQB2P2u4fCoYGY+RjQJ4E6aJtmPB7yXFCqxL hZkvtke64RNkBQse5TbdZPT7OfHGJynG/TD51/l4F4V36ozBeKzAs/VOV3TfZKIKuvTNKTZODJZC ZnYVYaK5sqC83Q9OvXr6+h86q4B5QfsnjJTv04IW/Eplb3AT7QoCBx2P0o4XEc7XNANAqSXyHRTi jqmDXqHc//hwjV3fgfaulKWV33sR/qEv6eg5QjYyoFogRG0y1YvK209N3BaHnuQ3+Rt0Z9Ofq6LL D7EtGi1D0zqOxMGoQilFsfTTs9W7Iv1bKDqvzCSnzt+pn0AM9o7NxGsFE3he1t4xm8OaLVDDeMDL bwMLTDZRKrY97LbVTsBL0Z4E/o+l4yZ8VUclG8j4ryhwZioYzlF2+F6C0SUAnY2u5VfsPNNSziYs FZ5N3IndWwwRuNCHXPXlbyU/8bA9ixTTbfqNH5YDYtwT/a4sFYJOEtBVmN2l8LHBh+mWlDomb0wa qpmuotmCaAADp3USNVuWADv9tRy6/WGj8QozywW6hkUA13sjeJlLf6B+BTUs+g1sb9tFIo16E4+C mWC3Irh6sitSOgi9x1B60brVqphDSZ3w97EhWl1gKl7FirWX6xTnFKY7PG2+pOPeo086VozkekWN 71tMbCRN3xFqleBS7Ly8zRnFgovoKPAdnD/VnvT6Eda7iolgCg9P05PxFvKSlFaBnI31zI1dut1s 8049GCgNSntrYK2TQEfBPJHBfLiy3y2UrUSmrrh9D1rv/4Z7rEWFRf/j7522aOQge42s0S5//4Ve H9ZpAWPzBsWyxLgRlCGufzbqVl47NNqJiehAkqEPNKBcEt08m7T7Qnw3EvcJquJwdJXWntjuEOga Dk1asskNBAtOzQhtGd4D8/2/tcs/BaXBsDNopT/9/fkvEKjV0oP+plWfQyEFzAXFP/MjZWLKWaY9 Cw90Y/PWaT7XsFytPnW1gaRUAL09pZtcFWikT6Po/ZDqCKNEo7sBAnzHy0VV2JsDZ9u12RlJuSvY Ec1dJafGdrWPKmD2sMv7W85rlt9BPjcuqmJTNMf4mrqE+4fcROZSwTqtktccq83/2hZJio6IWkRE xamJhdeqIwVnSv+Gw2QEbjgNWAkWxzMMiYL2uMjsTOZ7KN7X14lCw1G/2wDSxgbhzSsYSDQywvAW /n5DEhhkCePsJ4HSo8iEhPgoX41X3zMkcY2LYP31YYquAag8qTlZ+82dkVS22zm8d14Cu3xQRONK PFmXIFvpvTmmve1a0eusmv3hGUPJ5Bv+z3IKDlrDO1A4Bl+rO7hMx3RnK1Dpy//76MVueMPdx9Cp ge9KARc4yjC1ijMNiT+O84BJONltmMO6WiJaLe1FwW99slcqFkyoteaUwuhEEQbmPpc168FTiiWV 9/BuAOMUhY0ZvjyGnXQw834Bf48Y5Zi35I7dhVxYgWhSS9/knxGlTRA9cgAg7yTlYJQBlUU33r3Z QRMS0v0xkxUSe43JT2EwECmMsCY1f+KlBkV/bNvU6EZ43fJ90G5kWEWvNAFPj1ZdkscDLEkg87P+ DGeDXYay3VGJvhYRP54vpNrjA56SFcVBYrx00ROF33tVId/JwIqgTaSZbvKiVAkpDiqWr+dQOSo/ oagy8qs19R4dlIxXTvC8D5nRPmA4kGfHOpnvTJTQgAsjMJqnVMyrYPJS/Yr0z5KNVWwqSp7G8jck mVUWFPVoe9qcIN91VNFb4qdXMmDCCmVVyFDdCH11RIrhJF4BGh1fRxvHp+41OcfFhFSZgDyFLsyb yC4MnSD8OreCwvDcdMe5+98swFCIjYzYi5bsSLZ59sjqziNQ5m7/ekz/R6rhDOuaHPtiiAv/QFyU sAcMFAHd4dNj8MrYadceR4+ki7tWvddN1fc8Io9P8XLkv+vWfnL/w639G4gu3x0SkzOfP6DKq1QP pZcNlXYWTMDWSUETf7CmvlKmVaGVYjqwhmGvERyeTHSyLDkfjBRHefRAJSM2CbnC7xiYmvImcYMT 5zewY/dQb/mzU5ZElgO9bPL7IWrYvoE0Rdl+p6IxCZhc/wguPf7Nsy1dFDdx4NKA1RFEnD+cRpvb 9r3harIchZd3J/c5+9QHZvNLt+ayfCV3qEmU5On8hUm/nTSL4TCvuvUwE76kV9dW55iy77LYU5wA 0FyPGIUDRlPm9ppG0WkM0kqejy5JACvf7WK6R0nfDzd3u2jUmQlnYQ3xpgIsEuBl8QLC4BFjxw4d uWrcdNKpL34DdTOIqJCrY4hQO5/koz0RYQ7Y+zTavURjW4oyTu4oEmQbrX5lG5WdbJ8ajK/nco5y sMHeg5bFfnEWIgeUe2E46hdPXoLDAhfyk3U15hUL2wIrqEF32896aRSsFUI5gV2yY3dEyFACpqFq JY/Jpdg4LEjZPgv59MIKBvVETEGXWPpHZrASDNhwBkapIWHR3kYsKXmCY3D0P4ILffJV8QrXcq5r 14+MlUV/qEWaGpFyt59yK6exSYbC4iUFGLoN06avunvJmIkkCSp4g4gzjn3vkycMesDySkdZ3WPZ euqXZORsaus1yE9uavNBHh2qAI2/DkoWeA9FYDZvEwwKx4rFq77khKZ631nJu8ULeiRuP36razVP hLWpzUFyf6UfPbDIXFccwpL9kA3BmXZ+kpzQRTvm2haIQxR6bM+XXcohH61HCWaLgPuNbRoo+jMk yuiMBnWknyWmKYKaR+3N+5qUPpqxT3AtAbXZM8+QkoLw/N/hrjY+YugK5Ww5Mii2Sn7/7iV2COj1 2nljcXZ/HiPRTOJsfe6nOrMCsPryvg+itpiFuJ29wPCBY4o8dnHpOeSdTjh+IBtRPM2NozggLG0K lXd061UKOeTCfw8nBSkY8vas6yrg05dT1ElZKel1yoUGb0l7ED73Tm1uA5282LN2/rOqxUFw3UTZ zHd2G26oaOSPbPwXgKC+/G4D49nuw6fO28+GbjTQ7I2eq6kBG3o88uu/G9Ru//Ddxr9o8dKGY8Bp U/BEuOjnbdiipzSYCJF72AmO3VJUqeRDrHeksT/7FUSkZ+sDu0OtMAXDt0bhSwiJ4vidwx0G8hUz CZ/a1Z7oRy9OZSbklpLxLWZs+ORhnhnmOvgl+GL3q8AB2G0xezEw8hRTpFLp1OreSzocsO995qcS RQ57dd6f/DAvMX1nz6xNV4BVGz3nHAhJleWtUxoMROBsnqjanmjDotumw6i80xw7c2AzWja9E6xm ZlvKt+7PUYAdcn8LNqlbkX3bYMpze0vmKt/slDxG3pyZYpK/XuUPLmzDd1GVOTREufG3Vj++Ezvx oPbVrxcSQwAxv1DXZu70q86yjkA5FlFJCpJv2nv4E3jMUAcNu7LwZZjMXY+VEeCffuwFOn9+E/1R MuGybpq5mCyeBmhzDaPgzx/iae5mnu1/0pp7te8m3/uDOgtnvpZksausgW+WZavQxVj1G28EaYEo noWJdviuoNucQ6dolaw8jRKILt0qBG2wO7oz+NxHLmjO9BWBsWqHjA/RkVVnimogjxqBkF9wUJip W7qsZMLkS2NYz3QfeZ5zWYcoOX881DHxuFi+AUETINIF0RK1RamhwnHSB7XbKhgOgZJfC4oea73d jOLOgeDdva/h5nhJwjb8DnEX/vCUlyqcbSxlWeWqYUPz0pAjociHbHXkH8oLkKh6jF0fAuoUanqi PnP1e5LXYyL5WkUG/1AZqtPGBd4YnvRNkLDphy4DMKSa/HzZVFkv72RUyJm/x0Ep9Ch2DjQ/eAXn J9utHvvjo3ROgk4LX7eFdP3IkjEQ5b9O1kmLJJQjzFK7EvLNDeJT8ZTm4n8v4+asmuQS5lBh68nE 7881DCtiJXnpg1o22hj0w0hf5BJIuFrELgubR8uUj/RJKAb5bOUMM4AuOlCcc4pwMGTbYlU85ggI BBslx0yVIw0a7ATyE5YK1KHulxHsxtFiToHka6NSB4NMSgGQrTXaJlGvDFxO2Esb4KXX6ozFJQvH MDgWM3wLqo5EgKdAzCeVf2dEy8PgfKCI1TkujLfIrDik9uEpvPrpmgKUM3GfIYe6+YQEI3CWt9YD XQtoDtcdUeU6LW+GLM/FCOzW6iu4bkj9kG6u7i5wb8Clba0uFIrfnwrWpapoYSQSpnJsm+8hF7v6 2XfwvuLUl6caYT22tPkkkUKJz9/BdcDa3VWvfsrZW6R9nEp8qJay8d99Y/YCJ4eWMpdt0Jyf0GVZ ineSRQMX3vSWAqhHnM1yQPq5YUDahDDNHlEU8lTE6l8huAYIpPYSGRfxZf75qjG4FY719lY8005U 4x2ZyOfItorply5BWzU61H62S/Xh717/7gpM8CXJRsvGgxN9F0vLYicelD5JWu5/mfdoeYiRpx/a rgeB4Q/IICL3hpo5bnQ/smEE3wmRl2TryYVldZqLyxY/s+27JeWnYUmtONyMpdVb22dkorVF3XvT vtIC7aVf8hDb2X5jdLLZTngZhCTqv1lEjn61YenOAVl3E5N3OCurRFTbRRZrV3J2ca1FVuxKKzdR Pv98S9XY1jIRyo0VljaNlXmJajscd/iejL35LkSiv5EDwym8PjUmnQIcDhLs/MEtZb5Wjx9Vhpfd RZ+N7OCvOPBxQOAKYjfhgG7qpKfPCu2fZ7uhof/n4qBYnbqX3jkPtFl8I4mdywul1VkqhBT8lySx 8EkEYCZRWZZQrSQUQLFdDZOe1Rn1Q/aryFLcbo09x/6oYuPI8mVlzRMe+cw+P8vStQY7rw7mtoYI cUwSvdeuxi/4WNleqXZjsKxea/1Ordg/fBE8gvJLVYBYfAoADILPotpUaF+1Ot3q01swyG5jgfLE 5RVvOnXjs1gAhIPhdF+uKSOTzoY2+y0Qgvbk1SuDMHsFezUDOJlj8nrlf69OKIF4S9wI7PL706xg DXUwguyduDR2amOXzvP8QMHU+hR7oe0hxyTlV4aLG5Yx1kGqmvv6ot3KcF+M/kI7Vk8aAA87/Ptm 7mepVSqnx8hsA+PohlbV0VvfAdXCMm+Mtmh9TB4S8NdWao+2GdzFm/XsKcYPabkJAO4cvhUwNBRm 3X9rJ+Daalf8AKRPRHDgvHevrj+bSaAVsiMBQce6a6gHx0MPyGct7yaw+DNxHVzm79n15IKgG82k 0GcuijCpk/bK67V7bUcrAG8N00IiM/Pu5hT/hmNxmNoNruv/hqnYpyqH+EDZ9esfALnjcZuDlVpp ALRQL7w5yC0VHyaxOrRNfTcNrrtHpHwe17ZI2wVu9oc0P91n3MQ4Hj5lBPVW902IH+GfBCEap9J8 ZwflBPwW7K84l06IJYGFiA66PKg7BtzJiURYqNGGqQREJz8uUsNc3O0rmtZDgPP16CvS7ChU4n53 rCJVkoP1npf1AjB40Ok0NenzArFHT9QxACdsKOdvN85Kz56t4Ud8TT9VTxy7P/0iP5bN2Z8m/JEf v/hG/2OlEVXZ2euKJzOb6vQlNbqUdfcPM9U3yXrM8GYdUn4jNWUwQhC3a5Nzo/Cy7H+iS6ZJ5RfH cFlb6KDnsE2YjCfLamX6m3yatnJhgQd7CezFHsNx0tgkbyqSddX0mec8wEnacQFXQaV9nKNd1cNr DrKjhyc5xc1Q710XmWRIQlEk5FFahqHzkRzHSjkeMFQx9ApfUIH+cMLK5DFpIaeduAgkgTcp4xs6 hwXKcwGIOtANF9eYVrohFIoK2afp66nbSTE4EDZcxmi+I5CSiuC6yT8ovCwMHJZIyHbn9sR4TZV/ sRYbExiTSED5jsZRQllCn8O8/nAGtPOf2MTJV/QQncOdOOgTrg3k02mtuLJLZHOqEqo7Z5+kwTRy 4OnXUWsmE4vAyv/mVDIAEQj6vFjOJllvHNzvM1Icf5jBH+3p6qy+FL6d9Mno+i+lfGiB/1ksPqIx 0s3nI3rLEVM8WuMuG2vC8oop7n5eR7gGstZjQ8pLbpxlmz7v5sTJt7P09JRpEDx31t+twdDs8eNx 5Z9vk9D1AZ3R87O8oQ2euroZUEjUoGfvps9eKFrwVW/XmZRtxCKm1X0VlmEp+47rc4liymDCqUaT CSSx0sPTYFuCdFimR0zw7mEs6i8rdp7pGGFOGs7jlAcN63KfqGGEUMm8X+hK+RICNUHCVYJFHEzO xvFREsJnfBFzztuf1SMoYdA6mhvd5kHseXZnaJhx3jgUl/QdOWqPjwqTFY7xJ3hIOazX1V6A0snp yvUw8ugO7f/9YM+clZhkT/BmL0WIFjFw51+CrVxNOZqFrWgazX8/3xgvbpSK26Z6O9z6VX2twt5j pujpobYYbPoPpv+EFKsZkBhqqekqdmZF81gRBQ2yrJ0kj0pdabm9TP6fcTNaLtSZGjKdZ9cCuA/V 2hellKvuxpo4kMdlkUGrCLjsnqArBPQdR5Tgxeo5/o62egFczWMXEem10JKyMIUfaKibmDAAlmL2 WBBH/qRNesyGXSg6MXf6klc+GDd7lDOJKXc2ejY6SlDsxsnGcRcAgRrmkAWt6qRN41WkBlh7xkEe LfwgCdiTRshRVHAFq4dgknxNBS1RqrBJE9OLHmRmF4uDKCTBLPHk+4Z4so5F9tSkDNL7z2gg3ax2 7IBCSpx1/sIOukAh1e4DLTkrL67A8n4jaGiZFO/pf8bK8eyYB8id/Px4IlKVxf/ZWo6wjhJhVtQQ qjHIjpu99qXWNT4SHxDPSnAQ/brLjHrqKWNv4fJWwZPUJ3muQzV3TR/8u1mh5sQufH3WjyDi8UDZ AfyKQSTALBs+M8KD/kfs5WiVt+MWmg51qp+SpMct/uspXZwPxlvk/jxVUg2ZZTrBvGBsMjObU6W4 a70BtwZzgjY7N9Jv3t9vaw1Xjs89yG5QuF0CQzBd/rYju8htbyr2P/lIvFP3wU7F5FfJEkkvZhgf npVtkXKxy3oOqxNSwEzHuFmz0NiNHK6RDBnDvMnHKpm3nrCczgEpVFqJS98XL+inJtelCcRhd52n MDuDzLwh3LgPmKqxajOMNlOo/vDe1hViJWs0gExQiI48w6/NemuJqcOM5nkuaXe37ltAmXOMq+BT ddYSbSJAQl95c0SPxN7RepVgM7I2wWCILbX8LH4QRFLX1g4zdLrHWHeA2/gIa1V0jPuWL2gvNDPM yC2betvItX0vHn0tKYKJjQ+Y9/KVLtTZy3lxyp+Vxux5bg09tjy9ED+HyINlbp9HHp9HBnGi5B/f fTbmn1MtVsUHCcwfpHugacPAZ8NPiFS992PsYEzqxr3WYIlXXpTh7Eql7KOswgoiRbnhGskcKJVB 99aqzwLHjXwE2OqFpbCUe3ws66I0PmwxM2SP5/a5jOkuionedhZPbJXCeq2/nZ1Mjw7M2TxKvUi+ c9a8+HFY2eEUzST6bxEqeNDc4lKL7fH8WpfUkGqrN1Lvl0HJNIyIU9tvZZquh8W9U1NhJbbO2J4C DB0rzyHuOOcbSYsadpIrt/un48jVfkUtZACSKBo4ZM3F2KFMRaE7kftZ32y+umuY3sDMLQO7vdOx Z1DrpC0CNBxVZ+ToQ7Agg5mXxPNBw6QE1AwZxRZfxvY20CPcm9Dv1P8oLC5gnctPsxcvQ4KuAWPr zOZStJcesl8fMz7zxnCjwgJJcuCTZPMdLcj7mTk+LwLmErmmfDDHr/d/V/fNoRjc9pnAGEcdmIgM PRb74kBxj5YDSbTMNGKrpWoNCcDC0jWmqqFLXpmhxQNKfymPgR/T4Uk33HgyDIrIKVqen6ZJSXXE 5j+E64pof7hYOgD4lBT7yIKMHC4v6/Sh1icL3174gIguTovWKK7ypBJpK/rcc8nuAdEeMJIVKKVJ gYhXM0UWdeSfIu52ZfLj7q0+4uayUj3fez1GuXA+qhKBsmURRiI53KS1wtl+iWV8/YaeRSc4a7RM 8s71iu6DKzFDOF7SbGDY10ltggyVBd1YxYK1jHIm4xRR+llxvTBdgCExMl65qvcihO2tvsAINu8M Bc5oDJ59vPfvBObwNIfEkdrKi8kG/oeTHet8qp4YW/wzw4iEnwnqKoSAUCfGR7YPv+G0LW99UG1I EipDqBOmgGNeD1CUsh4Ckd0upxmY9xAMetGidPEDYqcQZnXBaP0bEoE8JpVGOulLT14OvnLS+RwM zOjsTLDG+Cvo/cEdxgqtglmjcvrUmhYZXjrTMoPUKPCiRTxcomzHAJjy6QVx6MSjpGZ8HVwIVbnw 2Wp+KMIkiKRKx4X5InTJB4/cEpStllaKxWBtn2Vx0VceUxop6lcV13RzifEcWWsxEu5SCR6AR4Hv a3uOCOR6DOwPkzR3+nP+IHLBndFwAMcAlIcHfksf9Br+QIAjD3w3AXXLPvy5b5Ipy520UwifyjUm ZuKSTmz3NTzoUMjuCf3X5LNdvS82TsSCes4MUkncB/uK7K/xiwnIJKrDCD6oGLO+s98iUhv0MZSc oy5TwJDwmI9dg9Dz477R6SnJcKmIGqz8GGby8BD6O7i+rZiEBWl2nx0PCLwKleV5lAi9bB0vMzTq NwX7rbrV7uUw+/+EGEIdxWqaGTE37TAdO/G60BURT5Zf98L2hooS1bylwIChTd+VmD8HLaHaKdOe T2UtrbuTrbmm0q+mw26faI9xDQctxo+42MHLBK47Eu8nLQ/DdvE9YVEPJqt2J2iROh4qt2zr77hV 2iyRKaPd/Db5MV8qUj8p4KIVhRxV+ImajgX/TtL6YMoBZEweOmXFoSvW/8XKCYmqOcQU+Y6zD6i7 E43RbmreKmFKh9tKyW2mWR1beT+ParT1CDNMxhp3hpjGq9IFYKucyIzv0e6mZeUikNif+oSUGzj8 yt51389SLuB2+EA8zxVieDHxXqJEAi11ohnJ+950lbl54ElhgzeKjNIdaXEMx3FtK8Imd2XTgfg3 +daiLXQMC/hNMLZ2lhTagyOuE1b30QnpV1GC2OTkRVtMKMpHfn0yUJmmX14jzvbVSxE7G9iEebYa s0zPPFS9w6TUptpQM19zVO+7l78SxL3Xif7WN9crxg0L9OsL4XLmpdVbWB77+I+3yz/8V3XEjmar cd5SWAqpJFvtdJQRdqgI6x87zv/J2hIFNqcp62eeypjymg7eHOIO3lmpq7A8iaN5g/WM5R1JZJX8 u183I4mbc/yzSH4Gr1HWHVEcgq3ym7bIKYMT/dnut9uNH4Viuo+mvbu+pUtOTvAHX135vchOMcTI aGs1f+pvFwOxkbdH4TutiDCBro3CMVHbw9f961x95cDdyuIeSPmYgm/ulbiHI1PiEgZHcmY+dXR2 Q2zyOaSYtpUm0SlzOf1zNXn02zZPgQbkhJHUC6uQeym6a8uhrgtpCP5fxgFkr1AQB5ToEjnSotND xfE4sOLfOgugYVka9N3tLf5i+WxmSiuHZx7c/TKX4iuYq4Gq+bvOhfpiqxhYOFrUJw7mV5kZien6 O2zhC95x24OLPzI0Af+6uouSbt4i+cspqSAY+VHilgRWSL/GQxG2zK9EV6OX6yIwE+Xbq4eoOd2+ nwis1t01gjD4YR6le3lcjzHhmIfPpI8gQwhCH7NmX6Y8ucU1pXfl60Ev8gVkJSC/DpiFLnZhUbKU TK3j/knNf2xK5b48KrNWLfmSQnOrsam1nTZZnQOKjoJmNsnbrGlfJObRvAfU6j8pTjeXycl5lC2N A6nbVk9EtEVjGnsiquVd9GRx91pdcZ9yy/1P1DaX7u7FjSimHtxGhyhSz3sIIUQl7kZR7DeszYwE QRYrCVMdONWtlj9M35TWpDpbKKa9+qjTtIjee1vVQ44YRoynxIA4Bx0ipaQ3aIEqYtRCYgaNGOEV o1KJXrBf6sB1ytr91EN4RIDtqvv+y61kizZ6fZRKbSSMgf9QvG1Kdv2fc8xrjznanCzeNXv9gOOl nk5C33B/lUjl6/HgX7eoCsGIW1tIy5YCjYaQn98VZ6cenWwCMJk0p1iHM/ya+ITnO6UZcK5iyp9a eGHRsJr4gzMn1JgORFgLLU+SPMJS0Ykt2wBVu60xAuW8YH7K++7e5dKugaeW1LRP/uPiy92sfvDt GOABt9YJZIujNphdgDglVoJpKnu6Yq2/wl0deZBtLKAU7P8PSIyB4hAaM++J3+VK/yElTgXgvS7b Ly1P2uSAmPt4xkaBliJmQABp19KSSxVS+wCj5ZGteBXmHdu//wy8YgIuerYkTvUYc8YMVsXaBXP7 NJbFiaujfbIst77QQ9r3dSjH16C7xiAjMQHg5V/DqxC5GOwbPRhWvmTF9Zr9FxwCjjYqofopxQ5X Zu38ZtP30jiiQynbhaN2mripCAgZjenuFG67HANJ7bnvAskEc3knXMe3mIofOqFG0k1tMDXZimlI 1ah2b/wB0gPHnlQTg+62nOy+I/wZrSMcVo+KmEHmXtepfOZCnpv47bS4T5ZGuCOmktEFbkBw96Bm Cmqm8+VH90HAARwYeJICzQv1kKBhpOzucH9nZ0o1c5Zz5WKJjnMqDVtBZSiXvYLdvBcp0KdMoLTE 7Vyd8+0Bd1J879MO+zteqENz5OiBVaB2sx7iNonobgiMJ6zS/R49QRwXOliyfARpf8Z5U9kW1z5f E1vSMkndK9ObtzbLaixP8zQUuqmAnFSj/g0SRZD0q6z5Lz7WgVNjUSdqudQClL5gcz901opuvCmC R5sCGdz2YVf5uzk08Vi4+EszSaEG0Tc5myCNiOc7BRs2Ww1cho0de5KYeB57gRfe9jtwW5yXdSBZ 9ktZjdmmRY6bmaA4ti37Co9lQP9dVyH1E1aI/8hD/ky36xLK5iScUULDMgPDciwiq8My7iBjpvAX asx8wRtFjzkSuHmnmNPBko31kQENXnMit9U3AcGCtYeCks3p/xYSe/CXSj27ZEhuPv4clPWXGxYE ZOR84uTya6YU2i/mzJE/RPDgvCj316/opmFHf+DEUcPZjMJpQ9RFv/lJAEBc3LlVFN6lMSnSLQaW cyp5n6U1WpeUGYCzJ2F9u10RhP0YEZHGgrdUvECr7w+hhApMejcGcZQVuTqM3hMSwcgHFqXZ8ITk M2mHlBgsCsEgt1waA+DN7tvIepAT1H/40ZeVDlLE5uHnXdqM11dMXTpqOq5VBorLVsIKcBjWGhH9 Cal2VaSTlZPJCswC4SGzxrG+DuFFg3ur824NeE+2woNNWdux1f7ASlmJvrwHr1RRHcw1iDaZEDm8 0+FsBe09X5tGIQvAGnUwZbKl2NEEVyLOheJoT13wJuXqLu6ZIrZ5HeKeiQNmF98K+ArPbkv1Yc4O lATDIPJa5TbHr95/LkLLQKJQCqzhI/nfWcFL/4iGbQuxeK0RAcs1axJrv+qUrw4xNQd8b016LuPc LZoEAS43KD82tuidrstpsseB5EdHq8gI+wPebpMSa6RM9VhbWxFvUPU2Tj3wveyJSnkYLTdXTZzo cKS1CyxEG58csvKlO4SuXbEKFJmgsMoU6vHbBxfc2MfdZB75g0SSH9OOsfuF4ZGFzcro4uvJmZXg y44BeiOQKqIjZz2N3gqOrHnUyTcxJjI5xW0+9hmp+3N0XIWN36VngcEEd/pBTGeSQpuVJY27lFw7 iMaJvref63bMmeZBYBLEEgD7/NT+XKvIg6J8C7Mm1/s3J8W+C5BYbr2DvdruTF2LhwuoXLW+uw3S 13HrC0vdmol57OI4WzyruzmN005FDsTIPpUiTX4WWnBOFjdyUV715hdJXe1XvsQcizjj/3sdOjVn e7A8/NPExqJzyoyx3kyzJZQvE4JDw5laUGTvMJUGHDtjokOgOSZMt3SwXtx8l5dJaUeCWZLuLstn xsxWLi8KwQMCHHTBpQjSvRqTpFH+t1lFX0FcB7YBsiu9SPq3KfEJEmUiLg7eTcv4P+bvvQ80BX2s 84M/Qblfp7hP21mHU+mNyRPltEfE9Qofi8P8bxXaeTxkVXmI4DpPwQ+0eloJobkiYxhJjpi36n89 Z67eWGUYjOc1Oz9kZYhtHk+4AqIwYt4wUVVEvN55drRcpupkG35KCv4XiqnCfd2keiHoXTEqnWtb aFrXJA9/9VhFZj0cXatLoXAkbdq92g/qqh5iQWEAXlyiWW9ODP8HzNapCNxwkGKgMFBpjW2x2WF6 wGNI6eSHlVxohEzxtnodHevhdCeVnB+RTXiMf0JDkWVY/8K5pjUEAdBL03nevKaCps8FI7qRcihm oP+4gMNuhhDdgbxpBqp6EyYtJImH1sTmzAMG65am15x7O8rWhFwJfFH9NtcvZa58HeEfhdUnlezy Tf2h1cGi/Q2z8uk1mf0aMCZHijEmY62lHb5cYPI+q5tZg12YYH1UEqOT4dVdCriBGUCQq27M+HVh HAoFnq0dX+/nGdSGzZ14Uz9fp9uuHbVcqe+SGr2YQmjh1fH/VQN3Kecry+EbqpmOKHDEVlE+Ql+z qMivvGeEiLMxKZNjS2e63qjew83xfmFvoEPNb/NvXUGPx1i/ooYdagMudtbS4LAhkvXnHrEscumu ufC3sEr1Fe9U+1R5vUe0CsPBlZdOSRDBfrUNxGHOl9Z5dgxe8HcCsOPqDBpzIgEtWBJyaYTj7+lh 0jMCyYyZQg1tDIord4TEKoRtDP4TbYqCQDFje0uwZfLN+4iq7lBiQYG59Fb3mLWXaCXwvjH71Zoa Z5ZS1qeEk8rDugMwjMAxzgPR+LtP2WVxhjFvFjo34uHWMnHQfIrfi6VRAiSLaUBBMUbdmJ+7QQVi AolfEascwtm5OLOvcmHuPOXWmXzXk8LSNMsI5vSZg77SydJqD1uvJ0Iz2JduSSz6aX6IzxsFY8mb nV7hF4MXae+7FWmj5245su8FVFQNe0Jteq1Z3kWG8Gttjnxc+HgiFO/LoBuYixywRhbWTMCUJ9bT jjBTLEXl4i657p5apegBEnwirhj8XHJQuZpFLcE8Pb++0UlUczX72htklAS2BW8EiYGQkJmf7r3c cAmk59lgmvSxVYJ8RnS1HirOmcH18UNPwFkSce5+kAJJbowxNBsJ9OTIAAMF6mhg9pzUFglpw1Sq 4GBnGqLp5JnMluAFv08yd2xMyDjEVdrmnQF3Bgs8h/+VpVJ2TtyE5vOGTnJv7UTBs2juBIsFmhji +0pDXcZX0Q/2kWU3cAJpBOSjX0coEl4S9Q18ZTwV7p50zg0crtMCH6HFE6ymdgOI/q3P2G7Hfwr/ s7GLmgrNUFTHchp2HDVHlR6Oybv1FvkGrGbVt+/clrHzfH3Mi2U0i/lf3mQKvyK0iZI93AzJmlEO djbOxzSho3kzdNoXul/c/agqsOqfT1K5ZA8B/ndY/BYhQD5u+1ychIGYy7lCmj6LUEn0EMHU1dob FB4Ur2PW21W0YSVxycNHxg/4qTmHhhAnCmUS78EVdlczv0SUQevyyPCFjRjocpl/wgwWyLluUO34 y3npFubFF6hMhtYAPwC1UyiQz6cWKTJ7q3XAMQVyEB+mfADlhWpJa17mt3q2fj6ecsiyQ3IxVrn0 QCLY41Oj4LZGBwvlhcNsmqX9Z0JgraClHe6D6wWGC7d+4d8CtsBbev1yyQEU1bNgxiMnVxNmkKKz m3qQHKPHXBI5YUwUs9AncBT3/DPsyuYjKQ0eUKMi1W9TfAjuXhk/81x1w+DxVgBNHjPIDnzyeo/P aS/ymBMiM2rJgT6tjtvkU/R5+yx4qZOZHwvq0GWAKa2hRNE+Hsj75XRL0TGO5gBzW+d9GmTXO3FB dAyVvCQEtPiDOLEiygOecqImAZcbc1CvEMWMgZ38fvScyAKYyV8QxOTusDTKigFsDxLpxGC5707w LJxG9qDqm/V04IRqsMlu0jmAeIQ1eMCVEv2+ffZs054kAFPqLgcNFVnk2QXyxnGfNk4q4PYQqvsi MAmJqRW0qjeTHtORk8TJA8T2r2OE4UYoOMUWEQFDAdgMdKkfwEOw6Ab5QpTDMj9sVfzoeTI7n/5p jktc5+30V9cyEsJLzP1LFszoMpfjkLbfxoLsQTdjYlCFnMV5AELS+g1offw8/kaj/Y5jIxFL+L7i Tt/SIAgmJfO9Ni2fClYeQj2QLd8wLJ14Jl0HHAqwk4D76BBoPTLLY8T6f6B7PCymH3fQNGnnnOE8 ON43bTvAfOTj8ZA+WLB3S3KvUuAXy82mJVisACYa/x7k5TL+gbUC8eWeAEvDkPytDTlelt2dlAro FmFmrl+GmtMSrCquB8H2JS6az7qXQFnF0bPLSnrZPd7hiW1CuxwHlyCZNe1/ukpEnkjB9wkJLBFZ O28pkW27OaG9pf/LQ9WrbetIRdAA4/Mo4/rqI4udovfDVSeBsZYdp09DpA2PCk8vl9IdefrAFwsR htn6ikZuS1V6r6yUgo9uaLhJn1im/WR2FNrdlSGAlQLe9in88q3a1lQaIPfzvHQPLK8iBI/GNkAl 3AKGI2QX/8fO3rzkA8aqxk7cU2MGGbnTqGniAY6r14fqq0YMpfYFKQxlub2aWfsXst95XUJAO/eQ 0ZIDv3YVfbEuW9FLIEcLymaLKBubpUPF4yqO+ZrGWujT7I6rlnK3uwBV2TiHJtLXeZEBb/GsAlxh /S+a42oaPUAGyLu4nnTYX8G8zo2dc6p5XPI75FOlG5M6VarSNGrpJTgRdQoxDboxHkK1qOCUkvWk Sc+D7o3OH/3nam1b/NYitUEZEjILNNHB32izB3dUP9O6t20420TE0eisNSBIPmzijpdlZoQCmqj1 J6yFOR4dhMMaAeor4URaEQVBbFyv7qfywHE4sTKCcJ2kUvBC+JNLUj445DRXn91JLi4y/XhRQGyP 9U8i1vq7kdZXBCJAwcrKCx5MhZ5rikDVYkuCTIDKYMyIo+bIR5Ge5PnUn18BBlCz5Uv5SdSQHq3Y XMnTVI83p1GF3VXOmAqfmLE78bBVHv9sjnxoyGhhQ492zpQnev8Fu8QwTy7YIZCJMYRkEURbKSbp t1xab6LDBsE3n8toSSmIiwIWaEMhc8sbf1eqHWe01wfZedcwwDevh9ainUBrsOFuFwXZxbVQEji1 pSaU65FW9Fw6WJ8plySVCOh+eqR8dyDzx4ewpUGow+o70nPiHyY+2o+MJGJSLbGX2AUDjP0N0Zem MonmDcRdrunVHnDcZCusmgRhrMsPYdChtUJIByXSPmIwwfSqt7xymSeYgGQcJMpT7hRazBaDX8sn ZnYyLaAZX+FviUL+oXIxWCWg71stc7Ti8zx51nVJl+Hp2wxUOv656ZS7tcO0OQPfPvOOR1rWL+Ll 6MW/QNlmL1kq566LApz9/HnfWzpPoip/1cPsq43QJ0mTmKQHgpPGEhA2AHHaANHFnBARh0nUUQXF R3wWfaoOxo/v4P/N6M371JWGk1XdlQ9gof8PtXOE2hW59DDB2sEv6Md3sUUirHsjzGVbwyE2TeFD y4kzFTjbDumAKk7e6FxTC7bwBZBZSKR6M/FVNNnBYuN4Xl7lNzVSLTW5taA2qKS4bw6TXqgG04MV VaE2JSQWp5p3jh33GqorRisUBM2odHbNZeYXcLVu33Z68fgCG2V97i9m+3/5x/gXlScTcSAAM7pp nVW7JMbNb1Jii7j4loN86tQxsNuyLIoW3aUE0PHLxA6Dx0GJgEtYHLtbzQG3qFcNUg8Wwk3C1+cC GY5U/YxN13I66v2vqAFrO22qw4f3NaeOZUNdTEfAwE2Rk6mgxfbS+TomV2a4fibmq4Q965O8eP8i Zxclb32HFoSvqJHqDzqXDAzgKezJdBzxwWsfR6/Ra4qzN0p0DTFlpbeF/VximXEW9VTGuseritkq ZhCYzXkf1YoZ7xxIDXrElmRz0it7mU9KV0Re2wRma4tGofPmMk766VkqZrtX6ju5qaZLYVftHgGS Fjpyr2A2+MVrFKUsqVJtihUiXcgFZ8SNX5IO8hHUyW6rWjVTXUofNdNNJcRQREssP581l/FeQsuM vv8Zhc2dTn/zWLOhF+tl4MfkTOrXReaMia3P1qGRPELlul7M7Ra6N7ygGZmhxH0FU42CNIxzE0Ry Tb3ez58T6BpWJJU0qAVXiatUDSGdO3r6Eu9SpAe9Az0JkV5Lqjcvti8vbCAD1U1I+UWRNjdJ8l8h 6Bsxf9gQtGTShmsYqR100Ya6n+US/OQl7GIcMDhhVtLdk2cKPX8xlr6XrO99/u0rEyNT9oRQH3G4 uW5XGhHkw3s5UrjUynmvV13uNP7GR3u6VajpZxkRarBZVnKZAp5AxcDJ8LXCKN6EaHSxoV/+GPoD cGCBIqxCIdRsTE9qiBsuAGaCKToY81g7zuldS+kMrdP2X3ynM9QN3apr9kSp7dbQE0vRCMk9Qeo4 Fa9mrfqFG4lp+mlJqBnvHkdKGCOw7j3xR7CdujgmqcbPb4HKmVIa8eErYIleC2eL7cH02ovHhdeF 1+pYJwZ82BM9yvNJkEjjdB5VuqJXHnpZtal8x+dlC6Qrk88DzKmhfv5N/PhiHOLnEP1e7B9XInnI X6lDG1JkG5NjyBhyQO9km/rPH3J+hryx+ht5HMDsB+3YlOl7cbW3N//ZB86zdx5HAsfr4Q913iBZ 0Jw0QiRv66GmNDtnVJxVLcxMlWzUoooZCzJ3IUZemos3/Xms9tIJ62hVhkBns8UOVGXWw3Bx9uzS wOjQKGpO8FzLUbo/fHq0HDdpXTWzYLttr8F0LqaSqDa9/e8Sd2EYF16wbyfP/HDAgR8/nUqHwhhR oc4trdQFe02eTahIdk/JBbuPMjEufoiQqr4qURnYJ2sF/DVbFW19BUIxv8YH713/mIrmRVG8YB1w RQlowvLKvsX+gZQ16Vpqv3G19Plx8tEn0jnOJOtdV2B3SQkOjUL81NHID9DTSyNz9v3vSVmSDGwI QR9Rs86W++6tbr7DNyhwsEqC/on8qqg4f83XlXuYeE+MlG5MyXRvDz0Y1LXHzT+855/5QDQl8U8f deRIK3wUYlRfapcfF5QmaJhR2QCuIuovFudgZcsx7/qJDuKM1lHLCWLP69RzDZMjbyYU9vzkoczi eUMhMlKfN7qPiX1XDbm+Dt56YNCAxUYAQAl6xMLQUAG50DSDdhMdRNmjNiY6lOsOcy8sekm+VUNQ IJhHeQDZ1oYS6o+aT91meRrJeDB0alS4YymfrKrNuXT8Q0H1RdexMQGAyPP9bKe80808X63E0AgH pK2Bo5lyd4el2aeln4pPXJ8xg6BSKaJdhdejFEJ4S+sKjpgG66u02LbMk49a4wTp5KDnB8MbBnDw YoYBYhuwGRtoqg+SUdEjfaPdSjzGgq8LDtqd0hMPV8PE+Ulhfs74Bh5lhUAVgQUOgAI0O3AS06B5 jNJYoTBKglklOu+cb7EkDay7OdnFtgF4j3rY7f4+VAF17XDrofexvKSkhlvQuIdk9q6WeoYtYNcd VcLB+qUCEZivp8Q6YVeStUGnfsV+CNBBqa2wp8trAGYCKRwyawDGA432PhJ3qhEQIlHP9Nwg4Hmh UU/8yNm5QWG6beOWcQ8fi1QAWQ26iD4n7BZ2EMl2OTdImZyqcMrN4qXJxcmpf3/TdfSNKmicxLnr kxT+BPxkl0/gTDDV3w6a7B497lJhv9iwczRir3nazntGsZ4IG5a75rwab2BoL473eVmA5UKMKqlm QFUlXpPlbk9cXdqsR93eMhnyouGf4icVpm2DOsO7GIZMio836sc1LZUBaWR13wqPWYxcig0UpXZK Lk3BEbGSlrn2zm+wZXIxWGzm99+TCWCiYaqFgTbNAB0OnziSXihjohgtWWDNm34UUBtG4UX/lYE8 5VTXqjdJnS9KXKOL1PQxmPMzsI5yrWO2bxtO7tHAE9HvkCtkev4Pw8F/1l4ClrnhdMu2yoBqwxsY Pnir+AgmDpE97A38QuuVCntCQxN4jNk6H4koG9Cxz924DS4AU/U6LZRdqAmzrrjd1nGj4A9x1iAL WVgOwdQxN2OZ4VFVTYg31ODlhPwviJGX+E63oJpHPGk4wsFB4FKRVoGjNAhTXe5PmZddZBhqu9vK Cr9JVu7vv1IOdGFlIThNsAmpCIc7JZ6SkuZZ/LYTh+bT0Og3TN5ndYGbU+TlaawwJWDdsuwIFmKu GE4po+pnTu3SR4PPwn60VNCyo9Crlnu/WQvjXoS1vPoyVoGvEXDQ76jCL3SExYp+VKNR4WK73hIg BO0XmD4xPb5H76KHgZPUuQkEnxtpznSgIovufo9HP4593qQp1qtDUIbtofUo306wyW5qxfZ/lbJ0 gSNPLAq+p0iUHcEjVDC9w13X5d5IgzYMe0pINJtRJhU+kuRdiA/eDaWgmmBg7Q7me2+LwLjRAlds 05nl1Fi3PfSPYY6h/jnGEcqanwdUGecKhwGFoohlRXJlqtux+KCSanI+4dVLOEbgTuJmE4rgc5U1 0EyhH6o7D/ZbI2LvIkL/weAchZj/pyRowZ0VJahQ5ZsSFjMqi2pF3CbqVldAncxGKq5MyBmjCLmu oEJwoaNpLC30cur/0bRr7sR+E5Y2MPrEys3XAWrhvoAsE+kPMjGCsKOUL8fb8iQyzEdRPSPOCQ+f 2lMy6yTgJDlhSjkOMA8kvHbzKVtvgKKT8UIGy8NSSOC4YNBY9l7ry1aR5g3RqR09Hi8+YzG4LFCQ dylQPBkjx1iyIOavSkAFXCeJS5v96qehQv4w2YMN8kCqsQX+skbQpUWMZ7gXHWy2YGQoFvMCbvm7 7blFpIz1SjVByzgiTV3s326Yy6g5Aki6+NlnLm24dgFG+YMuUvu/jE48VfB22d+WvW+9Ed8khCW+ /zU64eQ1FpIT5Sg0yzJ0+MYUDOZKCVLywia+PQ3H8SXFiubajfixlwArHwyxw2BH7M5qRHoaJMxq iweFcKLs3wiyEdBEpdtLhSWpsmLLIGiXt0UExaSndb2MtdN0jDhMVkP7Z1+mDvD2E0OhSCy284zi qVw5IKic6dVM62d6nFWUdSoRj6t+IhSyRj6sEVPM4Fw5QkvMqeHSTObRc+j3M59pvCSQ/Hx0L+/K C7hHPp2gAIbpKtTmuf/UnkdMAPdtvzXkon/Nj6xTOgSXHkHaMh7gKi+1FjVCBXyAc0EMTsSJjYsK p+Y3tR9QNoSoCfGUU8dU+PG0+V2uLR40c+HvVjaJ0dbEPSkrejGaJLJVrdSBkBwv6ytDLhmLXX8G fF4iYPhFzLQGZ8YG4jOkETotU0WQ8rSm3f3p/7M9ptdJI+bBIm9blXWuzNeQA1xdDL/fOrZfrUtk XBiH9PQMqcE/FaqAJ7BdrGdLgf9F21I2DufN0/pJCtumAhXIKFqEn7JOuyi0jz6f0GfwIOGDrzYG TY726YGsUDD7c9ZtAIXlu7skG8++g2Y1dvBiV3pAezQIF+d0qvn0DfgmXBT6s2zP7yOFMtVsh8FH chHSN5tbdnnOF0yca+jaGlx/EdZiiLJLwP5+z5vlaa/BnPCCG9RAwey9d2WM0r4UDZTvl/5Sp0ud ACHiJvsQvx5194PTnInDHhkEeoGgKqbw7FsXkq5wfDvG4LxBKE6Xjem8VB9KfvXmtHbTPL7q1KGA LpjpmSPJPKKqIlzDvlB0SCM45CDMpZXxsmDP13VC/l33lpJ30xv40osD/14rxPC8lqujEhCakG72 bddiiJ6tWCYlg+OPc3/g+m+vUG0pJkb83Sl/V9oeMZkXY24Ri+ufo1UCDUOQSvxBb3C4gp+dh0Ko I/jRZ8TosALIl3+/rTXY0FuUm0hUyce0OLcigqSsznIgPFitNoIRxQ0Q4yn8TJg8BYmFaLnr3fbP IzOvtGOIQkMjGsHSEnHF57u+4iYZAKkeTBPhRAZfgMixakuk+6is+pt2Bcrykhfh4syt2ppp0QiS tFtnna8PxDNVb1h97FAiK2xagmdsN89HqjoC9kyJ2KjfIiS8t9owWm/b2CzJlaj7Y+7k2oDNWqvB 9s97UiOiGngSTJY/Qd/j5DYGOWX/B0M03BX4CTY+Q9r125VJQCcH0NVQG+fYqdh48n7ZrFZgpvje FFCsYOA707TY7rBNAkgY+2oBarUW5PJlvIcrXxpL5QOUgRru3hL51dEs7uhSEijJtOkp3m9ei806 oX5Z1wP34XkIUEqB5hlwjlaM3zVoRQnkzLoZp6VtG8kHWZpmPHhNG24VqRQ0X/+d8zQ8eLnW1dTB 5JaLBfubDCLXATl1QBL+WemCChXufa3Nqg7FnRVUpqIAsuR4F5NMqrbr4794cAyY+MhDM6EEf2JG n48dOqPedfi8zYs8w1MIRqesU5H92Rua10XF2b/Kq6sOmw89bnAE9QlEUEdRXWrcUJKBd7pEv3zq zF7kXnW8XXz3pOBIOmAfrAemaMcaiAWZmN+Fuyr1d5KbpA0AZOS8ZZVSVxVKJmCY6ITSnMyHS0Ni hHDAi9RMBQ9n924zcwmxv7BNYv3yNINJaF1z9IiSwUhcBg14wlq9c9o5t0yhLZmTU5MAfFbVe/ph S0VoCD4zgYeHClOY3oFLc7tWdfl8Cuz8qYhzCn25wO+qzY12nfdDlvGSnv1XJeubv3JCvPI3lMfi oO6ObkGAcCLqZHqcGi1IlqCTQOXAxxWcfPgV3IqrdEPOmGTC0sG7nZmC24C+efjFbuwmPVfXZ/pW qRwvypgACqtOnBbh7zHp+Ql8yQcz52MIPCNeh0vPLybFrgeZBdmc0aatMUZKZiPeh6X+kZj6IZdp Rr19hKrBL5X9gz5SaoX5MkxPhOfRfk5BKEBjxRIWKb63ZohTuLYmNWTJ7EDf3P/Wr1cm4xB763Sx HtQZ2iviYpkHRYlzNPrPCaFeNBinI/h2l4UdvRtLMwHVlrAuE8VXvcBgs0kwK8+AK+3OOCy6bLUE 7VLemVnQGUqbOQ6CGr9d3evsMTLQeK7NTR218Ud3qxNYrcVWlP4L0PCiSGatDQyIkU9UO1+0lbZO 5YLD3bNh43USvFqJKt82tqTsfUQhtG9337qo/SdR3TxXB+eAr+29CRAa5ZyL+86AUoHTaEOgHhhq OT/tjYuR9RcSgEmihUhNEBA6d4zSOzQh3EthOXyjeyEYMzNcmvEMx+0SY9xp5J06mg0V4EKzp/PO +GJ8c6pTGjBfxD+XWbiN8vlKXTcCqH40BSpVD97oj5HxFUWJ59YR11vBE0d7F1v6Ydvk8iUc6CGG 9dooqlCvjMtBJR+WAisHS//H0aMuhDSBcq+/OQo+w9kJsZO4azXwHNvLJecYU76PzZWG34NsEMMp vnbnOY6V1wjBSuEOeTyKxmWtpGsQy5mRVcQmSMRDstj1VtFXd51D/e9ix18D1rcdzHzItNEpXn+I 5sD0NfKeEprldQveDT7yrVJnoq4613RSdzauacJEsNFVrjebvtH32swK7t3QTLh7ZfxDuWUXnf0U RxlZJVmpXv8G2vsdjgLJYaW0Hq8DtVpwvd1OmtYbn0yfL8Uc6oaH3du/hDLlbAUsVgodCkM5gX8E LfEnoSyn6ayV1dVIk74lSn/jHjL3fYmlz0ysJIDW91wOHV4QxTSYz7ngiTc+HWP6apzpyEdU/4Hh tUe1QywvSBWXJ7j9AphtXl3bdnFw5RpAHlHMXkymM/IuFYTt/yx9BRspbcbf4VC43YRRULuH2hxy MQ7ZZrewU7JwdT0lsHga4BWkiTCwu6VJ10zc/weeTIUvF16HsjZ4EwG3LkTQDK8ysubOhKJ4iFS3 F8tbXDaA9mHYHY03Hdfqh1tyI5RcZqtZKMuSNfBYS4rbopKVHNa4736HBvz5h5QlwDC+1aFiNMPs UiCN4JaYjSvZ/KZIGZSkC2ySJ+0dWokGVdjmM0ek+PMSspvjG07w/71SDUUv18jX35TgWFPAc1jz A4ip9ovS3CarT5V2z9RIDNSyWafOu+mLTNrv+OKSoc7TjZaET86fv/eWDTHtc39mnjN67Mi3vwFH 4mej0BAkdgu5NdNg3mPJ2U1lYvkGcGurXeICMD8P+n9Ju0kmdtmY6C80Tz3MsC5qTZu8ef3Qttz3 GyjnuqTCT7Ey6IjqhhlkvLKTV8zvnV7aKZFikcTtwyV/MezZ/y8nGJfmZrG5Xw8uAtHjD5V5Xxoz YDOS204tavz38F2h2WuOS5Z7HvNaZTZ6ickVWPR1PAaUMJ3CB8p91zEIwFrVk8kRjfRRyLZXb94/ YiiVoplrEWyV7L3uwvRZYPKdYL3MlE4S/+XxOwihHVuZCCeF7i3c5qSDJpixJ/u9uckmw6/+o76S BrwOXwA7PGhzlkDwvh+gHc1lk0oCRQjoIpK3Up/Z9cgzRguNJHv21ZeQ1JGNqVsIyU/4FVNybcZK bCogbhGEnFseR9fVBVyTM2DBdGKfadYv/9pFaiKP2xGcs29h13gxRR8LGUfAFVTZqktNSLruDdMs Lr/SvurxYOparkwCWFpfdQuMkw6PRiHxjhOuimzHwtrhV4YOQuSsN6ThF6lGGWZLE5twPYDLvJlY bgohqzQTe2Lu842e4qfXJ7sP/hnuI/BZ0Vs08fw3kCP/SuDB4aRMpf9VBawkm+fKJIBb5PB2KoTw Kpkb4aQ1XJ0iG1NMOlfCc1Kab/dkJYLagepB/ZRbqV0hMt/ZwXKixlkQ3NBv4/v4E2Z/PdmaYGqI CBltfazDmqKvFi4qoKYkxh+uLTlMC5rK/O7YmGiFuXvX+b6EUOdoIEg4VfJ6xVQNnLc62xfJD/w4 1oxDhGIuALvOfyimGfyqAgnedDsABdIQt8fBhd4lg+emSbGVheEa/22SD8R2hVD55Gk3UqFR7dBc rQz5t/Q7KjfSlrz9UotLPDfONVOHkvRVEoeDnz87oGD6m1zbnLPt1o+EpuOPlWnBcWPCeWsQL68V gFfWFQJ8hRYaG3KgvIspE+YtJyN/oHQSSseVMvzszKuV9nMD/BBmENN+wsz1zkTJaUVgxgXKPx1m X37vROpxQ0lZ4OGGNhbgF0rlsxTOquVsVHHq4+WcbRvy4FyhbJA3bYskrfP3rd/ABbSEBuUH7Hk4 mkv6BSd8uMMbr0XA0FvNYzbsJoJQIcFGnTxxORtivr2nr23zrMwhCcnB20oDM5rtzxRfKpWRrKUK v/NuLoYJLdvGt01NBS+cdJiRfatuLZ2tjFW0z8Uyfi309tlQkOKbzflorEhBrHGvUB+6nnjeOzNL hthiyBmul040GqbqeAftdCCuwMQgkj0/Y10Glgh25nH4zmZ6R2Op0fl8OWk2blkDbBl+RoNfocyq vWCY8p1LCV7y+p2mX7dhrQNNGFt1lZnnscIwBYTEJQfn5buEQiep+t2GAM1sOBD41YpgM0rSYPB1 zPGmCQnSKfN5ZqdfIQnxx78C+cf9rYYqOapVfC6yPYFzDOfY37Iv0zi2UuprJmvApfEYN/NO/3bD EHguysfqRKlAAZuZM8IjsIdf6Ni3Gju/yk18NzNuEjL9ug/nJk3m1m8cwbU6t7tYvo9IyxS/baqO Gx0qigoBPqq9pv8QPAGSGFe9dpRm4KPYNWldJlUSXtw2bznDVmJ5nGkJ3bP/2vIRSSwC28RiO/HA B2k51zSN8+iSORdB3QfIvjAex/YmKcBhb4NUbo827bjzzxzqzo5c0gTxKxl2MMS4uhV2+Yx1iGVk VfCf+p4GL+Fd9xIdMG2XpIkRMy+Nth5PDYaxhyAZSSTLV9zcUruDwf5NfdoRaEgL2Itlvo5r2wZC OJqtUwnBfUJ7gf6YC4twp1lof1VLxyRV5OxJ+YIzQkK+uf4DA6BGNdCHmUqC1N6Tbrtppt25qJYS vyb0h7HegAvjGn8Wnd8Y/im0xW2HPK6VRKKVb/gNE0fB2rLmyN6cOUsMGfKCTSc6VveOl4Jcglnr OucfdxC/XT2pokjF7RueEGFVUDbm1PooIvg4hl4c040IXECI8pOFJnFfv0F1UQx7PQxIu45HyMfh fRxunkRUltCZoHiY8Z17ssFEToFdEqdlk2meQYnF0FpdJb2WeS1+zeMxVqF0WGOvm0ZguMpAfkZE iMiYR519toWh6uLqv8d2JfZdxoxFxu8Qd6T0ickEwedTqIOHwa+7rhjAIWJCN29cRs0veQkl2H2q S2BuDvwaxYvLhPSJiHDQUigHECmxfhRYR+WZ2TsV/FGEYNxUSKYr/CKOqevQJCFVlg998olI80gW 8nJR/PqNfJuTP5s5dyMASbmAlnhrXEGk3oz2tl38AIyaYiorf+0P7YmasiTJkl8cM3oLfewaGt0e mFhRG+sg0hzY5qHJmuN8z3ONzJquPmQAOkBTGQ0NrBi2sfm8jXl9nz2tSIxnfvw1dO5GSVU4OyUP wPFPX9uSKD7dm/ZsSp9yO7PbU9zJiwkTObN01jPqO2fpFcIlYGiQuDov0g7bB9ZchI2RngT0ZvEi Gy6jeNJptODjBcsOL7N4ZSyokcZOf5ulgKtoTYUtMy+tLgqFtD4RTjwqDhtLhk+0cVOcKDgmfU04 DY2aMLniecJ4Dnpr3fnSVRCEnanmTaX+ArMyRJBgqMaPTACbd43vmELozgx0bgfXbT6daXCJzGaf OaVvasLeH95gAeg3IVJVrjIoxVpp1kdghoafXUOSodxO21u9Mu2F5tRKAFcHQEJj32YIJ8ymVDbn WokUFOdhoLj4dwbXvQllFPL8eCEWhzeWkUO9ADMln9b9PXcen0M01PCMx84eUMG6tvcVovMBHCgP imEgapJL1F0n8rjzqywfPCZ7IOOyNYWBwE3vr6s8DPMsqjZ2IMVgM8+KQLAwe1qD1E5LUjhc4sq+ wOyRgU+Mkn0p4AfUXPTi/4pcH1sVaT0rnEH4imNVjUotCeAc6fl+Q0ojzEv/TCO0wTWDvIHKaI07 ozjmTiBA8T3wPHK3koX7gt4jlNrc+PmvHVR3UTVy0CN9PO2qpi1rfGm3JtcOVxk7GGQw5/MIRjyq HoJPDj/s4ExMNKMuS48mtoi2Ue+6RYmzAHR9BDnGGSlsQwt1ImK1mNKiu5i2UZDFa8WGRXOQo2E9 iftHPG/Ba1nvorQXUYQeRiVVQds6Vt6Ih3HmRv3SXq5lxcx1t4HZVG+mFAYz+xh8FGlGFUQm59mM i0JpWsInpBAELINsjRRpKCOtWnhdnYA3H9jGIOMKtBXp9BMZmzsghk034okNqO1V5fKNQtMcxWZT zJwELH7IAQGflsk1KaM9bUxIcJUI7NsGI8fh7X8W5SlfZnAmEl+tmehDctUCNMGsdXVXLmLPhlfb jsf7sCIGiXP/xN4lJxobhqvffC6Xi4JzOvJn48eqGMPcFodwf+gujbg6+x1mE5/vaQlS0ZbKtZsU wPsFA83nLGzTGX5F1f/s2GcrgDdD2r9gXw3mAcTbIT/5xLZkwtdpwIJ9A/O/tsMAHhL+7aH88Dl4 tZnqCtieNj3yuVrDUauAHYhM1gV7GVHNg89wvgTUn80bDs1oJmFtDDRQLYSQ7A9F3Y5eHDrWUJgW jPgKuybKDuYzWcQTUpyauhm78Zx8VCWWWQ9oInKImpoEHeeEBQxXvtHRnozH7dleu0SJB5G5aR5q u0x9g/k6JuW18XlljuYYB1YvXom5zm0Qo4wbriJgDRdWCuDNqg91kgAy577V0D+rQKNm82cWV/6W C+qZXzYSVoH0hordXmQpqa16N1QGfUGD6EQ+q+D5sGGvRG0kQgZB6uHs4L3Imd8ml2R4+iLuwy8A sHN5zXoHB5WgJDHCdR1E7Ixc4mGO4yTdc/28pUe0Oz4uHfFI/yMaHrDJbXjl6m7KWBeAJ4uJMTZQ /7CXclTFxMVP7u6m/D/Bc4rQpwe1ZJ0S1hI2DeDnt077l87KMCY3u/sUdAN9c1MqER2l+iGxnijy 5NoR/L5DK0+Sp/23p00EyqfvrnUfc5Z0T22lBhuQ4CMR/fC5GgPgEbCdds5YesB8OFqsXmbQI49i WMum1llFSXF8Mhm6cbmwgxXEyCPWeNOvHwNmhTEiEDdjqVquuhRWYmPEzBW4BYpJxA24I8q+fbxR OIPeA+EdWLdhaUOlLT0uhYnYwvr5szkMLIoGoI5WhkdmcUOEnDfsPNhiHk+1tJ/BZOHZudtXXGHi ZS7tspltWI37k9Obm4YRNW1FsQZXtaD1Fphb1CLgyETODi2Y9uPa+xVFvQECYbIBT9mKGDrdMkGh l2Vic7gzHyhqFJ52TWwQPKigPrPVRU+wgixDcE0qO06l/e0RDgKXLEya82UsQyy9gAI6rvnhJJSN 5R/Um8VlOdg+1PTvFHI28FqhDQStK/if82YfVcLZFBVWOM//IQ5s0Kq69OIvOF3FuDvqJWG+Fgqx 8a8cXy2Vr910d3ZTJCbol7uiLcppImaL0zrILAaLJ0nak+BTYY4qpEkmvbkOpaIsj9XERV+o0YrE s+V2+IAqVl5ZRl/K9tKiS3W46un34HLlaGNFgQj3HQDlSECyZiAZBMmKqk7750GAe/qkDVol0/IH hmpVJDqiwjtyH9OYehKXxd+BhmUxUNIi5I05Dq0ELZTgVT+mNQO//QMIEoi0xn0hw9PrqOYggYz8 gPRHY083B3SJ3J7/Jf1OsaJdiziLkpuuerM3tMBknipluaKMgO2UbyUW06y4SqEQvutKe6VHnegt dt5u3Fw72zEPYhL1eJuaEVe3Hrl5F+sps/kXDhxrEyugk4JbrEDTPv72JklODU+Lv2Y7zUF3Dgov Y3GGOjyIs4N53aBwPDg4fekBPjMoxptl0KWO/pVeevdd1ba1HUS7SB0lSeU9ztwK+g9Ni+cPS8IN 8e+55YsZM1iftqI1NRb59/wYEdMFJJAzsP62XtgT0cR0AksNG0Y/o7/NKhUlVWHy08GvUUA+hsQ5 T81mSEf2BiRl9R20TwMXNblI+8mAFfjfDL9bApX0iJyFrZkVLa0nFAKpfU1qtsh4VXlxyu9F5ohk R4pU4canVxEe/WOZJffEmE4jsgtP3x+mSFd1yutCNpM37snClAjwOLu4DkRSFjBqAO+QuP3KACDw Twevbq+221dVrIm6tF6PO4+V59yR6Gc9nG4ztiOXBXglzioZaXORCpFQ+wGD98VbENIEfkh3yJ3H esYbL5KXfCrPIeeL3IqqF4rW1PK+kwgWvG73FSunR9ht9Gmy1rmxY2OQRctkSJYDTCprHBy/qjti JTloODre3gBuXk+B1OBgCq7F8YMMFJqWLjXSjBeLx8ad7Q1+dobNR3lFgkLZBQfCztF0kZ+q5ZYb Xz7Uzxos62tO0YfFqkMw/oBfD5XW3qWwyFCmO3TSlOlAyvz7EJP5WysFPSacuKXpy8LY/oYc5SNH yK0EEEfOXjHH9wuQHdOkKAcHFLPF4zNPyseHyj4vjmiuTeMqocnUxa0C6rY580wciG4KqiTR57DD O93GAqtD9KrRX40JaCTBkvWkT4Zt9QcRRxZQEghZQgOE5FITaa5qI9pOF2XFy1TWN6v8HXSEL4wD tXiEazzBZgCObXA8W1YCJ+HUtqcpaqn06DIaB2rxpwwJU3stOCRJzbkePetw9G0vGvMoaw8c8AdD QL+OMtGyNn1oZEm26o803zBVRp418j+SFXYeN5B0nRS32yLhHsaDuJv3lwzCszh0pqcjaARciN/R Lp5W6tDeGXqwJ9/Yrx5MeQs+aKjl08OlnrJsFRq+02gI64PTrzpdKqlnA801/vCEuCe0GwYR6XmZ mY+NXqBwG8+UwzvIwHeTVnZaIUg/b3EuaTCLu4q6WOQnZgQbri2f4rUWhsQHzA2q+kJ5NQRy3Xx2 yX4Du6Bh4CJqiOEmTAvIdE7X2xsFLISg/rQZemRwod1Di7ZS7piKXVGjXbPjuvrygmL3c0VC220y uWahuqiemoqZn6OZeWLFNv3qc93LvhXS+5jb+iwFSUv0agzu7Kat2uYnIVvZBKYd2MPG1LK651xB d2PhPcwAiP+sOFQCUI2rZuuHrr0hGMk2ZQeFKAaUovTsT0H9fioM15RbFHb+oMW4EnWitPCYSiz0 o1EcSrBl7zQ/8ETKLHVpff8T2Z14zH9f1eaw+sZXiBO/SHojDwtpAfd4NjMH9/PnP3YlmIBCvBwK i38X+/lLaQlToWtVD8+2t0euByyR9/yuYerWzjG2mXF0nFlhRlIZcaz1pNyi3T2DCic5PjIRzKOF ixzOqitGMYnI6jU06PdfNmzLM2pMUOzjubcu8AhxcTEl+T4kBnSZtxF1KSvzB3Aolh/7/pZxp+Zu coaN5wgWGxDdw9EsHz5wflMDV2NGeKP6XKLweQY0pyNUmxLGIMGS4rrjhAeLn8CXseXjBU3CgnjT yIBxdXkojAwt1QUT3n1uQBFVNqeN69knZBIBGXFfNnaEhahxRIlxUbU6XnBTCSwj4zL+RroDmOEt p6RAtVIhQaKz2uBmcSaY2oqi7MISMZOeg4plGHK9ilpHpIFYToIqILkvV8taX4vAe6V0dwCx1hDj FUTczidT0uXRguJ327LWW5oRSkD668XWEX5N2aZDvoIkXQYB+tL+F/dl9XqsVkRnQnRJ5bRDbNBw yCPZya7PHLeVLm7EJhOaeMGm677IoNjM8osDwbxFeDwGUVTJyfgq2RGqKTlQfpDQNB3DOt7GC6dA pQjrywczorKg7Jn7SgYsgBWg2Xf3MQA5o9mvMKed+wD/M/RjU5bukDa/td0qCVRpioahrnAnswre OB6Nd6pyL9fGB/trJxlUZ+Rmi02vsDr+lkPZ02Et9kZfvL4GN1TB4emd0kV2gPVKKOXYjcdzRqLA UkiyNlKGX+SCL6MGpED1YO8Djqq6s7EALWTlayfCX5pD+R2OUrqzfA6F1GBtUSqJTpeaqxT8yg5z FIUi7FXQsIBEBlQsTgaelb4E25MqCdq/f82QqD9cc1Wg5AoPfQ3Pp+gLVVdim4E1xUksl55htOca C4QvjmlzIpFXVCU0yYbgPSEeOaQo0SyQP7+cf/vmto92dN0Cnt81d1hi1fSx4yCvGKlzAB/2SSl9 SZQF/K0boabEJu5nkqiZbMlWTZGZd6ttIilBchfWHAxfavDBFg6MSzzgkC34GeRD49nnxm58Whas jTRj1bpw4VoLDGRk97lRqiYDeO3vwzrHz1l/9T75hIZA1O0ObqPJqeAuPXQWAOKfHYJMFAr482tu 2xYhiKffZrnKLkRNKEc80xiWDDoB70810FMhDLWidpK/2+O4iguzFmv3O+VQhdhewBMGWzsdraDH FgJ0ayubopLw1fHdoZlfw/mKIQ79rzIS63+zMekF0rwDODXhhSf0Z/02D/5SG19oiRkbUumO4RhN mt7vHsjDAemmZkJXccHxX848BWrhSimOiqZR2d4mXcXqmn03TcZ4EQy4FcW2ar7EvQ76z5BLSqfK f3XONRdAHHVv3uGcInfLE8wc+9pRftEuAZdqlSEHWuj8rAnjRjnFtCrMRq0gGxbknRGiT6NEjgsw MMBQ1FY0HABXAuX28zITv98kiYBxUorvAnr2vWh39jh3v/yE/h+hY5uiVDgea82J50MKajV92LgV cRYAttbRXK4Op4p+Ouumm5pGRWp3ic+xrvP72sUnHMWz4eULy+qhCG9vLuRuKvqmb4r5Yyz9rdIw RJpf2L9ZA/BYJptVrUjIA8g+W0XtFOqe1mWjsiy/Bp0BBqjwcvaF4w/1jr2DKgPjyYJAD6cGqv0V +RG5isv4SjbZdOCGUA54iPaLL/tDKq2IeEgha77HaQKbtP/jqH7E0EVncZmaQwz7AoS5ELTKRnJJ Ayp/qO7Evy9zD0nmt37DLq4b8ciFrfrygzzK49K91TR53+mPr28ZPiarJbp95tvQ/tfsffHk2joH j7vPDb3r7VlUUQPVM8ozZ2NBdM5tYeKqhYRtj6B8mcveMSrjR03ohSJCQv8ooTTmcrNADVBiBZbt jxjvVYAhVja1qn1o9COzyoPrmMDcREYOPydDEsfPc5SDlNpd2iuDu7VP4AFSS0nY1mPcisuGePe3 AbAaGNZ7qMovuItLOAUclp+xKoxvSEyxQVRFPkSE9scBfRa7MCgv0cUOu1ATP48B8hUB6bRQa2kF UZFZk2lr7w/lIy3JJNlKhGP2LEPDVNrLdWv2bI4cuPh5ySev92rAUxd1sMIKiojuaD+7TZ+tyhvx jVgPS6kfzk6Melsebz618EM4c5sG/y6FTbpaRsGldgfIYd/fGkmZCWVXXbZJmY+hiOWZhb3iHFTQ LW0trAiKMZ5tB2b9HIV39kyo7e4MlHaArFw+8MaCNr07A9Z+ICjdxxUokXLs9/aewMdNURavic2d TU5wLHLs9UUzt2LDo82J2Kp0NEUB0DXNFiUo9JEbyjUUDyP7nnFcJfh/iSth04NDrDlITM1YFb3t NrsoURE3r//3VbRKe5BkpBGSdpzmmpHQcFFv7BQeLd1TjKUxPU5doJuGt+V8rGpxVikvKvAUxQfu zLocelntlLni2RepQVxx0JYnFryQfP0FlHqBzmIXboWVBKCgBXVS01qlQtse0PIPI7kEHjkvRJ+g 2d3pXDEjaIAO3C0k21xgpiA6Cm6iVdm/c20JDv8QqOqOMhauDLrnp+7xaWl1xyZfCo2ah2g2Tolc 8ndcH+l8u0xbKHN+MCGktus3plpFfEWiTil812Us8O9IWrRK1SHaDg2dRFOeMTKIrPOq71p9oKc8 Gmp/CSFYbHt6to8HzGU/Egs+IBNjM9M/4ysc5bS+/wWJMEqIYsECcZjidOct6WSmlt16p9Jb7N3w gOBtwUG4Q7NhIR+TfDe7C1tYjl9U+F2izBr+DmjYXWl4vKPJHySa/PiBhYl0agPKAk1iZVmkwyEi A5k9ehtxMC8uiMD6ixyT5cCRiLUiWkl/OS2KlffDGvSxyAWaCfub+oFCQ7Iqom9qSwKrSSLAUIlp AV4rJwDJkHtGIO4pB79/g5xudR4ZV9RFApywVUrZjUjrrhEVb2NFYV2xEYiiFklGe55Jgdt+vQHQ 0ICF/Akd5rKijr/Ihys1QVG+YicFegI+c6FOXDEfa9L2K4TGQWnF51ihfIHgJNTP7DbZsvgtFmnc A11QbxeD87NqfdMRbO0rmACBbE/jj3K/wbpFZvHpQ0YNJN29bfwgVFGII9Rj/8VI5icytkefdWTn v/+NHnaZsgH+S67ZARnXGotIYtgTlbayh1UpxQp1Ul0DO6VFFJqGSoWMouRmOfXFRAlmw8++fpWh kBoLKCYdxi1/AEd3RCcPTn7Ta3rSODKOu2IBRX7hpeufEYNZccJOSBnZYLePmvnUbJ9R3IjuKF+Z xr0IIXM6ngt1ezJAllA/oIEjDs49ymkvAG4vUjxBA4XPRQ8reexvaEsRo7OAbUgmNnDKbtS0FUex JN6qSPwlqj7npC0cka60A8b41GcL0E0y1IRsDW3H9ZpOLtQtjgHw5PrPJDsG4AYGGCpL/zmq/Kk+ 2aWqcwioXTZbZLnFuPLNwWni2KAmsXlfxWQgEBdasNvJUaXGy5KUnKra3pPKYZB1lJAGSnr0F4a+ fykbMtrgHLna2TgQIzKhLCJ76/68ChBeKWYxfDsnCSDsulMfDfsMYBDuyYXqj7uvdTtclZfjDtTl lzzspDrIIwLsHuAlFgyRHYtzSkxFthMHgN8ONEBAan2EFw/fg2JgFZumCuJfm2iLy1kaX7yITFMO Pdos8CAsgft1e3PQu29bVw7Ov0C06hkhcI/99Xs4CZn8nc3I/d3L8Hzb7tyx6oPDYRI+LvA0cd61 SskDISxy5256JPpxMvyTPM5nO92IZcTAyJarTWXi9wCV8fS3XYPo2JFHGoKl9oylEDO4wFvyYgch ThcHDmZeNHYzbTXrdM6s0Musdkvj59GBPSVtxPg0I/ux8EvaR0BOZdOVzv3Hiec0q0iSD25TJSX5 2krCzvevVUWnUVrwU8G5b5v6yzjhoqR1Jy30/MGcWRYKPJOm1DZY2uglj6/Zx2HmFzpb5iGmVhIq g/vSj7qL94N0Vi2ccPyWaRcLJwpr9sINSTgGadF/XKh6OAAtvxOePYL0x39T/H6QPP6cl6MLCULf 3chAPsNd2mAqBTkgZ5wbWOxIe5u9IhIg1h+FNCelEV8rb2sUTFdCrqXeM0m3nZZ2x4xmpxr+SqAt WbiOLaAd3k9L5J0+8FvNXEMzTx6bzFnnF++uWOl9FooRZFjbZB6OEMiCxehUeVZf094aaSeMLmmR VYyIZxvSCd5hNCvAe9l2n27/9c9weiZJWnIcKS1alW5LVlGAopt0e0+R7ykZo9MsdE2PvMsGTSzw DXMZvEG9M9Q0fB452/nBWm30R0xIqZQY13LejTMa8UrF+sbwIvu68Ni5KNmz4wXmS/BUnjxihLqA NtgOjJ9a+6tMkIDPZIp6zrfXfrA2pqSBD9VD+ZZ3Z/n7yYIKB1ysTFCb4Q2VlGuj6witwP1R7w9y /pKEMsbVfcTWZIrgGVDD4W+JWvC12N4bhg5F1PUWZyYypMIoWGnJe5FS5gJYC8gJNqcr5EtguDU7 UFfX8mj/IHyM14G//czM6I1MHwN486Utp6yedpnIkWy6Cav0Xn4gfqoqEx2a+/MZNVTRMPykwuUe STEmZHXI3nLZ0md2/wrNAxUkuXwdH5OOqImOxmkX0NRjFMEvQCm2Igwv/3DINvXKx/1+UZprFlmm +mGRa+4zkrxKwVps9kK+yYV/sXyXFEGQHszmDNbBNIhhZGE27NqdeF+tgx+32ev6P2J0dzSjP3tm NmBluuFeteA0F9mbIA2jvm9WH3CJNGShzzNJzffgZS93gyK8YD6VQu1v3aUKwD2Jx+O2AKotaf66 wiuaLs8bM6f9ZKSLM7DPS3heYSAsFwrHPW2zF61SBU8OCqF9V4ivtpFCGqPMVOdB8Xjdx1g2QS76 uEILtoQJzTVdJPGVw9DOxNgr0HMOi2p92IjeWkNMC8Iarh0Z982qnaOzrzsw4jy5OiVEiJkMc7qk QaNmMcBk9Y6VUktKFIwtf01xNNvPWMiadJ5kTd64LDmpMli45wG9av4QI3l2Y8hBdr12fJvyM+0P a3Vz0h6otn0Z4f8SFhRH9AhLbtI+y0O3L/CPAaHm6S2+EDZVwboSsw+LtaWlseU/ayhBT5FsyoFS TCDCVCYfSkXfNj/vROL2cQTIrsqxn+9YxaaVYnKHg90vkd8aiFu63mEd+3sF1+MZM3SlW+yxkg7d JpI6xMlqYDjfEu3QMZXkX2gwgGfz9GP6Au6x1inJMFpVZUrUIh8yXt1JoU8xgF0fHASrgs7rlof+ NCl2VkwF/YvA0/1H2l3wrmq2EbUUGp3BuU9HC8KpbALOQ7gwpDQveNZ65DguENBMX2GV1LzuOvbw 3H/O0R9m3DBpAWY4ggWS1HYH7favmOcEUTxeOSEswpnLGZsDK5F5KS+V+g2YG9uPX0fpcKYPJiXP 0f91lTJBG3fXxeJj4BnwaSN1yw080v3H6erSpzoT5EN+fxDt7zGt3kU6odWxgPUCdxRHvFwnbl69 JiuX5k3NUpJEydOcylrMgjoZorNT1hzmDNVGGv78iutGgvvtPIp2xIwQx7KSd3yum/Lu300GoeyO b1c5mCXdfs5dJOivMQGvkZvJJ/hw9jMJFlACVhVHaXYxXVMuWVKuxtgxdvgGciz8wZpBvcOWnqEZ mBUvUimP4rVlkaI3K8oKA8fxMcZBMAgQWuCYryrD5IQOFg1RjdAzKcjuOE+soO9XjPtLXRUhDL0L plOu0np6G6WgtNqdaKedFj2HpPq0w0qtCpfNmj3BclGojYCgatmhTs0SpKM6FTxlEfKTh4tDDfpd HnHEYcPjDN21h7iAOfEiIvPMRqF5pG7tyWHFH/Y1G1ftMjfheeEHTj96MrLvVsB14PYfFuRlVRtc G4rVbMV7ykA0qZVl+lp4t6NFuVpfEvyeeEhq6V/4jWPbBzhe8rhry2vFavD1NGRYQ+ItN1slB8ts v/LaAfGy4M1M3VAVY2EvAwt6S5WU9Hb3xu2gJ0xBkMGlBpEuxKttwB4k3K4LqL1Wh8j8r3QYu7fq Fh+viqgf+1jbWPJPh27wwU2PFDM+lbLS7PTHv8nWi5O8JPmbQKfA2fDSSsfn46lk7X79fXw6czk5 AwkRksqb0vylfvfVxCBgnTcfAtUJASt6SmqSO7HySIBkB8zA+Sk5DyoOUO9PYa34umCuYDOG4Xoy 85YxMp4eAChwBgfe7jD06hWs+8tPNVU2YgAlAYqRTM4diaVutRlxHgLj1GTArpisIKPwEvKLQ9ws UkR9kO9qB4LrTI9gOp6kO5RI9WgHPInMuNgN5O0zLOIqHW9+tgqkQrDOSBX12qnAiMlWnxXGGNgF 3d8aTHN0pZW8ljk1JHrvMzVFuovIQqj1xm+0lWdS8y2d4pBwTntAh1kOrXvTuHOiTSFnl7AzEVg2 lRTk+JxWt8SqM/vb0YDGOEabefHLoiopqmQJfFDB6y7JY9Nwb7xj5vG7Ado3mPoLf3V0vW6QQifC gSBTVcMEgxuAlqWgQjQHOU6TO7slWt7jG1GbP54yUCpzjFCRv5knFud1n+MQeglpK+DvW74fUuZ8 /jB1pW7livNRb0GRQ8iWpM6XgwO2ZW48iEqmtaCNk91iZpeEz4A8OUr9CPwHJXKNnVIvCDY4c2Ec bwMIRRUA6CDP6PusdQPBXXf1+J4AqQXKIZmDcKDccfIF+lJ71H/mtUoSqhaOOByA57JrY1JKDU2S gYKTVp4c2//Kd0Ir/kHalW5MaKhRZvl9WfV8oEUzHNd1TOgrmsCxlY1yDQDu6/JMTrSyldL7M+mm b7Dnu+bssWv+s4KTW9YAxeLDzQxZ7kAhgx+T0lG1wqd5w9JTa2arF8AW8TAvTM/jPZmzCzmukQkK Mnp5hHFC3vs855nvHEQyzA0gvQQn1owj3PcBdnS5KuHYJf7SIVsC6UKcvQr5TdjB4T9bjXP7tlTN 1wfaK+zZwxorajB5KCEt/J21+b74HyL9kFnzNA6Yl5dx5zQ+Zy4YIAj/wRKa90EAgzYAZ0jiN9i+ 1kMsZXnYu8a56rOYQN7hHISUQ7LuoqpwWT5MHzNMBfbftBYM6WhB79UhyLagR/iGzjQ9cX3VvgR+ +zeokvZW5p8yWe4iInf2p2bjhlPWKXlkuzLM/u1d7nDXxffi/bjYFnAJuF7NoRXVXBvnKxkAiAcv z06j4ojiFPYGoTX7TrAmtM8x9fipLjaMYtLy9P8LH1BK9L5Xpe4TJ4Ot1RdRrBs71GSclWGcpA4R lncGNQGfnYjNokw9TsmNVGH5E1sDYDx3UwjHzdA0IJ4XljvbyR1pufCwUI6b2U4taqOoTqtNbpkl 5+VGqreS4h4JzqGjZLGPs+aexoUAH/MYWyf92v8xJ3TYVDDGQjOgcO2H0V4B3JdkQb3KPRmKfb6H SHBF5DYxNja9NfSKrpkAbOjq1jIZtPmlPilfAlyjJSDc9XctR+uuS8F2oWonQyuleh2Ux3aAsl4y C9tmFzypCqH7/jUz+MND6RbhkabKLZhDxhBDwFNd0zu/YOKPxPWfxNRwTDX1k2Gp36iENAMPBpGJ vlISncPVNUV8TCY+R+oYD/oK6J++vFcuHwAgR2L6Bkajr0VodR68Ibx6FHi2GslZukuy/w1o6DQo yJvWfSAX5GMpHvhdjksT14xXK/aUlwuaG47kT/yw33JCEyxL+IO4yAbeS08+e63+OqkD3iMjJhne ud0oCxHqmYcD2KtSzXYcrE4ufFV8VGS5ocVg70SxAVxvgIwn+7fc/cxqFqTLKxKcdx6oHXwKcNvW T5HLUUhK+xzZTCKKvG0F/rEppRNVMVh7ZkYuILPvPEGYajG+WYKdOoy33i69ppn8ZKBUHIo8oq27 5bG2uD0ddiEe02Qgq66BvTxUPznua8QQP/uOqE81aMP+/gxKKoZZm3tdkHLIZP6ne35/UooQduVm hcgi8KZ1O7Jyg5gvUBSmCVZovDqltUHwlSOej/GGU3BvNcEOvUYMNMl1g15zrTOuOwym5fr6Y0+Y vFV0tq/NxAmtcBKBtHlwJ9ld6dDyLf3fC6f2qRLosNwNQ8F555HWEvt8qcMsvb7Wg5OOooiQazER lNycI7ac+cviOhF/sG9ZMuS601tZzgX6ovAje6caoWC0YCkWqgwiBDdAsZrksIA6r6A89D1qPx+C CGRQ/vjwbe8nmtAEiRJBGKnusN5Jgv88tEgRFRVLKRp07hZtm64/wR5Wi7GzQdxa0X5myXPpNOva oCAt1D/n0vf2Pdl8t5HJW9X70+6cM/VuzRT70T4de3gKyxjWO5Gbvx3RmuoEc5RshTK/LI7BieAH K9JHLi4o6+7BBZV8aiMMNc8DtE8839TSqrVu642+8LLnNGxG7WWtB7EMlpa+L9xZgMZNTw/+mnqr XWxK0fL78DHd6vu7K1TkKSW1N/ZQyJBy+RVk5fVIAPe2LyI+yFjJM+O3s8miIL0JrilynfWOjFGs aykRD4f5oZePgwuMe/Qca/yz4AuG2upapaiDsaSrXfRTnLIQMaqrvhG6i8UVumzxu8oX9BStJYMJ jsfR8qQylLZUGDWP14/f75vJAk8L2CvNIKvWyatlxwaSB40DpFSCOvJ2Kdl9ceE2Sq+QQWPiDXSg dsJARu7WN2uOwEsx8d4tFVufbaNAaHT3IsgrvOsqbV96Xws+SVM83Pf4lpJSbzdRiVVRnMhbk0l5 AmYh9ZLU9a0pWRbf4KzVh3onA/Nz0HaFdadDiskF5eVrVDFg8nb9iTAz6GSSSZEq4XVi8Piz3rJo taOuMfzQTibb08Lavzz7ObPIWtO4Wgy+1yMbd67KAYmCd3GZ8tDq+0Ro9Rp17DoYu9pwAzeIZFE5 kU9XvFOI+rdhQd6oyrOazV1wJmwx11LoIlvqUpkQgWmcaX5i/fq2eWTSPb1NG4cWGs2xIMYuM1vr yzYmw5GQVRuttgS8KgxnlGjT7IQi/N32Pj1XrfiRHzIGP8EWlr4SGId9QBBjm7xty5EHjaKbJwtz GxSr1dHMndsNLGu/12CPOuiSiBQl5/aS4j1XcwucYDfn/R82Oydk5zYu1VXwKTSQSk1kb02seKhP x6s++ytZHkfWEmpxHInrb0XCDuB9TIehuJeapOXI0TvkKxvIX6vhIC8sWN5bl6b+MkKYEsBqIkRe kXDKWrP0N8UgDuPfxnttTTc+sV1UcLdsdIsOfVUtK2dZtrufgEf9hQjf3RGrx64mn5S0AwB7yO/y ik1dtKGaWTOgCDwovUqMgOt4DLgbvC+ZjIs6yT+65bRaKKADyX/cJY6cr/P0T/kuHRnkQts4/Xsb 7igKHoHbo+GXzpKt7h63b9txW4jM51ts1GXSdD07p7yLyfZkDsOaY9t0Jg8Z5MQ1km3FqcX2RPYz FPhPf0Xn1mnyL7lxnRUR+nhEcJxhNJF+TUlEH16PqG7YbWvCMe8Yn4TkVNgBXw1eCcdsu6sFGNdc WKKOvcjKeJXn7PAX1AqnrxIJjiifsE/QH4ekQNmJOt6kg4Lj8ocBTFC9POUydyaMoLJtgUIRpCcn PRShVys24p0YjHn/4uigxl42aWt/pa6lhAuGijbFQuhK8uhZW+6kzgaT1psw4uaIxlH7UDspjVEG 9XPTZ7uRzdf8bDde4GbsdLTC5vgUKqt45UC4qYay4+M/IIfyelxH73AYz0Vdj9MQEVr/B7+qvLeh uJPPLnjJZJkBjtipvpCkkua30vZ6B47ay0KkwAjiqWMNZvglS5Y+tfEePV3cD8wOJG2yF/5dWyrY 2vGpgykXWvXEEs8R+vqrOVPIRvyTvXn41xebCz5LX8/SO1GfSAdyT95b3uZUySllXPQDXx2ztU+X ZUREk8eav6B0b2FhOynSDFdMq0Xw9hmnTydd2/b+8URKuRQK5flQHcC80dvUjxBnVSJUqZdvlIjq iA1Mt6+OmFgSx9ex1T5gG/coJX7LXvdViC0kW7eftBV38GiDsmfdqHjziBt79YriWiYC1bMtGh90 9TAN4B0XrqDF9+vFDMb+uzoyvxUA+rlZZv7pnca/Zc8wXzYB5lHAE7nys4BFbKPSSJlv9ErovDeM TJ48auHvMlBhhUn1NKzPiEqScKXMZDUzfx9/nbFx0Tv9RKV9OitwB//1HlMg0HFH2WRNqRTe/u1x JAidLwdv1UTbWYswr2NyP0VAoe7sqpc/z5+OXmT1GNMl2hN0gA4p7k9cNy8nZ5vPSMUtHWslhkni NbJdiVV/+4FElM/5johnBLKtUzTX8TpM2fZ3fV7TjCeptmULb9Tbgj2RKtJJC50+vqNQep9tm7bW 3swSaXsjtCkX+amL/s1Kihe+XQiY56C7pZjH2au9dALG1vaMxkofV167fCnjXKl/07Fk9bje5SXX nPt7Sr4Vyv6Gd2w3WJ2NyQRqYb+VGlOM8gVSN12g0gp0NOKpF8LyV0aHEZVNqlrc3lrC3P5YmpEw h+5RYEzASLRi2dZr+95HjePcz4OjugVI4EPveVXkEvFmNZu9ooYV5HTS1WGpnoDCRn5l1nSQEJuu 5TmZWNGrxmKxC3Tf6wYBhbI4vpNWKln2bPTEd8KBRzOhbXX6h/XZRFCIj/eCbXY6UZTxbeLY3DxF lTKO6DwD8xrSIXQJWmHXJ7f4OyXxT2KVzaufq/z1MKyj1HLpvcAXz01nKNVLiDSHUQvvUnh9y488 zCx9/fqsb5Ro/pbNy2O1uI0LIOEqhXTXuF0CM6rlsw4Mbe1bov/fZJ3DV8xTHRAtQ4JfNCrKNSe1 JQ6FqDbG0zuqHmsp5YhSSB+++Mr9FF9NrXxciXVFR76FUGdjh9ZzmfxqU3tBvCkS5pbCNK8YgRO1 mF6BAdWe+moiHnVekxVKmo7VlooGRwklQeH24mcfjaK8+OKUwJ0ZIwYKb9sAldIwaXrmEpzDSsk1 rNJjOonohEeebrgHefB2h+PzBrWGNPYb5CJvb/seN9UPbdbqHC1A16KHWYC+QGl8C8paUMlWXh8o 4MzltinMb32lvIAY6J1c1wKoaA5Ujd42Y2N/6rcmrPY9HuZ1OOHaiuJRXVBwXNlDGA5siZ7v8mM3 GkLdP3XUswLkj/bkRhxRXzt870d48X4Irvga+UF8leKo8KYjHibRk2XkR3bgS4GEuj6OY+d4sdgX OWoxdxS1wkFiTmwFs3jKPaMA+3gOeCsX32yu1BUDBE5mYSn4ObcH3mp60p5b4LT+8WoDVqOk0dBT jeGYVDEBWCXpaP+izQnQRJQEc8LSTUR8v0wRKp+oUIha/3YBqU+ltsLi4w8HMBFym4fzhXA6bnX8 aLADj0H9ql4+vitzHzGiGTwRcgNj8y+HIRPki6q5BanLricJ9cV95BX6L6gcE8WDssCAkQAsavTs Dx9JquUW0uCLDLnrz+8UJjuvNRGnF5dI0axxjy7TYtNGMFqCx6c3ZCZ9/ah5EgexUI6QinfCuqK6 p09xv0Gu458s58NKxii+lJZ0I/YOLNRao+2Tq0+2BgcHwBBYH1psa10cLrELSCjz2nfPd7JXZ7w4 Xv5dWV7bCAvrjiTtXVueVFVJ5LPFM5CrkjA3GUrk0kl8PBKBjHgvgf7f/K4SrRbZxaYtHv3f/tsN gKPkzE6uHWjK6rplXfF9cxjMMIdlns/9OLE+ttjwHVID+h2SBfO4a7Uqjmp49ivL6bRkOVHedU/F eNiOUhcb9N7fvyOwZCIfKI5pAnBUiM5/5+0YhBzv4NBnQssix04LBplQNkhNyeCvfjAHWnM6NDkT pSVjubgBvBHcj0ye5RXwzagpzP9hxXJE/hOtEfiIb+hmvgfKSwuy4BszlVid1llLu9IIpEobt4I3 8GADqDQJ/bcVz+QMWhniV15AoSYvn1nECt3KPgIw8J9ejejoczedRUBg+22jkYIqKwtCFpzzTFw0 yVH/X7LT7Wbrd47bArCSIny73g8Aye3gvdGy1XztmZpWdvQNqvgiI3shrezBbRCaSc9Pu68aoJ4q J9c1nH6dnrcy7aJLIgzJn8usRZx6iZ2y5k46GyTHu7+0BbbZUdgFLnaj+Qv2D591qMt2Js2SVAeF 9q/yWbQqulqy4e4+kFxVUMCGosKtgWldNsGyQL+8EL+a62Qbb9+p4G5zk268P7Ie6BXNb8j6otOS Z8fjuIZlacTAf3Uo+ZyBzcJmzDriuWfc8Uiaegrt9VICkcHs0nhp+jJZ1gA+EIMxvXThk+aN9E1e I0H1GPM915UrWnHlxkAwhugNaE8W9xIJC1DI4BmsqVg5mufiqSxbuZwLPshGMSZ3tBo2Abbj1IMD ojKDAzEkQ6l6MJOWBu5E1+du2K2GCZfV2Q7XjSbCJ6JI5I/lTtQ87zCYFx+cH6UX80niockupMox OhMoZ++ROxxLN6tfBpquNR2SQ9Q3CpSWUL26H6xZIz55oPY+ZHS0m6sYM/UjnkyuH/t+XPQNNfrx LWctnRgL+GBZVd8ydHKV3IgVpuOqxlsc9I+CPkusjmqyhcw78OpZUhubXRQT9R1e1ZBpcDjYdjhd IRpX64ECM6SxOVxjAPQfSX4KjtJwfsjBeW9+/Hj7HoKGFN3bsRWxpEpuR76hq8xv5TbjRPZGvwEH 1tu66AfSgB+l3m5epfGtpC4JZtZWsNXJLejA7X5PEs1QiAEo+aFqUfeqcEndiFARbk3BE78PAXJ9 me2H9TwfEhPBEtBeRUsEvbPHqXM0e90P7GIW7Kx4g3McDE0l8bLMnPBzp2PH88DVQWOFaPkG+NHD Avb+MkPNSxnT5WFIyizSsQIj4Gom/LIqhIXUYyc7mmNBCAd5FCzjxDvORimIlMGp9HfBrnq7ICMO 56BfrOlUDGz3cjR56NDhvgEXPWnm+GrRS0DwVJonhfaA+gaKe5c1a7W/JAnC4NH9L052AnA3tcHj wc3IlPd+TfZdGSEP/LGUCXz26zd3WkKKI0nxLU/ZNf6pR76RS3XJKI31inYmk/MuAb+ANz0yi3GL 69fWI8LR0/ljqJ8CgA3y3BWlmecRjgrBRXbIVJKzlS7aSU6lmpZQ0ANZzDynGUE/YJnPAN+7BPZB MeEK0gUTx0XyL07wgQVIwOwozYkN9uRHCTCHQbMlMwMYeGhWgDhKwPICwi413xyRgrS+WoTujkUz t+F4uUNnchnAo4VbmlWxRD286DtTgsoiDbBFGTZJZ6W8eBNa3waWUtZxpEZDxT/BHxAv6BXN7pKF Ptbu1dHb7qqOqqIy5eRZYy3ZryBx7O8pVU4yJIoQ2NA517EvCcYOuc+JrgKVv9Q81Ssq8DtdbCVD fi9FG3cg2M14pKQU7uvKCmtScXm+EFeB5BwEL4k0OZPeclhXjHOdwBcIq/dUvUa7eZr/3ElNnMCD lyYSmYd+c3RxDnaNrF4bAIkws8dDS7sgtzsRx5F5cMZbBDFP/fdJl7/Qqm4OuWTZkDti6hzxrEur mVPHQ+beL6Y9dHIMwkcLMj/GVMzJcrjP8niucGN7vQLt7PlJKRwUAuwAIDsK7OrmAZws5LC2y62g xy9Hqhf47+HQiKq5uWAkuz3NvnluMPmZRlBnCtFFgpFZUaZ4VPar4fPWw6zmptbZj9Lv/I6xDFIF V+7dRRmcnVyXPMXtqiiU2jdoIYwu8ubbIJoW7JrdthvfYRZT/4Ufz9x6UtQxmfqYC+Xf5lmGnPrm dyUTRKM7CGGPFQMw/x/Y+bJOUBN8+JmwH1BIFzI1ACjIJotU3hqdnJ6otnlcvnY5KiuNUmx5LpEf hm1D6doJg5mUFy2vQpAix2vIpG6tyuZbAw3H+uoNp0KuyrDfLpKBZTrveu4xU6ulp0Kc6uuRZUVC JHZjZDsSllyHZbA5ec1+ReeoPGOLOwhbTy2vRV8YDmCkewqGFfU2CAcgU4rAZeidCVxS5PwDybgh DmXsZEoxF0EKqAnfag68hZH8GrQU0S4hGDFVAxb1c4DYurI7+0J8+Ds4wuytsOTTwXG4z1O8uajr C9PiQEdhM1HaZieS0e5NSkG/8CQangLH6/cSxbOEfPy/hB2U+7dTcdnIF1qrsnAC7GZEBI0gpAzh 2KCjRTjVm3EF4DHDxJBTOFA2t1l3bfUWH4PUIEkChpQ/mdTeAr9E+bFaA/kYIqPAx82y9mLv6oYP 6Pcq+SfpCY+i5nKcUbqALm/o0scemUoRM5jcGqwuOdAqAGJ8HiBxu77HBq7ARLUqfhXNM14GwwPz G+V+27028gwOK7bKCYwupBfV1pV3MGyZ4G9V96M1Nz2JUlh09p9qTkHR0FEfO8UWlcRQXTEJnV1A A35WBDIxaxWQclYHRKiNoMVkQdcNco3R0WgMzSQtMUsr51CipCSuUhvlX/OGpL8htd+ngevKTkVn WEe70NR9FYC+KP/zflUlleB3IBIua1BITtpxkQFxMOwRwWdp9ifag5pGpqhdX+oO82rNCqd+CNCD Q7rEWfGaGD0gOf1/POyrgVcykFWIC50h/Zc4r4ODzYtrXqm6eEs88HZd21oCgB6QIHGTAa76esGS rY0LicP5yv8zpNHet0t/XkCfAF6YXfPUOtIl3Y4dF8Akv+555NmSx/zxy8agvhxnZ8haHXJfawyg nG19w4AGWbWp0F4DOtSJCSxWzv1hz1BSaX+08OaAVcp6QGPVm2P0YlC3WqzMRkkmXiNE/ifWa60b PIN8Er6tysA5kvVOpJGjMWBHWKEdie4R97L7z/+QR9c2qn0+R0w4/85UFuydNJDgdSoTDYKodimQ HYmFctUxj1hE9UJKx4S2yuGa5RUvlkorj1ftVoxprmUYMsWPRM11JZ1H9A7WIe1TYXWaSDUv71WM A4XPorISonMq6p0lVXj93NvSxhZYKDZm2CIk9wbFOuurnaGNZJnNgANFKBNvx0ZiLAa1R4FnN4mJ hBw025hnK4awuE/uCDD5W5DTuzLjg9noMgqYrHP7q9w2dxVZdePxJAGMCuErb1R1cO+xuTCE2uBZ 5s3oEHo3K9iENB/uYttAhLT2QBLcUigfEwMCC646muj586kNkI2ldCJ6O3dEkh3L2awwUJoDAGfF h3/quedgcL+UiOf8w931pdvX7B6cBV7i6951ApXX/PBXNPzJmO9fJ20PVqR1+4e8LwIc3llgSexC XZDxT7nYHxLgLJxfKqkgawg0lbPT+zPW1YRhBw3yiF2UtDTLcowfP7Myg0TZ12BEVHGUJwb0Eh6F u+3D8KrFZZhqfudu/7yBun1Hxa+a3kxEqxBaU0JNal3HcwH8Po9UgYmfKhXb5cXc0SxlFJxmn/T6 evi9jh94/xriZkr/CwquF6//IrBXX+p+W3z3TjAMc7LCG2RVCNpD3zl5KRF8swGMUN/NSidbXddb y6F/08x7SaMveugOUOj+sgHePErNeEMtQgd6sBZ3VbXiJRjgUG0DsFd6qJRcEyAsr4MWibI+1KjK FtNCWltJ1riRXs/1mI2D0vgy+kGx10eszsTWmI+9Q4SOzKgVH3CCrXo1a9Af/7IbUVMy+hTdwbHz dKlVJ4APLEJ2P8rwBHWLIowP6gyDzPul6Myarkliww9nBdEw49EAiDdKKsCb1BZvq29Wiy95FGc2 SelgkSZyAZPcAc7cDritFSk7+UiSXdXhhIt+gJe4WhrOXmAihOigaxIXm1WbYw4ZDkRDJW61inTM lmz1isTkzuK8w77dip2PlmBaF15vTIsQ2fIWigqxoka/GItkjc8PANxrGaRLqf8ZWusSlhW/9BDF j2q70yQqzWZPRu0bmEf297YIglcsXcZv7123jZRP5DyI4pS+0OUiE8x5DPpRERId8RqxGmhkKc4W lj+yR3S9yv6Ah/A2Bqq+WqL8yBgY2+0rUpCY8v1pTQ28kP+I8ix8aPH3iTJ5KO3gCNSBX8X9DtVr ebqJ8KStXHm3urssewfF+OSoPcht9RpggVTKcMaOuXbP/mDrUKMEuiqQ5064amXGAWirmNPT5Ezm JZJanKfCQJB4at/A6XkAV7xwSi8BiOqVYD9zXER3RJkZTx927y9catf8cz3nPHRD/nipKSv9vFe6 UnoZ6O8/0DDVZ+UoHnfQdxOI7R8Ih5rT/zS9uFdetc804dOUeZAmIKamOos8torNz/kGwsfXyts3 5ijoJ8A4ms2fQ8ejiuBYbmvX2szY6CRWrtG5jbv0633uXwJVDtUGxNNtFR6Da4seKKgg2kemD7n9 BiIjRteselEBxpLjhGpHp1tHc2kG+qNsGB8j+uNJTtmXRwOYdXkm8IC6cDshySaF0KR/7MLZCSfa gYb8a5/hOiWTLJfH436nve2EnKWepjlKX0fbv6jk+XpMsm6toMWh8KVkNY9ADXCcivltv1ptUZ93 F+vAotsJ5l1UnSwyRo7WEQ8pnAGWl5+ZBxq5LiJMPGhyUJ4e+W9D3SPFkcql0ELJuvx8zAkXI/Ue G/z1q5xje/eSGJsnHmcnC7wWqtCbSoljMEu/02RFMdxyZ/dXzYDGTUFgok15KusBm18Xm6Gd5pUW VxwvV8KuxhkA6hDb8ge0H0M7U+xBrUjUGxoRphl54t25PtAcdrHjZDLHdh0t4haEDg0UmQ2fo67a KQkbc+sPymeAH8WwqvZdufOcldUCVGxrAi9qGPuf9uJpbeOLvGC0hLdooaWfTfjeNu5y4wCjwVZ2 iDrIF3WG64dhW2XH/wWOp3kn/hVyfUGx2Fzu8ED+7/qgG7bMAMqYJ9bi8+8m36st/DPM6bF2A7LQ /PZ8va/5R2jWAoHZsys1JjH2vD3Sl94YVFeYECOkHUgr+m5DNygApcRPyQVDKRHqL8jTPOsKcgM5 r8Oz+eJLdaA2/RaEH/eFpdOLgV+wzNApa9bO+exgc+CcsUxn1TmrdgvI+H3Sef6bChLP66oupulP Ce1JAW21dwyNRLohm3Wcjn11nUu67xBNoHmBbKstAuh4M+AhzrR+rxRkeEGrGX7fDfQvGrgZkll3 +orU0gljpBKFpfJFt2apFqmEWf6TrTfPJrnJzcCVStE8IG506/vz3G6xeu7ovqBmYLOX2LERh5Qr mXubdQ7mBbqVRnfORoetS70XEcpUtyacqKHMFNPIJwsS7EENBgEnf8M1derew2fclOQQl6c9XiEr KL1E4aNu7l+aTAX1LLLzotp2VfE1c4AUu5ULRnj6ufj3l8FoHdsUOPWQPV83uAODcI+eR5DdbkTg cYb+iPLSF1WMGsAR92ETO10rF5HH+APEors+fw0Xr/BOJyqFJLnaTIYCB5pSGFtCwfp2qp/V4QrP VUacBUnzwXQI87ug2bNi1ocj+RWHWTPXj82m46ztTHmPV9CsxF8aWYu9HeucP5TW+3XQFK3AsINp FbjQkgm2c/GyJOy23yq/JkO9oI/pb02vTCFOzYOW57jNpyVbKifirttvvZciHcpvLJZCr6XY3jFY 2I6JPsEX0KLGm5X/1DiJ4jh8PvzrPJUVaFzLrySfYjLqfoeDnNVsNweuvpD8jpRk6N2j+mkfzKGN nCN8j0SbxYMCODdA6+uTei6HYNGzsFlpG7NRmO/rBzTDqCHnOU5Ebc8YMCc7aALCLyuhkQ1imdIx WBz6Ylnj79DfsQeGkcdIcVVyqEqt3PTUQURstY7x222f5NBiXi9vwZ4wcTpgmgxkLHzA5vaCZlfs JvSmXT5hSdRFxtWGoUl8Jmz4ZaTvMWzMokAv7yitHa5XZZ4mbLUwJA2hEJFguxsblFbLabpZVyB9 FDimk69rEzbEqvH4Ywq41mL4dGYMOIJh4qhTs8GhkBfWz/XvC1aLgpZYgDSXEfh53T7uv8xxSOpW 4rlKLY40/aw/verQ96u7hg7W5JT4tuyv87FzV8pBohgl4KUKqGcQ1C9u7khmIoVRnai2EZjlcdNI RgsIscH3JkKRGFE7FwNH9jqbPatTXzCC/Hj3yfHrsvo90nBPN+31QSo7hCu7NyXgVCf3cx16fb9D 5th4So7kXfjykjv8EpatbwCpknhTzwFCKyHgGNNBE0yGOxK/zLvP3BKdMSuOaXajV0Kee3pCtgoc gO4Hs+rRcDEqPw7rJQ5i028foQRf73wmaPX2jfkYqYjO6Dq/4+id0lWrHp0m1qymLtglr3CPmkCK Euy2o4w6nn32cadQq1Xkysn3EXuIdoptmkfGRP3Lu/7qAUSreNCuBPFGkUwhX5DfbScpAspLx/dO m9LfSGxQp5bQF9HI7xnxq2n5qBlcOsN4kk9OnZSvK6Gz+hphF4I+Ol3QXx0zjoKLjFWS1gEf3Tkc mXZaA5C1Jtt2ssZ6cTK6zqP2acRZkDyeVnf2jNG1o+4As2fSLTo58FFJO6n4G0K1NROujIX4G2bp tU8E7vSnPv8facZ1/ZqvJmcJIJmnOUCPoY+8SEF3TwGElWKvfHZhbbvDTCg3MYX56PkRDDRMVqOI cj1pmGSa+AdxREvuo/ukRGtlpcOuUS3D4hGIdqiSOKHwktgYA3lNQW+w+HcpgAFGOQoVv8BMYvFW WxND7OLXD6vjDUG31Z8x2IV/lLNZOv+V/kieEhv25nTjRmQsMGRSoRwoSsaVKhNP1UxCkwTNutUS tlLX2mbaZ6NDgrYaL/vSQX4WgtwIfLyDcVhMZJAHp2qqyRRzX05lobaPmw3Ub2MgsUCnRATqay0J gMDtC0visJ8FE1WLLAsYuMgSea30YKoykGiSrni9KTERpVD/sEhJMUHEWi93LgBIkZxglIt7+s2f cERzdkAvmZTl2khS8AblVyunK4iIBZ1RWPTEpPUKAeuhx5FDcScLFMDW7dgMIisIjnwTo/C1b8K/ L5Dl4o2PRtCaDMROmAEbrEKl1T0qyS+tRaDIyIBLxuzhcaE0BNlq/dcTnbkJHxmORguyiNTWLm1H +E8v1PHB8ueY9Co++pwwZxi5xnxoibWcH6s8UvnfcbYukw+mrMbyic0VdGaP0JrXx968jXrdlpPn fG5+hkifIOYzZ222+s/ZwrkuHTkMefDC/hE7fMsQVTKaCWOjLaHDN2qI4C4sxLVbtB53Xy2n7sIC DYkQ4s7fyJ9Oqcqw9CjZl/ZiO6Z9JcNS3uN59NlfEitiv9UzMhlnrO1TR4abRoG0r3JFpEkrd0/A z8SRw1xirWl/sdIINVmnyYtDuKYuJw4vIO3jmtP7ESRJCQ3UwaWbkSUYx8lA2TqerIaC1MlUXykQ m6kyQc3Y8qFIRB+sDg7KVmr+5EpZY36tIqYsB3cIa3mack0nHk4D7pZJlnBcVzh00KMZHL+YZRTH r0nRvb4YHe9sfpFb4QjWqr/c/ti75ZqFUQB8wn0y7nUkpqkBsqJz0dDqj0Tbu2CelxryjB5Djm/3 UIwpb4/u6z69jV/NoAoh+OnO/2F/0s9Xnp4q1cVCBTFTl4U7mTPoSrLXv6X7SYzzPQA6BT1y8mtZ ZrsQRVnCEx4m1dcTGmHLvJp3rAtWnUEQe0sNfm0nf1x576hrnGE1/QwMR6WXpDPYzZ1UYjjpxJDl hQk+9+cUNHvSa+VE2pzo1U5ADxK1zONV6hd8hAsHqGcdW5huwVP/gLDIKGwakpyHxuO2o8u4zT8Y qdlMYEg+jevp727TvR9cvikUArQm2jkBQZAsHTx3WhCDsaf4/z2dDtLmoauwh2YkjrZzKQdIlQdM +C7GQx9W2XXQmz4aHVUINGpPfO7IW/9OARTzgRHSxTbq1DJQA9usaZX1IPeSK92mhTVVi0JMJDmk mcpPDCCG3tw1UzjpAdxqJSyACWj3tTOlyw37ttnLJPIS6GOojZRFwSw6pHUtFrwPZsOlGx5Q+VCM CLXqDydhlzBF+x78B8RhgeZFBsLTFZ/QzE/SyUJQQaJfMbc/7Q7Ry/F9L7zY75puBHzocRE4SiGp 5z7cRPoph6sdQGz2M/lS2PCUf0a72JkN1EvMCex/cUHaYJCa5ZqdnqLLZfR5CRAMVrDG3l6m3L58 HsPaEtQ9TpNRQXJqYZOBu6jt6+d4jYGYBT6baioyjebSj5oWWfkYYM34zTi2UJJfUlBGKuYhjh3y 2a788HbWFnFJlAKWB4SLN5cMVhu6vKDaEMeG7bYgb5oeITnJlaQbkM985rf6CbaC9eaM8hVpg49b qykfelA1PalXyjpCwVsvcbdECIg7hYwuAw95QLhxZ9gcN2xnZHN0MVaHcMAtv0TkYYQQH3tuQf8V hdRthVVD702cyFHiouWyiGgTY92UinFQwWeUujgcLQHOLHvVWTPROCMY/E6KFunhM6c7+kG/ob4X ApAeOBciX1iIoYNzoNczQ+c0yCkYID4K/kLjHMFRhGDAw/tJjLeK9zxEA49JbzZPDCIY/1PAmhWq VvVutc/lN7ap/ILdhHpmRFQZ/HfK3sHb7zol3GWWGDgHLvhjvSWKEu7b1DFGdAdBAHYhUEh90oF7 jYmOKjsJDL4L675lCKHzogjeCfIAbeGg+7FXqRVTEHoho+Avu4X+0M4CaaUhYQZsWLxWAp/zJXrD sQgAqWyzavpZjySiCQs8iQ8wQitGtFZAH8EalSJLN+nO7oFc8OWll4HHo/Bq3doVmKt1jiqja7i1 FUcmHocpHtUA8mNYTHsKOLlJe2194M3YHp6zJJf7EDIwjN8UrFtaLTwaEWTENP/nkLYTHK9WQUZP /ERghEwlU8sliu0iB49NNvOFpKlATEL3qv1PCRx8Yi8aWCoa54Lg4tii0glEoLRK903ez1Ja+2ED zs1IqWbdnvSrLw0mOa9+lkWtpGil6E+tEFaEIJR5XsTMkRKvfsB8g6iGfumSk58BROC8b35u5bBG jrqti65D+Tl66Gh75PYG0GZjjLCY9g9ZDudYIUcZgu56O43N67wdgW7z2Nn1tOWhh2XIwWQussrn y7dDVZiIc3kcyXI+hxuY3mXqPcpXrtT/AV7P70iwXmdWi+yj43rBTVrhyMQGQVNp3ZqZplrvQY+r BoY0AmcpwNnurtspa7grfJJ+roE/5KOeh0CDVC9+KTStdTEXqIfbDsBwYcqo+Hxm9uzRbEgH4gAZ jm5YmROwfV+xoJOdElCfS/856Gr553UphwQSIUQ2IaJjOIX7deqJ1pFSqDBYg3N1YSAnyjGFao6k oa4sNRBPWdGKRxCPbclZ0mt5GBFr+m5OwEiGttf6YFRXs1Erpfpj08ClGdl2OIe7EOPsYzTvBG4T AnCS1xw5tskBxTWZbbUdRaqm8zKvz9SD4j+8Tykyv+4QPVhSfSy5KvEMGlENufOFjFjVdFZ3DvZf aaJroa/+rJyfG2f20n8Kogo+7hll/7mIzRJjgSkb0l/4DqgvvH4NFpgGQWriMiBKkrtyyaTyl0jv GgDx2ufdpc4WBxCgphIqc/cErGvzvJVcFxZMv/Q4BdersVKfdmvslM6+FXth2s26ueFuLa9DErxh 80HHaUP3kUNRUNUZiCbYoQlqtQYJOMkIBRn1nlcTeFM+NITERdG9pnQiChrZJLNrd7d/10D0h5X3 O8vu6qz0S/AJ9HlkuRaRJ3XmETMOGkgM6l8Ihkhsc3P3GJsnqvWDr5gcVWBRoa8CAo/GJrQdnJjA PhwKHE2syzcmfSbrai/Ep04YxnSndZFrNJ15EX09QGk0qohc4rd4I3FvH8Y7sZw0gjdOTC3oQce8 J/4n+fO/JJUIrx89aGiAcEf59WWDtVlpJqYBy4W9jWcc7wfbJNDXlO1Q/UVJeOjuRDh6eJc85nPw gYkxXUqm1mghvaNYeBF34rM5uvj7gkmeqGs1QwI1hwe+jsjhxkZyFEx9h47l12OYyBwVMk2cfW/1 oNzFMLBnmR1zmqcUdTfN2HMm6Q3sxDsTYZJkM0IRZ49KLN9ef9slJm3FEBj0cLgSJq88ksJO8o19 UYL4HyTyA+4c3u8vindjayvFJTt1wQ+OgukxU5HCetd7iZiZ1Mdii7eWbbTfbYszIpVwQbM3LLfY 1QaH4iIvfGPqmadDod6ymic3VnDs+pyf4xpJKNg47SNr1H5R5KqIxFjnZgy/uFDLs61CkwKK/zXh +oJM1Egp2H727VaHoWiREVWJYW2wc8Ch/a7/kF39NXSkfPmcea/Eh5VAmtEY+o+lDVcehAUM5kxF nBftBiR9YDflNM+3pLefYTocg/vZGDlDLnbCunMo2t5wvj/WgHeIh5y372XTuNnEwN4H8G8bBMSX M/kUiywpUjWb9AEnWtCGF6ltXvKRWV0LnhNTpnBU7+2WtDtJqSCTGO2X+qnUIL+xMLE9NMSQ80kp 4thEfzynnheea4vt5NIFr7zMIB3GJrO6EbIS8QddfLnJcA6g/9lWUX5LZlo8+DObHYkctZXwV0R5 IeFfyS2ow9QLwJGCG+9IFOYp2PtZWRsZp08r2eAvM4cCyWMGQadarj+xwBp1jCLmKhZtSXqFXvag AtRMus6gHESHn2W8GHO4h7N+Rg+MY0eScI0vWXMyZYmi6BkQTXbB+Fqnvo4khm6xtLyRWzOST8/E aThLg9i6o1Gz2ZC0+RV0pERjxNxrPCQyMe7hvqxW7xTl5sIsYZZo/6QcsK4kSBtR/2/3SDwKmoHP lma2eIC9G00bWcjM1w9mbfmqzV2kPCeTr9z8btkU3Hh/+89vk2mYmS8CJACCmnLuRPBfFYc6e273 QAogz/r1/j7Lk25h0CUEQV1D+ayQotkQu3ywazWkbG6yUXvZcITfXeXusNC6gbkZutq0DFQVYm7H WuwZGci7F6YuDM++54+xrQDdgM+F1mZcdf+ZspnwP7FFFVyAyLByf5gZh/q46CAyNDQ4/5DoNmE7 TIZI7yTS59s1CQRDFuCEBOy6WNxi/Lt+ubC7EnL4hV7hsccElr36lWbHeon06qyH2lGtW1a3tWva hIq4+oIYBelzepn6E0VAyV17kMhbChQ3pO6qfB/OV3wUo9smHW6RkIijivS8qxJxNs0SWmTz0wEw FtYnqVmAoOTGTPesyEGBuUjd9pCnVBBVwT0lvuwAmY/OdfASM02+wFRKmMC2EjE/2AVnlFp2N1wm RAvzyqB9mkM3kPdb+Bsy4cv5uY34CDD1QO4SBaOCDa1zjsed5PTIPda4JFhzxh5NxGOY7VcjIyb8 T5XZtklqS3TS+OTw8TqW7MG2oQEl8YqjrXntFFDzgzUBzhBX7G4LFh0pxsq/I2y7TrqsIAKOboQK el0lz7rVqMY/WQAVRTBtYKXmJlt4RxQ5U9J2GbBpr9tm9XuhW22BmAdxhRiW39ShUSdr9XbUfnvI GOau2pwworPEcOh3txDIU8+QWcbbAVGEhz5hOvriGAX/Dz5f8VpNyEVxo/oCgzm7mNL1085vafXP PH6+Aa3Ky23y1aw9ef4j49upRIEbxWeWqWrNnWzKuon5AK3FIte3MQTQXX5x6wqZhi0dhiAOw9TW uVO3EuVWyW4dJQTDgH5Dio5n5bhrTJ3Gi7baZL2zdCJ3OhdPsZUAJ7FE7fWIqt/HpVUGhDRQxDWR gNmjrr3tq4thE4TK4jO1A0uz8IvYkxMFS6P5IKbEcNErJHuFduM3eAcLPzMDL09PdoMCWMc/pzk6 TCx1xEhEDhQ9S5BHFv12UzQIpqEc2iMXrZXYGn02KdGLuqCAQcFWVVVUWx8KhF679zsb8RnpEbDL T6ir9LtDTbTG3M4plOyYAEQyxO8IzfqYysGjEVyREZ90zK58YtVgK48mDic+hXCs90wt1ExT/BbF WRPmjA/5tQK0ohmBoDD9axh8dP6+kDwn/6lnC+UhZmcnxq12cmc1SNwrUWo7jECydz/EydgzWAm5 cx36f0jYZWQ9+9I3Ri1vhnJ3QxFxYVAEuqM4kQe12uiP8YnswC4OERnsV4Jlbe3PXhRv9ddsqkNa HH051cFqJ3jViEJS88OJLxapOmFhIC4kAnjszhWGrPLugjaurfWZCH0pNiE/Jr3UHUubMjrKvkzM wOajDXjZvnUgfPjL5KRn8N4eUKN6Fpzao0ZyLF694Q4ooXOHAlHmMy+OcQete2mXmb3PGeXn5vmt ojEmUUQ397yz20Q5z5AsOY7LsxGXcMN3KsyB75TWwoIklMP0i2bVFXZofLHfHADo/ccJAbfc6rd2 hxc/OM1AsxccXk41NNzVuvtNnGCMUkek2+GZZmgxqcN4mXHoiUQrUlPMZQZ1M7O/dH8aI6MijImg fowd3TPAgKZZMAoJY4+fJz2yUQo12nfP7fSR1S5JR1bGXyMkcbVjSJY4SJOHLcpFRGyPoAe4SbJu 9yjaA5DacfqzKNZHLoaFkkpuzzL+JfboLyduGhDEf/KM2O7nPaUbCy1MvXzdprDrG9CMv3PJgD9o jbm7NVk9w/GtoNOBgAjEs4L0NRqiZ4a3oAn1UIzHzOan6fiVEgmNPbzQUAuspnPeetzCOQiBUXI6 OXY0CSRInQwxaQK2Qri4lkeD0vMTK07zbfR42z8hWSZye0fyQyUz6+Cc1y76hzjVozLylkaoKwrn oSC7Br5rctNVI3FSZgiyB+fV7PHMVL6SCkhJJD01FBN4nJDjQz8p1hdwG/GuZSX/7x4VpWce/dWi 034YV+/iRvU4qiOCUlVFY/1yd5iEAGuLgsZlYryr1UJzssOnipeUAvxBE+cQNp8Dl8hud6QaoHmw 2srhjEALbxmfViIWmIoaBLzE2MUr7YulTCzwZMsY7P9WFeLYKIDjXOz4Cy61yQCBOtPS7oFtW1V5 M7/W99TUiWXJG0h2P6gfsBCcOr/61pQfmA2vAxqFX69COMMGs6JuUkpB6jZIaJCEmRlNAMsuqWsE Fb30JKkVZCklPa0f41e2H4GZeqrzcS+3HC3OgAMflhe9c0qnTauP/JeRzerVZyv7vToVB4/JO6iQ PS9HqB9cuueAcYTsiKCC8V9LvhI2NYWagmuMQiwBFgjUYkXuAV3yYk5DpBMMVpyCDsBQQisPr73c fAzv6wDJmCfDCgMhf1rBAyGuUThjPEeqsRdPvMVcG6xJhBs57NizHUDInDYKxfuU2PlR7lP2T44Z M+Ur71i+zQruvxfR81jqeLBI8ykd5aTUEOZD5tnjz8H+AzFcYTXCjijTeTOSifXH2NuQgEWSrJeD uiXn4+0VQTW/Z/rDXYX+zrUNXlMlYnvx4qQPPcNA6mX8AS8EY9IFMGNKaLr7jsxulq6jtU5y1ixB M64t7th3LsrsbGJZeII2YcfA24f+1x4j4mugut8yCozYGZV+CfNohZoMuFSe7KuKZLi55A3vPxyx kZP7f+UoT8gF7A4KbaUh6gSJ/82N0GkLj4euYhVaaVgV5CHRWGP3KIjBwkvwqw24AdP96+04mCTC aEorjRTqgIVKa3mm1sXJDuiquNkJFeStWxDi/d4wScwaBkx7L2UifdX8ojhwszIQaC04UoaKpZI6 AKxBkErZ48TkVDxq4l2XSy1zG5FC8nI4kRz63EFH5nlYeEbj6D1TtoIbTxS0V1ONuR2WlFl5hCvo uItP+jRYRDcbVIbuN01RZ2JAEK898qyvGvJTaaSVxWrQtYxLDjA7btNEdu2k69vKWcPq6zSTHszY njskCDL/UqVFOBuu41pIEfrknlEpHkimU84flaReSC7OtJ/m4nXVeqR4y4EsnoAuqB2Uh04feECJ 6ablqR41NitP31Rtgu128DuvZiuGMmObnIQYHykLdhD8uUydndtkC3lEbakkTv5rjvbJ3a0qTyyi XARXCMMIYxbwS8tkMyYE7eHe0aGN53UhxsDzZ9iwGeVchYXh+oB5OPXngCuoxoKTkI2uzfmb95lI uFx+Y7lXOByIp2S4HD0jMlBkaXfaJvrqBvCsgUE0l5Tmczedr71nVjJfy7DLDJ/mWJpUiXgvSFn/ O3h0eWPjoYQ54ebFUpMI19fHprOS5Watm10oZNRXg9OPCdrVQKIa9uC6si/Fbdek9j8ljfdJO3ok cliXYzskp+mwowdwlSPlCKVRxdiRMti39K7tOV0kmSaFvzc0CdwluFCuxhTzZpMser6olvoEpyjn FA9XQnXTaC40jFeog2Dx3HGqBnfYBADsxQ0PYUXqZ0q4jNE87UkBAU4RSh6MnGOTjNw3V7D8PnAi q4+hevlLzeJai2SimJ7+dRTK24ZHgIPIzAmWYrFv68XgOIHROih65VQ0JAg44WRpTH5UTPxTdVaH qsX0xSknWDfZ8SVTU5bB3DdVpmUWR78iu8eppkkwwUupO/wubrmWvsvJqepDAqmqe+1x8xoDqXJ3 b5SCTRcj8eTrxPOTJF7iFgjzoi3CoybfzfzgD2ikF1tO1kcgO35hPGxXfub8kgBqCNA5tkizsDEn bfCy1DynZiY6kdiVnsI/oORVpjsF2yBqrCx7Xi+iJaIFYucevNPLLz2WRgHTmrU6yucDATOKANHD F6YQnoT6rYxN9uYrr8yd5OBXx+crgpeObGFukLZxrhLrpGWhy8dsTp5FHeA5WbqT1tR9ePqdm85/ 2vHenpHkJpGg+eSqpeL0U1mdJT2JSUh5MaIhTZgeYLkhrBJsC/gah5RiM2Y4M3duQ8cDUuMDfkPz /H3jskqLAw8esvGjUXl33a2p/7qCmIv1VN5jSUPf18TqodiOPWQ2salwh/3oVx9aPXYeXh3yGXim 1HBYnW6Cycqkb3Updg6uEKd2byDJHmJihkzsn0ye2NuaGS4uSDm+6M2HgUgPA3b/IwGvHb+f3S+8 WkkAEpbP43CjX1THV3R6gqD91x20TqJDRQxRKjHCbm4Za/0mKbe4AB+1j81/j02Wp3lrw8572F2X Lx0a8u14aAN0caiFjLqgfkG9YW131zmlIeu0zXgObg7RRPYsaydkqvUgEoFpkJrVK7atX5ZnGiHd 2GwczOhXQ1fVyjYrzTQ8JbWHB8HD5XDwgklB1y0d1f9MXFrltPnmzKcC0uxYPQ0A9D2ftyiLeWqH gr0ZZK6vsLB4H6L/zguNbyhULktBkSE9CjaVFj6TtsM3Y688qeLWt6Bdk93SFBHV9whrnwHO0Kvt GjM1c2sNHM4OL/htn3qqwxILta4Tzqo6n8Cc7pBdx7fgtSGAKn6mDcoqapE3DHR2YabKQs3BT1ef eo9F9wyfyvkj1zyUmFxGEYEDvMMwkWQ6Uf0pYkKXxs9+ZnwNV6cLRpDI4nXwt2moj2eO/OvZzL34 XnoJZ2i2OgYnqYfuVOSXVFEqUqKQjeZqh4XeJp1fD1FnuonN5moOLk7b90Kd/WKwQrXW79lsN4gA bmuIXmP+V9rczXz3cxG3ohulLirR77TpSRBr/Onm3pvN/8dp8uR03/YA6nvVO0I8ICHg+8c1xy95 S2c6clx+8ozoUdXDFJu3+rwyYKk8y1Tnpu3DH7JzqZtoxmus6xOulPRHyTdnOMQwPtJfE4MB6w9P X53WUv282yVKphaEFNg/QVDj3kIjppddTYDlH2IiYIE3pfrMGU0CRCj3faJr6Y1hE9B+dgmb9LDr cUZmid/UtrbXLMRgVNo/SyU6t/UuNOp+f8ffzQ3GJZtM6G3Wc+MOeBuPzQRH2gwIjKc6cHy5vsvA +PTBH4D1gBOwaK62R7UV47ZubvCjWzmmOTXrUR/2u/HYN0DpgMNImzlt8GSAQ9Vyf7514YXUvmvp HpYRnpjPFKPpJX1/hdsZi6o74DE7GUfYuevadcsPkYSRLSZkZGFLXvkBVQp4v3jc5MWiSsWyFTh8 Myti4uCP6cvK3fuhAZSZnr1bCUIxq2S9pcIAjCDU86xvVM2Sw/d7jgs790TC0xxWEq4YPfhm2fDL Mqqvy6+V4dqFtU4yIceiSWHSu9OyFLfqCja6zVVyUr8aFS1FuzgbATxc3NPxaA/yWTn8lULQerG3 NHIiJm1cwd0Vx96mgF+O98EtQqhGEg27FO+Qz5dDDmtHoCP/gLGPceJ28GOdbmkrlaGMH5T+HaAa 6JyfpGKigFb4HPYj7WfbMtC5ihQPlvItJXTYctGMuLKBKpeWiWvz1+41e41lgtEZ8T1aTVx2Ww2/ Ugr9ANA3Du7QFX+BqpWYsoqdFwTKY1hHUen0ki1eLNRlWekTPflDxOvU12a6NEP8r37j9XYgcK6E 1eSz02iXbBqIzCd3Jw923uWDJFkyOdH/3kIazwBWoyhEycRM17fh4/I9ynysRRLVRCuYdKZ/FcHu 4WxmY/2ZUyrIBsXM6pD/QzD2P32aZES41z1LtzAdQQhgQMuhJCifPtpT8oEgWul7Oi9/M7rctBER d6Cy/tnChTkfgBhGnRyZzVGHlPuB17/Z4rVmErs+zYbj4Q3ZzITB/TRIm+nDpPac0tkb3YcWqr9j zDa8EdNjiYsKnzPGvXcfc77c0FFYLWQ7yeJ1yu0Ql9Pzc7KJD/LIdlpVMDbmhdsrQoy2TQGgmTTN QQwS01nt1NTWMIsraSn5EIcO8nkw7Ulg4Qm1JD8qE+SielTuO8wRytt2IHxDpfp+DZxaTO31YEaR +q7bsEgXh+h8RHlRMkpkYUZ5Qddmnm6TDGNHr1coIbyoAHFZ1ui2s44v0xWyGmrMT2FO3RG5YrZf Qr/m3I1o0jvwY9kBkxcPL46ZpH7Yjx3mdvpOrHyAWkjoALshy8z17msRgXiB463qxG8A9Pu5RR9E kTQb937egbFcTzopQ68uNdrUBkSqSPJRkri2jo/IlU+AIN8WWnL6WiicDz+tRffc93TgvgDupE5I BlcsHN5WIyFwhrL1YdlEVOvjXaJB8iuLKIFdPkyCwsSxP7vTIZhYYVz//Bb0llsURmTPW7oRW50c hBNnWA1XwA8DyUsu/CyMmCxXX/oqEFB5Le9PJCLDnzQUrY5c5NhkCLvnyIXu1JmTG3XGxOUAK605 tVlre6e1x7IJxD843xZ+Md1UZ+MI19u8stSaW+gHzM80UyVVeXD42eDcMH0N6cjor0k8voMrjxsk 7SGkBhyalZ2PVjVvh8ZagFvUS/DEDi03lhgUSiuXzzMSbIGaRTouOP3fKhVvcyeBxsSIrsfpuCUa /dlujv/bvO7jMMtc7HTSe6dBvu6AVSteg1nrY7g6ZqIa2ey32q65VYJT9Xa7STbd9CY7VYFul4U4 /cPlo8mJbVb1vN1L6qDbF5yGB+WcFkazjeyPO+ys5jh4fWOglYX6h/PWLuzRjHI788J+VmEWzPAn s0H/ynb+nLOs5CosqGX1jGPoT/15oHT4pXAFR448pKPAn+Yjzkq6Gk552SRxcsk8mQNgf2WM5rfF VNzq7dDPS0z0knqQW8mnBBrC3+zL8fan4TSk3LJFmXh6rAbX2F4aWMFCnqwYR+ARZ9A0F501xAHL ZnhT6YmB9UQjdEvVtZNrfGNvWHV9JoWGlkMSksmcPFnDGoBhH3FgaOk8JI/QWEIlJ0WyVafMDQgW 98fVJEsCPxfzFnLaukBp1OBAeCsVfaCr3/+m8axeRqKEpO7n2issnQWQl5jv9JCztCp79UaddxkZ lMX177ztBZ4fjDYH2IrOUnccYC4kxo5TGj2nQCAYlA+QAavoliMCbd3RBZJXtAvuhvGXtWkBS85H 4FSr/EAkZl6qUbKBNWiG2kb4mBTAi3AsSaDw8ogzdbUykADwypOtqf5/TtGytbhmSZK3QecCKxcK AyeMv/2Osm4eqgeby4Ibh93lGokvJSclrTCY4VME6tyNNrRn8AibltjPIXX33W3zFO5uovXqpL/m 3Z9fKaQCsErR5KVMNMxWCcmuUAtduZ4EIJRehBvjGs+LVovit7UMNy2w1UuePmZcfyWNRBokc3oM DNm0S/3SRcs3B0dywD73+cGYpIUtMs0058I8nCQlSZZ+/HPpvZjOFDekvBTFROaM9CikvqxV+hPs StBfI/AIYMzprhG5sLCmclA4EaKERW2hVSTJKqYdcqAHkfK/4X9ih9MRDb2nSpYO3zzpi5uIROoI LAw7TL1P4jTDLbGq5E5O8rmjU8PXE6DGHxon69yT8PWkHot384JLJARAeqOETQZe6kpKszrTq9DP mG+mlo6GYfXWtJYWJ5EniQjk7gNinFRZ8Yld5VahvBg/c+8iVBYMncCYgVnktuC+ZFyueUP/5wM8 KGvs4Ajr4R7iSvNNkmy8oPAp3fov3T5Gkr9OBKMbgl2cHH+cE6YMGMN1wyEOhuNbXagsXVEDJlzf x2ylLTimhj8h3b6U9N8LaSqTZzZKork3W71iBM9YOZufGKjocez29Ik1O/8etbnKKdLXfRKMSHWQ VwzM4evEZmqIKIuml2dHzpTLj4mICFcjggPJl+aHirveDr7XX+HyXYwK9rzBIFJUo7zdq54RC4dp ytD2fopcau+YUbqm2CVl/Tq1Hz3dMXt6DTWexS8/yROOI0vv5bidVHAVGrL2e3BoHC6I0Flp2/6D NNoEhNw7eMpsEvQq2AXmoFzz7iwOKSNhldIxJli9dRdknisAzf4ikf86u2zAEWm1Drwq7HJgzXJh Uo7GPAFtGWSXop1yBbJ0yeciDqFQRgwkDTGBB+VnC98KO9FPEvOhqF/H2EmoYWasBAqvrKC4i6ml QS8Yydo7V9scIvTVvhhgNOFSumbgr4yXt/L4+6FbnwCmlyQYnJmSWQalyf6sXsh07ToHyHFfBYXL Y8qPYa1/GHlNSyfcgHPSgzYWtbCnNXmikaYmI1q7UFkdozwLZ7OklYfZGVChoBL1qzDnW80nnmRY UnnPkkzSZk8eOaNGsUGWGxCxUrolkBwyl+WWqAvXMzMH9gAaLO89KzQlnJeiTAutkZ/yFUP5yQP3 fDuPUH6RnpS0nHWEwmv1lB2NtpYeBq4FDoco6ctg5ELmvYT6Ndq6rKJaa4IjG6d9L/oVjHZyJDCz D9f0GbNUycYHn6a4i6Bq307NFJ6SUp46ZoSND1jtRSwW/tNBf2TXUXcvZnmxmUIq62W9C810YZyT hMAM9kOzYZAPACwzMjdAaK56U3ooi6VrEac4nW2iX6Lajde9xhbvVHFQJplN3UjubqKduJM3uSTL U1dUwHC/AhNA/yf9zZi4/P9f7JGZyDDa1rY5pOBupzGORQY+Q9+86Cd/A1ADcpi3vIfm0GMnifKc wCEwPleTWJxHEBbBHME6OteWXgaMhxFRWBd1aaBCIp7tDrbj1gpT14etj5qyF4BX2dCXltZZBbSl CiqVh31Rn2WyM68SqrfKHw3JrOw8KowxABhJM0UrLgl0FUT5wxCyC/JPbIRep4sRuSx4Id+FEY33 8sBf4Foke4qqz0a+1YDUCi4L9prtDsWkdw7m0tn8aXFELzyB1ARvSfiIb00gSzoLYOUqLnVP5NF6 5lyPQfmuEu9rSa+NepSYl7RDUNKmJLjQL6sM+4otoSuo6PgofB9DRzc5PjaHOoFWP+xawhodH438 dy4GkVVoWtQj/Vl6P8ZS2UxkNy4p/OSwxwNRZMVRSreuenJcg1/gr8+Ffzx0eRnJX58prSACd7BK piTynE/1zLQhjJq4TN0ttmjXGXhbDc49H5jV4W2gy8lCkv0VwI+VQ2R1N+zXk/dXviH5UD9GNLPW wQaMn0fF+2I2/gFXudDlwPmGLOPHae5lkRIJhbTYcP2DXVIKLm0oHMaCTSNTPkkkvRzgwdOoSuVu 1ArxzJmNHNpWmZv4P19633/RbHimu6Oo/WyR0GFQ/2opkIuGoIQ3reqgfFd/VMGQ2H/YhAGMiEb0 ogEs2y36DKX8A7CDjoGm7CGBFtxIX3Si+Z8wgEopISXrofnfe9q60DxepNTiMhGM8SPm3jkom/ED B0hTk8Rp41qZaV2cb7ZByz7wWeosw23jxVCygjs8A0xu2IOwmUZ+zbznSGDxv2qSj49B087mHer3 On6Rd7YaxrkMNNydGZtplWasgY/WCQFCSg7aZ7ref1WuuAQv6Ah277OtXOvT0namRix+14tMSiCx h6j3VlVKcamDLknJAFVNgal18jeJrAAs5SMYUQ9DZRakg8NxjgblgV/sIAY5EysdkivzoXoIOolO Xt3blWvDCg14cUpnZ+vvDG4xPKh0at1nUzkRt7B6eeprO8CfvZRWFfrBP1FMhn+EcZGmWPKbbxh+ iUoJkusvKQIeFX+20z1kEBEzdjRUUkEYKu3PDcp4ScJ+thsrAofzjmJHKSHdF+ih/KB18Zj9I6NP WcnGiOJ/xjl7oQXOuFk+LzBE7/9j52ETWzZRoPd3MHOWi0RrsmoMeRNc80BSZCBolfdG6ccT3l8D WXWnvnPwylY8tjhmzpmgr6xTbw3NuAlCe14KeKLtalfGhFCEexUE3gxLfP8FzHfqfE+4R0mDhO4m yE2XIWjI8HtJYzPe4VTbQR5TGgj9nGjxS5UIX1Fs1KKIuMDKofF8HtiFo7gctNWvZ/5TC7yDkNjo VHVgWqPs4py5WVnUJMtzgkYd15kWG0atL0atOV5LTi43DlkactBLQ8RETVwcAct7xWW0CzagicYW e7S9EE6QkQG8B7DFtBZOxY3gFh2FvwBAMa+oUkp7j5VzUd5aOjJf86zWtlZnrc9Byg5aBCcUtyYI pDRHSftk4ASTg6MrZpmybIs619+yB5bCvM3R2bwVoFwKWhAppbyoU/fSLpF0kVsZalm1ZnuncCiW 3jDeEq9clREvmdC9k92ryjcQFaHhX4YAUMViscI6ZXoDCFVytQMmQFPUfALPGuYmUu2OiFf/XSXA DW4sbh5bvRmW/omRZ0Y6tSMfCtb+sAk9nCQ+xFb/ng1gXnm/fdV1fGa8uX1y9pXOiX9XxbDLwB6w 2p+e4mweP8EM4H7IEOXjx2gIZ2VnGPrqCmwJIKzugPdJOtD7o3ml/slCp6NHaZLp+yY+XBYPvJRI nChoRAhMciEHsEa4OgccuOadSXR9F8av6r2fvSVPNiIFb5DGW7aBtiUtEILNE9b1f+qAjEJqJ/yF LuU79DWJWQHt6KPaNoZMfOApp181bAq+BmU90MNAslonBBuQ5mnor3jvN9vmBwQ9TAf03B7m8Mhs f1tT2DdaJvJ0rUj85rfXgOhF62jsTj2uKab0NzUamdLBrQAyBMM+xhBAK1wz+xBZ9xa4Tmz3Pywl iaSqoFgKVyG6u5sufqGd+N7+owbxjzCISmeBcqvsHGPOmkf+gytqTBnDOK0zrPzBNjEdPSXhF58h F8WQKfC/VBtA8ZLIav5eao2We73ODxPXx1Ka+7XArRvkyrbIAF6D1FH61nwAi8L1d3/zl26g0NvT PEV5lXUVWN30nvQ4ymioet8aI26TY1bTX1t5ycK4h2LEqShwjW63eAHIBBQ+QO9GroQHagV4yaLi MLEMDZrTQS3tgSjyK2sg43nImEjBvsj672DFolThlU3Xksoif+dmz/2b7ny4I2NXVNZyz5DRGksP y2JSMCvdRlPAUFPc69NG+tkfX0EgqAoVT9J1sagLYHBe2SSM9q23DsZNNPd9KzItsdji/AbJcbDv PuLlu7hB5CH9rbJDuNBKklTT6B5uJMDmvPxi5afbTR1rJ3Ju8I+ysPs4ttBLzJfCSfqf68LFd47I l52piLlTD+4L/nKj4E50owGPP+lwFUS8YZWug/KnbduFcKaDSSmSO6AjTzxcd4qMue/H6+lmpr1d oDn/F623+DpV95bYtT24NKpkwqhxDJKRW8qI2ScRaUGPVcFlGZ+SUNTko+FMci5O/YSI7qD1ioau jQWrQWOQtI7Nw5RZcqivQo+2cfjnqCwQddydbJ74gt02UBDBawJ5j1YmVJYDkgHt1MhAJITS6JC+ 9bMXSJeyoZWOx/t3v/5hrwVPHKIiX+aF6lF/xuuNkssU19kwacdjlgaSfNxhILAmuRLwx1/Tackg 9AaMub8lhICI6OZv7jHT1L/C3aQvGJo1CLb9hrjdbXNiDRT1tsz7LePUbK1G6PEf1eP2xSzwTsK8 kT2RY77lgdLPCARgbh9PZn8knKOoAyXjCCxh0YgqBjSPSdZaS/ifDnosIlwcUyI1wjhdfc8qSCYS f/uliz4UyydKVgk3X6qA9GwGUypFy5/FlmfEHxVJP4qTH+I1F4BeX6J57ZgpDkOer0RTPqbYH5pL g825hOGKMdsDNUDJrdasTRUYvE2ukm7B009wIO38HkcNPnUI5ipEjwMxaSym+tq0GEmtyc+SFYxs WUTJEDgMG/jq0PRDsXT+c6rxUmnBABJ5SR4Mnf7Sjm/hWOrxZB5rTPoNBTYoIg8JwHB9/DIuX7Ok jOWCgNZSnhyNATdSk1xSloeIK2703nfbswgtqRqpOMVAelDxffz/roqaCAywcBNtZWn4XWUHPgxE MiAsZN11h8jr2+NQOonmzMHDNLTVjBe4ZWlgU0yuLj3wAt1qm7wdYe/EVyWhz2qHbMKLwnA2anO8 CVI/aWlomg6byJYHVW4mZLZYKtaovqfyI/+XonAKDgLYNeQDpHvSV8kIYFb14n9kaYsgFOSpnOV5 nBWm5wyhWdO1AX0In0sijA6kMzaO1ecDFGr2cAq8Mc+seEsOeZwnu+Dgkjt/6oWwyK3P9khv0nhm I9H1AlzdOxaTM4YHXkH3MOs56klPKAfp0NgRLQ2dCirNq9J+7RlGf9j/FIvkZl1Yc1atxKNAha7y CP8pEB6SOV7cW7+XC0CS54XyxT8bqCDK3C6rO0MVoUUh0+TeCR44/sS23O6aAYILt5aRMXJTnaT4 dF3tKQkELZ3aZ50UaO9wkWK2IpO0v3IWNy0LKrRUg57JaCAbI1j/xpsF75e++IhwwMQ0NGLJZghg xXPEGsW4V15l/PxJ47xwzTpqGnuXar5ba07AGLWpGORy1CHKqOctgt5Y9wqIHGS9eP7rzY9Bofuc fOpKyqk017uXHPiwFEERXCTXWgCfcxhNos0+R4SFfuDL/XJMUTltz0yqcb+s4JZSZBC7BWDTjoTf LtC/1KMnyvfDg+DlZNIv3k9bOCr3kMoMOtWoouqLIklecOpi3SKZQlGvfg7tps/MFpGibDY0Z/Ge pEdRQIaSZW6jpeh+FdLRE5/hDG0f6JLxKD2DGp0xiNLxoZ9yC+6RYFQ0HCNJ2bQcjXTY4L2YMS3P s5gGg/p9z2yn9R+BHl1/QN2NNTMiRdJ5JsEvVzGZrLCuMiLk+WeMy042jVp6fQ2P7aW3ayJmPvYK 7Ec3arl4adfzKhg7nE+patvzaV44/f2N9pbfEwUh+DOkboR3vomvl9HFb1/X/lsZusVjQfg9mt/D uQaB+aX5/mkiKrx15XxqO+Jbsr23uO2MKOuqF3W6GWWVjP76YhVS6rbEky9n1YJfjw99CNmhkOPP hOh+3eyozv3KOH/yU4swszLxDKc/oYorbdy0lIGDop06xF4p9jIBrEqfi1cqm9W+yZ6DzEAcbvZU rnaRnBFbgEOSEOFikzBOKBr79vf4Q9uYEwxWYxGiEQfvwgf3rHktjJlGEbjdpMfPfrZVIdc9Unch +qL9unMV93BLoTC5l6ac4txI5oPWxeH+0nvEkYWUzWJ99cUqpxFYXJItxPn9if8Lw/c2Hv1NB2es anP2h1kWBNKQLWApSvrzLl7zuZnwzt85Bch+D3cb0LVOD/vu9BASlVKUW2y/yJLzmQLzyS2rSUOg TD+8tWk8cEJjbXNKDZIWmYCE2gjrLrJasDD+Lff+JNsjAFxfgyu8vYJvM+cxjbbR27fDSqN/76J9 erg9YPbngZfbYH4eyejTtYuwCEyX2quUn/L7liqwy6Qsa4xq7iylski6hp8Z1KT3jZzNj5MfMtfR 73pr83aVHNqeEilohMPGqTggrpo2ySoL6RR8fQUAoMFWpopPHNXQdCYBWzl7HECH2mFMcg7m00RQ mpB4dWRVi4SUDaAmBPnmTXxCwnHUH2eZj3dJiunsf7f2Ny1oRVFcerQUANfoAhgsC6WNF3GQHTj4 SmXp8y5GK2gfP963d9yIGtfVAMb8nUTkB/OGxmiOIm/BijJ+4XcChETkRCl3CCMBo9piLQV6d31/ EAwqyJ/4k6BXytVyw885363FYGBHE9hL11fXi40uOQRacFywp3L8rljnVDfNJKmgXZ5GXk1ANZe/ Y/MIZxpIKtnqox9ha36rDX8gNqTqdHfsbowoZCXyEuolSVQbQGf0ReAZTOcTXYsbu20byx+j8af2 MOJ9dIBUiTTD0+fcgoLjOhwCG1CwZkL45U18LLf5ukH+wn9u5NzFxoDcl9bZh9Zt+afPCWAiBPII UAi6SaGIkdP5Lv/7GtEb1KrDUSSuuJ6I5SKAy9iU1fmHeZp000Qb5gtVymEJdza2cy6nC8LjqKbM FdTYEUFmdr0PKoTUewa4Ru7TfuyZ24daVLhbIr6r+eYbk5diBuFaysMhFpiudIIZXb3DBsrKLred JV2A62m6J0WFN9rX0ALxuUuR0vbDdJzhCLHIyLXYnUuoAkncAw533pfC/gJVrMWk169SWgL5ZL/w +nHX9/YV43zwdx8k50idyI8t0zF2Pal3kIILzNj0ILtzUekSa5vn1BkcjaIuOy7xr6GliNbdFxHX qMww+cdUPaPIGzVAvuOxNSpvFZOTMD2C8GX4gAK7Re3YFit2sgXk+Q/kX1wJfB1YTELKSD7Smwna IspGhlujlKLUVys7Zk45AAKZggamriztb+3QnjpOe9QwI/BtjLb47SFvPwRcbgM70FZsxQO4ElJP Un29FFw8kqmPuejI572YwC8aZ7JOTri8PrRC5Zt6PFVLULrVw1BMssiM0BzukyBQukVXRXf+8FaO K/TTRjvT9prvxGHavrJ4323kLZxK+DxRRO/B88pzbTlzGKqINO3W2yMbAydNspDwfWvpYYb3gvzy yCvaDNeXwqJerjp3njpMORmBeElB0umarxlWuwLRU0HhF3iBYSREmNSNzYXFiPMmysqyRKHlx6Eq J6ltmLzWeoGXKd8c3n3PttQ9kKHlvFU6ThzkI1Lmr4aWbb8FYto4zW9DCGuohE1b/xlv6ERDV6LW zWk9AWfV7LtgnOkCkjZCKxBIDsPOiSJ5WdF7LQWdpZVFNEBzhklkT3kG+iXtTl8b2ZEHBpg9GPHt ZdjCdik0XEqTJs2i956thRau5OvN5cmDvOBNRSCuY3pliWi+gWUO4afC0YPmSC+I7IceSpSHKCdI 5nIDBAkpxNiS1OAj+n0uGEPb2MZtspzPuh7fQjDb+1UjzWysI6AIyEs1whaaJov47gusuxN0bgKn J6kxf7LP70X7MlLQzo5PUe0E9+4XgJiMMdXsyt7+Dwjwsrl1P3VxM2Cjj0BopCATcKTqb/JnWgOr uHbyBZiWL2yoorP+t6YtHYtw1VfuEqt+xSoB2XES+GIpyVydP0XQzIk+teA6G6BoXe4ccA7R430v DsCl1L0t06e5NT5qy0lt7d2aXQzBjRbyMsK4jDvxnUHCZi3YC0wUsEiS9dy+NAxCVEopo1NLlxgG R2WmUDuVlm/1OJVjVYEDXbukuu2nQFXetAJcPPlgzW/itf9qDhbmECv3sVK6c+fT8boNXM48ZyeL 20G5pIKATlJWjWCdk9GDHQhD0AnYo5aR4ErQoKWZKt6Z4xprZWb/aBLjOYZTuRlUO/mwVODRfeDK G9f0fVADWEDv8qu6P5Z0L8F33NIZg0jZdAcjYFQA+3rTbJKnoJ8EKmDeg4ZEAdgIALDa6/aOGE/t 2L8V9LGW2Jv3FPFgjCQl4a/k2fcY1+nYP7S224JqSLze6cxgU+F2rTVbuC4AbYTPf7fM4G6Qejfc dDcL/GhLhrQSQff5vKgKnv6pedZC6+IktvSz3hB2hYeiZjPfiHT/cFz3dpQfWJNAEjTyhrsZBBEb L9SkVmrDKx2e6Oea2pgR4JLDwNj5OxhEOqYJ/t7EsEb4jjZGu7hGtrTFSVhEe4SBBMxfuvGqT/Ns OMuqgV7/xO0Q7QH4yhp9rMS5qF54MNfvBZRa7s1QRTNRHmAQeEJCilH0CSQaac3rejl19dkDMZof jXTKEquUPBsWI9BfMIyukQtVg5HKbf6kbxV3RhtnCsAjOwo5VyuH6/u9ipT+EUA1CPRpxAAsO2V0 41t6pL1miE5Wic1WHb44/DXyPECNf/oqIPLucQiYiBbRbM+Ybab55/HHFJjeAp9kTSvM7v3q9SCW 3KawyQqLm+1gQZ3BKrDLsqFO7YvkJ4kp6k0a3ZV7Xt+ITQ73032/cDw0pPjxcAKGHpJHUBINwKGw EslRktaPJt2WGovi/vwoQ3MpLlwYTf66ShoIqiZyaHXDvyo3mAW6knPETx/QJBbX9XFxuVa9NgwP gRWqIZ6oXgc1kkdfbM82pA36TwjP3gK4Gi9jAJeja+iwIZQ0cAxsSawEDH9ZB/b5kEu8fu8TY7cq w1nBcmA5aFk5Dw8ZgaGwJAlJv5wh9WI1kGjIso7qmjIxepVHuAXDKQ95leo6St/OSe46YEIKmxoJ Tzp/tMkYH2iLXpz0Qe6hE9idcr7KagDsbjPzJQn2xz8K4vnLTN2BwkNhvIzFO/ztp06D6Z8FTUyd wiXgAgR+eyYNkahECkol/yaxqHZ+kjviGJXqOt3BJWfWIDRDy3SBrYzoyihz/ufUUzSsIVLWwmji N5wd0EynlPR57riQ4rYWNcvQNI5CN0jlj/yg554V0Z/0S5+nkxYqDh/zHwNUDlILFul6JY13h6DC J0gNtyCy6rw7f1z2A2z8pgQBOKsV4R5fOd4CMExT1LJ6y6B/alqrkoTnpnRlmp90bl4kiMDfoiC6 gNcIjUg/DIvd09cczrT48Uyokql2Dk0KwmLbMeefqAwPp2KVda9+FdM3R7XgBO3JtnqnKi3EuGV2 ov6xSt8o+S/c1RtcIIk2Nk02m3aZD78kouhR9/8FTVXmEltp7XvUSDSqHyMsZW/ae3HWy7CBC306 aBL/ouESGKtNbElZNzf2E+7q5lckhud7GKDpjBq0IjDUddxhv4EX6VNLvS0Fcnt0HU9r+i7vEYxw zcHjpX0cCfJkHdCkaZtlWSyAQ3pjhlaZqQ00V3AECvujmk9O/295n+yP2TDPSaLiKWaXBb1w5t9Y cbkCEgL8vHeqC7NsHxF68t9EdH1sAHIryQFkJ9m7DXAR8QNfFx0ReQoBhyOG2s9tIj/SBCG8nUGE 7x0vz1X2uVUKPd8TzHcXWlwp++TInry9/jJ9Z7oKKmVGhmGjDa09Pt72ZT+yqrJxA/kN+jNAPxRR 9Ss51+o4m4El2Prx/4jFOurQ3AwBwju8Koh81q0agYeJ+QEty02BFArDn44OB6uBNfGn9XwkAEpF YK1dagFmn/CnHXlrhE0BveLD5g4kwAqWRft6Yol44vRRi+seYdmBiR7pUQFB/kv+kylaWbvu2BQn hkbf/9ZJjk7cchH78NRTEvkBHKAJgeSWhocUs+M8x4asZgMRF4A20fTbCxb1rVbtFDm+3VW2ljNw dE7DISns+/cdDZolsiErHjMJ10v/MfapW26VZWhv2QD4uHg19mOAwn9M99QDdjZB6WMGJSY8Mn4m 4GgBXhq97wo6JNn7wYb0rkQiVjfSCvjt7K718AKqLiyrRpa170TfKXvMQjAgZPZpP4SgMw3+OZJ8 ZzhM5iml2CogtByN0NVM5KoGeK/9gutWcoEQLj8D9jLZo0mD/kP/d2mv8XXZ/nwcsr1PztCBAYyI F3kRAuSb7sKOCnjG3/iH+B2S6dk1dvAxR5YSE0+0VcWQajBjxjeV5bfZckU9lhml/vs651O33eIx FASoZq/052ejxW8c4LE7Hg+eRk5ylJeyqpPu5tDEqHmH5i3nkiN/maze7BX9BEPMXhZZ1iv/IS6T my/ves+Z9OzXP1O3LLfTQWtDzRUJuBjFxuUapcg+MQiQwFyfJ5pgWBPH7a5xCAmRkGf2DdKnTNOt rwCq7qKXMqWpnrTgSnecVRXwozUue8pXDRq90NQ0Da+29w1RYlUra0RRg8A3+ytzSzL4ibBqGAwF kIhcpn3pJHsNGHuDtgjpg6BgUDzzLvHS0L1LPoVSl1Y18ylhso/kNIPoNfz+s4hCJ/R7D54XP1zL 8/kRSt4pi/8Z01e+noSd9y3ueRf2da83R0bSwplaWIKeD1OWgcc1E6hEF5droj9rGSc1oNejowQk xAp0R+9Ai4P8IOJK+xPi+3pxJxlLwnzYyAzzai+D1qYrpT38GlFHzmLAmBxwR/bxVO0wqBM+v87z umBAYxvACWT3ATdJkddD7Ob/azKpx6QJ4tmdVWLjpYtxqqomWYcU/TeNThdORn6y00/XohuxuT8z 2rgfEDSfE7h4pHbKiQ730qaCcJQulP8zl/gZmJqaNmf++CC1rth08iNuriPl0jjGRi8EGk7mQ7kT 6f5nXDagznM5csCyKYyj7kwovPXEvijN6YKzbYZTx8EJhya1ew2JNdq0L2HRj9Jz2GRiNDNZ1pDR Y4XCLQmsKqR3xHIdvFyILKDKk53nOCoR6i3gcUEpTf+5MRc5hovRbwvXda+Vg97/A+P/bE1rVtAT t5vRDbRg6FRKv+AtmVv7l6U2g+ec/ZhFsgvUcELPsUse2jD/lzzpa5+Ai/cZOUbtJsZT9GjHGYOh qqE14G8Rk4j7ddOpCbAed7n+hulP6WBjJnv6z1DHIULglWvWfldgmK4I3BlXEbxo0HwzggmdCOTm yMml0VT/rUPuporVVKt0ZAyB88OK43ow9Ttb1qxCPqf1CGckwf2c5mIzReM+gbDG6jcJUFeaeEb1 gJEhZp/TzeY5ASHb4IERWVl6nYL8ZCuatq9D1hraLEaQqt0axpR2dSkMUJMLzbILsec8wZ6d+MW0 WlehcXB576qs2Nh+sQj3XvxjJqdAMHNmP6M6+EY0zkq5CAFWJ61wZ5dBzAqQ4NNhLcSZaAqORhSG N+f92T1Xvd4UFL33U2f7AALV3Kx692krDMgbPhA97DV79RFYGfpgB+vRNy3I9juJf+XU3c3qpzdz tUw9hVjpMqef0cQ+qEXKzlLY7PPTM5p211qIUOcZYET/h2yLvB4gSN2nWWrtqMWl0fXfLIj/Xx00 B98Ak3xz4rKjMah8xbDwdPVRxx0tNf/ic2n7LCegDIvgRb5k8xFZioKFr9aAWPBuBA2rthw3YD3i 5zqWsah4ys9UValy87RhasTNLWF09pn5osqUsVTwbKxokHsFl50nJ/MaM666vhzH2iYfBk1oIpky e0smzg3ZkZNd8bxC/BVf0hbUvPPo+FquM6m1bn5exYYMDJWjOyKaRtTXQ172vIprZxt7B3qf5KF9 RTfA+aNoyzy7z1+iZmTwTyrvKOmgAglzjf8PNe9uDBQIXEDedFhvhSgg0AR31tz7QSP7AfHSYmYF Ole0zvqG5taW8uxC19NobT17nf9rUCjftV52xtld/g0WVxQLUxCqTCiM0pekbAE9aJmfESv1Ck97 RxCCs8jMJOAQ+gCSSjrZ2p3fygOFYfCWaWJ4ieBqSejYXtRYshvrP0yJP9tsg5LvAFlFX/Y/AWn2 EFhe96psb2DZHAD7vdPjLE6y7mgoVTkMWlgk8YXKYZDVZHjTj9bwdyq/rUYjLCcM/MCDpEPDox1V zVbfAF1jNzc0z6uPQTGi3rqzCX0sfDzXx4TUJJjdiJrhlV+DSJYevRztAPXOGetmeLCb0obnNRQn zfGN9Nig8LrNzhI3oluWuUILYxETC4jMMy0zpRTTz3IwB2wH0+4uEY38ZVqbAisSRC5zObKittnE MGMcc6YuDTTnibgE2m60xZmTluVk3lqYok5zLyGove1Ybd4yi5633gjmIR0wz94Oc0Bt7MmtLYh1 ZJH+6jU9mci8h/4AOOKt+AmM9nMojWOX0PcBIaHG/JCDR213UD5siwFQGTWirGtxKIV7vpKh+xFE wg+Fb466woxGC6t2kItoddQH8vxS/vyUJYryeQppFueuuTqu3bccJNo6EYFR+nE+iBdj1hRY4Mwb V4K3jXd7/yKEwtwBghcJ0oW6iOj7ShNvfhGMXHdQuGfzOMgPAGMuJjXC6CiSD7bsr5q/nv3XUgvs QkWEKnZ2XVwvq2iPXHZnSUAn3cP3h+sdAN3Xu80hiCQ0fa/dmsKT7lWxVGlvT04bs0nPXINEQRrh H54E8PEz9ph9Ihodi057UJsrZG7zFd2lzMVryklb9sXL5oY5b8FWwKQt/80EcGet73ewaVS97Hzu soBn0JNwxTaR39PSeQTTC/A+TY6M1Q9noU9C6HKtJWi5sDf4LgwxNv0Hdmnsqp1GU+Q19H1q/4VK iQE1qwoNKFwKO7m+F3amXy7jr2bYVdfS77LvHmWGmVVvfoBbgrl1bEtss33AmmptsRbDDPZxvy1C Fp/dF7O/2MhVN+cn5xqf8o9ungRLUSHeTLNu2qSTc1PIsDSTz1XeuiIYgoVy/MbpHJjsOWZCd7ed 7QgvJmqfk02elFy+VQoahvE62i2rOIFzh60nXFhhtDxE500ssp7EabWgXytNK1aVl5cRhGh3TSgI 5VlWIXA+ZdPJFLf3hJzIY42hCCMkHdV+yMc4UC90HuFaJQkkTClLmPunPSpjR67HjLhUiZ/OZuvU ttLOxpP9DJfL+dziLo2IfknHVFejBB0HJ2bAYD3J3ptiEACF3jbPjNmlR3v2WnieVbwDYNgpRdKC SSqCt2XeZajG49LbL+tJiQIrhad5lXVqyj3lfgdBGKPXtdCmgnF3J1Ud/hH5j4GWE1KpTrferFuF ngKbnv/uLgWTJBVbMYBfKuVO+gfYZmNlWc6CpodsduG8QAPy4HUHQXd3VTFvZ1/9HI0DAvPPk+so vcBY5sSi4XhERld+DizeNq0QN0pYNPoOybBW/y9FvRNhVg2fg1OuYAfzO5mDdips9pH7S6heQ/kO mPdynXfBcobIn+EtOUM+EuSrr2ehlxSS4hfWiW5oHBpwZyJZLB+JFVbra38wN2EV44ej0cJDKHNK zkd1l1QiDYOIauIPvAi4fTNdQ49TCSunPDhNtmZ/B6jUxPq3efU33gW2onN1fYqLdil3yDDJU2Qn Jdm87jiHLzCYEk1BgfIEEfA0AoUSrI3DCFP69jyD/bcn/J5daCXjHKWevYCvQe+LRU0wCrVBPQBG egk31Rg+xtYtsswUJ1c5PH52h4d4cqMIj0JqJf64V3chskc00y8bvy8liTSIYCszbFhlvdGV06G3 HEo9ic7aXb6JdQLEgSmulTH6w/Gg5eI/Xus5bU9xMYVXaEkEwDD4hRX+rmXW71pZa4tR3FQBLz94 3hQ9BMo+4Aw+cOTsjvJuhLuDwHUpFkB1mQyHn6luW9pXZT9iXA/JThx0m7dks2WOSVPtSbWhGNqs VeblaIA8sr87iMbtpaZP80r5aNzLIM7UqCMkc0ZF726nQSTloKbqmQAlNQe4XLiqvNCgDCmJjhjn jrfBmbpUOLoijhHt1yFO/OvjrCDCLgOPztx9ZSXr2wSYrnf4JCDlgnaWgSEd2ttC46idUjSIJ/j7 eIgDAQlKCWVIOfDH5/9bHXefN6XjhsPkXt3FDSobG+ho4BLrmXjSWTVfiu/x2t80AT8WoaAEUvua cFjiur/8Q9czZ2f1szb2m+hU28MMY3vgyV+Q3FANfApQ0fE2zYYkw09h+OK06FMkFn2ssE0MRMoq u9cXZeB6J6nTc//l3FepL9r3tzj8gAdDvuzxxigLib4uMfNznRIn+m/iWGFhgIQbgtuztOSU/GSh oR3br9qbs9p9j6kjAG7xHL/iFcrszQQuN/Wd9PRm579knq2qZt4QywqHHg5dxGXmDu/4dAnaiDhY pfaAbQQ+FYbdHlg7kmaY1If79szd1rD/ZNaW150riPO5th22cNvpjPVogumME97aAae/cn2GLICt IxH7/C4nKHeIizTpxpr8gwOHpM8UFv5ZmfMrpBZhiOE9+iuYJw9o8B+rGM3cJWq1HKp0UCGJqaOI 68TXq/z/QazWsdVh/aHEK7ybU/c1ZxnqBPHRs13IlyDrIvU/Zo5FawbZnjNRP/YKzisire1gEj4w rTkzt7kjWRhtYKaOVVy57DyqV8HPBrCfu4H/hJl9hEgS3l+FeW94zi3lFsjlCIwJxVUAR/fYEIgw 9Rd+1OdxDoouAwekQOwwj7NvSFW1vWIyE3XNAA6KBrB/UHBBRZsbDd4JTTADJ6ghQwVxSjIyDi0z HFLM0imFgSeMY+MpO0qpcTR1PrnUTzwF9W8PnRd1hawkIgeqQuSHty1Etl0jIwqzmDqj9KYKjKEv qndYMXjhcj/GrHRpPWs3jFG5VQOTLqHy1XOLBhJ36KxXYn82bCjbTs8Fl/5Puwgqf/LjyQFL+uXJ DZvT4mkyN7C848rGvhAEh5WNGprk53ts4orjgndNjttT5nnIrZhfSkzRyQVc0FhlAyU8H46NF1bf rtDg6cehteOXWmyTzpzldsYbNl3euxcCSXtu90kdLgxBL9cumIQE0+ORVKt+coWc/qaNqc0bWfvE W82kOLLL4EZ6fI9fPok03nA5Bvv2WK0N1f/7ORDhLjcMURzihUrmsQ292wPxgFZXe5R6CVUabu5d jSIaKpGUwLLyQXH+UJl+ZpJ6S1i1J+J3473oT+8mCH7UC3PqQaaYP/6zkv+5NRNuvDDTbENSaTOA EsAB1QennSOZQEmNKbzHVnPXgu9VkA66netKmTynwtmuj4TEMthtcUN2gc1nMOK6Nlppm1stNQTp UJcd6rL9KNVmUcigxxNriNYnT6NCCIkI2eCQqvmxDQnxYlKX+zkv63zl8HQ+NhQ7bs8L3Qeg6xsm EPZsS6icfdBwENNfX3TKzCRHUpE4wsthOJJZGpqv2kNS/7Cakjvo7Do3ClfDB94mGPU0jU3MSZ+A l1QEEcgI27dfLg0yZIPPal+A3spBLi+orpYYJy6MYQDKUaJDRhEJIp8SWvEbL45VVPK/BEHmSgOw gNq8U7seRHgVwQ2h6yopnXAVYgSUBgShfmjpdO+K65Mf23d6nREPOXamkY2hypPVXwZEL/W+9bqf QkRLQOJH+iacI7y/SvwSYpNMfexfIfqm5Eu44JmZ3EFq6boutqlVyZjFRVkGcWi5Sg58TOIHWLjI w1PBBdLAg48VxruFYelejo8QR4IvDNpJPodT7TTXYnvoGYZ4x337mq+xSz6l2NbQwrl/DEsPFnkW q8OycqvwKLzJzRlsTF01Ye/UiBuC0lz5RasJhWW6FJ3Pq10cdeGPah6/DJAcnnyVHOg8l1kxz3Jc 8uV14/HD+bnI8Lf3W5o9RfNpLTxxHZwpmm4Lj/RC5VUwnqJZILUNtzu7feFzWNPTXpOXvOD5DZjZ UnwrmOEtaLsewyHg6hslSAuAZZPEfVxrjCgwykcGWwPckCMMSDRJJeuraGAFhMZTMU0zaESVqIGN 8Y/bOuMi2HsYH2IiOtheoQSS+1UQ1QsSCNv+SHZGVX77sF6geqJj9ruIVeaitC5BVKVmdXLUtJkz e06XOHS2oeAKdzgRB6lIxHWOQA+RpIIuRXwQ3M8ismhWvCDISNllj6S9JqOaruLUpXK6Zrbm4ZP1 gH3DiSADO/X5yndvwqvrv3Aikbg/4iLC/zl1qcwwvOX2BEmwMepcYJUeOq+D6gYA3YXIS/CM8hrR 0zmWsgYHX1wOylffcdjUk5CqvKd0zbhiIyuYvDNYGBGAzIyMw/Rd9N/RUEtf3H6NLmEnY/UbneSK 9FmeFHfo9Qo/cruQmm2bgQAjvzCnVPPHRz0KkESj7dvxP5CYQoZxaa2uYpLktSEeHLZBmCnaywbj 3H3lz/FBTvLWXSKn/P/+lRdCLH2GIgdFdSx4HlF2LNM4i5JvzVQH1zC7UsMQc45MkT3mTwD4eF4f 4dDfIqFJLLPCsNxPZVFdidUbMbqclNHX1yZ6Ab2OzmYO02xLfr+wuGHy30MmzTUyiTAGurCcyKqU ECBAL0CRRwKLri1FF+/TL6emIvpEZ6R4DCZwrO5Y7xeZ/s0/pXaRk2hl/ZsbNVOi17yn5QkWs6bj JEmdIsC+10u+qWhRbUf1AU2WUFjD+y/mW9lZ/fSGahUIoB8GtdyrnK0l1a+rAKfzNLr+2yOD481V MCyioGfJYCiQPB0MELK8E0G5u31zicGgwY/W15j29EPKHrD8FNsL1yuvd63dIL5M/ShYpkV1LbSS xM7Y3fyWdSeJqGjuCqmeKT9d4Ik6uhETcMnjXnV73hnJnvC4eRtMq7lK1l58xYSnzBTxjO+ZR1Dt gVXKmnMaYLtdyuJ8/9+kB1ZuDTGdkmU2wiO3yk+9e/DUP3B0jE10sc2Z1eSUu5NHaGLeIBagU2Kl agW3XZbcBSztHyhOcipxVIioYGYLTOg0bJ4unxYwOOY2kq9phY7DyzfsYuPeIOTv4llHx42j8V88 q3sdQiME+pFpAGiRwnY7A3cVIPwcVFgi683KV5r5pR5BpXVT4e+hWOaQBMncCO99o3O9+tbKJGBA hxzHXKUFhkjktPGjRzm+HgHM60IbIwASyvkJ8+Cb0KFB4QXtupbD1rw1ZRrkiek5/c9tqIcmsqtn qVs8JGrkczTXkvMpv3Xmvcs8fiwZAgta5x9PWNFRZ5/CUBg9eQhTVNK5CjBtbmee1qh3N55Nx3g+ Rp8f1CBXCzTq9Xh/rxyYGG6GUQEUGpECvkJUV9q5uCo8B6PgCve7tr/CegLNEgLFy8kh7zxTAsJK AZYYVTqWtq2XTO5MA5hNpzoHu03AFafJWQgjlJ5kFlUifTA+eXoRN8JOn+HqAqGUa3H6BmQ4pla3 1EjzrGMEaCd2/PAaf/K2xbbZftXpCJhZtTCo/BUQ5kOd4dnNCpQOT/w4WLRNK3JsxvWib23LLyf4 wC7O+6Cemw7379QM0udp9u4Om38xoZaA6DH6CLWKVBkrH6G/x07hWPPkR9Vq3FHAcCTYlrgWIp+x U9h2dUlrRqujXCfud0aa2vX5HJpEnu0fmOKhrgqDasbkBlOUpszvMGzM4DoXlosSTh8+FwXgSAwg jPj5lSL12J+my4r/eKypjMktDA2dKFQg0QGBBQ6Jah2caiCwkCfAnKxyenBhg6WtsaTBbsolMI0g haedIJWhbRam3LwmM9p+kCt79Xem0JCWE/i7TaD2JOAj22lJxSVgNjIOEt11mDX75AWxKaVtAGxe aaNCIcXOCmxb0h5rzauYmNDQwHztY13mHj8VsRuLOpBa3yHoOWeDejX5z6E0zaAIKDxjm9UZMPjN KUfq0t/tS0nI+QqqPH/Qn2amgyu6phn5CEVJcnwCYnRrzWW9A7BtdSfpc7WbmsvwHqdrujoVjTun v+4PGpjsKoyB1ZN1cUS1w+bjTGdJCVUiAI8p3HTeR6hrRUyNrthXQivQHnvQi7RCGaf9RThu6j/F 0c6lYYFZU7RO46ogIjf70fn5K9K50z3c0fCotczLiisES9Zv1ps++4GBPd/W7PLsoeTMbclJM0l3 osBzuKR2Rb181n8K/UtIgCat05FyvDgJP7EBdXuIblelqVcONMpytW9gr06tnKJJzuubIGW0BPWu gh8snAfLY7tVHE0CVgKj0BY4Wa9Oku3VcynCoBDIIWAuLnwXSKGqHO54tga/1xLFyTblw5Hj5wr4 /YKYGMQgCNpcCYZ4cATBZakHFMghtyoyWcIbQtJZhsDwvIoDD+WlS1Bv0NxPnr01g9V+yR/Na8Oq 1JtFZN7IFrnlTcIkHFgulWU+iTw46KM5V58ntBZVmCYH8WE835cljagNChOsqN4m1Ry7eQaXewuN JwqIjHeO5mCiTA1iC0uzajOVx+8b2oyOOaHsfynhlt7U0+JYANGfcxtPbicl7GzGFWAxkB8NZxkW JklGh56l9EogFzlj63Izu3WNcS4ZZNRr7Q1zMd9F22pdt7T91jqolM/ZRLX3SSXyaHL2Sik1NYyQ MV/LHX7gAk/0pPhDEsJ/jqwaDcQMFKAy93kVoH1dV2cgjErJsFdzeLLhHG8qmayHBM+NAEwAC9ev ZrGgctBR5Xe2s4wN88mKGNGYYL/+83FAvRQnqG7Rrngo6RR4zUuIvCqT1/+NdOeTzsGDevfJ9pQn q/xRfR4yI7p7dnA7V5fQg6OhOn+dKQPukLC44ykg1aAuPMXKPJ6tB6i8B7f3M348HYnOuBSVwWvR 6EacSd7VjaE4btELdnU84z3DLW2qkCK+pQuUhAdr1xEH32PrZ2RmEmrq2tBFJMS8fTicOM3Bbe52 okuUgqSeXjpC4U3gB6OYerUZmro80KyZowHyIrhX6wffP64SRXzyUgURYTbwtlKQePqepqovMcmC nzbM8CMZZFKvfSvNkjCAz2oEjqKRXHdkyr6WoEAP56SRbn8brjEtPbVA4ueLEi55AG3tCwUxo76B c9YHwIOGJJl5VM2ZjmvdCvRLw6lfJPTYAB8n9hHBZPryQAzmUeJrUN9uJ7corJ+KlcFGNhbqLnqO +dJqyH48pFbppIGYGirW6IZICEQS72O7f1Su0FZceQcqDuIa+an+IAwiECawfQqazytGja8RZLbC BKgJI4nN1mCKYmD1jjWi5MsnrURVBVmcPywAqnasQ7fo2+jy5r5mEw3pgQ4MROT6gzHioD+xhHog W2HypkQhMkTubEb29GmAp40uYif6jE+YUxAO5ziwAqPfqswwXbIl+U8TzkxDAqTE5n02pPCiBNmr ooefH/VAl4HP2qowL2bDJGncu8n4uy7kcj7O1wNfyLs2m9kmrOIgTf4yBClLA3qImYqXyjrak1Pl 07u00T8lqJFRPEq4L+0ZLy0zeiGvYxCnHvKWrTGeJw+HFcoqi6pWbBwtj8J1aONpBtaVZC2bCG0F f3fhkDrYbQRDwrbmAMhO5vJqFBltHBdi3x4Nf+x1Qp0dlYx0OIG9Vd0hsh1ds12gb05P0tFXyP8W oK5iwlQq05hlI+u+By+QwE9VovjXGQMZnn4Fykxz7X8uCA81+LODPJOIBaKTU1UOj2L6FkNd5qXU RoMWdi8tejcwxDTXbQ6P1s/oWbIcNdHyV5dCwXvOvtkeudzM4wDmGJLynGceMy9vvpZHEYz7Ldzz 2tg4WTstFV24hOrF4K/uLBA5VETJoogwC3Sd9IXeWfShUgKypNo/4N++6IGDrcUnm/NlEHan4v9Z 42aaZ+sLMBFjskM1pTrXLllNNVcVNDngZ7fiFdMoo+4vB8ZNDx2k7ahvZVbJYoRUTjHMvQ0Az+o4 kI6Wg2tPUsfyKGURyogPbYPGarT+2uvrwwYsfzwrgCATukiWouWxe7wfTr5FOvTEvYFv29fwiuLA fkfzRqPaR1CboJ6GSEFixEd70vp+CTjy3WEQ5zvbKZEFdvyfPqzauy23DmLVpoBjvm/kzUX48WBy AF+40UyB+MJ1SIYHx+DZT0WgJOehYqZUCr4M7qj5fAVZKCV09lUT2Fts8N6ge1YAFc1IYLLmQq81 x9LXK4soarIEpTeikcHP9QfUBrETzPJtrsY6xA2PuitSKEUBrt/FL1UrKFGjPVOHnmNBtTCotJFg YLeJra0acNAwxW3B5Sgt0KTjBRlm9FuaAyB0LXr32Vt/zaNKmU0e80UcmVOuKbET8hCo2xQ20EWW MtoDLmOdQOHNkMIejtStGtl5CJSAYCW4mtcT6ew9CfmvZd1OMEb83N0uZ5DibkbvFdb9L2eno3/v //ECg3DvJcMsDKUVPN0TiiBf5WfKSUHg6pgRtrWC6XTsoMv9fd483A0X2SRzNOZ/oG6pL+EjyWYd scruGQBoNeThB+jQAHCbIdibmqAeH0i+qO4djxUwKu64jz2oarG4oIKpROqeYvivyPtYqVUn6o2B fDXYda0EiY49HiHVm4J2W/hOKjKNzcyUZsmmZtSkKPuO3WnyjZnw6u7E4K77xd1eEIhF1wCEdm1f xAoyfdDmg4YvDlp6rkx1u+XV7YNRfUsyGtdgZYXtcNcx3p/fvF6k5Jtn8xtt4SFm2M3YZilOfULi nuK6VvMcb2Jzoj6LOmptUqaJrRj52wubzInlVbxkGByaYS8+1CXWQaOOwUle6YZS6E+zLtBgPZZp 1212c2srGn/ipk48Lg7/I4fP2/kqox1uGh2HB7C5tHLgIDIRKqeNKnmwWLDAr0S0Pii8g6bIFoEE 1Hh+kjV4v6xBor9s2Ix+BlcHZ8eP4DHqXe/5iW2NHh9QgNdbOE+9xV3+mRYOlx/oNbXfsdJjzGDy Om3u+Xxvy4EnLkCTvAgS7x5fj+LKr0V8Gedn1VUjfNBIcHae3ysj1eh/N/3se5xpK6b/DmOWJIm8 iLDkeeplIHKlxR4jdlOveqAb8UhkImRdb/1Ka+OMJ6Mk/lvUbpU2yfjFFCNZCBBJO/lfgD8H8bOP SvEVjUOGg1wGYnKGi3qy7Rp3X0YFTnGsST3PfndGH5hpxKyDEY9vKaeN23uMUZm6WAMX2OLqEpiO 88f3U3AwJzWsQciE/w8GiANYyvjQer782hI4Y3w9w5mgx/gNGSJ6lJ52zx4XT+06hNP49ZNERkcQ 5i0GU79PsyKNmPprHWFab/ekEReaICi8pAn3Jf2ZP4RsRWKQXk9Aey6CnXrx7GVSmR+D7V5u8KdX ycRQcUNnnDtaxhj/5u1JgdBTYE/vEKgsFZJMkP6J6P0tNmktU+PBERpLcLn85cEO2dmex2bTjn02 JfUvVvrxsVXb7iJTmTatmVCHow5Vg0/VgyaXFNCQ/Aj8S1N0l17QyFvqYJOdZ9xItSqJyZidYpFG vWF/X0D5aI4muSlJu3umkQJ+davuOvokaQgx4u8TPCMSEsQEllswfKc5jgmr7wp06AKgRV9Ekgnp oTy34JpzDfxm0zyv0dpSmXIQWl7mKovjt1ktnhCJ7+IYbkhsczZIvEmtw91K9z3uCYodpYr2SItl lYVx6kU2hxxM2N34FRLgx24c63NdgoAL14AE8P1B0Mh+VfQ2Ug5AtAV5LWfCszquEZkD1ZPetBm4 AQ0tEmTvMvo6l4+c+OgwM0QghbyYKN3iUtZNFFZfOS53S6JtWmnftnKm/2NxdQ7HNTPEvh5H8qBC PJwTDblbQDu/q+uuFHaCST1URdniFeYHuY3Fj6grOWkWMo/E+caDVnWB0Y87nwG9939lgFjlNlGw TiKxSs6PL1pAq3FHKX0yjZRKZjHskiQvyRMwZfmodXzQy82hYP5aDYf+qiV8Jdom5fplvo3n/9M/ OHkXmwO8Z4YBKF94wwxsYbH2i2tggjl38nG2F0blHplDCxBb+Nhe+Y1MhYLW3YgsaGNljSQ9JHDJ 4VXOLnG7BVUrmTfZoJ/cIywrGxqNX7nLcRgP8se+sGgVbPakWQAW5mmNXOBInvOWyiFPhZG6svz4 6/r/jzn8fOeoiOxyvauQDcb/HrzIhWvOxvRTig5H3EnH9mlvYu1sTgXYsQ/20Ucgp0OWTbUoqaEw 7JEXY3aYIj+NsvG3B7B++cGLThXS29AH9h3eY6YJDjK/QkhgBuBzlYd+c6sJEx3IaCRT13QN6qfC 40InfWvgw9YID+M5Qyqzsdl80DSYZnUI1bdoMHQ59SFu3BL0+IRwjDrpZ+e6VK4pW3tGf47YvGtW 1QmPuTgs5eHwl/JUUbUHQn6VPhKXbwBAIrqKWUPvgocpnmCPl41s5wjyWYxK79JlQedxhYwmygGr 6m10UXyvcQ+sXb9ZampUGEnGzaQAfxBFYg+VCbMl1eucTLnx5YfypPNu/SP+OcGlHk/mTxBcsnE+ hFGwPbPscJjDTBDUUewFRT1VkQAqX8qUlCX0s/6IooG+Ty5021wxPnoNWGy2V92YuQGX6YNKSk7Q ww9Tgxnzvq4EovFJix0qO4qOP2ypuezaqLYkpadpHZEmuQwfSP7bbXHnsnFP+/JKwEHAbA3eTK5b BYsL8ORk5b7gbL0f9qaJKj1Skd7YTJ8o99n5mbxZ0XW5DK1drNSPB1i0tNMa2rXG4RVQ+vcgZ9XB /7qpZlGfZ7YDHW0Xl/7E2vF7rhlfP1iA2W8SKQdnlzcudwVjlq/1u6AdaXfIygP+m80xfxDX701O kiHC4PPtzzD9i1c7UuNU/Vi0R3wVMwfuKTfdObYhVAD/WRYDiUdvFZa/+33NrWyWMd/nwTUcp+NT m7enyLClvfpUov5+os7yvgzeF5I243rOfImzTC031LE6ON6+/Ivt4jyONHcsBBwwrnT9t2Bukeud sUM60RV5kuld1ob7Sd5L5CmjrE0w7d0kaWQieY4U8ciVQIPwxdKq16gMehNID5Wei0NNL0fR9Vw0 YUz/IZVZc5YRNyskb8N5q2ypRjrRkX08BWg2SwwmQl9MXm8k/xWFKWD6Aq+ioXEK7EmYmd8/uMZp R05aJVaCl3CqaTP0ZwcOFDxFsFwRYa5L8Ke0A9dc95F75QKVZEmsF88Jk4c6SKTQpts/FugdKZjP mcdAMEGNEyZ7j3+tiigAFeRx2MNIAcD/JTCDFGXCKJFzZhRdSVo14e0HOfS9PI7yZvC26Eeiuj+s U+Z5Mb4kQ4REOiurKGfNi25VKjUGDTY984yQLrFr6lR4TkDa8kxcr/dLxan7EuH+aiR/W6axfqM1 TU5GtdfzPq6pZQJP+CRNVvq98028/UPmvxK5aloJdpPycsWxSDdOQKlswdIs1f+x7M+09OyJV5y7 5xjgh1dVVRdYseczxpy4L8HhFJrNgVY0wwGAkUun9ke/hCofjiNUZguad7DVOaQEJXr4W7L9PTM3 tA5y6/nUtQM4NqK+rDqo0HJ0qQ18ijc2XMkoWQOwCxgdbTov0aU8d6k2+Yj3R/wHflQotbJ2+Ie7 XHdGGqi8QAfxn55gghyArRtgVHoFOr+7y7dFrJ9picBR44TUq1v0deEZy4/6DZCqhOtHljccoTsh 1CQ1yN9KXeIcuiddRxyqzig0gMAPiEH9mp1u9jrrTVqFETADrV9/Yh7dkGXysiTrndcxdL9wT0Vq N8SiS9072TUtq6vzSCMxgbQ15ig1HyyIePqPSEVTpDWzoOp6KGiTW6CP2j2ABtW1I89HEtXqegxO iuLjztLiIrxnDzaZiN2+F+kFeX4KW4b+2Ir9j0L7E+UdwS0SLILPjEE3b/6rPkU+VrdkuDcVU7lR 4BkyII/n3NqGuU0Lpm/r58ukhac1nTbN7N97WBmhGOg2bUqRAxgVVMor5uW8XUJJ+Dw8AVXruRU2 u5ZMO17R/bjdTG7k1cX5VGk7DwIlW2MzH23lJ2R6aJ3RHB5kdDLGo7n/Zksx7+fU6VMTC461sXX5 dTk0ul7nvSZHuyrTiRaUNps1aP+6nS/aDme+JTnY/hu5376w6HnnZZMMeFpyAxAJNkrKvLK9YQ5X gBL03hiq8ARN9j0s5zcxise4+Rb91dZe9MfRWwaNxCc96HqlAIYgcBox60WzXsuSrX9p237B7mda LtPSXn1ABAn+y6AXROU4yYymcJ8SHYgy/p4TVjoiryyLzInRZE1xfW1p0Cm74Pe545zeo+1MTy7Y Ern+p1KMRjnC99SqANqpRHNEnSNBrbfHvx03SlIchHArxJxzDZBgHqctniYM3IpqeP/jZTYZVD/w 2sevlVy5LuABtdHXDHNPJ7LPDeISjPfO5D+SlBRM0XyPsFkbzs0Y4TZnRx3FEznChD+NWjtmvUAK urWv9dkLBFrS+VqH29G+qMYx+KBxeQfigFMPmSxWdQQNLwInr0pG+958NT+GKkXyDbE/tzG/Lq50 muFmRSERYrKQOwThNoUY91D1c6hSfTQux+5cgOC2ojQ2Fp+lQnvJVz5ExZysZDNz9Qz8KjCtpdg0 UNAhTxBwjNxEDAeYnyQl4fkDKfBwtmm8vcu6yX4MGzBVrkvN0gvz0vJrY7e5UaT//lUMqCVoUqEp uF5tetZ65g1xaNihZp+BWjqWJLNPm1w77In5+ZuINLjKxmY0RSZo1NpO41mgOWC1S9RunuEU+DUT 9ltrrKaveZrnrrrUxMKPmr67V7G23p9Ao5ag+ycbz0eQXWtzFW6h3I0NN06dt/GfK4zjrYUytv5G 8wHETQZAoRH2xlbHd81IfAe6bXBxNBUkKup8X4HGE8LfgjaItvfTboHFHgDc/OkPeaGsUUarKza5 scHy7m+BSb8HFugWqkHcF28Wf2brIUjvwgoMLfw1OjwacrxmJJknSkaxBy9r/sED2288tPKENIKx 6cMNlwMniAfbzHbbES3//eSP2EPs6oEsHsHGi+sttiksl08TSikiB+zHKEKE8HgoITrHTbUIMPgX tLP9q9U/Bf485TOt/jpU09ui8mtfCGCECfJqtTEfZXRDqlE+y1Q7qcw3B6d09Yg6rUZ5GyftDqqY AWzbC31fw7obRA3e1joavxy0qXhySiHCGCxlLHtEtkpNC9DsR+9yyHYkm+be1D2bWw9oNqb9lPY+ A94POTE5eQZAfrcZw1jkKL7tFk5qatCk+glGCjKS3062tNoQ2WX6xW+xTSI2kTBTR20qiExZbYjQ RzCpX4tQO1aTLJGVbtTP7oGO0BZNj+OSo/SOPH0WwLH1inv9gnz+/4nVEBRHspVldn0wazFQNb5U Fj//jpARL+1GfTpSEt87CMrDEcpNk096lBYMCX81zJE0qeEv2DLjpRQap0QExP528/1er/8d/0d/ 4oaZQw+JPaRkQPHyNTb4cTHO5SFi1DvSTB5uab8u15Y1tKISh3C0+AZz3LagOeOPO5dJ823VcinB XG68EWQJgoMB020CD8nTxTNeAzrjeC3E2Zhs22IiydUubC3HsJjvHgItJP31X1gNqQ+Vm8HFdYfc I/Gk/BXXIIg9PD3qppvWaoASAGnL2BZtvXHtnF4OUfLb3pOWgi4hl2ETR14H9QbHXjpw6ZJqJBEK ovus9/0ydlEMJqQs54neYfcYUlwAcd+GHnRsPJ5oAUIRMtAy2M/ZEgHTiyyO9INPenjiH+KOON0J lEdgGIquHGLjlv0JqN3zauMK/KVNZJ8SEKWSVQeTep8nfp83ZNHHAEZvaINR5G7DX6aPm50A4kHT QrNPeHgDQGErTxZ/ZyV4AjCep08vDqrATmnfCvo/Qmh39ALU4vzoV61A2FsOYyElduoIziUQs/dg ZquPeLN6DnZUWTJPhXtesSmwd6Tekg+gdZER1iS90e6JLvDK7VhReewB8fhyxZEbH0IUh2oBEqEh rd93OTw4PGvO6U7DHssZOJY2uhPEJkia+iZ6btNnd2h31zCYuBQ9eomf3xG6kKB+ECj4cLoTehA4 qpYRDPmNpq1IWqyR0Hv2lwOauJcZzmRWvf6i9VZcCkkL0COlqe4cDz3o30DBylgxjRsnwsKSxn8w w1XjcLaGYaK5SmZN9vpwl7WUp9oSPCEGgGbZokxf4v9TytjV74q9w+lX2J7/kCDpM9B08yPfNbf6 NGdkHhEaToeY7dlVP3aslnpEfgluPSEemsaw7hWpI5FcU8SQ01K30n9VknuOq0VOVFrj4gzCsIYG CeM0DjaKYSM7bnh2wHr0lQ8s6Tp/HSeGHgjjuk1BoqxQni54yDf+bNJM7T23S2a7uUEpIKoEYyBL B/tbsXeySZcyNFpvEbR0e8UTmYVyPe7x+eexjz0FIbDbASkMCpwb6PeD3mBJ27dNz8Ig/M+DiuBi ARcM65jOgOwIkmAlu5cHMJ4YYEVCZ+fgt32tBSHKTSFXGcbfk3d6H3bwkDmaT2gSpGE7rSNgj4sv FS+BjtyCHmLFqTyrwj3DHgCzAvF2rKc2r87GsimuEA26UiRIc2dZIxQoPIcgp/7Xc7JQOahbZTG3 /4nUQCGNbZ0KWzCJjyZKTrHIA+/8kO4kuQqILYjm8v2m33DOdT82K5vTKknhdB9bO+pxSzTuQSDe O3a93g57DAF9US14BDVW7hkcdkmuHMOpn1VsEBCimUI7CbQiTq9emWf1vZlixGiTOcMGpE2PSJ2o Yzp836/4xCLUQL7NtYkKYzrxvx/C/Sam3Gpi3xxwuSo8XlJKqZuQxOyNqZFDCUipMNKLJ2IApY8P AgGkER3CKr7V8vf5dWXUBFLbjJWX5AcTWgCbULi7ado9A7Ufgq+2KyRnVPQ01xdy2MOHJdlMQU/n nedqKpc2wZxLRRwA1YRQeXs6OAU3fbQcW7wUtURSwjDTcjyoFZl3InVa3JOcUeNbxodZnKBxyqbS Mrj7Bd3XaRTs3IrECPBclZSe5mLnDewri/LpZhikzudn3t717yD5suG7uD/LxNGGggTNoOIhcEMg chMjeMGldG9GAbV/j8mIhc26xsLIighnoeIPsgnxB/Q8SaIsY2TDl8Z7zy6aZryR1En/yatR9dDk PznMTFbY/SxxV7gEaHWBPjoBisW6aJly0w3xNUAGVHW5QvVDgC/S4h39ojGiHwBSBB0WMwWhs6zD 2mpNcPGSTF8BjhB4XuYGtbX6fFUGeZ/zylilCeqQF09iYjfZN45WK3s31RdrdTh608yZzy31w0f7 PTJVj3QIOogNtRrk/yOJ2vNVIScsh6lDqKOocLUBNry1nhr7UbyIaS2IbpPtlqimWajuUQwtvQmM eq9AO7ywWVSg6z9vDw+w6yOIzEx9EX+ixW1LdtMWQr33v5ClOXDB9+jYQNdwO/QNl3BCHndYyG1M VBaB+nr1eEUZWLOZ9zo3M5jSObTDSKTqpmfN3xqRakBNsRAoZoGGxk3YehbR761c9Mz9tH7IfJIA /l7uZqm2W1nc5y9zwFtuZzl8hoTu35fscmPmuuvrUz2aszZujA+DwcJbCBOgRbbvZGVm6VAGflO1 /yywA4qSuzWgCdMWoXVI2JBcnQK125zhvcN4SR/SKIUWiu+owystFXW+u/vLNIiVcZsnqJ63PuIi 6BdQDNuf2K1WspDWnKoVXxJWIkmd8hmiMZoXzCNa44rMGC9VjM+CKVaszu5cIMAqZoA7soSIe/Rc jGYdDIGuAOM9xhf/yVQZ/+MlW9WZUsdIaNmokZZXMQyn3LdYkoGpIkDHaEhm6Ka3TIvkrdL2Hrbh FlBQimtIQl5jNiSHhWkS2Cz4GpJYhjvL5oAwtnG9Tg62yiJaKrC1MRPoSUydw0FvuvCayxstJr8w oz3butH5Yqy5081lmNu4FT0icjwYvQpVHw2ihNUmrU/seleG2Z1KtjoGPnj7jVTU1cL988uNT3XW ZWyRk+joFIW8pWtwT4wLPY+rBmSr73wRI6gtRAMGlMOIEMnVIL8QaBbuNnxJU3agyUQZVT6m86e1 MqIU8RbRGF92151ka7M6y8dyi/yA/vMySa7w+hv/TK1kilryY9w0J1QpZiw2WDdLkPwpjdc6Ziz9 il3OhwrLJX4IHPtcwAA7oHWahSwWbwsKA9dfXJac0e+t8IQP9dVCdTyjOLvqBxPbraV4Tn9rzvkx NhfIL7qd/LefP9rKeategnLJofT2Ifqu0NTBfzZedMYIg0IX4OoCw1u1VnNHYTH9yT6ld0wwhURa VYFJkYeKT0XymyvxGFPMboEtbG485H77A24IuZH2HUMc5PECKTgCdhaxhkEYpCV0uAgYmsU7QGnj SkIY8Go57a6mu9Ub7BGTVxV86Han2R/CP4Ei214vfnJfcNKn08RPqxMZn/VFm0qRih9a/i/wXpuG v9VfB8rLu93wZvTg4nzjxA/DBau9mF/Eq0PP7jxgZvvVjs1AtsxOeDTnOxUROeB0Q6jDi8hXhutR HeGGVV2Kt1F/5O7hVD3m6OyR9FzcxYWEPdObzLrr0dRMV/RBCtThY2Gi5vpn8ervLTq+U0Xjmyhv 6qZEMW0ISfIs0BNiNQHjewSLqaS3tpXi9dSC8bWYUH6dcBHa10vOOR2U9939wmA7m2IEDvB3zeRh BXEY8lqpGiXk8xzv/ddCjUfmD9mN1XZQ7gE0L7JwQCCcaLJGYIiDLpvApTVRTO8uYXOlNVDMubF/ 4JabRa83ZNHuBBkFrsvD7uoO4kFU2l8NZwJxMTh4xJ8ugKAHmExKFEn/YDNzJLZDDDjJrXFOO9rP vt0QLcGWG5QOuCjH9QEUDxfQkRhhcNhEx3TO0AE4ugICWnYmouDAZKj5gikp09goNXJFTsg2SJhS nSviRL61ru4AnZfKJTqOK9gcKHZBJWAOPhmApPgk3jT5h+3fB96NxlGH8mB9ObZGJC2QSYECGe+M /orkQByLkytsGBiv/HnKKMDHMkaOVyCLDTmgWBFGlaN/GZbAZGaTif7HGgDCtS6WQcsV0XUU/t66 Y4L6tDlzHd0IkguvYdLcbBJV9YMAhjJ8Hk4lDPAPzt+ytgX9ypC6nThHnaPA6+Xe4vcE0EuY49uE vj/ZxQdxsHpvT740OPDF2jtFDhjKpC6nXBkxfu4esJUniEQp7LH0lg8Mg84/ilZff6trzLlaHNL0 SjJ9xyhlzVogB7CvndnrZBWg6rgggOJmmRXNudWa9GIsPkrlUDPX+Lg90/Bd3yxYi/6LQ6rhVPT1 0QMIxSG+olMRGTbjg7AatmUnu2XdeIk1acO3mFDNiKFyYNhLOqbZztUQVbJFd6tz+rwWwrbqNiJ2 wONcHxAqH7ZUj1d4jgisigA1AzvBwKsXjzDYOtPLCDZtLrmK/pBtz23LdInqdG8yzUg6Ga3wKMpv Uzn+ilrSlhIa+pxBGUnEKfo4ey4AIEfIWrFKAjkdJ480MCtEftFypUT2a7uYPdqtyJl8yAmI4cz6 +DXMgd7CoDx99f95ffYGiVdyqKCSq2WD0HKjgTIDVFNmHSzG0Bx7zRRKGV2dO9cJI6b/I+BLAqEP dtgeMQMDP/Pf3MG3H5qks9/e5DKcJ8Sxhe+D0tqGFOUR+3GJXIe/KCRKDraXn+Xapp6ST+1Z07tV gXKwC1igeZF9uHhGg643UvejhcQVhgVlz/PxS/eehCGYtmD412I/2n2n7OiJw9n74fVXYu8ma3EF FR61pDsuQtp1qPlCpji4yUe4nq4Z6saGaujg5Sxke+Rfr8eprYgtDs5IZaOaxuFrR9dyFOmKr/w9 P3aNuP0PC3itx2vSJ8X2WJQvciBMxqFlbEhysKFCR9ma6D9S7BXDWsGnN3MnHaCiHrMmBj1fU38D D3dSQl5hJQByPAkuWPxU18RQ0PyaQdmgtLSZvQwwTjL+bzStww8vzxGCB09LTfMyzbnLNSyVm9yx 5AuGOwOon1NNjJfvj0mmZxr/R1lEtt97YwZ3Yi0I6HX43w5NjQEs/VYIC0VbVVGAW4bpAqtZf0ZB ZffyWQwCWDmAotwqebORaDOLbnPNRC7C4Z4vOcDmFKDzRIMGK9Sde9Xu/NJdxzBKvpyy9rOhvnBH dM0vqeFAllacggMhK9EHitr9kFWqoh7oKuPzFl+jpfD4h708N+7TqizX9A7qawt9M4vZd5mG3gCB aYQ/bpMVpxOtfy8NgH0lXbgIai8yaio3dPpLQSPiSmG25iN+kKYYpxvPPDBDGmzRJzzzChNKcDGE eiqpvuLy/PbVOa2cU+DmSRCAX8j9aLATAQMuYlaRO6v2M7YsVcSuZNphtrgla4hw/uXNf44tuqzG 9E9VWJOa+wCr87FyqS/0zbpXJIfkQtgUdJ5CYsX2ObBGKcVjCVwcLVoOorZa6eu21gc2x/OiS2PW AbobG3xQz4a5dMtMDPVuKRT5kVuG4rESX3XqGaYGkN2KJ4Ufd0Z5LrF2vVDl3OjeV/D9WZ+H2DBY UlRYaEZKXKdxa3u+toVypg/iXNSbvIgdapjQcDR3gp/vaMteueAySVmfloz3Hg/yfXNvP4APwOYQ ALKpG53G2hkQqxssHX+SuWJYeGR9FyjVZI6uEClTXE2N0hLenEX2K3+BckiDHllSUkIK9CpdNo/8 gq7rnSwaWnagM74iS5HCFfud6KQ7Vlol4BQPzAqWqbAaW+rcVMBew9TZQVW4izxe5PpxSB+wmXW/ ukwotLgjOqPuDP73auH224cNoR8BPBHctXy9p1STx9xpdCjRSXfpw9kRlcajI7W1OTcBfhph/Uh3 v03jSBrxXbgegX4tnnGCCo+CjvJKioQp20aAoFDT+EJgLgB0GOMwTdHhrBUqZPmhFAXn8t+fIU95 FUPWDHmLFjcUbBpOd6r70BEceBjRql7IzhptIqxTqBDUEBs0GN3o2IPzlwz3XIuWyevmUis37R0g iVKyY6OYJfY8Wx8CWaqz8CJbtdqfw20Czlir1HvIZ2J+fqRl9O+LP3RL3Hz05mvFRy9kWMmJjWFm EpYp4QcyWe1M1FKkzs6VtasDa1eFheJah0S+ruzm2bNbOYpA+Ug/qXIt8Ok+cPK7KkKnYp06MTWx GDRZdA1NFEIIZrVV9taylTderWdF3L02haiTUf2wcFih5mkvOnAu+0kWmVrRpf6DfA6iAmlRRNpV ea0SjjJH0jQ0ph8TtrG2OgkKBqyAaEmRtYL7WX3Bggr8jOccU7gmaSmrby3muNpTQrHidmIecqRn 5Eu4LJDItzPSrKQU0jHWT675Wjj7U8JAvdLH50n0oDiU07wK6JiyQzbPoNbZulJ+lgD/ffmU9ro5 OCwM8U216aUv4aTrDdh6KfrllrsOMGplDsiKsOAhJx7hSJrmtUl426gRJrf5BiM4CSNP8R6auy/7 ekjtBFsKNgRpEGOXa0tcdy41sdqbEhqe8y3BT3hUhnlJx/CDUYjGPt40YVJMDDQ1dDeMdiwT0LSd ej65ZQy0IPIKg8O3mkj6ANZvZH+VKq0szrezKHyXDBCWnZ3bLOEbH9lX6UxXhBISRTA7QEJmygpH SL9ohAbKymGLBwrWk0QfmQZeUT7ZCwY9/oWQCurOTGMSdrC/f4hDgpW6bCrnoZP3KJ/9kAHVQPkT FhhjVEVSGLYBfrEy2czE7UBZig/a/ijvbgLMBIZbZUr5UDxrpPrUoIDz2Xb0APbrKgltLbxbracN PNnejd/Hp7CbO10l1iWqK+qUIqxnfMWbvUTpNiFKjOa3jaJ0WcYszjqeEaAiFNCxp7Gj0ax42eBe iJgBO9PN/HnHLT7cGv1REWfU/FcqrdojxwdWNNogxbw++OfP/kQIG4m0VpfdWjGoJ4NQNpGOZHmI JxvyQuxxq38OR9HdrDpa7eAnGwgzk1CDHm5R6AgTqG5g1PDFyzf/tiJqAN9PE/mkEcJTDLOVoVJ/ n/U/Lip6OBs2U3Fn/JRKh6JOVQuGUqwygeZmEA0fm+reglKX6vEMzc8TL3QiZzNRq/fgmjf0I0ye I1wuEZQPHuFXsYPadc37gwOXG8WJ00Uz+zjxgUPKoEjW8fDid6H8WGJD+qGvXTA+I6BAMPd33vgr MZJaB2Tkdf5Ju2NK/Lg3SH1k0oMDMju6GTJwzatGEDF8EnyRr3FYEz3vBheOA6DniThx3qrQlB0k JfDc3TqwLAUfIbHZn4x3JmRQ/D1nZyXyHEodszHuyTTf6D7FRz5LBGF8IebJWtgRm8+hIU3v2jwV WDcy0kNX7NmTnyXoArzKC4IW5viyN1tlui6J9NVhHDfgxZP6TNcuPoAUAro+rp+4epb9+OwdXIC1 tdeQwdEXhRC1MTQMpO4mJjf4R+vgQGBFV3XCdUO3rR6ozVenS63QUFGzdWj9VAQ8tIhCkJUx3LJh QH+vFvHUO0C7hi+zjykJUfDkwmSIce/PV9beBNvoRKOk4AkD5LWaQNFF6Ks2zVBvuuoBaiU2IRzT 1cCYYKDeuNlH/sg2Z5JD73PwzFq0saoN9DCj60WdIDAed0op/67zg7L8RIZ0/Ff1QgRuDiVZgeYN nqQ33nNWxUMjTTZNw6F8dvjGhqEtoh5V6UswZK9h15bFKx4tVgI8AAidxrjK/9U3f7czQgGbq97w 4cZsxBV7N60eVuRjowKULjQ8EkVEVD1wkZV94BQxi8PQOfFvHVONPCc+vh48U984oWI445M1DzEW /l1zu0ZfpDSjrNI0KG3sF1kPFeHxkaa32iXxfuQ1wDrW0rHpkvyqevntur/BVFucdemXRuYafQJi O3unjRY+dXVhxj57BufBVcbWcZ3zOatG8TkAaInnYIQ6MHtroYWI6ExOpvgg6/UemWP8pFlvEZ10 oFoJ+be7g7iNQyHAJtQx0lRqUsRCQ5RlFW6ZOJ9+6VuTROsxnM71+v5U8dHAOL/AxX27N9vKn+ef OJPVEukwN7jy/GOLRTRVmciDgGHSw81aC2eUbFA5fkgLGeSfu4cLmCxScoVwhTpXtmSyxwyM+AbE 6+2STUEWbR+VbO9V891cMNe20rVEKMaEXaMk1IHAr1Z3tDWwaLRYgmEmcZp5vlzwaFehzyemVapf QaNSXKwkuxplls2dGdK+G4v7OuOlaaAjsxHi0AntuG+kT/UU8rj8mbdWoyCaspOYe93lLakEBsHr K6lwUjb836OKU7oPlnFOiwF7NhLxWYb+ZcupTbxd5aaHjr5q8Tg6QX+vqg1yl+W/ylA7HayMHcvj 8QQw1ZkCvkBsvFwl0M/UHpmCpvX9eyGVSuWXrrBYBRUb+MOT6Ep1Gay7YbygNaeg9EWW7W4/QhAR CyOkmOde9bptXu4HwnCWLpz7Es3JYssIW9fQAESvzRLJNzXX3PYxhoC4jLIWQ55OhwjAVgMcjtdT wnB4Xz8NHMjlCyiYSe/TEV22smGmuda+VhIwIcBUp8nixHJompfsldm1bkdedTZ1GNWICvwwOsks +UHe4oTNe6hLJh4ZGGPiNiH4fsxW9+yJh7cK8Re4s3PdmVJEw3qwnVGkge4d9EUXhby1H2QJ4riU XzQ74hL3cX30XKYVwAeC8mxA/PVi67c1xbm6eD0mtKFd78CxfCOKHw6xDMJUsPzxyM0D4kRT04Sr fVzvNDyUfDPPtyEtQv0920QGx0BxGzxW62+xq8KDPm0BHQJ6cN3Vy4oKbZ98+cFe0TQGjXNJUnJi 5ferKIS/8LcKg8UCiigedGpfROgM9hqS9+aRnYb73k5+bPIKGYppPj41DyncUYmcIRasIv+hjwKU CFgXDq48OdtsU14RLYiKRrtRQ0Wu6JXwdP14JxsO1RYQWs9aRrcAs8VCIvevYRMqHgXRj0lbURIM EVfnq1aC8P24Kj/LuvT22Qs/TZG3vLQ1VpTEaodld4aG9fY1MTmuVRYuCOgNzVNrtw/eyjByv/F3 mdq1I5VL8x8ZfBLwDteSwTZkWM5DI+2uaTGm+bafAv1EqBoqgdRji3cIXla0Dtos8SXyC1UO6llK E+Yq394CyP+idi8XsH/6znhQoUihYDRCyF2u2KPnYHrIN+/z69d1ACzFzjswiFUuEF1iahnWTzC+ PZb6jsieBRhC5hcFmYF5h23thb6jfWP+UJkRdITm7o2LGMn2J7QSxbVOp7WmrC5kP2nwz2qfbaZK iLJ1rBSEkSOVOH1LETnAO5gNqJ4MBllZislzw/62/f3PpsasiwVP1/6VLPwI+wpZXSNYiXcq5cA3 5jz3kKXTu9QmpBX4lKW9U4uxftmmg0HU7yh0lp7dUo/KbaYlkIDgpQ2Hxb5THrXVCTE/kei5+aq8 HHOoHSrzs3gYPwAWudxbsThkk1gRWXNO2zGpeCda66EGmwNjFXtI1tRuQ3Fp8nKKLzcDHloCpbeX pAjDOndKt7qzD67+mg/Gl1IdCuBaaXopBATQ1gMEhYjYtVgyXPOIJeJVIjYLUjcXm1SD285Qdm9A K8BEL4WHA6tRJRlwUMosquE/boi8F8n6ka+5kMeZp6Ug9WwyqkcgGhVtS1B0fKaH4K+yJqQ6rCHm p4YGxa4lu/hTMTQlOsy7eKDo6wDVE5sT2eqec2O1RVKLI4Cf+kAVDUU5c9gYwSf2aXrMir4r/VcP W9K5FzRsh50HSB9v7/lcuiop1HGXnkgg/ZL9GvEbAklShGYqDzopuyQpICbfW+AvVbwz2h26HD6D 5XbLXk/fjpobWzzKOa0FoqD5DnH0Vvs2EZvHGc35epdKYoegFJpF5APYnbLt1Vd+NYVIxHIh6qMB FJoffVupev2hsh/xcGTFn08iccSgqyLEjlN2uLfABAHmcWaxbO64NSlWaEGHHr8W01OPUVezko23 YgCVaxQ7+PEM1DT4pvNZlKqqOWbtFica3cNAIz0ZV5GHScTN7pXupLGh9EugUwsLqzOYLspLfdj5 kdstjlO51JW1le7ELK8iPXiiVFT4jgnvKMZFTsTpAXr4A4MSfBodL6KwTobFiL6m2Xf+mJ/m6BUa e6f7dA6r1SeuFXyjZJLUrL9lKckZro1+UcXUJacV92sAznakBs8F9nK23t+ETMBAmdCEPV+PgOWs ikEpMUpjcT53FriK/aQDOFPNMwS2KH+iLpyktGu3W+YM/ftsUIev3mAQp6XvDOcckRS5JQkCX1pY EWqXU6q0HIeN5Er8TgpawXRMr1+DhDiHm+1E+f8QVWSPdNNL9DF8sahAfwfEsaza4R7A7tHQdHJm hqNLAmHlmqoqilz6WA9+cQFnKBbJUNCyo6BPnI7dmjoCm2c9jJkN8Z8SNnttPAj4gHKPF4y0lO/x 0dyynOFuRgrzLJ6Rp4+gDVndTWz4WEN/u3cvv5yNJqRAf2T5W3jVXc8EduL2NKS9jdNAoSnUxHmV ER7lSkt53QNEDSRcfkUCPxM7Qe3t7Y6lNZucUok5G3j9w73X36nr/xtbZBJjIDOVuB4AAg9F6nEz +y62Tu0lCE/GNq+RNGFy1xQWq25+JC+ZI43hwO5p1Bht+0gZCkUYehtqSyaqfgx2HPhMUcXQBAVH MKLfHIlhqKQUEtumJupsbuI9w6QbkDIJYLtPGeV+ajYFseaZpbildVr8AWJYn2xBEYtY88feIERX uaQ5jOb25zzQJBcb1XY5bOb5T8zNVJT6Yo4ydLrXIHpFrKiOXl7UeY3mUIK+CCvqFaI+I0Bg9X02 AV2kGHvDFj/HjYONlboLZJC22dzTz6OpLWlo55E2968X7sHFq0mKokm4bSQ0531Ki554DqtWTykQ x7YZkhHYtWCkXZ8oEge+wQqbvGb7ffmEEibygUKM772e0GpZ7iUiFmRM3EzLJCO8haytRLlXfOdP ouHhEmDz6FK3GWuxKu8yNF9DgQ9QJLNVWWotqG6/W9+ZzcAsuVlZs1Lu3D0sDPFcHG35PCce4s0L EAt3iICjWpz6alxCe1TgxQMFWISavyixNGizlJjLfJJl6Fh+oMAIpx8zM+FBG6/6Rwi5C7h50CeD qJz/6+ALiEksZiKCxih4VYB/xQto8KLnCecHNv6pgfO4VYWqSfqRJSDEz3UkXCSuASm2ljXJcr0D uvWgzXLkYNENOdH6uJHYVhGzVyE9ctrG7tveAMf8uJh91gR4Mjn/B9GQVPWHEp25ispmNBVXkOlb szC//lvDVQW46XHZaH3BCgkQIgkzy1iL5PaNTMGer68E5h/dd0fV4CyyWvfoPk8yTgTp/f69fRku dWJxTbf3ICJzTtR48WbzJG/PJOgfQadAlju01aXMhiC2pMVEkYcrOHcTxbK7b0wVvgsnlz4iwCzD gsP8EH81Ly52jsK1o92+BoqLpGoCJq1XLgMrCyv8fP1Xe1BQ6NcbknoSFooXjplsY/r3/JqqyMzG /l130c6uAIWF58S/68GwpkTfaUMkKW2Kb6ep5MpgWiH59Av8qJyqH0bl/j1hSh3Rv34z+6/3dlG4 8c5BodAP5LvUfJRjmgGFfwjQFq/8Cbv8ctevbghT3YI3OvJJtW8jRXCC9XdHKcxHeJ3l4Kcb5oPe sTbWkF9KUynRs2Jrp6I3WO0B01CV5Ab1YCnM6IHnHZVv0KfOrek8JMCRjc1VuknlFm4a5fl78wXB of/Xc1cEbCF6AUGa6t/4XoHBGLsLghz/FhM4wcYlQcKHxBODMqPvGvXADroKWqX3zfC3vtzJvULh CvjzR1CJiXx4f2cKRRmNuIXvi6rEOIgLmylLVq30nVfx0io9AsC7xJNl7pbz4VQBI7gEDYDiFxnF 074FSIsGypvEzegyAsOhX6ZcjgxGDubdt6zLCMUXrdc3ah0WLVrhVNJN05LvaZXCorOT9m2trJ71 SrcOTRI1RCgNwr9mNUDXMz4E9vhDOa9b6QxaTjT/WUOzWsDXoWpC6azmvk3us/Xucva6mGFmQ5Ey xyHAPXFMLVNv41uiycnrR2YzhzylybTrhCU6qqh+kUbX2BLYAYlx6bkoUU+mswLi2N09bYtgRwzs 3ohOL9mhpgG1kI2kESxHAyMdHwI9vpslwOH59PwrqLxRbwmj5Zmuy5yXq2919aWMigSiaqvQKcPp J7h8LMfa+rsn/iEiGskYfTKBHSexU13Ztga9CksFVyD49xRvnqpe0tVwUOkcMgl94rkznZGsMdo3 qRv1NHhdCdT/81rmV57eaYpURh1MbYyFu0LnPdOB8XMBp5MITZ5fa5eAtMY1rcJp11M0/VtoiOqo 3Ed28U9K4XfDQuVcIAoXackOWabu6Q1KqSWmaOOaSSVi9TC2z35R4gwzxLeQNRxKx1EC6hD/joak p0X6Mlw49SKylyb0lLhfgaWJruBJm9vPbcf37PXaBPpo6sTTVOYnPv0jZRd6Bu2kQUY5j3fPVx94 AqoYDe/Qkt3eQGnXRrlKYz2uWVmlimvNxdIPkoCPY9LT3nn7ccEJJxPzjCRdYnQasdLbU3nH5+nw AttSUrG00zmS+uXwXVEDOhMRf4KidOqvWi2dqCCj8DfRcVmA+tcbAfEXuFYLkWD+6vxiyqbCw+GK NCeU87Yx2dM0NadQ6UbXZk5HcZ3KaV33alpN+frWELuttnOTkabi1foRbR98wjFrP4YcovAssD3Z EiPiFzH7jF6LLTM3qHz5LK4RqdEr+l3FdBxAfKUQX6YCsM8iDx8GS9qRA8JAP6GbLS46X0KcOgnH bIF/k4gS/A15rETiDbD5FMyPr1UpCtKwOISZNF4w/C48xTK10pS3u8dzepucIivq9WJspOKAyIqR MLmMUfD4n7N/y4FbSQSDgqArFBHXfZCyaWnnKYCgg0AVC+48gr9o7O4vAF8tXQVbSLKsZ8cThLz2 bzGWM1oYLbCpgo+n8bxP6yhx6dvHBP7L+Z6Pjgq2zU2+bSbBNnJuSVYjJiDZdx4ZdbPSkbSwhNBz hYkeaCPv4s6ApfqPsxYOKMeOhHzP0tFoCfE5miHIaBcgHtX3JMKWmcYa71aS65vezcurPuR6rik/ MrYAvK0JKrjeDwGS3PJ7zx5U1s1A7f7yW0ITpAllw3G26dsALKebvo+d5NLn7GR4Vy2B95qBD/AC IA65GTavazJHpSDfcr9BX4sbUwVaJpD1QAJ2Ybvmsf1yvEcE0P9+t2NcdbCEFvsUfsVeeCGFWZQS ZnhAY8q8QCEuYERzXxC5rvO+CKbEYosUPejdmVl0jeClysl9YSnwQFIOJ/SbF0wosREIcToM85yT ap0GZHrSEaZPvTxqcDR99bEtxdvHTXPnOyCsJ+vWkS3AByS5uT8+xpvgxLZ/cjZA1NiXcPndDHzX Q8Glb/Z1Qe+wiXMm8OrHquPVWrvqlSTiIDdykhPqoQc+pnT9c/qhFGSN6wAxlgHr8XlcdetBTxyf l6GMc4eF2GpEpMM3VFhAh6itvtHNxyk0UbUzFHRBqHYWXvuV0HJM0X95Awy5jDpqpXP1oIDPRbj8 qvtpYJBeyyy3w2EBqZ0ODxyiDpJbYirIMlI0ORgkkp1PAYyoJfuBxRXwUk4dOLlP9eahgjkR5y13 walKiJDsC0uFG3KsgBicP9JYtR4gPYbQjr1m65yhkeyeH9p4NjJgYSlmciYW3JZDFrypqwBrfVZG BlFPShgGEY7j+xHOney5U/oIMtxVIB/CWDKrFjh0Jk67IaYzPjZXLnIswkqOmThTzDSbPShzlpbz 33714jJ0DnSdv2bt4IAq2CAgtpt9aPeojlZanpphwZHW3qpUBbE/qHMSovlD4H1wAzM0nClS++GV DBneICpPvuMilGG9wfw3sKgSy1YDVJwgm7l0Tb3SR6Ak/MPNtJPzTN8uuyFBeJMZJh68maIqxqcU R8hMqBAaV34gAtrkUzk16Nljca82c59ojC/8UdATxfCupk50a9z0NLB+aLQntClQxeYe2Y6ANuPt 2VfjgU+rNw58vLlZDuAygwxGkB3ZETQjasSMMH07sPw4p5g8hBsfVC8lPHS9NBozu7Ev5xVBlN1l CxJlhHJk14Z+cagnyat3zM+9b1h6dGo4SeW2hd0nowXh9HK3fxUn0MTVJM/YVt125GUlETOHMt8i IqTjcEuebcGsvB8CCCb81vxjkd5T6fSBvBEH9ev7idwSURkoRo1Jy0eu+e0GSURBMC2nnnve86bR JL/nJLN0ev16MWGlcKQWnSkkkYKpEblyFIAkyRb00rF4x3mNFunnOFim88IkRVsWQzvb6VfRzCSt xgrcQwfAmtIeYltJREtLOlrw/oKkHwV90vhA3LfmWOxs+G5IA9Qk51IyWyO3cu9jAzQCEQqHKsRp d7qLgdjsiOBY3NVsN7wIqMNqpguVcr8k7oLF3Q0lKNJHJdZD/WwxYHWi6h/GZfXphqQfZRRYgdJs 8eap5kF7UfxhKoouI5BsA4Bd8SQc31m/vM8tsn1KY6g553B9RhND63uzJrk9d+dpX/zxXzdSGo+z +uiZTAeOGPK/cPl+qI4DlmSq7EA7Dyz/bJ4O0n4QvZQSr2Ndz+qSzVLxpWt9kYkvR1k3b9xNkJ0p 93FJtwVXGOUZ2zFrBv0OtKAEhixvuGPZtWgfirf8Vq0laOv2E8vLgx9E+jQyhX4eInsKn9SNcdm8 cntnG2VoVtx5p7AZoaBuEJTTtAO5ySr4b+Lrcf2xaAn/0cwNKB4YCh/q7TFdB8rySZSG1aNTyxy7 q6gyPfAgl1NulySXVP1At59fr5wym0B7tQFX4mWe1pupj5LQ0JJvha79L0U5zjQpcmG8+MQuanoh ozbLP96R+URLpZNjYCBHjLEKxvydp7mJp6hD/UG4CVc3VwiDe+rq0HcFkzp7hnmNap4TnDyyprbB i5BdmO/PTcHa3SrCptuluRYh2LaRQV+3wJEljEEAm5n1tArQFJ/VOrmBkbsCQ3YUr3ppyVu32Jae TSqeB/YYDuAyNQzs2kwQi2Kxf+TYrvCT5CICD0EbSkHLx9G/RxnosxOKhs6yz8U3T2oQWRwsvUIo 8RCiUGMs1N0oRd9C4jDLwWPP9XTdsV4IHZYdmyZQj5tMLcBQcREhp2MXH+AE4mq0dVMKwK9++coG 7IA5ls310p0ApUNnHclBYDfeIQnu0TCKXotqXP3sJRsJBKQ9Z4fVbEwdOF7W3G0dO7P9CqwVCeh5 WV8yXSl303UcX3YAZ+LXBXJ6VyktuhgK47bOC2hu4Y5PFBNZiEdzAeQGoImQDuFmiZi8wGl9zbYD toWvtnL9zAMr13Kuht8fiOZd5DOV63YPZ+tDYNvL+d/ntU+MR4OxsHfkfhOyT1xxKXAjiWwWlvqp giqXQUnC1ayTdwYdpM56qhR4AISlLMByqoDMqXEO4a+zcL5UpW168IYGAJwweI66wS04qwid2yzi Sn0CaqT3TZ3quA52jUkyedPjf7FbA873kwAXIyk5b6xYxb9S36FYPYxZMQcD1TwNKT4et+ZWwlH7 QnT+1i1wDiAe4mGti7oYeptaHZlRNMNEQulAIZDi8i6hwIfNmsTA2yyCzewDaV7rQ5U8vC0Qg8Lk /h/XxyOPxhaRrI7phLk7yP3Ta/sKzFggIxsToWuCX+xsZvIuFp12AIuRC/WBSd57Zetk+dcn6Xw0 pC1kkPXvbE+PhUbAuodExW4BjXVkQQ4So2yLJU4YI56lq4D0swnaMzbtjxrMxdPCcLif/wQE+B68 4n7Vp8r5rGG6YMBGfj4B4XDuAW8b6nVYwXZkKC0UuQmNVTWVZIpDQJnUuO7R0tZ5qGl2jLV9MN5i Zp/t7X+Inn/HHMORu6JdrGCzjBc4KPzSAspjXmFu2t36omYFPPlriq8YYREeXlrHSj+poJ+/WDGe cGUqVmPdQl28udPUtK0juzbpGk+mlSHOQ5oUtJu0nHIdBbhj6zeSMMETs3Y2AXoWLeZtNpt1DHlj Qs7hW+tZ85ENnNFo3+7bEjDlzaZIqtwYxunSYlkbR2InPKUAdlSE0vD9c91uUYL2K21BOUlNtgkD KJkWJL2tYMz1WTIS0F+UZNmpcFBo8AVbTbE65xENY+vDZ64eTyAx9X4ilz9XKqN0FcwcX+Qv14yM OsViFFGl41DULWbaGKgsJCAPMP1OTjCY5nmfWDPzQiPE/FVxh7P9TUcwsbfCzmqI5hK8vX3PO/B5 w6z5k7YpPcOzYp2RuEfs6SjnI4dr8l3/Hl3+SxCVafsGobOmfh2VAUf9G5rund75YFDSRfOGVWQy 8k3nO8qBIo4cbNeglDFXUISZ+aW2tBrTUJ4ho7D0CGi0VfBOes1FPRsSkvhmi7AM+/RBFQA3BlJL r8BOet7RcHFYZ+3DLrKlov8WYJy4x9+UCqozSHSXLM4uyFT5bT8ar4cXy4Dt8qZKFct0iRTzcBXO 6DJz6Rt+5J3z7HX1vUEbzCW8zTswqKwAIVmZgqiXGn801P+ARUTqO+EnKXzBesOgnRMB8EW5GoN7 Sp1tQlCM8r1a7pCyWQm/p76HvbhiklLW8iMn8yooW16bxvG7lo8jSHH2N+JN6vbfg6xFmpKvDGn6 DJ0MIfdg55BwsOM0hgk6uG3ONef+OmmVy5Lh+ZplcnsdrtxUe/Cngbd2aCovfJwEC3RIAbkIUdM7 enAX0/wBUnR3lvgr7DZVhlEzQqwxxwZLfhd0jYqvBl6kqFpGKd2+WnXmL1CmmkLbqsjw4JmSszyJ IJIdMgc49YLLkkfsUg5yYTWUXV72IdSjFj6Om9WdGrEV+XNzChWloqIlpGTjpa2CEQS9cZVQ5dT8 d4mUbcNyqpXWUYonvqJ46nKmGPwwTGh7ybJjxRJMUynlu6qkq7LZQp40r6pdW9cYVIE/I8NLcV1E 2y+yrV1nQHk99TSigvbMJYbe4DtaLhE+dGHcun6rgSBkRyuwErV9Bs5LF8hSD0E5i2UWFbs8pdFi /d2OvdqZdh/yOpkmWwUPg3Cuh7lzL3d6X4hC3J473F1EhiSEpupd9tNaYFAt0ubqj8gJS+KwEsH0 w3ikf6KbnH9+RAt66uqJ77QxjZ6cmKKr3UrZqdhXQZYLsdLPVQzi9JlGcFVp/hkFgShWZzl4FvZ+ nQU5DUCs23RyezBHF7E5AGRtzmUnmLoIOlAe83vxaW6lO47W9HQ9duIOZe3qqKeigz+mU28ynCd6 WKAYD5V4/QTGuhFEMURV0eFc5Jo0jmdKLxfDmVdsb82+S7fCsz2kCkHAVttvY+uhehfp2JUXZuH9 wNEFV3sTS1ymasEy+MGLRL45OgNF8/l/5HXLWY+dJ8vwmqjFwdZPKJjlQjQnn4sV+3/siYmgPMiX fcxjoWfMLqBswVqkE3oEZz5JRvoPROP13L6eCVWXeHNWv6HagVkxcaRPwGXz9MVd/DZsbO04ntoj wJJwok69RdEIuEqK90m1/q2SttQ3t5zmIgSdPykExaLCRUhQEhV5iAUsuEI/DzUVOAW0wVpLUSNn y8tKdx4yhoTCW+FEzxN8J8h+ciJyro8p4N7jFBhgvx8DQfn9qOUP+WHdGCzSo/VyF1dFMFQNIbCJ iVsoLh4H2h+c8iht0ScsGVBO3rc/dfxNhcVdF/3lI6Ip5uVGmXnYOqvLScvvCclCcP9H13GVL8XH /g5FVogHh5H6X6bZ5M3YbV/j103cXBT22LFjTzXJT0Ggd4UwOK+I7CzeY/A9FJxzca6IglLTgo47 TA4wy0eE2CiJl2t29uquIRW8EMb/vx8XsU3DQREP+kk3L66BuLVhJGd7V6ZJZJkHyLJxGmoh/inL nkwcgtRaHr0aXnkIJktq3I52rE4LZqJno005Opi65npAUvdeyRjYEQ1YCC8E5FK7i+NA0KR9jexn ZIeDZO/YhT7vW9ZHA7Jl2llvgI8cqZw1BAObipW2nJ6wODOzJ/5jTevAbnxu87Q3uXRyTjkleq0N P8ws0XkyLkHf64XFLLh+PuP+Rvv998Vec+Nfy0A2HhmH/8zNW+aqzrSuEMkKA+IZWKM122KlIVFn h1bwUhRe7pw5f2j4JwTnaZHkJyTRf4SsbvYsU69N70nGIIL/rpKRLkiPBoqbevIsWABpnBMpTfqk GkwKVeFjo9AUC79XSyypgSgKAYt6I76eZOxhjDNMhiYW0Z/CRWSCpjA0AJIgJiiu+x31urcnNL5E eIrad2r41k4PurwFIpj6LhRslnz9588HhYmcrNuk0EVoQj9tUAtsV+3uPqIWsFTlwUkkLWQs9rt8 y9rhCzA5oSdmm+TABMcYZ67H3moMeut/9fiqKXBlrmlCRpzcxkEpszyjgjMf+hQ6zoYauUfi89Sy Vps6/ydMKCGAI8YT+f0v2rY+1pe/Da7oUxDxImmRITq9p/Bvouf191v0qykvNOmGTJ4u9oArAJvT 7/tyTUKhumNgei17Zufwcw1THoIP01Sguqo83uRRvbVmujWKTCn+zQxVqrf6PQ1paX+93mVbwk83 1P/SYdU3WDdJ+leWiB/tqQgFFwEGUZFGss9CgSM9melIDyo1bQfxjlQyWddSnyExzyIjj2ZWEDIU LQnpP2dw8MMafhSE0QRJQb7FSwePaTOfcHmDYU9lhhRmO2uczlqt4SwIx/L7sg3TI899MX8lRCe+ A3MUSLD0sRK4/roS4lZ17r0vrsBhEPZ2WcjDIM+kQH4Jai07ovfXqBh8mKjW7xnQbQ7eZhdlxH/2 9Gn9HDQ32/xftZYbQQZM6AzNoxDx/EUXIN8iJPlV+iC3HBUK2xYNP6nX5ZqKS7YVKcp2QspKlZiJ MHKsGr3yR3KKAPSQx+aalClSoFw0AOKSn3bQlE7IKST0LMUX36ZQpXoL/Ahj/INOhesQwHZyWiPf Q0yaCpQM9pQ0zrEmAHUZbqJC3W1EqbYuKJbi/6RI7xPff/Vwd0y4mbY3cO+1UCAlaxQ4ibPgiaM5 GBTBoRt0yh0TNLkwztI0b53pYnxNW0ZzS50oNvSNrG/G7YWjbOpLkP1v0Y7xSWzzVvrOulo2wDy8 7vMgLYCezerq0aQ5czYzsAmkmdHzXuJr86VPsAubM6ZCb0pDLaX8gk2RjU/RrtO+bpe2UoCp/oxb ED5YXszvS4ni844bdbVLRNsvpXLo7NlfcHa0rWnNlbjBquO5BSUQAKEDGh6KKmF5ydhGnxWYev3e Aq93jfJqspT2x/1MXWKvAXtFz7guhB4ypWjA9vEuf4A3DkM/52bXZHLKQrwLcdxON8LheANUexnS ycUDCEYvFzfnLRPAhJapyr4zX2GiCv/F3pAgoBd8MYEJg9aWeNfQQbrnWqez6wSfusm6NTEpYyAx pgQ7DbreIv2YLL6bUaYAXMwmoRvxntl4bVE0/CCLjbW21N9oZE6Pc4AwZQy1d0YVtpWKR4/1yOWP s2Slt/toVren4IMaMcGWwUqlKNKXZX9HDhSUPgVMOqV7yXG6jGQIzbUsl4b4IWE67eO6Qw1oHEcQ RiY2KT8HzHXcs5sXdHmI/J7UXsVZZ2kXbUp30zg8AklDfDGszbgByBaU6miJ3oLwURMTgRWQmq3Z HAtChePTCjoY2tzCpNumO87nyUNI9BZ4aZapUoVVjf6eG1d+uzAxBoXFrJK2Qj1cEzdIxy+1ILFJ 8INfH5DFDs1GMnFyLDmWhS05OHCCWTcwdfjg8v1ArBadoTd5b0xXuaWubck4iwEyOtH84U/PfGGF 1KXALuEV+u0iOuM8sSlblEfTo7IzAcZzMM2z9PGKLdN9RYf6NR2SDduN8vyVM5RtJrfk3+PNfRY3 A7tsNrVKT718IaTfTAz+CaxfoVEBiwgpYGPYeXtUR/dxZvX93GPwBNiYTaLCrH9glNJ+E3Sz3RKp 9LFNT3iUq0zfWsiABWBUizR8Xgr1A5Sty24jOLGYlkqCpzO0NMAEWhxqAsMayYi/19AK5aSxSSgX WT33e2Cnwmku+LSEPc3PltMUYM70Uqt4gPQ3EX6X/3sA5w3Kz6uPPtAkOibLK/duxiM8SUXSnIfE pG4M11vA/VNGmhIdzfKBEIxTrQm7f9MF7qP9i0oFhc6lYnVwUVMfUKddJpI8b+6TzsRXTt+XwuK8 WAQKWCgGVValVNxdcQ0qMbSQU+REAPObzsclUxf8rEEEp/GoDpBXBj3hHBykViSge27wYFG3H1To Wqo6Sf8CiilAj++A9FJSB2p2Qm5g2gB9JygGpyR7ixDuycQPnZH9zR7TzgumkK/wZuJPGaYuL598 8Tjoo13KgKmupRANWfsBRU4sIiYs/1SM1Q+icJZM8+I8mXzMF1gd7MWGtrq/mLvctlyMaHLHUpX1 A7ydUbvDT4mKDhOoeZVq7Nf7Fz+qjiBRvMX13mQX168fIfpKw39dAIIRvipK9V0vmPQha0fCk3DT BUvU7mEsayfjZ4aG1ZfrYduX/AXntDLOkAuFhPFXubm8JIKEGkMvGoLBsXpF2gH2eEoZEnm9h/aU 1h7i2UiZZ7fpj6+fbLQqXcvtM/ELTLPAXyregIQil3TsJOgEJv2tb0vq67ukIUUyAoFSk7vEB0tA rmTllXMz+UXJfZKutmqsBMHDEl6/CAOmN5AzeyhcA0MA7DntsARLWS077Y6b/scD80HAgfKaXPu5 7WSNhs7yVcazXliYq0xAllFYf1jbCMDkllGkiH88njcF2m+EA5hSqBH1E1m1NIRiAZL+ByAteeZv rbOoauYVu6yuen/G8voEF9X4bvBVVo8wIuYTUplRxJOdtwDv0MCUxPh1fkkzKcbhFqvcjrq85Y0m FG7aFPvUygywYRqkW/NB9FVsmHaZCxAU4AKVMATdEna9QL6xYCsk+vCUfLDZ8RN6jAXgwrIu/kUE Hja+PiIHsr6akPxJfKRecDLgHy5Y2mPK40NgHi2XrJeJXpJzOzN98q5yexchMt2JZ2XvWIZs+TxG wzfXCAlr17/6PWa2nsnLvYEBgJQOeTOyjhj1AkEsYP9At1mkICeOrkewQa3Fyw4ze3igJpDoDBK4 OjbWbmhkbeLMdI10zqplaJyzfWcOJAP4Q7jO/YPsX3zrYpSlofyhAC/vAbJ78UkWXqp2e6aR7ZE9 HS7cYDFXuT0MNNm2KBXKpigrUAwO0Ne+JDzUzai3NwL2ZgI0Tm/fwVvzkn0kJRhhISYiDsDa+5gz AUcaSRQoeJuRYp84cjDuOhlxqJixAkNhwfF6gk7+sp9eDrr6pinyB6prUUrzqUoDJ4yI92pPdTJZ pCRJ3tvBUDVSGyPzCdk9mxKRKtHUJVJIBJDEHqEADKmD8u637gBgnRS12xHkxwh3Zx/kGNlHaqsH bjzdXOI0CehTKFmeo34fSqLF7XjxZ5wk6nMwVdEojypLGx4OaXFaZfU/LTjQZDYIDA4jjWjG6/oK 2n/y/Ue+kF2qvhKP5YOXd7+zSHDEB378hp/+equvlioTGzDPQ7NmMU/8YrWuUlPShjAkxYRA10X5 mIz8f0G4tz7mGcdjBXhBebNUB6DhlDvV4tWCMzuil6uIMDnpgrtokah+4XHx2zf7QVwv43ne2A0n ZPAHHQCkT/Sd3huE1P3bK7ZV7/khkB9ovXzCvHbV8XD18KbN+OLv9cPfJsZt2FnL+U4+iJuWo2bX T/d3HNYCV1KaaGPwbF39lmHY//XXXPSZ9opFbPX07Y23VDW7gD2TD1wR4o0GEd16hntpu9/K2hY3 6ikEl7JBAOtN7RLLtqUyRUx8x2RsBjLt6zKOJnzf8gFAF8tINBMEMtHU7y7825X6+eiY7IrqnfGQ lmMCWGAy5KPzRYgZGLkigpSooRG4MZ6eMSUBt0AAec5SJ5cRmQQo4Xwz3/ODkydtCdu5n7Gs+ny2 z/5HK7YgTHS1rauEx/NlRIANhAcda+VoZIvfTlqEUWy6oU9nkJNxaBDIzeMOyI8tHsWAcvs26nMH zSMiWA0pF3vBB+1aXQkQXJet4VOCzl0ufRcwuqDEatv0a0EJXZ3QpmGjT7WCnElrfZQ61e8KFYCT pAzDhtE/8F8H1D/yguGzCohoT19N8Ui0nEpNaPcc77VKJ7XSO34B5Ag3hBEXKxY1iJX1ErzR09xN E3xIkPK/V2KwD+7sHsBvu/JfayZKGxAWXEMyrssBgI9TOT4mPFm3TPEMk8oAClcnNL77SfvFtl9H nmmYlSAAzM3VBrRzsd1Kwc5pOpP1iOUG/ZmhhMMB4eSnj+xPJPIYudkEfiFlM2RId4dBW2FePsTR i5ponfaUFZyffqnZ0XL751KFCVYo8HMkFPxmsmcVS8kee/ybsAn8KMa2yJ6Rd7TofJCKdLln+ccl 0goQhhRpK730iK9z2BuzrdK1lLXkCqzftvnlERoDIFOWkFkodv/ghDmN03umjrdm521bwG7Rfy2O tTGLNEhcK7lsAeACY0V4M6K/MCeQsggBrCEKnOPIHj9OKfU41dko8oY7EuApyjVE4JZT5LHoo1Nv RZ9nYqP/GsSkdjNS+FcIfeL/bbdw3dwKBvj9wXog3SX2CbKsbFQaH7m3Ld8vyMWn+IH55wNOxcZj rOEn1sv2o4GwSu59cilQnGtTrpNekc5/DfUWAf0Z9pI+dwFFf2AOkbQnKuhmmP+o8C6Co/iFIaky JJJ/GoJWth+a5XOZz86LI0ZtYsdFvlaK3JWgomUoNzGa2NFoSs5kAUhX4fcbNVrfjgxVHZtu7kUU xq70e5RvSjRBYaUFDNQvdUJjYQvyldgsZrur4XHPlv5Ec6VhXYEH01x7DEIMJT702QY213VhPCoh pZSHneX11n8wIseFMMn8D/CVR+HDI6ItjaoEbeiFBpM+qqZK/PXJFasinqfT9BVpha/T3Q1+5fFy 6L9YkgX7r5EuRbC+j1SR4Xhm2qBQOQKfGVzBwYQXT02su8ggO7GInRdYjq5VJGEt3nZp7MyJwFWa OJDgGLexCZVaAZOKAznTpr1a3Rb0KDTD7g77XsCJbkQGcWwpw/sSScPc/f9O6yXdhde7s4WjmtDM GLrwnV2ntTGVEQcWAgAtgyxR2rj+zO3jqypizgEnkeSujcW+sgSk8Orqv3mH202g3YO++ZKcYmPr BqmCOCEoTIYqVKcyb4Yhot8Cayn49AUPXPNI5elSejlm9c4AALodJy0vBhuEBCJhEE8PbIgJ6Jju pBR0Got77EnaK9J+aJy/LP0PQChOt/tMSveoWTN6rcnmXHC+4E8EErPPHd8+kJX68Lkqh1XwFAKp SOBAaI3z/ZURIBbK2LdfGtoglGfvZZeaMFKWn3Rm2RHrlHg4sKUaNP+stnXWohDHkbEfH6BVtmFD 9SnRMOWnZgVieAxn9KKyihZCPSi0NddFwOao4+IlpaDX0M7fKDEbWKizddDDTRJCqY20K17cJ6lW nVHmGQGSMVKls4IDjtIggTa7og9I4AAfL0+62rkahQlkVRVmy+tHpaRHemxYy0lLkTcERleHmZ8D 9/1TlARMoLR7A+XMzkGyWyVdWkfzcGYuvMec6lfdCzUSxdazM4eMRXAAbBTt+Y2ILc2MDLyw5Sj/ aGfEmq1cTPfWn0uCM4GqmjuX/8+UDZCEA5NWIBhvD2RvTA1AZZuDLwQ4ifVKB8etdxjuSwH6I4lL B3RF/KWVJ710bepqM4ZSXoCBCRHVK9GSCf0u1cvWF4pBu6HjkeyWlvC1MQqfZu3Ld2x9Dpo14QVN YAKrtmrDU4XC33iFAEzDTHjl1uVB407LsK+xGPn9PVZUhV/vqumlbZrMqkdhL/pI/YdOpO4DUsRm OdErPN7sFXglBgiviJmjedl0QfMnvz4RDpyHjkdYKnUZw7BZSPoDONWCcHLrHVhAWpUp8rtX8+cH FxouvmpgzNwDjg6dhjJFLRDHnOXBbffmP/c/qGUDWtZdLnXQO09tyXCWByIQlwJ2okUK4evExiAS aSkxF9t1307ltyMtnzGorDwG8QtZIu32MhBUu5tDwJ+VoLyOnGTc5qkCpCowcPWzOKHEXiOxpfOv BZPRdMn1PD+c9xqrNmsfKV3TvjMqR577SnFcH5o7n95oKTaHl4NUp5tfKi4CaSY/p8hItxxsoPjg d2J8KuSOgUbLe9wVs+AlT4MtyL9LJ0GcfhcuQq1FwLucb0dVK5CVPrwlOover+ly9xNyhvQMxuP+ HT6Q+HnJ0R0JYTa6hfZ2XOzacY+RihNGJrKWBVNTj03jW/1k4el1E84FJwxwwp+hAApGe+RkrVFC rNnrGUOPeUSVHxVT2mH8WymdkYq6l0Pf4yEC5X2f5LYdoNNZHS0LZ+9+qvjqo+FcI+Rqm+zJSWe7 0IcqXipXxSkftEsxpP16kL0SACeIA4RhmpEHlpERVHLdXgGMkfRLVU1xyEEWeyDTLckXwHPhACnP fvV1AAsEhna/m/exY0XmrdD6Upg8c3PjomVQgps8Ow7nWrvS/KGW3zyzXVaXKz0xAXcXPDctgEql bOS5SiEK4vPAfOjkhRxOpdqvzO/6MUfiEkC3LvEJ4VwiQXUGTrskkWzI6GM20qIQ0X6/z1mfR2dq De1MFVFH0df4r/pgg+lXyiedB/8o6/1rZdwORy3WNMZcNf1HRAnQqFygecW4HgtBtiOPMgkGYFzm r7yB6vMOnjFuHFeDDOOVFn4huAYXPa8Wd+HKc2y+oj7dVDjm2tH9cjKjuXHkJc6u7ZssIGlsQnYl ZS2AHNP4yxBxPUnPLC5u29u24OK6nbMrhQi2YOMgVxk/oezcZI5Rq13oq637Dtj95l3bhjwXGZW1 S3gXbv7juXofmPbhk6aJbK+AXzkoj0cgGSgkR0pCBNPb+GdHA1NqJ4p20RGrzBpxZdknmGSKdZqQ gVQZAWTpKIlK24ntXemVdTwNf7XoDIBILhsdxGOm5JLQS9uTwAks8R3u6Dk2g8iUqVfgjcOWBysP C3m8NrIFb8iuNJIoz8miISZQbNw7BoYB97Khf57+Duda9E5sJbVz6N0OXYXTxL5iwJ9qVGgh747u Os+PycxAbywxKp4GOlN5hd3hWABSQ161RHjhy2idp2HIGl/xmhxFea/lOPoyPJ/Nuixl0Kq0n1zS Q50/m0Ta1tUyk7S8CwzNC/ikdIMCKULf4Noa3W525qqdiDnCgzzaIUdQ7Iiaq41lpZz3Izd0+s3S wl9xe1buItanRlo88ZSVaFKW9V29FIqNPiROtJTRyKIeAkXCncrvXXlVa/R3VzlLXXIJZfEj2USK ADuQ4NWdBue64YvlZ/ahqXHooy3YVrLF789yr3behfd+SWdKrqs1HFrctz9bulLH5ZWeewAuzG8h Pc/p7WXDVsVqKsVCqVSy+rXhAxqlVxfrJWepzraown1YitVqQZHSLJd06kGwD0L76K8V54pT/t32 OgmWI01wrmV4Vm2m+WhPtaW5ikF0NrT43gnxQ2SDG3EGYgmWXXS5nOiClvQtFvzBvY02wU2mMO0N YM7TM3se/NlwMnxHkf4wEbJIrOPA9ztjzhs0wuNZSjaSljzbCty4vKY79tchqMe8iXdlSLgr/mCQ Moquix8rHjRar42FVHSl785HrWehTPN9XlwN6m9XHDgMXUzYpBBroNb+i8p4eUI7SPTQHTxPhZDr emgH0Rbk6BCqwDtpt3dfiTQrTkogBFEp5VK833IASR2vN7f2rL08ImqyTe6t3eGcZNw5dtMZ7TVv nxImdI/SDfLpR8JY78s2czlK3Z/GlxW+FzOz38ptQ6kVjOi61G/4tturMFcFkkpP8fKcxTID8FPN 8W/xaXUi0vNArNNYTLP34CuAQzNhtEs0SoDaNhLb0q/Z1FHauQIBfDOGRBoWEibeFFw/yLgVtlgU GjTOtUXMZqBSDasPuNpVaSS3xNhZi0H5ZIAx5mktGhkmhCrANb3dcYzEe1PytemTytNoY+xt1JuU EM1Pu+lAL586rJ4/HnsDkH2D0j3wyPChubYY3qaM2OoawuJ/RFKMN1+FoP/TMqO3DWqRoStEdOxz sVonYYERJ+1LUTH5oS8ewTZgfRWpgzJ620bxFYPWkXBgvnm5CJZXzB1RXS6Ee4vFLKB7cGSCYDxn ULMhYUHuVWtMNcG+ukyDzYF9PWgdou9qfR00zgNUFufX8IHb7KxcZkTgdlZI62n0DzlGBQQxYNzH vWLefzcEnsTiQiPpeuPGnnIUo6PdAuStguoq28+vimot/2hYDrL+zWNujW3Wai2/Wrma3eQbSRDG V/mv0ruvuf7ICBCQ6+14qtc7SWLEKVPLGaC7UdHu5APp4O1fkCu7wpoz+b5OSE0z+UXg8G5y7748 1TQH+lEYieKS3jKmD9/VGIGxwXSJiiO9IrDPxwAmlLi8cz1TRj29Yq/y+HT4/l6+iQsY+QxDVKIA 5fYw0RPtPiiEQiun77H0ICufOxDSvOaHujHGoKZ1sao5UbsINSHzEtwUhf2wb7rwAId/JV6AMWQu uDZOUXTCd3b1NoNcyKaTxpxHzCMot7QKA8UpZwjhD2dRhB+qbSfzzEjOk9J7LGzdq8I7eniifimz sW/4sBG7vuG+Mrf29oMMGEV7ug4hscFBKNlr7Ltgz406cwQ2YA59XP6rPgI+yV2qjR3CtMXskmY2 3NUqHBGQPaZyTkyYKb4keuOY1blNZNd99jd9tT6Ego7Wt31h9IIQ6oAbeR0LsE43B4ZnJgP0Ag6X WbGgs77ByTEbPI2eHnZdDTvyF3LGGyf2E1UoRAMXeiUxvl58T6+mXp1/2CjUuE3/AdErbeTfBB0P 1ePcjxV/TOvhlPXkct/zl37FD6j6xw7TbcRIJrWEi2/qDjeKqPDw9OFn+iFe3a7b73DnlPwoId1F d8m1RcHzSz4IH8FwjdFG/3/t+VJLO/zyEnIUAza8tRjEvPcwsa5ZCO3z+XSr+XbPSOD7+eWhk1vb oZI//LFvcvEugsSJ34lNKgv553QrzBkWgpHPldWwVtq3oudGSdpA4GiAIsqTcKsx0OVA0l5Fm1HO +Rx6G1UM4OqiIXePNHsjOFPBYw34+2ABQlUvpicimIRrf0IIVHME8WKp64r/G1wVoeuXmbLKL8Cd UuFRo7ru1B0jAQPRe66Il8+FhfVREyaevA4iJzetAPQvjO31dTNJiXR1svSCeM7impkXPx01ruRk wWFhB4XXoPQZ+ozsg2Cjb5bMrI5s9aBVkfk8IKI4D8PR4bbHzsce2IeHpwlEOVmRWEhbtS9pq5jx /MA4nwZvziaDK0lVruJvOcAL6kgDahpLbsgOREZZE0DODJdVbZr+B7+X/Q75V1TmB51GzlfBO8Xq 1Mu6f81NXqMV3dOCdC3D4B7INfaov2tlXlDk1Xf+uWZkhlS/WwVbMG7A2qYlfDte73KZTir5qAHl 6k1RRu+phtPjolAKTd1SgOGuim/i7trDdovjetcPBSEJF41+T5yGGZ3hMSq0RliIi4Z5yJtPhEbQ rt0AyFirHLfkZ+I1a+xz15LcoEeRM7U1SaNJ+xWZbmlso4bPzDKxZkRNxX7QFlJWIjCUEfjO5KNW e3TygevWxME2fCmdD1vH9bBTCNZUMVBdd9z20k4Yp5nVCOkwlW9skc2njbel2rGWvubUuIQusS28 nmcD5EA/kGAWVHFZn5sNEjYYVylfL6wTtidM2eQRgzIWUJBbEBbdh0iCapDLAzi5IF5kb289PV5R /8iVs9dhcdaJKGxWYqYiEmiB1KOWkJC8cJZZ8FRWO7Nt+DgAnrc0TIRZgds82xhxTd5BdQymJZ8N jLCjYij70Y1U0OX7D5aBzUxyeJRJn/ZBvXehx9uGfdMdxAGdBZnT6bU5ftbtvSibURdVt81igiJ3 xnIo7tO8JK4CjVVOfYiEa/50WKjVFoK6O9wOrPARUOUgTYHtk8c+1ap6FNjvXWZDXB/lf0k8p5xR Cgo+JC2XmIli4JWNspSrVVkgu1xIP2zZ6vtkMzNG5sf0xDr/V63aMnM7WdXwj3OnV9teSL687KfB LBu0IX5zeEl/tl1f9MIlOLS7IGlO+ZdmX/G9Nl+S+jjS9qn5NRCveoA3YUhg7QOgbiOj/V8AszSx 0ZyXTa3DakzclGs8F/WyeJwdyWNx6JfZKpzALS2Vd3at9QciNrUxDFnaRBZY+1ypZMWHODnszS7d sKLZBSj0ESDu+Z/FsylD9JuZQ6g+uOdsggl2OcrYVq2p0ch9L6de0zUIL/7KWNkJqyQf1R/sfZ7m 28rD+TsQi/KRSfT8/nfNTlgR1fPqae3CZ366L+vmjfmZA0tyzKOKsPaaeJ4Sv2YkaJ+ZQ3K1dbwT XaWuNezE2sOS7vqUS2YDXFw1sIsRQ8q0L3UYsFjamonlN5CteNLK57zUvtH9ZLUp+tRQliqIZBt+ A+vL894K0lALl5WCWgEzNd2TP68snZW7/sHWgRBxW+nshP3kmBMV2jIXMrVs/sWfHUGDbkIl3+63 NgPpxw7XswNbjQFoIZ0LNpMbv7PlFyHpT6OQx9fwlvePrAr/H1O/MTleOy5zDpdUISexO+JDDHQn hdTzoKCclvjCG8GsEmwDmqhcEl4aWawVY7ZvOQ3Urxyt3nCg/OI9KKSyvKtZbo+EQI5yltQeqFrd eNgWoLEYl7zIWaRzFOpgpxs+f/dalEJ4lyH77Ajd5nrmtwlcFUJUk82wE5E8qYBgCyLfNVy/GbCS 1IgDxgpFgMkNiKSZJi6FExq1ozqmhnJyspffewG6am0O9MXVjIXc454BmRlh6XqllKokF7eBNfaE typPb7LKRhU/FZFDErTMZODDqWZ9opPBuKzzjsRESregcraT1BZDZEx8sBqwV1dleix2zf//NEpH L8jeHxI7XEDSi5yrljHQ+j0riXr6jvoqammCnxDtHBusfnzhUsjpta3MYRrnGxVkW1hXXbEp8VRR C6yqW5BryZ7a42kG+E2RbFGOafroYwBuqZ6hcHx/P9qd7BpbvH0cWXYSfVUQhjctV/64sFK9LBF+ rifvuQ4rJhlp4/rLPHs9qqFj2vFADvnuaeKMjWCKiQbY5X2nu4LhvuWIOBwlwGTsAGlultXpMtAM MqDMUrocnJfIJxcF2yHzJxPq8hp/OlobBj8ia2yHSD1ts6fyFw2HVI/zy3fuawfcrgj61/v5xVvq WDiDWANJ9LUovti0Sa0kfTK1Wq98T8gggi14toWsXMipozHuusMdRNKTrIXRV4Uq9Hrx/pX8LdaY kKIhZ4Jh5FYI9BdyoDC8p/8P1lLd8/QX7xTFgAff31tlLIEHhoWIVtIphYf9Zk4MmBHw38WXgas8 fmYaG4b5pfXKXZiDB/Si7aBfbichRw/LiSBlngpFs5nBhSOWYhqXT1sLtZjSdHV/C9zdyHJNSOak zaavIoWTqYm3Tux3BGPXIGk1R1qBTF1YGXrfd6omZrMb+78kPCrGV9Oj0m/3iIzIMseB/H53k2mE gSvJM3PvTQ8lsXD1pJvbCriOtkRrlYK03xgmdS9aWPGbBsZION5ClcxtTqSnoMdnYotMrMKK8P/M h4Yc+7bCP9nVa52CQrW/plD0FT5YF2qgpcVlO+UQE3AwPhOzlsK54R17+xswMvJeC3EFRovqYs8e vpC/KNzren36KXgC88xpgbbHGSDVn1SPxHxZLJQVKBaUocJCq9zRXJ7WlNxyPzJHMRzJWb9VNmYO yOrTZCto590evDFfO1xswr8bku4pVQkQlnjez0V43AaNYAdoUDzOm54zid+nyu27OY/Cp+hYYFIM 6U1FqPvSRcuL/N5sb+dxZMoyyYLj7FqJYj2KPi/YAiydoAK/BYsTJprIz698dkVgtSv2bYdjaAaQ p4BeiZ062RKSYAQ4wHPKC9JkwwhOfZwS4d9bKQqC5BuY7rGZxXHtKQDUAIz7SCKu26Jl/UcRLCSa e3aZ9wLpwg3qx2k8mT9yvhGAcZY3uT1Nv5uEzIOQhct93Bbx8vLZiQwNZNzm9faK4QmnNrIZ4ZvG 9ASb76UTTho3g+1ehp4IQLtVOm51f/XH/967vpG2g3IHdRNdS8QuBvewoPlepOgbMhWRI5tw6gOu D/6EA2pEdiPdS/elM07VZXCwpf6DXi6P2GrTEclaoAzNUMMu9s1JSsx08hs/1/VEkdXV4syJb4Gp /27CuAuYqVW1EzakKbLVOJDh7Vr9+UlPaXgFSDHbI6WpwmsrfLkNDAeh1qjY8/OtxZGIvMOuwJLV 09hVbQspIJyI1Kl/fx6kZjt0boPurWmAUbgCDvKQ8YJRmJraQ4eWCJGfraSOIK61W4lnkOqmWo15 D5l9WbIwqWQWuECy3M9nHe/f9yye9iTRJIxUtu1gi8CwhJrtzfOh6ircEzUvkm8EUdI+drpEJEGD Ng4mqRqwiocxO2y4APVqkPUQ5JTRQG5xdH3L/ANeTgoYxNSJZiMt/jYwQjXtl1C5STkh8t2eBkKH u0b9rLtimPPAPDGQolsshXpX5UX6CC9lhGtU3z6AWp72UVCOaJHLjiNCYJhVGXdWHFm03/fSUe8B d88yfQid+tLUiYl0WDYV9r0QZAmB2NrSzWZbxF4HZ+mKRE4PkRth9BLuUjy1wrQ200IZzz5B81hb b0YwTBHL4Th3MoFWb84H6b0e6NZJlOfPFyq2XvJq6c9O3jIxIvjYmbw0aFVcKOb6lrR8nq4F0Wy6 69SbHHpv9yf8fm3A2l95M6h+FJgeaCf0v1+UoGFmr2Keq4wOsd5gQDCZTLQTSIwDjnbOk7UaZkBU 5zcCzVKbthDL6qBzlAOll00uuV11Q1+Pajtex7+3VIeZ75IxEMkL7RZFZENyqlYdfMRRy5F/vkqw H9QkLXs+bHXjDRa2t6IKH9Wg0yxgHLErGMutHEtL68xHAneDhEr337aHAbzFuXMXG0kZ6j/1ojCt 6kWz+w04/h8ijvUfchnzYf7C+zZX3LGkyozll6ZkvpBWLqTgW2nvZUIl1/cF5RA3b/NnQnlPFFsl icNH4l3aueyaIQ0r5O+HC4G8xzdmKjMN7JVmhJiWg0iJj52euSKFiPjeNF0hyGsBdoX+IuKZyuEn tcj8LL4LHGy9mKP9asr+qEqBYltHWWVQJHvosYEpELBD/d4LoJc9mHD6rnXXtYWqchDDNpeCAa0/ kZvSkcN315ygwvJgvSq7qeAc2COoJpF7ywTAaZ8Bu/b2m07oNeWXC6/JogxdFq9TUuBN6CorlCT6 x27nbZ2hMvvXWlNL/c+zLFKFlAHEe5eQOdQSUz7zcFSt09J7/bxPWHRQUcvAE0T/xCiAKeO4Eczq sPhxyVCN5Ze6jZkgs9GrlHpnzfmO/50ybFvVDq8BS8uTEJsse/GW9wDEfc2NjJcy47MUolU+tlfT UdyPk7WS9PAciHJmai9fPj644+SjUiSfA8TX0fBNPBKCj4J6NG6To7NolJvWynICZKWCDwP23FS8 GOCPrCj1nhk3laz7nzhNeIste8Q1E923u39lqiCOqJAAGSMuyDe6gyi3ZubKvtdH4ptwWRaOCSuV IB0M8Yk9nLn3DMwQW2RIYQYR8NzV9LnJ1KTrU8erl/bzzZnrcE08atfvc8yLYHt51olAym99IfMF U4tcxgSNg/dglX18ZKKrlhl/71aS2QnX8PpO48EoYPRNhyuTmkYNKCCVhRyAypzxKiGFS+sGWIDq sV7lVfTnAPaiDClBqdUYaRFZMMOuacH0armnjRaD1tPysbvnXE1ZCoOt+/23jjKYEgYerNirmmdH UoNwOGhw9gHnd7Rtiu8SFhZ7QJpg7TPO0oWzRD5IOUZoQCg/epquPVrwEidWjVPhqLZ2Hl/N7Oav 34Ni/sN8jaGQtVRfywM2Qv/3S5/+2HGrGtZK19paojij+6h3XVIphZxZHLDPepdZk8Gmr62Sawlc ydKUsEC8bNZqT5pw8Pv2yigsWCnb4/kBzSRLLDjfYEU8Fawv1jkBQFq9+18kjeNp+2Z4f+EmKXFv y2I1KFwf0XRVQqmlYqVx27jeSahhOg4kTe1qtQcL8z5a45veJfZKWAOBZCJAWhv4s9N1lTbNrLH8 d1GjWEaOkOxBrBh7T+Aeivoc9zZ9lhkjX0SS/w1YPMI6GZ1xcxe2L0Jnhyz3WoD4t3RJhH943vQm kc8atDPKp1O3YNbs4AJ98eksZBrwReHhKN5/1UqDArlGqoJHQohIPzhzWWevoV1vKC9O7OilfG2V KszoadGNorsxvpqUqioeY3Vjsy05JY/OsRSOf965cmvUYGZPQvKiSaKKaG6tj5Qu38B4SSLIhHLQ Wpl++RaReelzo8g+FnO36GHEHdeB9/sII3264aCFI5jDZUEQR1P2NjUz61tuL8N3/yjczH6Sj3de MxXQxlom4hc+mnR+E+tkIV7tMUOiCnsTj4rp53rSo4LO/OGVuyDDVMXOsNV6ZXdjNrIlkuGoQh9r CJrI8h83ucvhsn9TGixI+dpmf4x0QwmMPsrJamSjV354a+Qu0yHC7yKl1zvq+kSyncHXofRWBIj2 pOWnShKnX66QnVB0q1P0YEDCGJKp9Cy0yoeyNaNiMGGUHKJue5eEy8BgBR52aKdexSVbXLxsipL2 Tjr13Ms1UcI74odN/vP4dJqZ3yUdHFU34Xm7n+xnJzpho65PVWRSLeb8GvHio+H3sq9vO+ZkIIOH FPp54RHcDRXLgeSco8agaOs12orthwS7DQ2KOnGW6Cs4fAyvVo5ksES5ywGiXhupPY4VzR0fxnAk NnzkqrDX0VxiphxeswjPwVIrNoT9TIiVrN74o7UER6zwbnQVVzr1kMvUvd1bjspuWtaLBDjf2MGD ArCf1M5r1WPK413iR0zib/sKGgzkLuDNIoAagO5s1B/OcvWNrL/NbsD4ZIdOFKbtTzNTeBM5dKCx sc+hwsJ5ShB2H2+CvfVT3/VvGU3VuUi/90zrhiOKtKgMt/kKrpOb19hs6O6+enOQZW5+Xldg9y7n h8w45xEg7320IYwBlRLaAxfMgv77odsn6/xc+QY/0C4AmVXAjFT08p6yP1YzRNMuvucKI3iYnR22 OfLLdxyEoB0g6pKzWo9fuanPmL71q1IOgDxovzqj94sFctOpga3H5FJuxFGhaAf1hBwXchGHoYKo +rQ1YB6RFKNDFtJSXYXcYBWIIjE0mPLkPVAOmprbZR0uiACKwMsOro8BSF7ikIAtnhlhqtEHMQCi wv0EwvRHLBStph3vgHKHLp2ORMXQNIyOifJZW2PGEf5a/PwJ+5NbYa4U0jHr7EYulO1mNe5uSlkf qw7NpXpC+2D2wSne3aY5y+FZGlZj9Ny9MRLBQ6eJXwG86Q6/U320v34qhvFQA9z7m5AHmU/p7jM8 8YzuVxHSJe1ygbjqXGhQvnucYlTgknG2wKfBIgDiIVgcJLxf6udZtCri8FCCABIRSo9PDlTZYWoy wc66GhfdEvpVNzodvMHu5wCUHMB2brfExunaPBMNFLaq9a7vxtaxMt73WK/N4Ej4wRu2tH896CdT swcoaHcNvFGZNFiow2CxqQMejPtgrn2wn52SUwsp2JIJix1hPUBuQA0Tm0zLeWWjggjygHcpT6Mf tizS6J03IWu7QMaDrUpPl5Wwa9u9Mvtslb6A1BqQ4M+rWZjLNTzRLd0UuR/lQ/8EplGdtmEEE38v bbbYY20AldoLbYPBknH9oVkyDnGz/oCGguSTvqPjlRiPosOWdU0nb2JPN2yzPsuSGOlNsGGZT+Mm E0EBseiQWRMXv0ZEFdotducos3S93Om35otAmHnw/xi0oNgsV6B0+2+4hlaaSt5oVSbmdkLNew0p M5YliXbVWvdZaNXbgxw8A4VWXQ6An6zfaH6mF4DfgCOgTo/FZrtwBQcLPAAT8dnS0OJ2MWycWSHs hC1JUIO7Kb2Tcqc55uPJ+FIDb/tmQKqeUH1NePd959LCD75StCRz71l4WMOWp4SVh579+GZ1oVRM ujruzyeujbhw09awaTVZz8Ou9iNyBOYDh+28YAtyj2ReL33fProrDFyYdE5VN6Qbq2hvizQ1eLy4 I9tWFFNSkzsJVd8YHI9T9YEBIa17Uv/uJ+ekt8GRfbaLnux7oPvy4zh7+f3AWFpGiFpfx9+LlM8d UJKu18dZSIkjMg+IzLN3c9ux7uJHNYcjeLh4Pu2hxnJ5xdlTWC68oSEEqEO5ZZ9Jl1AIiSBqFuNG M2ksqYgglS9AcOzFlaB2yUrRB0LJoADclYpdfxFVApVeU3Xfk4lApznQR89ZLHr3mGfOkP+ilNOW P3zncdPVx8Uyp9dODL1vU/oIOY9+8FwnRGvQqJmpBkdV0QTnb12rt7l6ARWlkpzIhxIqLiRlK3LA UgE/KeT4KhAQAhqnrXZsQQnm5yR2TUYpB2/RZNMhzoCwTZvnMunNzizKOg967GhW2D1FXlIJ74AL HUIqqcqe6dSeaJVtiSURaxxcGtgiMHRaw7gkQFfd04sEXIjbMwxuhfQckRChikq5o+pwsbqA1Jt0 nFgy/VaTWWReR3crIuYlZvWxHnzGB/wbgnXCngi9QSpw1NP8FgwH2jp4m6jxpZooiEzLEO8LFaYW LmS6ZLUmqfWK0z+U+VQbrovyPH439h0JADYHglRFJgXjUyPEzmgplOpv93cO88/27myPhuqe7Ion p5xNAmKUjRLua7H9le8r8mNsydYA/B3LZUmdPWfPliIGJMkokIr5TjfU5O6kAvYcerJuJ6LjwSHq abm9WTqquJ3gYNUorzIhqkW9zXca9PpHnc9u579DUv9PX7BjrRdqd7vezwl38EHJwKAVpiqGq9eW HLqbkgSD/EA7Ih4i4PghiWtB3l9vi9619KTO7TmEj9oML7W5nrQP8oShsrfUOfI/EsL3ucFkVNWR PN1fANZOkwS43e2DnBAt/uw3bWKW2sCMr27LRQ2DaUmGcwvAy3KfyinONlT44o8CcS3YpMfnQPh7 P6OLe2/VkvGlYyzQ+LNq4rI79T4osfAzmfssb0NWOBD/PcCErL5e+dE0vok5N5fD+V8JBRumM6re q7GSk/oOKi2gWcTpRN78Vx04AaynBl2p8kygiaDgHtlY8zOIuS1vmVXHZ6e3Zm/t413RAbGK8H68 nWDlRB12Zyg80ZkVtr+G5hyL8kc69P+nIZ7QNJ0TCOjHwLZ4CfdksySFdZh6Gtq6hAoSQXgWPPY6 Fskrl7qtd1+d1VozjfhwUxcL83XTxk2am8YGXkxSXamPctuydRAyKvomKM819UJtI9V9btMMjs/L 0n8SItxQ+b8hPmLul0mABizjWSDU80iCpPocx7dlyQTp+btjqcTq4DdO1jcs2bPuOIaTZ+HcW5Oy bkKSGDGW4az81lNefH/XQKH1TLNGDJAs9yFwCFwk3SjGC1GYVcfaHUZDUZY27OwOGsHecWaBZ5XM VNMkVEbh02DsXD9QEdZVTq42ZwZliWKY+OCflMKEfS243fkV84cCXXinkNcrfSUCLKDXq8uytests/test_data/test.p12 0000644 00000004765 15025013616 0011174 0 ustar 00 0� �0� � *�H�� �� �� �0� �0�W *�H�� ��H0�D 0�= *�H�� 0 *�H�� 0��h #$�� ��ę�A�bz��AtԠ�<�Ύ9;���5�0�g�?�/)�����ŨW0�S(>��잨3����IЛ̼(n�]+�g����t�K@�L����� �RD+�>�u���4�C~]���\��A���I�ͱߚ5KS�ŰuMDo��.ƥ��Bſ���� 6����N�*��/TV���-�8��y�Vޯl��R�%��g�Z�7[]A3�K �3�x#��h���x�r�� ���J��D���>�^-���7֬ p@rp��wtv7b�� m�e�_� �㝝�p�P�,y7��p����"A����7�I'���PFӁ� cp� 8�u]F/ H�Q;�O��볐t5v�óL�L���T�����I��*vT�g�[J�f�)_Hפ �tU��{ � Z�u�u�r�d^��('�!���P(�m�*H�_.����p�;|�l�^����> w��z�g���' \���V�Z�"deavs���P���X �v�t��^>�� [3��[�����87�Ҁ��o*\�a� j��^g�< }��/�B�����4賑�Q��� � �o!�0E#2����EE����o,�m�|^� x�k֝}ƫ���U��B�q`V�F��(>��\�N_�� ʼ�Rq;O ��"�CeF���Pk�XZ�[��昶�2��G���[<1hl0Qe��c�k���i8��S[���t3 {�A��F4k�<�-_97� ^� !$ ����~y�c&f���7 F��/���b\��;�R��{��'oc5�s*��ԇ�2��QG���yf 7�'X���Vꄉ�8��%��f�k�tT)� Z#��mX�w� � F�&�@E��7�9E������<��~�BD�J>G�'�ql"��sg��U� �/�(s���s?mZ����f��aK�]W 9�>�:���X0V�x����l=tݍq�rb��6{���G����� �#,7� @i �z���SxN!j��W'� ��2+"�;B%�U��]�0�A *�H�� ��2�.0�*0�& *�H�� ���0��0 *�H�� 0�~j^�� �ȓ$��'3l��h6xp�Mo�������.K�k��qlF�ҝݬ�<�"ֺ�U<NH�iL؝����.�>!Ѡ2Rǫy!i$�ue�\��-��ji7��q}�^]1k�V.�c=#���Y�V� ��N[B��r䘒��㮭`��>C|5���f����n��8��J�\�.�R%�|��08q�� /C�Ɯ��co�*;�ۈ�vCs���s�^��]������b�H�%�3�\�!�K����4���@�����k: #��I:�%Z���K� H�t��H���>��ɑ�D�_�>��\��I�Aܱ�.3K�� ���@�����Ji{ rG��`���Å�i����ɫ�[B�a�B�{�pgmd��ӂ٧k&��؟&/����m�}��>�,2�췹�J�.q��)�� 'r�@|l�M�� ��=�r&�iBԽI�Ԙ/�-�������� �����"�� ���_`����`�gn�U=p��e�����y����J��?�GNp�� �آ��KU�2h�WӾ����e� Šm������u��i�g`|��5�~���j�W�t��Ӂ:�����o�SEV 1�ګ��C � ����3Kˈ���=�0�b��>��7�4z؈�4r�H��:�j�{��o�S�;$����JҪpi��/]�9��(e�T�G�?�� �� �����B�d;�Oss���&��"�0��MM�M��`~ � �^a8��4��@��� 7YHl���<��0�h��?67 �+k� ��^L:�� D2WָB�e N?���+� ���[�\��P��e��{��P����79�$S@� �$�kY��x\zՄ��@��h�w$�Q~��`!�ce�f�L�Թ�'��iL }�~�s��<��n�J�ֱJ/ssq#>���ϗ�n2(�݂����=�2��ǐ<yU rﶸ! ���w�1�D>�p�#Ȼ5=�5hL(3t-p�� �@E�@)�7 ���d�^'?�%�ǰǁ��"�|$=<˃!�Y��Ky��@����Ħ`|p��`�hV�� ':S�^����-֗L nV��^�P1��J�:H����0���}dSY�58}=��|��=�� !j?V��R�?�e0u�Y#�b@ ���$#��:"� Y`Ue��(�{7O�8�b.�/�4ʑ�V h�2�h���Z���.�E�2�N�� �� �:1%0# *�H�� 1��٬tm��4�ڃT$�010!0 + U����T�W }O�^R�%o ��g���y'\ tests/test_data/test_external_account_authorized_user_credentials.json 0000644 00000000536 15025013616 0023022 0 ustar 00 { "audience": "//iam.googleapis.com/locations/global/workforcePools/foobarr/providers/bazbas", "client_id": "foobar", "client_secret": "foobar", "refresh_token": "foobar", "token_info_url": "https://sts.googleapis.com/v1/introspect", "token_url": "https://sts.googleapis.com/v1/oauthtoken", "type": "external_account_authorized_user" } tests/test_data/test_external_account_credentials.json 0000644 00000000512 15025013616 0017520 0 ustar 00 { "type": "external_account", "audience": "//iam.googleapis.com/projects/12345/locations/global/workloadIdentityPools/foobar/providers/bazbas", "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": "https://sts.googleapis.com/v1beta/token", "credential_source": { "url": "http://test.com" } } tests/test_data/test.gif 0000644 00000001452 15025013616 0011325 0 ustar 00 GIF87a � !!!"""###$$$%%%&&&'''((()))***+++,,,---...///000111222333444555666777888999:::;;;<<<===>>>???@@@AAABBBCCCDDDEEEFFFGGGHHHIIIJJJKKKLLLMMMNNNOOOPPPQQQRRRSSSTTTUUUVVVWWWXXXYYYZZZ[[[\\\]]]^^^___```aaabbbcccdddeeefffggghhhiiijjjkkklllmmmnnnooopppqqqrrrssstttuuuvvvwwwxxxyyyzzz{{{|||}}}~~~���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������, � H����*L ; tests/test_data/favicon.ico.gz 0000644 00000013074 15025013616 0012422 0 ustar 00 � ��Z�[S�@r�@��@��Hb�P!B 2 P�/xm{EJ[,C��R�u�V N8 �"�"��j�w�{��xk�s2������{�ܰ�;��Zk����k�@� ᑄ�����L���ϧ\����BB�que?o�r#����a���J,͎���|C.~;@.�k/�����ƕ+d|d�L� %w������75I榧�½{�}�hq�<^Z"?>yB~��g�h/���`�&�M�?�/4��P( �N�����8�@s�?.| ��5�R1b� ���b�ST�3 �0T�Y��%Ě�S �� `X<#`,3A��10 ���&���gX ����m³3�X��[`��V��Y8��+j�R �ψ@�8'Rf�~�����ύ�<l����C�h���3�|�eNh �!f��4� �� X�퉃5��!2���px��z��[{B ��jeY�,�h7v�6 í;�YV�� �Y�ŶK��<���v ���+˚��p��� �x{g�G�� �g� �� ���c�f���,x;���ϛ �� x|k P��e;;zS$��*t���g����d��d��� ��x���o�aB%00g�&fX��� -���6���}hv�����ʞs >Ht궆Н(���Ņ04��G�A��-N�" �$�)� pa L�|V?C�|S4���[Qܥb��L}� � )\��S����o��f� ���G�q�Q=���J��@hv!� � �� Ǝ��7E@�IXbF%���nn&<��=@(����*����Q@� � ~�X<c�_(��Fgg1g��B0��z:2�c����(�]0!���pW4� �a����h\s���W1��.>�Y4�`%��q��:�%t�;�ˏxWz~9:Z�O��MGtl�dP _����`�=A�rB���le\(�x��r� 1�7� ����ȁ�E�q.� � ��\��d B�[���(F��g�� Gp�Mx� "��9H�&�쁆����\:"�MHp芙�Tk>�gh(���X"ZK�x 6���f<߁g�_D��K�J�����,D�������~���e�C��3cI��@���G�"��� � ���� h�+��� �~z��h�E%��>B|<6/�X<� /��gl�nZC� �؇���[� �R�K��AC 0mi� ��d^>�O��f} ^�0槚Y�gL���)�B���<��Wd�����"��v6Bs�xg'֛ll������77�éЄ#�� -x���@�;@� |�i��9BK�a���Q W0�; �_�C� /40�,� e�D6��R+�� ��V.�p���*�c�+|E���= 9�&�B���oIO�C���R`6�d��K�<{��陣i�9��ߜ�[�-� �g���!�O�Vp�)n�$x�ǹ�� �gL�Y��lFe���dƛ�.�g��(�'+Cm�Y͇�X�X�>��$�͕m�9�R�·�\����~<�MRoZU�~�<b��,���+ϒ[ ^��p���;��Oȷ�.Ϡ�� ��~�M�h�:�~�� ��C��<�S����c$v�"n�]%����.�-N��Æ�o@�7|O,�j��X��{�3g���3d蛳��sߐ �Α����o�%���e���CHCd��CH0>�t�\���H0�p�\�p�@/�g �/^$נ�w�W/������|�\���/!]"�/#]&#Џ����}���a聮 ]!7`|��0}�= �(^� ��_� ��52z�� �oµ�א��1�]C�{p h ��_'7G��g �GF�u2>r��O��-�v�7��A�Go���Qr�߆ϷG�F���ߺ ��&h�� � c �����8�8���ݤ��'���q�ǡ��t�L�F�M����-�[dh �O�� �M M���dz �w���63q�~70 4�g�`�&�&�� 2;�4I���$�$����&�w �SS0�F��kS��4�g�f� \�����?����g��{�H��y�ܟE��]��d ���"͒�y�v� � c~�9���?G��"�q|�>y �8�~��`~�<X@Z a��G8^�~�E��ȣH������� �� !="��{� ��K�/��T � y����#}O~@z����� �� ��?�H~��G��O?�_����o)(...���۽{wS�qO3�}���om��藧� �t�ĝ��9h�)����⢂�Ҫ���ZD5a36�?�����W�?615�� �I�-�����EE�L]m]�n�� �~��ׁ ��/.>LB�>;3=]���g���*PUS[o���}���Ӄ� �YX|��HJJIOO��g�g��90�R�T���i��> ���@��ȟ��-=?{KAQT�B����(��~߁�ݨ & �S�x��Y�_T��:, �e���"�YMӾw~��YΠ�Ň�E%$���⢪�l�c@X�B� 9�`WU��w �������� ���⪪�Z�=�=0,J�T��U�9�5F�p䳯ϣ��s ��#��k���)�N���$ ��ൂ� �-�~x��GƧf�.<X���G��4�:) x�e^�!I�Ѥ�gd�K��'��NL��->\�A��S����-_� ��P(��T*�&�l'j���+h���5�77�6��(t� �� ���B�Tis�w4�}��gg��ə��ή+���6�57��W�J#����0��MN�6 ��G���+^rZ�gO��M^�%��\>eBDhX�<*&&&69����2:vk�㕕���{Z[�<W{�&�k��5�$ET�L&��#�Y��O` ���^ ���ͭm� ��0%�Q�KW) QaR�,T �6����w~���txXh���m�ū��4ٹ999�z�F ��`Ypd�:�d{c�_>>yy|�$�"������C� ��Bh����j���ʼ�YYY���>|~�{NvvyW����N���T�ǭYZ�[P����jԛ?�p�!-��ƛ��~�������������3J� ��),-///�Jrs Ye�w�N�T� ������k]ḾCڻ��e��M>���� yyegg ��q[���~�������Ξ�V���|�)�����$o���_~;�����C#3K?�ֺ�@O[Ooo���4&���������2li� ���g�Z������;ڻ�u�����I|e ]qu-*��!� ����]o����'/�L�uw =�����[,�e�PS]Q��0����w���''��� �:u�����W'Y ��S~l��_Q^f�5�ӯ7�]��3p�� ��ށrevium]m-���*o.=��аcgC�۰M�~{�^��[��.�����V ���'k�����WV�'���~�������+���W��S~� X����f�,�����m��:5p������� ܉��Ԡ \�¼�MG� �-�(++)+���5p������zO���9t�0ײ�����/��7� ���s�xW7�{�#M] �~�#�B���!�k�����߯��&X�W��yy9��ʯ����ʅ�����u�X;�&y�g��N�Y�[Ԑ�c� �}:�`0��d0� ����#���U��[,���>�o�] Ц���}q��;փ= }!� ����σ�,-�U�hCal�Z���7�����BW@O1� �1d7��4�W_��T����:� ��W���o(� � s Y:�W�� ����S&'��ǿh���W�� �3#� ��#rk�ɱ~>A2y$@b�ʣ�t��ˁ?!=�K��);H�(��������ytL�ݭ��E-���B��lD��i�jeltd����Tur\ttdd����~��/��.��^� aF��h�Ap�,$65 [j�Z�EG����6O�հ4x�3pC�S%'ǂ�h�6- ��� bCϞ�� �������ehT�Q8G��c�ݳyso[�W�]^��F%��i��ғ��1 d֦� 9t�m���� ��B�r���&#N� M�"& �a�t:��v[^IYYW[74%#r�KP�1��R�9��c�9� �/*��:Ջ��&s��{�N��� H��t��G���Cϳ�'[w^��;/\0מ�u�m���:����K���wjJ�zr��?�)�~=i[G��~|~����u�����p��/ֆsϩ ٺж&�ԃ�:pΦ\�Zк \��-� �}�:ж|Dk�Ǐlk?S͇�W�5 �w/� o���))�L�a˖|<*��>��Q_��_�8q246>)).)%%U���ɰ���ָ睎�>��ĉS�e���q�q���e� ��o�}x�:�u�L&�}=))00*���P��~ρ��>�g7�6���/^ (�d��)�n|�������nL���� %뤡�q�d`ؾ����@��3>~�6fem�&�n�uA!Qѱq�Ԝ�ꦶ��c�O}���;t�� ԁY���/P* ��c�aO{ϧ�N�/_���(�������/ 0H�ݼuG3j8. � +�9�� �I�|�% (\ ��c+�] E>�itttTH�������p����v�~��XY�����'J�����˂|���'���`��r�����h���8�����T��DF&f��?1t!EUT��.���A��T-�$l��|h�&2�����y����2��d�V���U�T��G�6�5� *��USoLY�%O�̄Lr�&���[Kv���:WS���h��R����4�V��(�2x������б��{�F�+�ѩY�R C�V��@un���w�jm�i��[�.F���>���z����\X�]��[�<�3 ���Q�:!��o���B�V� -�$AI� ��mUdb��4�Z�n|o��e��D� ۏ�W�##ھx'11Q��hl��Ңs��������q�(���}�����-K�d�KU%*�=s�m��ܴi��8=�2�,Y� JMEG�� �I&������&�)`w�����֬� �7֬Z&�*R��de|L�<x��k^�]�>����X�L�����1�A���o�_��o�ڵ+��M��»��r���k�a��a U �7���x���c��A��a�t�;�d|� y 4�� nFDlJP���5u ��<�U~�(؝A��K��1�6��j��<ŕ�U�y�ݳy �+gs.נ�/ټ�����ޚ�'ls S a�C���?���3�>����'��`��������}o�t.�y��'��س�}��{�|dz �t�y�h~��`>������Z���h/�?�$Le�P%�2�˷U�{zy�� � �7�U/w�w��^'�O�/�V#^ ��\�=��gVl�s�^k�����}C\Vq��-���\��2H���;�u���u����6��Q��]eR�xYN��Z�6�m�,$x�4��06�p�N�� ,�(��- �H�t����l �I*�2�DkBB����n���t2�D }_}�m�+�4Z/�g`���Wܜ�ܖ�5n���\���GJ��a���+V�z�5_� ������-�&3 tests/test_label.py 0000644 00000027065 15025013616 0010407 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for label command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import xml from xml.dom.minidom import parseString from xml.sax import _exceptions as SaxExceptions import six import boto from boto import handler from boto.s3.tagging import Tags from gslib.exception import CommandException import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForGS from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import ObjectToURI as suri from gslib.utils.retry_util import Retry from gslib.utils.constants import UTF8 KEY1 = 'key_one' KEY2 = 'key_two' VALUE1 = 'value_one' VALUE2 = 'value_two' def _get_label_setting_output(using_gcloud_storage, bucket_uri): return ('Updating {}' if using_gcloud_storage else 'Setting label configuration on {}/...').format(bucket_uri) @SkipForGS('Tests use S3-style XML passthrough.') class TestLabelS3(testcase.GsUtilIntegrationTestCase): """S3-specific tests. Most other test cases are covered in TestLabelGS.""" _label_xml = parseString('<Tagging><TagSet>' + '<Tag><Key>' + KEY1 + '</Key><Value>' + VALUE1 + '</Value></Tag>' + '<Tag><Key>' + KEY2 + '</Key><Value>' + VALUE2 + '</Value></Tag>' + '</TagSet></Tagging>').toprettyxml(indent=' ') def setUp(self): super(TestLabelS3, self).setUp() self.xml_fpath = self.CreateTempFile(contents=self._label_xml.encode(UTF8)) def DoAssertItemsMatch(self, item1, item2): if six.PY2: self.assertItemsEqual(item1, item2) else: # The name was switched, and to a more misleading name, in PY3. Oh well. self.assertCountEqual(item1, item2) def _LabelDictFromXmlString(self, xml_str): label_dict = {} tags_list = Tags() h = handler.XmlHandler(tags_list, None) try: xml.sax.parseString(xml_str, h) except SaxExceptions.SAXParseException as e: raise CommandException( 'Requested labels/tagging config is invalid: %s at line %s, column ' '%s' % (e.getMessage(), e.getLineNumber(), e.getColumnNumber())) for tagset_list in tags_list: for tag in tagset_list: label_dict[tag.key] = tag.value return label_dict def testSetAndGet(self): bucket_uri = self.CreateBucket() stderr = self.RunGsUtil(['label', 'set', self.xml_fpath, suri(bucket_uri)], return_stderr=True) expected_output = _get_label_setting_output(self._use_gcloud_storage, suri(bucket_uri)) if self._use_gcloud_storage: self.assertIn(expected_output, stderr) else: self.assertEqual(stderr.strip(), expected_output) # Verify that the bucket is configured with the labels we just set. # Work around eventual consistency for S3 tagging. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], return_stdout=True) self.DoAssertItemsMatch(self._LabelDictFromXmlString(stdout), self._LabelDictFromXmlString(self._label_xml)) _Check1() def testCh(self): bucket_uri = self.CreateBucket() self.RunGsUtil([ 'label', 'ch', '-l', '%s:%s' % (KEY1, VALUE1), '-l', '%s:%s' % (KEY2, VALUE2), suri(bucket_uri) ]) # Verify that the bucket is configured with the labels we just set. # Work around eventual consistency for S3 tagging. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], return_stdout=True) self.DoAssertItemsMatch(self._LabelDictFromXmlString(stdout), self._LabelDictFromXmlString(self._label_xml)) _Check1() # Remove KEY1, add a new key, and attempt to remove a nonexistent key # with 'label ch'. self.RunGsUtil([ 'label', 'ch', '-d', KEY1, '-l', 'new_key:new_value', '-d', 'nonexistent-key', suri(bucket_uri) ]) expected_dict = {KEY2: VALUE2, 'new_key': 'new_value'} @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], return_stdout=True) self.DoAssertItemsMatch(self._LabelDictFromXmlString(stdout), expected_dict) _Check2() @SkipForS3('Tests use GS-style ') class TestLabelGS(testcase.GsUtilIntegrationTestCase): """Integration tests for label command.""" _label_dict = {KEY1: VALUE1, KEY2: VALUE2} def setUp(self): super(TestLabelGS, self).setUp() self.json_fpath = self.CreateTempFile( contents=json.dumps(self._label_dict).encode(UTF8)) def testSetAndGetOnOneBucket(self): bucket_uri = self.CreateBucket() # Try setting labels for one bucket. stderr = self.RunGsUtil(['label', 'set', self.json_fpath, suri(bucket_uri)], return_stderr=True) expected_output = _get_label_setting_output(self._use_gcloud_storage, suri(bucket_uri)) if self._use_gcloud_storage: self.assertIn(expected_output, stderr) else: self.assertEqual(stderr.strip(), expected_output) # Verify that the bucket is configured with the labels we just set. stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], return_stdout=True) self.assertDictEqual(json.loads(stdout), self._label_dict) def testSetOnMultipleBucketsInSameCommand(self): bucket_uri = self.CreateBucket() bucket2_uri = self.CreateBucket() # Try setting labels for multiple buckets in one command. stderr = self.RunGsUtil( ['label', 'set', self.json_fpath, suri(bucket_uri), suri(bucket2_uri)], return_stderr=True) actual = set(stderr.splitlines()) expected = set([ _get_label_setting_output(self._use_gcloud_storage, suri(bucket_uri)), _get_label_setting_output(self._use_gcloud_storage, suri(bucket2_uri)), ]) if self._use_gcloud_storage: # Gcloud may not have exact match because of progress spinner. self.assertTrue(all([x in stderr for x in expected])) else: self.assertSetEqual(actual, expected) def testSetOverwritesOldLabelConfig(self): bucket_uri = self.CreateBucket() # Try setting labels for one bucket. self.RunGsUtil(['label', 'set', self.json_fpath, suri(bucket_uri)]) new_key_1 = 'new_key_1' new_key_2 = 'new_key_2' new_value_1 = 'new_value_1' new_value_2 = 'new_value_2' new_json = { new_key_1: new_value_1, new_key_2: new_value_2, KEY1: 'different_value_for_an_existing_key' } new_json_fpath = self.CreateTempFile( contents=json.dumps(new_json).encode('ascii')) self.RunGsUtil(['label', 'set', new_json_fpath, suri(bucket_uri)]) stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], return_stdout=True) self.assertDictEqual(json.loads(stdout), new_json) def testInitialAndSubsequentCh(self): bucket_uri = self.CreateBucket() ch_subargs = [ '-l', '%s:%s' % (KEY1, VALUE1), '-l', '%s:%s' % (KEY2, VALUE2) ] # Ensure 'ch' progress message shows in stderr. stderr = self.RunGsUtil(['label', 'ch'] + ch_subargs + [suri(bucket_uri)], return_stderr=True) expected_output = _get_label_setting_output(self._use_gcloud_storage, suri(bucket_uri)) if self._use_gcloud_storage: self.assertIn(expected_output, stderr) else: self.assertEqual(stderr.strip(), expected_output) # Check the bucket to ensure it's configured with the labels we just # specified. stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], return_stdout=True) self.assertDictEqual(json.loads(stdout), self._label_dict) # Ensure a subsequent 'ch' command works correctly. new_key = 'new-key' new_value = 'new-value' self.RunGsUtil([ 'label', 'ch', '-l', '%s:%s' % (new_key, new_value), '-d', KEY2, suri(bucket_uri) ]) stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], return_stdout=True) actual = json.loads(stdout) expected = {KEY1: VALUE1, new_key: new_value} self.assertDictEqual(actual, expected) def testChAppliesChangesToAllBucketArgs(self): bucket_suris = [suri(self.CreateBucket()), suri(self.CreateBucket())] ch_subargs = [ '-l', '%s:%s' % (KEY1, VALUE1), '-l', '%s:%s' % (KEY2, VALUE2) ] # Ensure 'ch' progress message appears for both buckets in stderr. stderr = self.RunGsUtil(['label', 'ch'] + ch_subargs + bucket_suris, return_stderr=True) actual = set(stderr.splitlines()) expected = set([ _get_label_setting_output(self._use_gcloud_storage, bucket_suri) for bucket_suri in bucket_suris ]) if self._use_gcloud_storage: self.assertTrue(all([x in stderr for x in expected])) else: self.assertSetEqual(actual, expected) # Check the buckets to ensure both are configured with the labels we # just specified. for bucket_suri in bucket_suris: stdout = self.RunGsUtil(['label', 'get', bucket_suri], return_stdout=True) self.assertDictEqual(json.loads(stdout), self._label_dict) def testChMinusDWorksWithoutExistingLabels(self): bucket_uri = self.CreateBucket() self.RunGsUtil(['label', 'ch', '-d', 'dummy-key', suri(bucket_uri)]) stdout = self.RunGsUtil(['label', 'get', suri(bucket_uri)], return_stdout=True) self.assertIn('%s/ has no label configuration.' % suri(bucket_uri), stdout) def testTooFewArgumentsFails(self): """Ensures label commands fail with too few arguments.""" invocations_missing_args = ( # Neither arguments nor subcommand. ['label'], # Not enough arguments for 'set'. ['label', 'set'], ['label', 'set', 'filename'], # Not enough arguments for 'get'. ['label', 'get'], # Not enough arguments for 'ch'. ['label', 'ch'], ['label', 'ch', '-l', 'key:val']) for arg_list in invocations_missing_args: stderr = self.RunGsUtil(arg_list, return_stderr=True, expected_status=1) self.assertIn('command requires at least', stderr) # Invoking 'ch' without any changes gives a slightly different message. stderr = self.RunGsUtil( ['label', 'ch', 'gs://some-nonexistent-foobar-bucket-name'], return_stderr=True, expected_status=1) self.assertIn('Please specify at least one label change', stderr) tests/test_command_runner.py 0000644 00000051244 15025013616 0012333 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit and integration tests for gsutil command_runner module.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import time import six from six.moves import input import boto import sys import gslib from gslib import command_runner from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.command_runner import CommandRunner from gslib.command_runner import HandleArgCoding from gslib.command_runner import HandleHeaderCoding from gslib.exception import CommandException from gslib.tab_complete import CloudObjectCompleter from gslib.tab_complete import CloudOrLocalObjectCompleter from gslib.tab_complete import LocalObjectCompleter from gslib.tab_complete import LocalObjectOrCannedACLCompleter from gslib.tab_complete import NoOpCompleter import gslib.tests.testcase as testcase import gslib.tests.util as util from gslib.tests.util import ARGCOMPLETE_AVAILABLE from gslib.tests.util import SetBotoConfigForTest from gslib.tests.util import unittest from gslib.utils import system_util from gslib.utils.constants import GSUTIL_PUB_TARBALL from gslib.utils.text_util import InsistAscii from gslib.utils.unit_util import SECONDS_PER_DAY from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock SKIP_BECAUSE_RETRIES_ARE_SLOW = ( 'gs_host is set to non-default value; trying to fetch gsutil version ' 'metadata from the wrong host would fail repeatedly, taking a long time ' 'for retry backoffs before finally failing the test.') class FakeArgparseArgument(object): """Fake for argparse parser argument.""" pass class FakeArgparseParser(object): """Fake for argparse parser.""" def __init__(self, name='Default'): self.arguments = [] self.subparsers = None self.name = name def add_argument(self, *unused_args, **unused_kwargs): argument = FakeArgparseArgument() self.arguments.append(argument) return argument def add_subparsers(self): self.subparsers = FakeArgparseSubparsers() return self.subparsers class FakeArgparseSubparsers(object): """Container for nested parsers.""" def __init__(self): self.parsers = [] def add_parser(self, name='Default', **unused_kwargs): parser = FakeArgparseParser(name=name) self.parsers.append(parser) return parser class FakeCommandWithInvalidCompleter(Command): """Command with an invalid completer on an argument.""" command_spec = Command.CreateCommandSpec( 'fake1', argparse_arguments=[CommandArgument('arg', completer='BAD')]) help_spec = Command.HelpSpec(help_name='fake1', help_name_aliases=[], help_type='command_help', help_one_line_summary='fake command for tests', help_text='fake command for tests', subcommand_help_text={}) def __init__(self): pass class FakeCommandWithNestedArguments(object): subcommand_name = 'event' subcommand_subname = 'set' # Matches the number of CommandArguments in argparse_arguments below. num_args = 2 command_spec = Command.CreateCommandSpec('FakeCommandWithNestedArguments', argparse_arguments={ subcommand_name: { subcommand_subname: [ CommandArgument('arg1'), CommandArgument('arg2'), ] } }) class FakeCommandWithCompleters(Command): """Command with various completer types.""" command_spec = Command.CreateCommandSpec( 'fake2', argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), CommandArgument.MakeZeroOrMoreFileURLsArgument(), CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument(), CommandArgument.MakeFreeTextArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), CommandArgument.MakeFileURLOrCannedACLArgument(), ]) help_spec = Command.HelpSpec(help_name='fake2', help_name_aliases=[], help_type='command_help', help_one_line_summary='fake command for tests', help_text='fake command for tests', subcommand_help_text={}) def __init__(self): pass class TestCommandRunnerUnitTests(testcase.unit_testcase.GsUtilUnitTestCase): """Unit tests for gsutil update check in command_runner module.""" def setUp(self): """Sets up the command runner mock objects.""" super(TestCommandRunnerUnitTests, self).setUp() # Mock out the timestamp file so we can manipulate it. self.timestamp_file_path = self.CreateTempFile() get_timestamp_file_patcher = mock.patch.object( command_runner.boto_util, 'GetLastCheckedForGsutilUpdateTimestampFile', return_value=self.timestamp_file_path) self.addCleanup(get_timestamp_file_patcher.stop) get_timestamp_file_patcher.start() # Mock out the gsutil version checker. base_version = six.text_type(gslib.VERSION) while not base_version.isnumeric(): if not base_version: raise CommandException('Version number (%s) is not numeric.' % gslib.VERSION) base_version = base_version[:-1] self.old_look_up_gsutil_version = command_runner.LookUpGsutilVersion command_runner.LookUpGsutilVersion = lambda u, v: float(base_version) + 1 # Mock out input to trigger yes prompt. command_runner.input = lambda p: 'y' # Mock out TTY check to pretend we're on a TTY even if we're not. self.running_interactively = True command_runner.system_util.IsRunningInteractively = ( lambda: self.running_interactively) # Mock out the modified time of the VERSION file. self.version_mod_time = 0 self.previous_version_mod_time = gslib.GetGsutilVersionModifiedTime gslib.GetGsutilVersionModifiedTime = lambda: self.version_mod_time # Create a fake pub tarball that will be used to check for gsutil version. self.pub_bucket_uri = self.CreateBucket('pub') self.gsutil_tarball_uri = self.CreateObject(bucket_uri=self.pub_bucket_uri, object_name='gsutil.tar.gz', contents='foo') def tearDown(self): """Tears down the command runner mock objects.""" super(TestCommandRunnerUnitTests, self).tearDown() command_runner.LookUpGsutilVersion = self.old_look_up_gsutil_version command_runner.input = input gslib.GetGsutilVersionModifiedTime = self.previous_version_mod_time command_runner.IsRunningInteractively = ( gslib.utils.system_util.IsRunningInteractively) self.gsutil_tarball_uri.delete_key() self.pub_bucket_uri.delete_bucket() def _IsPackageOrCloudSDKInstall(self): # Update should not trigger for package installs or Cloud SDK installs. return gslib.IS_PACKAGE_INSTALL or system_util.InvokedViaCloudSdk() @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_py3_not_interactive(self): """Tests that py3 prompt is not triggered if not running interactively.""" with mock.patch.object(sys, 'version_info') as version_info: version_info.major = 2 self.running_interactively = False self.assertFalse(self.command_runner.MaybePromptForPythonUpdate('ver')) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_py3_skipped_in_boto(self): """Tests that py3 prompt is not triggered if skipped in boto config.""" with SetBotoConfigForTest([('GSUtil', 'skip_python_update_prompt', 'True') ]): with mock.patch.object(sys, 'version_info') as version_info: version_info.major = 2 self.assertFalse(self.command_runner.MaybePromptForPythonUpdate('ver')) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_py3_prompt_on_py2(self): """Tests that py3 prompt is triggered on Python 2.""" with mock.patch.object(sys, 'version_info') as version_info: version_info.major = 2 self.assertEqual(True, self.command_runner.MaybePromptForPythonUpdate('ver')) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_py3_prompt_on_py3(self): """Tests that py3 prompt is not triggered on Python 3.""" with mock.patch.object(sys, 'version_info') as version_info: version_info.major = 3 self.assertEqual(False, self.command_runner.MaybePromptForPythonUpdate('ver')) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_not_interactive(self): """Tests that update is not triggered if not running interactively.""" with SetBotoConfigForTest([('GSUtil', 'software_update_check_period', '1') ]): with open(self.timestamp_file_path, 'w') as f: f.write(str(int(time.time() - 2 * SECONDS_PER_DAY))) self.running_interactively = False self.assertEqual( False, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('ls', 0)) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_no_tracker_file_version_recent(self): """Tests when no timestamp file exists and VERSION file is recent.""" if os.path.exists(self.timestamp_file_path): os.remove(self.timestamp_file_path) self.assertFalse(os.path.exists(self.timestamp_file_path)) self.version_mod_time = time.time() self.assertEqual( False, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('ls', 0)) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_no_tracker_file_version_old(self): """Tests when no timestamp file exists and VERSION file is old.""" if os.path.exists(self.timestamp_file_path): os.remove(self.timestamp_file_path) self.assertFalse(os.path.exists(self.timestamp_file_path)) self.version_mod_time = 0 expected = not self._IsPackageOrCloudSDKInstall() self.assertEqual( expected, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('ls', 0)) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_invalid_commands(self): """Tests that update is not triggered for certain commands.""" self.assertEqual( False, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('update', 0)) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_fails_silently_when_version_check_fails(self): """Tests that update is not triggered for certain commands.""" command_runner.LookUpGsutilVersion = self.old_look_up_gsutil_version self.assertEqual( False, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('update', 0)) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_invalid_file_contents(self): """Tests no update if timestamp file has invalid value.""" with open(self.timestamp_file_path, 'w') as f: f.write('NaN') self.assertEqual( False, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('ls', 0)) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_update_should_trigger(self): """Tests update should be triggered if time is up.""" with SetBotoConfigForTest([('GSUtil', 'software_update_check_period', '1') ]): with open(self.timestamp_file_path, 'w') as f: f.write(str(int(time.time() - 2 * SECONDS_PER_DAY))) expected = not self._IsPackageOrCloudSDKInstall() self.assertEqual( expected, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('ls', 0)) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_not_time_for_update_yet(self): """Tests update not triggered if not time yet.""" with SetBotoConfigForTest([('GSUtil', 'software_update_check_period', '3') ]): with open(self.timestamp_file_path, 'w') as f: f.write(str(int(time.time() - 2 * SECONDS_PER_DAY))) self.assertEqual( False, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('ls', 0)) def test_user_says_no_to_update(self): """Tests no update triggered if user says no at the prompt.""" with SetBotoConfigForTest([('GSUtil', 'software_update_check_period', '1') ]): with open(self.timestamp_file_path, 'w') as f: f.write(str(int(time.time() - 2 * SECONDS_PER_DAY))) command_runner.input = lambda p: 'n' self.assertEqual( False, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('ls', 0)) @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_update_check_skipped_with_quiet_mode(self): """Tests that update isn't triggered when loglevel is in quiet mode.""" with SetBotoConfigForTest([('GSUtil', 'software_update_check_period', '1') ]): with open(self.timestamp_file_path, 'w') as f: f.write(str(int(time.time() - 2 * SECONDS_PER_DAY))) expected = not self._IsPackageOrCloudSDKInstall() self.assertEqual( expected, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('ls', 0)) prev_loglevel = logging.getLogger().getEffectiveLevel() try: logging.getLogger().setLevel(logging.ERROR) # With reduced loglevel, should return False. self.assertEqual( False, self.command_runner.MaybeCheckForAndOfferSoftwareUpdate('ls', 0)) finally: logging.getLogger().setLevel(prev_loglevel) def test_command_argument_parser_setup_invalid_completer(self): command_map = { FakeCommandWithInvalidCompleter.command_spec.command_name: FakeCommandWithInvalidCompleter() } runner = CommandRunner( bucket_storage_uri_class=self.mock_bucket_storage_uri, gsutil_api_class_map_factory=self.mock_gsutil_api_class_map_factory, command_map=command_map) parser = FakeArgparseParser() try: runner.ConfigureCommandArgumentParsers(parser) except RuntimeError as e: self.assertIn('Unknown completer', str(e)) @unittest.skipUnless(ARGCOMPLETE_AVAILABLE, 'Tab completion requires argcomplete') def test_command_argument_parser_setup_nested_argparse_arguments(self): command_map = { FakeCommandWithNestedArguments.command_spec.command_name: FakeCommandWithNestedArguments(), } runner = CommandRunner( bucket_storage_uri_class=self.mock_bucket_storage_uri, gsutil_api_class_map_factory=self.mock_gsutil_api_class_map_factory, command_map=command_map) parser = FakeArgparseParser() runner.ConfigureCommandArgumentParsers(parser) # Start at first level of subparsers. cur_subparser = parser.subparsers.parsers[0] self.assertEqual(cur_subparser.name, FakeCommandWithNestedArguments.command_spec.command_name) # Go one level further, checking the current subparser's first subparser. cur_subparser = cur_subparser.subparsers.parsers[0] self.assertEqual(cur_subparser.name, FakeCommandWithNestedArguments.subcommand_name) cur_subparser = cur_subparser.subparsers.parsers[0] self.assertEqual(cur_subparser.name, FakeCommandWithNestedArguments.subcommand_subname) # FakeArgparseParser adds a FakeArgparseArgument in place of any arguments # passed in via add_argument(), so we can't check for anything beyond the # presence of the correct number of arguments here. self.assertEqual(len(cur_subparser.arguments), FakeCommandWithNestedArguments.num_args) @unittest.skipUnless(ARGCOMPLETE_AVAILABLE, 'Tab completion requires argcomplete') def test_command_argument_parser_setup_completers(self): command_map = { FakeCommandWithCompleters.command_spec.command_name: FakeCommandWithCompleters() } runner = CommandRunner( bucket_storage_uri_class=self.mock_bucket_storage_uri, gsutil_api_class_map_factory=self.mock_gsutil_api_class_map_factory, command_map=command_map) main_parser = FakeArgparseParser() runner.ConfigureCommandArgumentParsers(main_parser) self.assertEqual(1, len(main_parser.subparsers.parsers)) subparser = main_parser.subparsers.parsers[0] self.assertEqual(6, len(subparser.arguments)) self.assertEqual(CloudObjectCompleter, type(subparser.arguments[0].completer)) self.assertEqual(LocalObjectCompleter, type(subparser.arguments[1].completer)) self.assertEqual(CloudOrLocalObjectCompleter, type(subparser.arguments[2].completer)) self.assertEqual(NoOpCompleter, type(subparser.arguments[3].completer)) self.assertEqual(CloudObjectCompleter, type(subparser.arguments[4].completer)) self.assertEqual(LocalObjectOrCannedACLCompleter, type(subparser.arguments[5].completer)) # pylint: disable=invalid-encoded-data def test_valid_arg_coding(self): """Tests that gsutil encodes valid args correctly.""" # Args should be unicode; if they are not unicode, they should be utf-8 # decoded. args = ['ls', '-p', 'abc:def', 'gs://bucket'] HandleArgCoding(args) for a in args: self.assertTrue(isinstance(a, six.text_type)) def test_valid_header_coding(self): headers = { 'content-type': 'text/plain', 'x-goog-meta-foo': 'bãr', } HandleHeaderCoding(headers) # Custom metadata header values should be decoded to unicode; others should # not be decoded, but should contain only ASCII characters. self.assertTrue(isinstance(headers['x-goog-meta-foo'], six.text_type)) InsistAscii( headers['content-type'], 'Value of non-custom-metadata header contained non-ASCII characters') def test_invalid_header_coding_fails(self): headers = {'content-type': 'bãr'} with self.assertRaisesRegex(CommandException, r'Invalid non-ASCII'): HandleHeaderCoding(headers) class TestCommandRunnerIntegrationTests(testcase.GsUtilIntegrationTestCase): """Integration tests for gsutil update check in command_runner module.""" def setUp(self): """Sets up the command runner mock objects.""" super(TestCommandRunnerIntegrationTests, self).setUp() # Mock out the timestamp file so we can manipulate it. self.timestamp_file_path = self.CreateTempFile(contents='0') get_timestamp_file_patcher = mock.patch.object( command_runner.boto_util, 'GetLastCheckedForGsutilUpdateTimestampFile', return_value=self.timestamp_file_path) self.addCleanup(get_timestamp_file_patcher.stop) get_timestamp_file_patcher.start() # Mock out input to trigger yes prompt. command_runner.input = lambda p: 'y' def tearDown(self): """Tears down the command runner mock objects.""" super(TestCommandRunnerIntegrationTests, self).tearDown() command_runner.input = input @unittest.skipIf(util.HAS_NON_DEFAULT_GS_HOST, SKIP_BECAUSE_RETRIES_ARE_SLOW) def test_lookup_version_without_credentials(self): """Tests that gsutil tarball version lookup works without credentials.""" with SetBotoConfigForTest([('GSUtil', 'software_update_check_period', '1')], use_existing_config=False): self.command_runner = command_runner.CommandRunner() # Looking up software version shouldn't get auth failure exception. self.command_runner.RunNamedCommand('ls', [GSUTIL_PUB_TARBALL]) tests/test_update.py 0000644 00000024567 15025013616 0010616 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Tests for the update command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os.path import shutil import subprocess import sys import tarfile import boto import gslib from gslib.metrics import _UUID_FILE_PATH import gslib.tests.testcase as testcase from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import unittest from gslib.utils import system_util from gslib.utils.boto_util import CERTIFICATE_VALIDATION_ENABLED from gslib.utils.constants import UTF8 from gslib.utils.update_util import DisallowUpdateIfDataInGsutilDir from gslib.utils.update_util import GsutilPubTarball from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock TESTS_DIR = os.path.abspath(os.path.dirname(__file__)) GSUTIL_DIR = os.path.join(TESTS_DIR, '..', '..') class UpdateTest(testcase.GsUtilIntegrationTestCase): """Update command test suite.""" @unittest.skipUnless(CERTIFICATE_VALIDATION_ENABLED, 'Test requires https certificate validation enabled.') def test_update(self): """Tests that the update command works or raises proper exceptions.""" if system_util.InvokedViaCloudSdk(): stderr = self.RunGsUtil(['update'], stdin='n', return_stderr=True, expected_status=1) self.assertIn('update command is disabled for Cloud SDK', stderr) return if gslib.IS_PACKAGE_INSTALL: # The update command is not present when installed via package manager. stderr = self.RunGsUtil(['update'], return_stderr=True, expected_status=1) self.assertIn('Invalid command', stderr) return # Create two temp directories, one of which we will run 'gsutil update' in # to pull the changes from the other. tmpdir_src = self.CreateTempDir() tmpdir_dst = self.CreateTempDir() # Copy gsutil to both source and destination directories. gsutil_src = os.path.join(tmpdir_src, 'gsutil') gsutil_dst = os.path.join(tmpdir_dst, 'gsutil') # Path when executing from tmpdir (Windows doesn't support in-place rename) gsutil_relative_dst = os.path.join('gsutil', 'gsutil') ignore_callable = shutil.ignore_patterns( '.git*', '*.pyc', '*.pyo', '__pycache__', ) shutil.copytree(GSUTIL_DIR, gsutil_src, ignore=ignore_callable) # Copy specific files rather than all of GSUTIL_DIR so we don't pick up temp # working files left in top-level directory by gsutil developers (like tags, # .git*, .pyc files, etc.) os.makedirs(gsutil_dst) for comp in os.listdir(GSUTIL_DIR): if ('.git' not in comp and '__pycache__' not in comp and not comp.endswith('.pyc') and not comp.endswith('.pyo')): # yapf: disable cp_src_path = os.path.join(GSUTIL_DIR, comp) cp_dst_path = os.path.join(gsutil_dst, comp) if os.path.isdir(cp_src_path): shutil.copytree(cp_src_path, cp_dst_path, ignore=ignore_callable) else: shutil.copyfile(cp_src_path, cp_dst_path) # Create a fake version number in the source so we can verify it in the # destination. expected_version = '17.25' src_version_file = os.path.join(gsutil_src, 'VERSION') self.assertTrue(os.path.exists(src_version_file)) with open(src_version_file, 'w') as f: f.write(expected_version) # Create a tarball out of the source directory and copy it to a bucket. src_tarball = os.path.join(tmpdir_src, 'gsutil.test.tar.gz') normpath = os.path.normpath try: # We monkey patch os.path.normpath here because the tarfile module # normalizes the ./gsutil path, but the update command expects the tar # file to be prefixed with . This preserves the ./gsutil path. os.path.normpath = lambda fname: fname tar = tarfile.open(src_tarball, 'w:gz') tar.add(gsutil_src, arcname='./gsutil') tar.close() finally: os.path.normpath = normpath prefix = [sys.executable] if sys.executable else [] # Run with an invalid gs:// URI. p = subprocess.Popen(prefix + ['gsutil', 'update', 'gs://pub'], cwd=gsutil_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (_, stderr) = p.communicate() p.stdout.close() p.stderr.close() self.assertEqual(p.returncode, 1) self.assertIn(b'update command only works with tar.gz', stderr) # Run with non-existent gs:// URI. p = subprocess.Popen(prefix + ['gsutil', 'update', 'gs://pub/Jdjh38)(;.tar.gz'], cwd=gsutil_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (_, stderr) = p.communicate() p.stdout.close() p.stderr.close() self.assertEqual(p.returncode, 1) self.assertIn(b'NotFoundException', stderr) # Run with file:// URI wihout -f option. p = subprocess.Popen( prefix + ['gsutil', 'update', suri(src_tarball)], cwd=gsutil_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (_, stderr) = p.communicate() p.stdout.close() p.stderr.close() self.assertEqual(p.returncode, 1) self.assertIn(b'command does not support', stderr) # Run with a file present that was not distributed with gsutil. with open(os.path.join(gsutil_dst, 'userdata.txt'), 'w') as fp: fp.write('important data\n') p = subprocess.Popen( prefix + ['gsutil', 'update', '-f', suri(src_tarball)], cwd=gsutil_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) (_, stderr) = p.communicate() p.stdout.close() p.stderr.close() # Clean up before next test, and before assertions so failure doesn't leave # this file around. os.unlink(os.path.join(gsutil_dst, 'userdata.txt')) self.assertEqual(p.returncode, 1) # Additional check for Windows since it has \r\n and string may have just \n os_ls = os.linesep.encode(UTF8) if os_ls in stderr: stderr = stderr.replace(os_ls, b' ') elif b'\n' in stderr: stderr = stderr.replace(b'\n', b' ') self.assertIn( b'The update command cannot run with user data in the gsutil directory', stderr) # Determine whether we'll need to decline the analytics prompt. analytics_prompt = not (os.path.exists(_UUID_FILE_PATH) or boto.config.get_value('GSUtil', 'disable_analytics_prompt')) update_input = b'n\r\ny\r\n' if analytics_prompt else b'y\r\n' # Now do the real update, which should succeed. p = subprocess.Popen( prefix + [gsutil_relative_dst, 'update', '-f', suri(src_tarball)], cwd=tmpdir_dst, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) (_, stderr) = p.communicate(input=update_input) p.stdout.close() p.stderr.close() self.assertEqual( p.returncode, 0, msg=('Non-zero return code (%d) from gsutil update. stderr = \n%s' % (p.returncode, stderr.decode(UTF8)))) # Verify that version file was updated. dst_version_file = os.path.join(tmpdir_dst, 'gsutil', 'VERSION') with open(dst_version_file, 'r') as f: self.assertEqual(f.read(), expected_version) # If the analytics prompt was given, that means we disabled analytics. We # should reset to the default by deleting the UUID file. if analytics_prompt: os.unlink(_UUID_FILE_PATH) class UpdateUnitTest(testcase.GsUtilUnitTestCase): """Tests the functionality of commands/update.py.""" @unittest.skipUnless( not gslib.IS_PACKAGE_INSTALL, 'Test is runnable only if gsutil dir is accessible, and update ' 'command is not valid for package installs.') def test_repo_matches_manifest(self): """Ensure that all files/folders match the manifest.""" # Create a temp directory and copy specific files to it. tmpdir_src = self.CreateTempDir() gsutil_src = os.path.join(tmpdir_src, 'gsutil') os.makedirs(gsutil_src) copy_files = [] for filename in os.listdir(GSUTIL_DIR): if (filename.endswith('.pyc') or filename.startswith('.git') or filename == '__pycache__' or filename == '.settings' or filename == '.project' or filename == '.pydevproject' or filename == '.style.yapf' or filename == '.yapfignore'): # Need to ignore any compiled code or Eclipse project folders. continue copy_files.append(filename) for comp in copy_files: if os.path.isdir(os.path.join(GSUTIL_DIR, comp)): func = shutil.copytree else: func = shutil.copyfile func(os.path.join(GSUTIL_DIR, comp), os.path.join(gsutil_src, comp)) DisallowUpdateIfDataInGsutilDir(directory=gsutil_src) def test_pub_tarball(self): """Ensure that the correct URI is returned based on the Python version.""" with mock.patch.object(sys, 'version_info') as version_info: version_info.major = 3 self.assertIn('gsutil.tar.gz', GsutilPubTarball()) version_info.major = 2 self.assertIn('gsutil4.tar.gz', GsutilPubTarball()) tests/test_du.py 0000644 00000025163 15025013616 0007735 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Tests for du command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForS3 from gslib.tests.util import GenerationFromURI as urigen from gslib.tests.util import ObjectToURI as suri from gslib.utils.constants import UTF8 from gslib.utils.retry_util import Retry class TestDu(testcase.GsUtilIntegrationTestCase): """Integration tests for du command.""" def setUp(self): super(TestDu, self).setUp() self._old_environ = os.environ.copy() os.environ['PYTHONUTF8'] = '1' def tearDown(self): super(TestDu, self).tearDown() os.environ = self._old_environ def _create_nested_subdir(self): """Creates a nested subdirectory for use by tests in this module.""" bucket_uri = self.CreateBucket() obj_uris = [ self.CreateObject(bucket_uri=bucket_uri, object_name='sub1材/five', contents=b'5five'), self.CreateObject(bucket_uri=bucket_uri, object_name='sub1材/four', contents=b'four'), self.CreateObject(bucket_uri=bucket_uri, object_name='sub1材/sub2/five', contents=b'5five'), self.CreateObject(bucket_uri=bucket_uri, object_name='sub1材/sub2/four', contents=b'four') ] self.AssertNObjectsInBucket(bucket_uri, 4) return bucket_uri, obj_uris def test_object(self): obj_uri = self.CreateObject(contents=b'foo') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil(['du', suri(obj_uri)], return_stdout=True) self.assertEqual(stdout, '%-11s %s\n' % (3, suri(obj_uri))) _Check() def test_bucket(self): bucket_uri = self.CreateBucket() obj_uri = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil(['du', suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout, '%-11s %s\n' % (3, suri(obj_uri))) _Check() def test_subdirs(self): """Tests that subdirectory sizes are correctly calculated and listed.""" bucket_uri, obj_uris = self._create_nested_subdir() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil(['du', suri(bucket_uri)], return_stdout=True) self.assertSetEqual( set(stdout.splitlines()), set([ '%-11s %s' % (5, suri(obj_uris[0])), '%-11s %s' % (4, suri(obj_uris[1])), '%-11s %s' % (5, suri(obj_uris[2])), '%-11s %s' % (4, suri(obj_uris[3])), '%-11s %s/sub1材/sub2/' % (9, suri(bucket_uri)), '%-11s %s/sub1材/' % (18, suri(bucket_uri)), ])) _Check() def test_multi_args(self): """Tests running du with multiple command line arguments.""" bucket_uri = self.CreateBucket() obj_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') obj_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo2') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil( ['du', suri(obj_uri1), suri(obj_uri2)], return_stdout=True) self.assertSetEqual( set(stdout.splitlines()), set([ '%-11s %s' % (3, suri(obj_uri1)), '%-11s %s' % (4, suri(obj_uri2)), ])) _Check() def test_total(self): """Tests total size listing via the -c flag.""" bucket_uri = self.CreateBucket() obj_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') obj_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents=b'zebra') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil(['du', '-c', suri(bucket_uri)], return_stdout=True) self.assertSetEqual( set(stdout.splitlines()), set([ '%-11s %s' % (3, suri(obj_uri1)), '%-11s %s' % (5, suri(obj_uri2)), '%-11s total' % 8, ])) _Check() def test_human_readable(self): obj_uri = self.CreateObject(contents=b'x' * 2048) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil(['du', '-h', suri(obj_uri)], return_stdout=True) self.assertEqual(stdout, '%-11s %s\n' % ('2 KiB', suri(obj_uri))) _Check() def test_summary(self): """Tests summary listing with the -s flag.""" bucket_uri1, _ = self._create_nested_subdir() bucket_uri2, _ = self._create_nested_subdir() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil( ['du', '-s', suri(bucket_uri1), suri(bucket_uri2)], return_stdout=True) self.assertSetEqual( set(stdout.splitlines()), set([ '%-11s %s' % (18, suri(bucket_uri1)), '%-11s %s' % (18, suri(bucket_uri2)), ])) _Check() def test_subdir_summary(self): """Tests summary listing with the -s flag on a subdirectory.""" bucket_uri1, _ = self._create_nested_subdir() bucket_uri2, _ = self._create_nested_subdir() subdir1 = suri(bucket_uri1, 'sub1材') subdir2 = suri(bucket_uri2, 'sub1材') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil(['du', '-s', subdir1, subdir2], return_stdout=True) self.assertSetEqual( set(stdout.splitlines()), set([ '%-11s %s' % (18, subdir1), '%-11s %s' % (18, subdir2), ])) _Check() @SkipForS3('S3 lists versions in reverse order.') def test_versioned(self): """Tests listing all versions with the -a flag.""" bucket_uri = self.CreateVersionedBucket() object_uri1 = self.CreateObject(bucket_uri=bucket_uri, object_name='foo', contents=b'foo') object_uri2 = self.CreateObject( bucket_uri=bucket_uri, object_name='foo', contents=b'foo2', gs_idempotent_generation=urigen(object_uri1)) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check1(): stdout = self.RunGsUtil(['du', suri(bucket_uri)], return_stdout=True) self.assertEqual(stdout, '%-11s %s\n' % (4, suri(object_uri2))) _Check1() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check2(): stdout = self.RunGsUtil(['du', '-a', suri(bucket_uri)], return_stdout=True) self.assertSetEqual( set(stdout.splitlines()), set([ '%-11s %s#%s' % (3, suri(object_uri1), object_uri1.generation), '%-11s %s#%s' % (4, suri(object_uri2), object_uri2.generation), ])) _Check2() def test_null_endings(self): """Tests outputting 0-endings with the -0 flag.""" bucket_uri = self.CreateBucket() obj_uri1 = self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') obj_uri2 = self.CreateObject(bucket_uri=bucket_uri, contents=b'zebra') # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil(['du', '-0c', suri(bucket_uri)], return_stdout=True) self.assertSetEqual( set(stdout.split('\0')), set([ '%-11s %s' % (3, suri(obj_uri1)), '%-11s %s' % (5, suri(obj_uri2)), '%-11s total' % 8, '' ])) _Check() def test_excludes(self): """Tests exclude pattern excluding certain file paths.""" bucket_uri, obj_uris = self._create_nested_subdir() # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil( ['du', '-e', '*sub2/five*', '-e', '*sub1材/four', suri(bucket_uri)], return_stdout=True) self.assertSetEqual( set(stdout.splitlines()), set([ '%-11s %s' % (5, suri(obj_uris[0])), '%-11s %s' % (4, suri(obj_uris[3])), '%-11s %s/sub1材/sub2/' % (4, suri(bucket_uri)), '%-11s %s/sub1材/' % (9, suri(bucket_uri)), ])) _Check() def test_excludes_file(self): """Tests file exclusion with the -X flag.""" bucket_uri, obj_uris = self._create_nested_subdir() fpath = self.CreateTempFile( contents='*sub2/five*\n*sub1材/four'.encode(UTF8)) # Use @Retry as hedge against bucket listing eventual consistency. @Retry(AssertionError, tries=3, timeout_secs=1) def _Check(): stdout = self.RunGsUtil( ['du', '-X', fpath, suri(bucket_uri)], return_stdout=True) self.assertSetEqual( set(stdout.splitlines()), set([ '%-11s %s' % (5, suri(obj_uris[0])), '%-11s %s' % (4, suri(obj_uris[3])), '%-11s %s/sub1材/sub2/' % (4, suri(bucket_uri)), '%-11s %s/sub1材/' % (9, suri(bucket_uri)), ])) _Check() tests/test_perfdiag.py 0000644 00000031167 15025013616 0011107 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Integration tests for perfdiag command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import socket import sys import six import boto from gslib.commands.perfdiag import _GenerateFileData import gslib.tests.testcase as testcase from gslib.tests.testcase.integration_testcase import SkipForXML from gslib.tests.util import ObjectToURI as suri from gslib.tests.util import RUN_S3_TESTS from gslib.tests.util import unittest from gslib.utils.system_util import IS_WINDOWS from six import add_move, MovedModule add_move(MovedModule('mock', 'mock', 'unittest.mock')) from six.moves import mock class TestPerfDiag(testcase.GsUtilIntegrationTestCase): """Integration tests for perfdiag command.""" @classmethod def setUpClass(cls): super(TestPerfDiag, cls).setUpClass() # We want to test that perfdiag works both when connecting to the standard # gs endpoint, and when connecting to a specific IP or host while setting # the host header. For the 2nd case we resolve gs_host (normally # storage.googleapis.com) to a specific IP and connect to that explicitly. gs_host = boto.config.get('Credentials', 'gs_host', boto.gs.connection.GSConnection.DefaultHost) gs_ip = None for address_tuple in socket.getaddrinfo(gs_host, None): # Index 0 holds IP version. AF_INET = IPv4. if address_tuple[0].name in ('AF_INET', 'AF_INET6'): # Index 4 holds IP tuple, where first item is IP. gs_ip = address_tuple[4][0] break if not gs_ip: raise ConnectionError('Count not find IP for ' + gs_host) cls._custom_endpoint_flags = [ '-o', 'Credentials:gs_host=' + gs_ip, '-o', 'Credentials:gs_host_header=' + gs_host, '-o', 'Boto:https_validate_certificates=False' ] def _should_run_with_custom_endpoints(self): # Host headers are only supported for XML, and not when # using environment variables for proxies. # TODO: Currently this is disabled for Python versions # >= 2.7.9 which cause certificate errors due to validation # added in https://www.python.org/dev/peps/pep-0466/ # If https://github.com/boto/boto/pull/2857 or its analog # is accepted in boto, set https_validate_certificates to False # in these tests and re-enable them. python_version_less_than_2_7_9 = (sys.version_info[0] == 2 and ( (sys.version_info[1] < 7) or (sys.version_info[1] == 7 and sys.version_info[2] < 9))) return (self.test_api == 'XML' and not RUN_S3_TESTS and python_version_less_than_2_7_9 and not (os.environ.get('http_proxy') or os.environ.get('https_proxy') or os.environ.get('HTTPS_PROXY'))) def test_latency(self): bucket_uri = self.CreateBucket() cmd = ['perfdiag', '-n', '1', '-t', 'lat', suri(bucket_uri)] self.RunGsUtil(cmd) if self._should_run_with_custom_endpoints(): self.RunGsUtil(self._custom_endpoint_flags + cmd) self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) def _run_throughput_test(self, test_name, num_processes, num_threads, parallelism_strategy=None, compression_ratio=None): bucket_uri = self.CreateBucket() cmd = [ 'perfdiag', '-n', str(num_processes * num_threads), '-s', '1024', '-c', str(num_processes), '-k', str(num_threads), '-t', test_name ] if compression_ratio is not None: cmd += ['-j', str(compression_ratio)] if parallelism_strategy is not None: cmd += ['-p', parallelism_strategy] cmd += [suri(bucket_uri)] stderr_default = self.RunGsUtil(cmd, return_stderr=True) stderr_custom = None if self._should_run_with_custom_endpoints(): stderr_custom = self.RunGsUtil(self._custom_endpoint_flags + cmd, return_stderr=True) self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) return (stderr_default, stderr_custom) def _run_each_parallel_throughput_test(self, test_name, num_processes, num_threads, compression_ratio=None): self._run_throughput_test(test_name, num_processes, num_threads, 'fan', compression_ratio=compression_ratio) if not RUN_S3_TESTS: self._run_throughput_test(test_name, num_processes, num_threads, 'slice', compression_ratio=compression_ratio) self._run_throughput_test(test_name, num_processes, num_threads, 'both', compression_ratio=compression_ratio) def test_write_throughput_single_process_single_thread(self): self._run_throughput_test('wthru', 1, 1) self._run_throughput_test('wthru_file', 1, 1) def test_write_throughput_single_process_multi_thread(self): self._run_each_parallel_throughput_test('wthru', 1, 2) self._run_each_parallel_throughput_test('wthru_file', 1, 2) @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def test_write_throughput_multi_process_single_thread(self): self._run_each_parallel_throughput_test('wthru', 2, 1) self._run_each_parallel_throughput_test('wthru_file', 2, 1) @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def test_write_throughput_multi_process_multi_thread(self): self._run_each_parallel_throughput_test('wthru', 2, 2) self._run_each_parallel_throughput_test('wthru_file', 2, 2) def test_read_throughput_single_process_single_thread(self): self._run_throughput_test('rthru', 1, 1) self._run_throughput_test('rthru_file', 1, 1) def test_read_throughput_single_process_multi_thread(self): self._run_each_parallel_throughput_test('rthru', 1, 2) self._run_each_parallel_throughput_test('rthru_file', 1, 2) @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def test_read_throughput_multi_process_single_thread(self): self._run_each_parallel_throughput_test('rthru', 2, 1) self._run_each_parallel_throughput_test('rthru_file', 2, 1) @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def test_read_throughput_multi_process_multi_thread(self): self._run_each_parallel_throughput_test('rthru', 2, 2) self._run_each_parallel_throughput_test('rthru_file', 2, 2) @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') def test_read_and_write_file_ordering(self): """Tests that rthru_file and wthru_file work when run together.""" self._run_throughput_test('rthru_file,wthru_file', 1, 1) self._run_throughput_test('rthru_file,wthru_file', 2, 2, 'fan') if not RUN_S3_TESTS: self._run_throughput_test('rthru_file,wthru_file', 2, 2, 'slice') self._run_throughput_test('rthru_file,wthru_file', 2, 2, 'both') def test_input_output(self): outpath = self.CreateTempFile() bucket_uri = self.CreateBucket() self.RunGsUtil( ['perfdiag', '-o', outpath, '-n', '1', '-t', 'lat', suri(bucket_uri)]) self.RunGsUtil(['perfdiag', '-i', outpath]) def test_invalid_size(self): stderr = self.RunGsUtil( ['perfdiag', '-n', '1', '-s', 'foo', '-t', 'wthru', 'gs://foobar'], expected_status=1, return_stderr=True) self.assertIn('Invalid -s', stderr) def test_toobig_size(self): stderr = self.RunGsUtil( ['perfdiag', '-n', '1', '-s', '3pb', '-t', 'wthru', 'gs://foobar'], expected_status=1, return_stderr=True) self.assertIn('in-memory tests maximum file size', stderr) def test_listing(self): bucket_uri = self.CreateBucket() stdout = self.RunGsUtil( ['perfdiag', '-n', '1', '-t', 'list', suri(bucket_uri)], return_stdout=True) self.assertIn('Number of listing calls made:', stdout) self.AssertNObjectsInBucket(bucket_uri, 0, versioned=True) @SkipForXML('No compressed transport encoding support for the XML API.') def test_gzip_write_throughput_single_process_single_thread(self): (stderr_default, _) = self._run_throughput_test('wthru', 1, 1, compression_ratio=50) self.assertIn('Gzip compression ratio: 50', stderr_default) self.assertIn('Gzip transport encoding writes: True', stderr_default) (stderr_default, _) = self._run_throughput_test('wthru_file', 1, 1, compression_ratio=50) self.assertIn('Gzip compression ratio: 50', stderr_default) self.assertIn('Gzip transport encoding writes: True', stderr_default) @SkipForXML('No compressed transport encoding support for the XML API.') def test_gzip_write_throughput_single_process_multi_thread(self): self._run_each_parallel_throughput_test('wthru', 1, 2, compression_ratio=50) self._run_each_parallel_throughput_test('wthru_file', 1, 2, compression_ratio=50) @unittest.skipIf(IS_WINDOWS, 'Multiprocessing is not supported on Windows') @SkipForXML('No compressed transport encoding support for the XML API.') def test_gzip_write_throughput_multi_process_multi_thread(self): self._run_each_parallel_throughput_test('wthru', 2, 2, compression_ratio=50) self._run_each_parallel_throughput_test('wthru_file', 2, 2, compression_ratio=50) class TestPerfDiagUnitTests(testcase.GsUtilUnitTestCase): """Unit tests for perfdiag command.""" def test_listing_does_not_list_preexisting_objects(self): test_objects = 1 bucket_uri = self.CreateBucket() # Create two objects in the bucket before executing perfdiag. self.CreateObject(bucket_uri=bucket_uri, contents=b'foo') self.CreateObject(bucket_uri=bucket_uri, contents=b'bar') mock_log_handler = self.RunCommand( 'perfdiag', ['-n', str(test_objects), '-t', 'list', suri(bucket_uri)], return_log_handler=True) self.assertNotIn( 'Listing produced more than the expected %d object(s).' % test_objects, mock_log_handler.messages['warning']) @mock.patch('os.urandom') def test_generate_file_data(self, mock_urandom): """Test the right amount of random and sequential data is generated.""" def urandom(length): return b'a' * length mock_urandom.side_effect = urandom fp = six.BytesIO() _GenerateFileData(fp, 1000, 100, 1000) self.assertEqual(b'a' * 1000, fp.getvalue()) self.assertEqual(1000, fp.tell()) fp = six.BytesIO() _GenerateFileData(fp, 1000, 50, 1000) self.assertIn(b'a' * 500, fp.getvalue()) self.assertIn(b'x' * 500, fp.getvalue()) self.assertEqual(1000, fp.tell()) fp = six.BytesIO() _GenerateFileData(fp, 1001, 50, 1001) self.assertIn(b'a' * 501, fp.getvalue()) self.assertIn(b'x' * 500, fp.getvalue()) self.assertEqual(1001, fp.tell()) @mock.patch('os.urandom') def test_generate_file_data_repeat(self, mock_urandom): """Test that random data repeats when exhausted.""" def urandom(length): return b'a' * length mock_urandom.side_effect = urandom fp = six.BytesIO() _GenerateFileData(fp, 8, 50, 4) self.assertEqual(b'aaxxaaxx', fp.getvalue()) self.assertEqual(8, fp.tell()) cloud_api.py 0000644 00000121736 15025013616 0007066 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Gsutil API for interacting with cloud storage providers.""" from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals class CloudApi(object): """Abstract base class for interacting with cloud storage providers. Implementations of the gsutil Cloud API are not guaranteed to be thread-safe. Behavior when calling a gsutil Cloud API instance simultaneously across threads is undefined and doing so will likely cause errors. Therefore, a separate instance of the gsutil Cloud API should be instantiated per-thread. """ def __init__(self, bucket_storage_uri_class, logger, status_queue, provider=None, debug=0, http_headers=None, trace_token=None, perf_trace_token=None, user_project=None): """Performs necessary setup for interacting with the cloud storage provider. Args: bucket_storage_uri_class: boto storage_uri class, used by APIs that provide boto translation or mocking. logger: logging.logger for outputting log messages. status_queue: Queue for relaying status to UI. provider: Default provider prefix describing cloud storage provider to connect to. debug: Debug level for the API implementation (0..3). http_headers (dict|None): Arbitrary headers to be included in every request. trace_token: Google internal trace token to pass to the API implementation (string). perf_trace_token: Performance trace token to use when making API calls. user_project: Project to be billed for this request. """ self.bucket_storage_uri_class = bucket_storage_uri_class self.logger = logger self.status_queue = status_queue self.provider = provider self.debug = debug self.http_headers = http_headers self.trace_token = trace_token self.perf_trace_token = perf_trace_token self.user_project = user_project def GetServiceAccountId(self): """Returns the service account email id.""" raise NotImplementedError('GetServiceAccountId must be overridden.') def GetBucket(self, bucket_name, provider=None, fields=None): """Gets Bucket metadata. Args: bucket_name: Name of the bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields, for example, ['logging', 'defaultObjectAcl'] Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object. """ raise NotImplementedError('GetBucket must be overloaded') def GetBucketIamPolicy(self, bucket_name, provider=None, fields=None): """Returns an IAM policy for the specified Bucket. Args: bucket_name: Name of the bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only the IAM policy fields specified. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with the cloud storage providers. Returns: Policy object of the bucket. """ raise NotImplementedError('GetBucketIamPolicy must be overloaded') def SetBucketIamPolicy(self, bucket_name, policy, provider=None): """Sets an IAM policy for the specified Bucket. Args: bucket_name: Name of the bucket. policy: A Policy object describing the IAM policy. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with the cloud storage providers. Returns: Policy object of the bucket. May differ from input Policy. """ raise NotImplementedError('SetBucketIamPolicy must be overloaded') def SignUrl(self, method, duration, path, generation, logger, region, signed_headers, string_to_sign_debug): """Sign a url using service account's system managed private key. Args: method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. path: String path to the bucket or object for signing, in the form 'bucket' or 'bucket/object'. generation: If not None, specifies a version of an object for signing. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. signed_headers: Dict containing the header info like host content-type etc. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. CommandException for errors because of invalid account used for signing. Returns: The signed url. """ raise NotImplementedError('SignUrl must be overloaded') def ListBuckets(self, project_id=None, provider=None, fields=None): """Lists bucket metadata for the given project. Args: project_id: Project owning the buckets, default from config if None. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these metadata fields for the listing, for example: ['items/logging', 'items/defaultObjectAcl']. Note that the WildcardIterator class should be used to list buckets instead of calling this function directly. It amends the fields definition from get-like syntax such as ['logging', 'defaultObjectAcl'] so that the caller does not need to prepend 'items/' or specify fields necessary for listing (like nextPageToken). Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Iterator over Bucket objects. """ raise NotImplementedError('ListBuckets must be overloaded') def PatchBucket(self, bucket_name, metadata, canned_acl=None, canned_def_acl=None, preconditions=None, provider=None, fields=None): """Updates bucket metadata for the bucket with patch semantics. Args: bucket_name: Name of bucket to update. metadata: Bucket object defining metadata to be updated. canned_acl: Canned ACL to apply to the bucket. canned_def_acl: Canned default object ACL to apply to the bucket. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object describing new bucket metadata. """ raise NotImplementedError('PatchBucket must be overloaded') def LockRetentionPolicy(self, bucket_name, metageneration, provider=None): """Locks the Retention Policy on the bucket. Args: bucket_name: Name of bucket to update. metageneration: Bucket metageneration to use as a precondition. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ServiceException for errors interacting with cloud storage providers. Returns: None """ raise NotImplementedError('LockRetentionPolicy must be overloaded') def CreateBucket(self, bucket_name, project_id=None, metadata=None, provider=None, fields=None): """Creates a new bucket with the specified metadata. Args: bucket_name: Name of the new bucket. project_id: Project owner of the new bucket, default from config if None. metadata: Bucket object defining new bucket metadata. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object describing new bucket metadata. """ raise NotImplementedError('CreateBucket must be overloaded') def DeleteBucket(self, bucket_name, preconditions=None, provider=None): """Deletes a bucket. Args: bucket_name: Name of the bucket to delete. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('DeleteBucket must be overloaded') class CsObjectOrPrefixType(object): """Enum class for describing CsObjectOrPrefix types.""" OBJECT = 'object' # Cloud object PREFIX = 'prefix' # Cloud bucket subdirectory class CsObjectOrPrefix(object): """Container class for ListObjects results.""" def __init__(self, data, datatype): """Stores a ListObjects result. Args: data: Root object, either an apitools Object or a string Prefix. datatype: CsObjectOrPrefixType of data. """ self.data = data self.datatype = datatype def ListObjects(self, bucket_name, prefix=None, delimiter=None, all_versions=None, provider=None, fields=None): """Lists objects (with metadata) and prefixes in a bucket. Args: bucket_name: Bucket containing the objects. prefix: Prefix for directory-like behavior. delimiter: Delimiter for directory-like behavior. all_versions: If true, list all object versions. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these metadata fields for the listing, for example: ['items/acl', 'items/updated', 'prefixes']. Note that the WildcardIterator class should be used to list objects instead of calling this function directly. It amends the fields definition from get-like syntax such as ['acl', 'updated'] so that the caller does not need to prepend 'items/' or specify any fields necessary for listing (such as prefixes or nextPageToken). Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Iterator over CsObjectOrPrefix wrapper class. """ raise NotImplementedError('ListObjects must be overloaded') def GetObjectIamPolicy(self, bucket_name, object_name, generation=None, provider=None, fields=None): """Gets IAM policy for specified Object. Args: bucket_name: Bucket containing the object. object_name: Name of the object. generation: Generation of the object to retrieve. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only the IAM policy fields specified. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object IAM policy. """ raise NotImplementedError('GetObjectIamPolicy must be overloaded') def SetObjectIamPolicy(self, bucket_name, object_name, policy, generation=None, provider=None): """Sets IAM policy for specified Object. Args: bucket_name: Bucket containing the object. object_name: Name of the object. policy: IAM Policy object. generation: Generation of the object to which the IAM policy will apply. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Policy object of the object. May differ from input Policy. """ raise NotImplementedError('SetObjectIamPolicy must be overloaded') def GetObjectMetadata(self, bucket_name, object_name, generation=None, provider=None, fields=None): """Gets object metadata. If decryption is supported by the implementing class, this function will read decryption keys from configuration and appropriately retry requests to encrypted objects with the correct key. Args: bucket_name: Bucket containing the object. object_name: Object name. generation: Generation of the object to retrieve. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields, for example, ['acl', 'updated']. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object. """ raise NotImplementedError('GetObjectMetadata must be overloaded') def PatchObjectMetadata(self, bucket_name, object_name, metadata, canned_acl=None, generation=None, preconditions=None, provider=None, fields=None): """Updates object metadata with patch semantics. Args: bucket_name: Bucket containing the object. object_name: Object name for object. metadata: Object object defining metadata to be updated. canned_acl: Canned ACL to be set on the object. generation: Generation (or version) of the object to update. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Updated object metadata. """ raise NotImplementedError('PatchObjectMetadata must be overloaded') class DownloadStrategy(object): """Enum class for specifying download strategy.""" ONE_SHOT = 'oneshot' RESUMABLE = 'resumable' # TODO: Change {de,en}cryption_tuple field names, as they don't actually # accept tuples. def GetObjectMedia(self, bucket_name, object_name, download_stream, provider=None, generation=None, object_size=None, compressed_encoding=False, download_strategy=DownloadStrategy.ONE_SHOT, start_byte=0, end_byte=None, progress_callback=None, serialization_data=None, digesters=None, decryption_tuple=None): """Gets object data. Args: bucket_name: Bucket containing the object. object_name: Object name. download_stream: Stream to send the object data to. provider: Cloud storage provider to connect to. If not present, class-wide default is used. generation: Generation of the object to retrieve. object_size: Total size of the object being downloaded. compressed_encoding: If true, object is stored with a compressed encoding. download_strategy: Cloud API download strategy to use for download. start_byte: Starting point for download (for resumable downloads and range requests). Can be set to negative to request a range of bytes (python equivalent of [:-3]) end_byte: Ending byte number, inclusive, for download (for range requests). If None, download the rest of the object. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). serialization_data: Implementation-specific JSON string of a dict containing serialization information for the download. digesters: Dict of {string : digester}, where string is a name of a hash algorithm, and digester is a validation digester that supports update(bytes) and digest() using that algorithm. Implementation can set the digester value to None to indicate bytes were not successfully digested on-the-fly. decryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for decrypting an encrypted object. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Content-encoding string if it was detected that the server sent an encoded object during transfer, None otherwise. """ raise NotImplementedError('GetObjectMedia must be overloaded') def UploadObject(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """Uploads object data and metadata. Args: upload_stream: Seekable stream of object data. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. size: Optional object size. preconditions: Preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. gzip_encoded: Whether to use gzip transport encoding for the upload. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('UploadObject must be overloaded') def UploadObjectStreaming(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """Uploads object data and metadata. Args: upload_stream: Stream of object data. May not be seekable. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. preconditions: Preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size), but fills in only bytes_transferred. encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. gzip_encoded: Whether to use gzip transport encoding for the upload. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('UploadObjectStreaming must be overloaded') def UploadObjectResumable(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, serialization_data=None, tracker_callback=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """Uploads object data and metadata using a resumable upload strategy. Args: upload_stream: Seekable stream of object data. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. size: Total size of the object. preconditions: Preconditions for the request. serialization_data: Dict of {'url' : UploadURL} allowing for uploads to be resumed. tracker_callback: Callback function taking a upload URL string. Guaranteed to be called when the implementation gets an upload URL, allowing the caller to resume the upload across process breaks by saving the upload URL in a tracker file. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields when the upload is complete. gzip_encoded: Whether to use gzip transport encoding for the upload. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('UploadObjectResumable must be overloaded') # TODO: Change {de,en}cryption_tuple field names, as they don't actually # accept tuples. def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, canned_acl=None, preconditions=None, progress_callback=None, max_bytes_per_call=None, encryption_tuple=None, decryption_tuple=None, provider=None, fields=None): """Copies an object in the cloud. Args: src_obj_metadata: Object metadata for source object. Must include bucket name, object name, and etag. dst_obj_metadata: Object metadata for new object. Must include bucket and object name. src_generation: Generation of the source object to copy. canned_acl: Optional canned ACL to apply to destination object. Overrides ACL set in dst_obj_metadata. preconditions: Destination object preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). max_bytes_per_call: Integer describing maximum number of bytes to rewrite per service call. encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the destination object. decryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for decrypting the source object. If supplied without encryption_tuple, destination object will be written without customer-supplied encryption. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('CopyObject must be overloaded') def ComposeObject(self, src_objs_metadata, dst_obj_metadata, preconditions=None, encryption_tuple=None, provider=None, fields=None): """Composes an object in the cloud. Args: src_objs_metadata: List of ComposeRequest.SourceObjectsValueListEntries specifying the objects to compose. dst_obj_metadata: Metadata for the destination object including bucket and object name. preconditions: Destination object preconditions for the request. encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for decrypting source objects and encrypting the destination object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Composed object metadata. """ raise NotImplementedError('ComposeObject must be overloaded') def DeleteObject(self, bucket_name, object_name, preconditions=None, generation=None, provider=None): """Deletes an object. Args: bucket_name: Name of the containing bucket. object_name: Name of the object to delete. preconditions: Preconditions for the request. generation: Generation (or version) of the object to delete; if None, deletes the live object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('DeleteObject must be overloaded') def WatchBucket(self, bucket_name, address, channel_id, token=None, provider=None, fields=None): """Creates a notification subscription for changes to objects in a bucket. Args: bucket_name: Bucket containing the objects. address: Address to which to send notifications. channel_id: Unique ID string for the channel. token: If present, token string is delivered with each notification. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Channel metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Channel object describing the notification subscription. """ raise NotImplementedError('WatchBucket must be overloaded') def StopChannel(self, channel_id, resource_id, provider=None): """Stops a notification channel. Args: channel_id: Unique ID string for the channel. resource_id: Version-agnostic ID string for the channel. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('StopChannel must be overloaded') def ListChannels(self, bucket_name, provider=None): """Lists object change notifications for a bucket. Args: bucket_name: Bucket containing the objects provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('ListChannels must be overloaded') def GetProjectServiceAccount(self, project_number, provider=None): """Get the GCS-owned service account representing this project. Args: project_number: the project in question. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ServiceException for errors interacting with cloud storage providers. Returns: Service account object (with email_address string field) """ raise NotImplementedError('GetProjectServiceAccount must be overloaded') def CreateNotificationConfig(self, bucket_name, pubsub_topic, payload_format, event_types=None, custom_attributes=None, object_name_prefix=None, provider=None): """Creates a new notification with the specified parameters. Args: bucket_name: (Required) Name of the bucket. pubsub_topic: (Required) Cloud Pub/Sub topic to which to publish. payload_format: (Required) payload format, must be 'JSON' or 'NONE'. event_types: (Opt) List of event type filters, e.g. 'OBJECT_FINALIZE'. custom_attributes: (Opt) Dictionary of custom attributes. object_name_prefix: (Opt) Filter on object name. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Notification object describing new notificationConfig """ raise NotImplementedError('CreateNotificationConfig must be overloaded') def DeleteNotificationConfig(self, bucket_name, notification, provider=None): """Deletes a notification. Args: bucket_name: (Required) Name of the bucket. notification: (Required) Integer name of the notification. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None """ raise NotImplementedError('DeleteNotificationConfig must be overloaded') def ListNotificationConfigs(self, bucket_name, provider=None): """Lists notification configs in a bucket. Args: bucket_name: Name of the bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Yields: List of notification objects. """ raise NotImplementedError('ListNotificationConfig must be overloaded') def CreateHmacKey(self, project_id, service_account_email): """Creates a new HMAC key for the specified service account. Args: project_id: Project ID owning the service account for which the key is to be created. service_account_email: Email address of the service account for which to create a key. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Returns The key metadata and the secret key material. """ raise NotImplementedError('CreateHmacKey must be overloaded') def DeleteHmacKey(self, project_id, access_id): """Deletes an HMAC key. Args: project_id: Project ID owning the requested key. access_id: Name of the HMAC key to be deleted. Raises: NotImplementedError: not implemented TODO(tuckerkirven) """ raise NotImplementedError('DeleteHmacKey must be overloaded') def GetHmacKey(self, project_id, access_id): """Retrieves an HMAC key's metadata. Args: project_id: Project ID owning the service account of the requested key. access_id: Name of the HMAC key for which the metadata is being requested. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Returns: Metadata for the specified key. """ raise NotImplementedError('GetHmacKey must be overloaded') def ListHmacKeys(self, project_id, service_account_email, show_deleted_keys=False): """Lists HMAC keys matching the criteria. Args: project_id: Name of the project from which to list HMAC keys. service_account_email: If present, only keys for the given service account will be returned. show_deleted_keys: If set, show keys in the DELETED state. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Yields: List of HMAC key metadata objects. """ raise NotImplementedError('ListHmacKeys must be overloaded') def UpdateHmacKey(self, project_id, access_id, state, etag): """Updates the state of an HMAC key. Args: project_id: Project ID owning the service account of the updated key. access_id: Name of the HMAC key being updated. state: The state to which the key should be updated. etag: None or a string matching the key's etag to ensure the appropriate version of the key is updated. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Returns: The updated key metadata. """ raise NotImplementedError('UpdateHmacKey must be overloaded') class Preconditions(object): """Preconditions class for specifying preconditions to cloud API requests.""" def __init__(self, gen_match=None, meta_gen_match=None): """Instantiates a Preconditions object. Args: gen_match: Perform request only if generation of target object matches the given integer. Ignored for bucket requests. meta_gen_match: Perform request only if metageneration of target object/bucket matches the given integer. """ self.gen_match = gen_match self.meta_gen_match = meta_gen_match class EncryptionException(Exception): """Exception raised when an encrypted resource cannot be decrypted.""" class ArgumentException(Exception): """Exception raised when arguments to a Cloud API method are invalid. This exception is never raised as a result of a failed call to a cloud storage provider. """ def __init__(self, reason): Exception.__init__(self) self.reason = reason def __repr__(self): return str(self) def __str__(self): return '%s: %s' % (self.__class__.__name__, self.reason) class ProjectIdException(ArgumentException): """Exception raised when a Project ID argument is required but not present.""" class ServiceException(Exception): """Exception raised when a cloud storage provider request fails. This exception is raised only as a result of a failed remote call. """ def __init__(self, reason, status=None, body=None): Exception.__init__(self) self.reason = reason self.status = status self.body = body def __repr__(self): return str(self) def __str__(self): message = '%s:' % self.__class__.__name__ if self.status: message += ' %s' % self.status message += ' %s' % self.reason if self.body: message += '\n%s' % self.body return message class RetryableServiceException(ServiceException): """Exception class for retryable exceptions.""" class ResumableDownloadException(RetryableServiceException): """Exception raised for res. downloads that can be retried later.""" class ResumableUploadException(RetryableServiceException): """Exception raised for res. uploads that can be retried w/ same upload ID.""" class ResumableUploadStartOverException(RetryableServiceException): """Exception raised for res. uploads that can be retried w/ new upload ID.""" class ResumableUploadAbortException(ServiceException): """Exception raised for resumable uploads that cannot be retried later.""" class AuthenticationException(ServiceException): """Exception raised for errors during the authentication process.""" class PreconditionException(ServiceException): """Exception raised for precondition failures.""" class NotFoundException(ServiceException): """Exception raised when a resource is not found (404).""" class BucketNotFoundException(NotFoundException): """Exception raised when a bucket resource is not found (404).""" def __init__(self, reason, bucket_name, status=None, body=None): super(BucketNotFoundException, self).__init__(reason, status=status, body=body) self.bucket_name = bucket_name class NotEmptyException(ServiceException): """Exception raised when trying to delete a bucket is not empty.""" class BadRequestException(ServiceException): """Exception raised for malformed requests. Where it is possible to detect invalid arguments prior to sending them to the server, an ArgumentException should be raised instead. """ class AccessDeniedException(ServiceException): """Exception raised when authenticated user has insufficient access rights. This is raised when the authentication process succeeded but the authenticated user does not have access rights to the requested resource. """ class PublishPermissionDeniedException(ServiceException): """Exception raised when bucket does not have publish permission to a topic. This is raised when a custom attempts to set up a notification config to a Cloud Pub/Sub topic, but their GCS bucket does not have permission to publish to the specified topic. """ gcs_json_credentials.py 0000644 00000050471 15025013616 0011306 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Credentials logic for JSON CloudApi implementation.""" # This module duplicates some logic in third_party gcs_oauth2_boto_plugin # because apitools credentials lib has its own mechanisms for file-locking # and credential storage. As such, it doesn't require most of the # gcs_oauth2_boto_plugin logic. from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import base64 import json import logging import os import io import six import traceback # pylint: disable=g-bad-import-order from apitools.base.py import credentials_lib from apitools.base.py import exceptions as apitools_exceptions from boto import config from gslib.cred_types import CredTypes from gslib.exception import CommandException from gslib.impersonation_credentials import ImpersonationCredentials from gslib.no_op_credentials import NoOpCredentials from gslib.utils import constants from gslib.utils import system_util from gslib.utils.boto_util import GetFriendlyConfigFilePaths from gslib.utils.boto_util import GetCredentialStoreFilename from gslib.utils.boto_util import GetGceCredentialCacheFilename from gslib.utils.boto_util import GetGcsJsonApiVersion from gslib.utils.constants import UTF8 from gslib.utils.wrapped_credentials import WrappedCredentials from google.auth import _helpers from google.auth.crypt import base as crypt_base from google_auth_httplib2 import AuthorizedHttp from google.oauth2 import service_account import oauth2client from oauth2client.client import HAS_CRYPTO from oauth2client.contrib import devshell from oauth2client.service_account import ServiceAccountCredentials from google_reauth import reauth_creds from oauth2client.contrib import multiprocess_file_storage from six import BytesIO DEFAULT_GOOGLE_OAUTH2_PROVIDER_AUTHORIZATION_URI = ( 'https://accounts.google.com/o/oauth2/auth') DEFAULT_GOOGLE_OAUTH2_PROVIDER_TOKEN_URI = ( 'https://oauth2.googleapis.com/token') DEFAULT_SCOPES = [ constants.Scopes.CLOUD_PLATFORM, constants.Scopes.CLOUD_PLATFORM_READ_ONLY, constants.Scopes.FULL_CONTROL, constants.Scopes.READ_ONLY, constants.Scopes.READ_WRITE, ] GOOGLE_OAUTH2_DEFAULT_FILE_PASSWORD = 'notasecret' def isP12Credentials(credentials): return isinstance(credentials, P12Credentials) class PKCS12Signer(crypt_base.Signer, crypt_base.FromServiceAccountMixin): """Signer for a p12 service account key.""" def __init__(self, key): self._key = key # Defined in crypt_base.Signer interface. # It is not used by p12 service account keys. @property def key_id(self): return None def sign(self, message): message = _helpers.to_bytes(message) from google.auth.crypt import _cryptography_rsa # pylint: disable=g-import-not-at-top return self._key.sign( message, _cryptography_rsa._PADDING, # pylint: disable=protected-access _cryptography_rsa._SHA256) # pylint: disable=protected-access @classmethod def from_string(cls, key_strings, key_id=None): del key_id #Unused key_string, password = (_helpers.to_bytes(k) for k in key_strings) # Cryptography package is not bundled with gsutil, Keeping it at top would throw error. from cryptography.hazmat.primitives.serialization import pkcs12 # pylint: disable=g-import-not-at-top try: key, _, _ = pkcs12.load_key_and_certificates(key_string, password) return cls(key) except: raise CommandException('Unable to load the keyfile, Invalid password or PKCS12 data.') class P12Credentials(service_account.Credentials): """google-auth service account credentials for p12 keys. p12 keys are not supported by the google-auth service account credentials. gsutil uses oauth2client to support p12 key users. Since oauth2client was deprecated and bundling it is security concern, we decided to support p12 in gsutil codebase. We prefer not adding it to the google-auth library because p12 is not supported from the beginning by google-auth. GCP strongly suggests users to use the JSON format. gsutil has to support it to not break users. """ _REQUIRED_FIELDS = ('service_account_email', 'token_uri', 'scopes') def authorize(self, http): return AuthorizedHttp(self, http=http) @classmethod def from_service_account_pkcs12_keystring(cls, key_string, password=None, **kwargs): password = password or GOOGLE_OAUTH2_DEFAULT_FILE_PASSWORD signer = PKCS12Signer.from_string((key_string, password)) missing_fields = [f for f in cls._REQUIRED_FIELDS if f not in kwargs] if missing_fields: raise CommandException('Missing fields: {}.'.format( ', '.join(missing_fields))) creds = cls(signer, **kwargs) return creds def CreateP12ServiceAccount(key_string, password=None, **kwargs): """Creates a service account from a p12 key and handles import errors.""" try: return P12Credentials.from_service_account_pkcs12_keystring( key_string, password, **kwargs) except ImportError: raise CommandException( ('pyca/cryptography is not available. Either install it, or please consider using the .json keyfile')) def GetCredentialStoreKey(credentials, api_version): """Disambiguates a credential for caching in a credential store. Different credential types have different fields that identify them. This function assembles relevant information in a string to be used as the key for accessing a credential. Note that in addition to uniquely identifying the entity to which a credential corresponds, we must differentiate between two or more of that entity's credentials that have different attributes such that the credentials should not be treated as interchangeable, e.g. if they target different API versions (happens for developers targeting different test environments), have different private key IDs (for service account JSON keyfiles), or target different provider token (refresh) URIs. Args: credentials: An OAuth2Credentials object. api_version: JSON API version being used. Returns: A string that can be used as the key to identify a credential, e.g. "v1-909320924072.apps.googleusercontent.com-1/rEfrEshtOkEn-https://..." """ # Note: We don't include the scopes as part of the key. For a user credential # object, we always construct it with manually added scopes that are necessary # to negotiate reauth challenges - those scopes don't necessarily correspond # to the scopes the refresh token was created with. We avoid key name # mismatches for the same refresh token by not including scopes in the key # string. key_parts = [api_version] if isinstance(credentials, devshell.DevshellCredentials): key_parts.append(credentials.user_email) elif isinstance(credentials, ServiceAccountCredentials): # pylint: disable=protected-access key_parts.append(credentials._service_account_email) if getattr(credentials, '_private_key_id', None): # JSON keyfile. # Differentiate between two different JSON keyfiles for the same service # account. key_parts.append(credentials._private_key_id) elif getattr(credentials, '_private_key_pkcs12', None): # P12 keyfile # Use a prefix of the Base64-encoded PEM string to differentiate it from # others. Using a prefix of reasonable length prevents the key from being # unnecessarily large, and the likelihood of having two PEM strings with # the same prefixes is sufficiently low. key_parts.append(base64.b64encode(credentials._private_key_pkcs12)[:20]) # pylint: enable=protected-access elif isinstance(credentials, oauth2client.client.OAuth2Credentials): if credentials.client_id and credentials.client_id != 'null': key_parts.append(credentials.client_id) else: key_parts.append('noclientid') key_parts.append(credentials.refresh_token or 'norefreshtoken') # If a cached credential is targeting provider token URI 'A' for token refresh # requests, then the user changes their boto file or private key file to # target URI 'B', we don't want to treat the cached and the new credential as # interchangeable. This applies for all credentials that store a token URI. if getattr(credentials, 'token_uri', None): key_parts.append(credentials.token_uri) key_parts = [six.ensure_text(part) for part in key_parts] return '-'.join(key_parts) def SetUpJsonCredentialsAndCache(api, logger, credentials=None): """Helper to ensure each GCS API client shares the same credentials.""" api.credentials = (credentials or _CheckAndGetCredentials(logger) or NoOpCredentials()) # Notify the user that impersonation credentials are in effect. if isinstance(api.credentials, ImpersonationCredentials): logger.warn( 'WARNING: This command is using service account impersonation. All ' 'API calls will be executed as [%s].', _GetImpersonateServiceAccount()) # Do not store p12 credentials (Supported by google-auth), as google-auth does not support storing service account credentials. if not isP12Credentials(api.credentials): # Set credential cache so that we don't have to get a new access token for # every call we make. All GCS APIs use the same credentials as the JSON API, # so we use its version in the key for caching access tokens. credential_store_key = (GetCredentialStoreKey(api.credentials, GetGcsJsonApiVersion())) api.credentials.set_store( multiprocess_file_storage.MultiprocessFileStorage( GetCredentialStoreFilename(), credential_store_key)) # The cached entry for this credential often contains more context than what # we can construct from boto config attributes (e.g. for a user credential, # the cached version might also contain a RAPT token and expiry info). # Prefer the cached credential if present. cached_cred = None if not isinstance(api.credentials, NoOpCredentials): # A NoOpCredentials object doesn't actually have a store attribute. cached_cred = api.credentials.store.get() # As of gsutil 4.31, we never use the OAuth2Credentials class for # credentials directly; rather, we use subclasses (user credentials were # the only ones left using it, but they now use # Oauth2WithReauthCredentials). If we detect that a cached credential is # an instance of OAuth2Credentials and not a subclass of it (as might # happen when transitioning to version v4.31+), we don't fetch it from the # cache. This results in our new-style credential being refreshed and # overwriting the old credential cache entry in our credstore. if (cached_cred and type(cached_cred) != oauth2client.client.OAuth2Credentials): api.credentials = cached_cred def _CheckAndGetCredentials(logger): """Returns credentials from the configuration file, if any are present. Args: logger: logging.Logger instance for outputting messages. Returns: OAuth2Credentials object if any valid ones are found, otherwise None. """ configured_cred_types = [] failed_cred_type = None try: if _HasOauth2UserAccountCreds(): configured_cred_types.append(CredTypes.OAUTH2_USER_ACCOUNT) if _HasOauth2ServiceAccountCreds(): configured_cred_types.append(CredTypes.OAUTH2_SERVICE_ACCOUNT) if len(configured_cred_types) > 1: # We only allow one set of configured credentials. Otherwise, we're # choosing one arbitrarily, which can be very confusing to the user # (e.g., if only one is authorized to perform some action) and can # also mask errors. # Because boto merges config files, GCE credentials show up by default # for GCE VMs. We don't want to fail when a user creates a boto file # with their own credentials, so in this case we'll use the OAuth2 # user credentials. raise CommandException( ('You have multiple types of configured credentials (%s), which is ' 'not supported. One common way this happens is if you run gsutil ' 'config to create credentials and later run gcloud auth, and ' 'create a second set of credentials. Your boto config path is: ' '%s. For more help, see "gsutil help creds".') % (configured_cred_types, GetFriendlyConfigFilePaths())) failed_cred_type = CredTypes.OAUTH2_USER_ACCOUNT user_creds = _GetOauth2UserAccountCredentials() failed_cred_type = CredTypes.OAUTH2_SERVICE_ACCOUNT service_account_creds = _GetOauth2ServiceAccountCredentials() failed_cred_type = CredTypes.EXTERNAL_ACCOUNT external_account_creds = _GetExternalAccountCredentials() failed_cred_type = CredTypes.EXTERNAL_ACCOUNT_AUTHORIZED_USER external_account_authorized_user_creds = _GetExternalAccountAuthorizedUserCredentials( ) failed_cred_type = CredTypes.GCE gce_creds = _GetGceCreds() failed_cred_type = CredTypes.DEVSHELL devshell_creds = _GetDevshellCreds() creds = user_creds or service_account_creds or gce_creds or external_account_creds or external_account_authorized_user_creds or devshell_creds # Use one of the above credential types to impersonate, if configured. if _HasImpersonateServiceAccount() and creds: failed_cred_type = CredTypes.IMPERSONATION return _GetImpersonationCredentials(creds, logger) else: return creds except Exception as e: # If we didn't actually try to authenticate because there were multiple # types of configured credentials, don't emit this warning. if failed_cred_type: if logger.isEnabledFor(logging.DEBUG): logger.debug(traceback.format_exc()) # If impersonation fails, show the user the actual error, since we handle # errors in iamcredentials_api. if failed_cred_type == CredTypes.IMPERSONATION: raise e elif system_util.InvokedViaCloudSdk(): logger.warn( 'Your "%s" credentials are invalid. Please run\n' ' $ gcloud auth login', failed_cred_type) else: logger.warn( 'Your "%s" credentials are invalid. For more help, see ' '"gsutil help creds", or re-run the gsutil config command (see ' '"gsutil help config").', failed_cred_type) # If there's any set of configured credentials, we'll fail if they're # invalid, rather than silently falling back to anonymous config (as # boto does). That approach leads to much confusion if users don't # realize their credentials are invalid. raise def _GetProviderTokenUri(): return config.get('OAuth2', 'provider_token_uri', DEFAULT_GOOGLE_OAUTH2_PROVIDER_TOKEN_URI) def _HasOauth2ServiceAccountCreds(): return config.has_option('Credentials', 'gs_service_key_file') def _HasOauth2UserAccountCreds(): return config.has_option('Credentials', 'gs_oauth2_refresh_token') def _HasGceCreds(): return config.has_option('GoogleCompute', 'service_account') def _HasImpersonateServiceAccount(): return _GetImpersonateServiceAccount() not in (None, '') def _GetExternalAccountCredentials(): external_account_filename = config.get('Credentials', 'gs_external_account_file', None) if not external_account_filename: return None return WrappedCredentials.for_external_account(external_account_filename) def _GetExternalAccountAuthorizedUserCredentials(): external_account_authorized_user_filename = config.get( 'Credentials', 'gs_external_account_authorized_user_file', None) if not external_account_authorized_user_filename: return None return WrappedCredentials.for_external_account_authorized_user( external_account_authorized_user_filename) def _GetImpersonateServiceAccount(): return (constants.IMPERSONATE_SERVICE_ACCOUNT or config.get( 'Credentials', 'gs_impersonate_service_account', os.environ.get('CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT'))) def _GetOauth2ServiceAccountCredentials(): """Retrieves OAuth2 service account credentials for a private key file.""" if not _HasOauth2ServiceAccountCreds(): return provider_token_uri = _GetProviderTokenUri() service_client_id = config.get('Credentials', 'gs_service_client_id', '') private_key_filename = config.get('Credentials', 'gs_service_key_file', '') with io.open(private_key_filename, 'rb') as private_key_file: private_key = private_key_file.read() keyfile_is_utf8 = False try: private_key = private_key.decode(UTF8) # P12 keys won't be encoded as UTF8 bytes. keyfile_is_utf8 = True except UnicodeDecodeError: pass if keyfile_is_utf8: try: json_key_dict = json.loads(private_key) except ValueError: raise Exception('Could not parse JSON keyfile "%s" as valid JSON' % private_key_filename) # Key file is in JSON format. for json_entry in ('client_id', 'client_email', 'private_key_id', 'private_key'): if json_entry not in json_key_dict: raise Exception('The JSON private key file at %s ' 'did not contain the required entry: %s' % (private_key_filename, json_entry)) return ServiceAccountCredentials.from_json_keyfile_dict( json_key_dict, scopes=DEFAULT_SCOPES, token_uri=provider_token_uri) else: # Key file is in P12 format. key_file_pass = config.get('Credentials', 'gs_service_key_file_password', GOOGLE_OAUTH2_DEFAULT_FILE_PASSWORD) return CreateP12ServiceAccount(private_key, key_file_pass, scopes=DEFAULT_SCOPES, service_account_email=service_client_id, token_uri=provider_token_uri) def _GetOauth2UserAccountCredentials(): """Retrieves OAuth2 service account credentials for a refresh token.""" if not _HasOauth2UserAccountCreds(): return provider_token_uri = _GetProviderTokenUri() gsutil_client_id, gsutil_client_secret = ( system_util.GetGsutilClientIdAndSecret()) client_id = config.get('OAuth2', 'client_id', os.environ.get('OAUTH2_CLIENT_ID', gsutil_client_id)) client_secret = config.get( 'OAuth2', 'client_secret', os.environ.get('OAUTH2_CLIENT_SECRET', gsutil_client_secret)) # Note that these scopes don't necessarily correspond to the refresh token # being used. This list is is used for obtaining the RAPT in the reauth flow, # to determine which challenges should be used. scopes_for_reauth_challenge = [ constants.Scopes.CLOUD_PLATFORM, constants.Scopes.REAUTH ] return reauth_creds.Oauth2WithReauthCredentials( None, # access_token client_id, client_secret, config.get('Credentials', 'gs_oauth2_refresh_token'), None, # token_expiry provider_token_uri, None, # user_agent scopes=scopes_for_reauth_challenge) def _GetGceCreds(): if not _HasGceCreds(): return try: return credentials_lib.GceAssertionCredentials( service_account_name=config.get('GoogleCompute', 'service_account', 'default'), cache_filename=GetGceCredentialCacheFilename()) except apitools_exceptions.ResourceUnavailableError as e: if 'service account' in str(e) and 'does not exist' in str(e): return None raise def _GetDevshellCreds(): try: return devshell.DevshellCredentials() except devshell.NoDevshellServer: return None except: raise def _GetImpersonationCredentials(credentials, logger): """Retrieves temporary credentials impersonating a service account""" # We don't use impersoned credentials to impersonate. if isinstance(credentials, ImpersonationCredentials): return return ImpersonationCredentials(_GetImpersonateServiceAccount(), [constants.Scopes.CLOUD_PLATFORM], credentials, logger) command.py 0000644 00000335116 15025013616 0006544 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2010 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Base class for gsutil commands. In addition to base class code, this file contains helpers that depend on base class state (such as GetAndPrintAcl) In general, functions that depend on class state and that are used by multiple commands belong in this file. Functions that don't depend on class state belong in util.py, and non-shared helpers belong in individual subclasses. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import codecs from collections import namedtuple import copy import getopt import json import logging import os import signal import sys import textwrap import threading import time import traceback import boto from boto.storage_uri import StorageUri import gslib from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ArgumentException from gslib.cloud_api import ServiceException from gslib.cloud_api_delegator import CloudApiDelegator from gslib.cs_api_map import ApiSelector from gslib.cs_api_map import GsutilApiMapFactory from gslib.exception import CommandException from gslib.help_provider import HelpProvider from gslib.metrics import CaptureThreadStatException from gslib.metrics import LogPerformanceSummaryParams from gslib.name_expansion import CopyObjectInfo from gslib.name_expansion import CopyObjectsIterator from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import NameExpansionResult from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.seek_ahead_thread import SeekAheadThread from gslib.sig_handling import ChildProcessSignalHandler from gslib.sig_handling import GetCaughtSignals from gslib.sig_handling import KillProcess from gslib.sig_handling import MultithreadedMainSignalHandler from gslib.sig_handling import RegisterSignalHandler from gslib.storage_url import HaveFileUrls from gslib.storage_url import HaveProviderUrls from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.storage_url import UrlsAreMixOfBucketsAndObjects from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import FinalMessage from gslib.thread_message import MetadataMessage from gslib.thread_message import PerformanceSummaryMessage from gslib.thread_message import ProducerThreadMessage from gslib.ui_controller import MainThreadUIQueue from gslib.ui_controller import UIController from gslib.ui_controller import UIThread from gslib.utils.boto_util import GetFriendlyConfigFilePaths from gslib.utils.boto_util import GetMaxConcurrentCompressedUploads from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 import gslib.utils.parallelism_framework_util from gslib.utils.parallelism_framework_util import AtomicDict from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit from gslib.utils.parallelism_framework_util import multiprocessing_context from gslib.utils.parallelism_framework_util import ProcessAndThreadSafeInt from gslib.utils.parallelism_framework_util import PutToQueueWithTimeout from gslib.utils.parallelism_framework_util import SEEK_AHEAD_JOIN_TIMEOUT from gslib.utils.parallelism_framework_util import ShouldProhibitMultiprocessing from gslib.utils.parallelism_framework_util import UI_THREAD_JOIN_TIMEOUT from gslib.utils.parallelism_framework_util import ZERO_TASKS_TO_DO_ARGUMENT from gslib.utils.rsync_util import RsyncDiffToApply from gslib.utils.shim_util import GcloudStorageCommandMixin from gslib.utils.system_util import GetTermLines from gslib.utils.system_util import IS_WINDOWS from gslib.utils.translation_helper import AclTranslation from gslib.utils.translation_helper import GetNonMetadataHeaders from gslib.utils.translation_helper import PRIVATE_DEFAULT_OBJ_ACL from gslib.wildcard_iterator import CreateWildcardIterator from six.moves import queue as Queue # pylint: disable=g-import-not-at-top try: from Crypto import Random as CryptoRandom except ImportError: CryptoRandom = None # pylint: enable=g-import-not-at-top OFFER_GSUTIL_M_SUGGESTION_THRESHOLD = 5 OFFER_GSUTIL_M_SUGGESTION_FREQUENCY = 1000 def CreateOrGetGsutilLogger(command_name): """Fetches a logger with the given name that resembles 'print' output. Initial Logger Configuration: The logger abides by gsutil -d/-D/-DD/-q options. If none of those options were specified at invocation, the returned logger will display all messages logged with level INFO or above. Log propagation is disabled. If a logger with the specified name has already been created and configured, it is not reconfigured, e.g.: foo = CreateOrGetGsutilLogger('foo') # Creates and configures Logger "foo". foo.setLevel(logging.DEBUG) # Change level from INFO to DEBUG foo = CreateOrGetGsutilLogger('foo') # Does not reset level to INFO. Args: command_name: (str) Command name to create logger for. Returns: A logging.Logger object. """ log = logging.getLogger(command_name) # There are some scenarios (e.g. unit tests, commands like `mv` that call # other commands) in which we call this function multiple times. To avoid # adding duplicate handlers or overwriting logger attributes set elsewhere, # we only configure the logger if it's one we haven't configured before (i.e. # one that doesn't have a handler set yet). if not log.handlers: log.propagate = False log.setLevel(logging.root.level) log_handler = logging.StreamHandler() log_handler.setFormatter(logging.Formatter('%(message)s')) log.addHandler(log_handler) return log def _DefaultExceptionHandler(cls, e): cls.logger.exception(e) def _UrlArgChecker(command_instance, url): if not command_instance.exclude_symlinks: return True exp_src_url = url.expanded_storage_url if exp_src_url.IsFileUrl() and os.path.islink(exp_src_url.object_name): command_instance.logger.info('Skipping symbolic link %s...', exp_src_url) return False return True def DummyArgChecker(*unused_args): return True def SetAclFuncWrapper(cls, name_expansion_result, thread_state=None): return cls.SetAclFunc(name_expansion_result, thread_state=thread_state) def SetAclExceptionHandler(cls, e): """Exception handler that maintains state about post-completion status.""" cls.logger.error(str(e)) cls.everything_set_okay = False # We will keep this list of all thread- or process-safe queues (except the # global status queue) ever created by the main thread so that we can # forcefully kill them upon shutdown. Otherwise, we encounter a Python bug in # which empty queues block forever on join (which is called as part of the # Python exit function cleanup) under the impression that they are non-empty. # However, this also lets us shut down somewhat more cleanly when interrupted. queues = [] def _CryptoRandomAtFork(): if CryptoRandom and getattr(CryptoRandom, 'atfork', None): # Fixes https://github.com/GoogleCloudPlatform/gsutil/issues/390. The # oauth2client module uses Python's Crypto library when pyOpenSSL isn't # present; that module requires calling atfork() in both the parent and # child process after a new process is forked. CryptoRandom.atfork() def _NewMultiprocessingQueue(): new_queue = multiprocessing_context.Queue(MAX_QUEUE_SIZE) queues.append(new_queue) return new_queue def _NewThreadsafeQueue(): new_queue = Queue.Queue(MAX_QUEUE_SIZE) queues.append(new_queue) return new_queue # The maximum size of a process- or thread-safe queue. Imposing this limit # prevents us from needing to hold an arbitrary amount of data in memory. # However, setting this number too high (e.g., >= 32768 on OS X) can cause # problems on some operating systems. MAX_QUEUE_SIZE = 32500 # Related to the max queue size above, once we cross this threshold of # iterated tasks added to the queue, kick off the SeekAheadThread that will # estimate the total work necessary for the command. DEFAULT_TASK_ESTIMATION_THRESHOLD = 30000 def _GetTaskEstimationThreshold(): return boto.config.getint('GSUtil', 'task_estimation_threshold', DEFAULT_TASK_ESTIMATION_THRESHOLD) # That maximum depth of the tree of recursive calls to command.Apply. This is # an arbitrary limit put in place to prevent developers from accidentally # causing problems with infinite recursion, and it can be increased if needed. MAX_RECURSIVE_DEPTH = 5 # Map from deprecated aliases to the current command and subcommands that # provide the same behavior. # TODO: Remove this map and deprecate old commands on 9/9/14. OLD_ALIAS_MAP = { 'chacl': ['acl', 'ch'], 'getacl': ['acl', 'get'], 'setacl': ['acl', 'set'], 'getcors': ['cors', 'get'], 'setcors': ['cors', 'set'], 'chdefacl': ['defacl', 'ch'], 'getdefacl': ['defacl', 'get'], 'setdefacl': ['defacl', 'set'], 'disablelogging': ['logging', 'set', 'off'], 'enablelogging': ['logging', 'set', 'on'], 'getlogging': ['logging', 'get'], 'getversioning': ['versioning', 'get'], 'setversioning': ['versioning', 'set'], 'getwebcfg': ['web', 'get'], 'setwebcfg': ['web', 'set'] } # Declare all of the module level variables - see # InitializeMultiprocessingVariables for an explanation of why this is # necessary. # pylint: disable=global-at-module-level global manager, consumer_pools, task_queues, caller_id_lock, caller_id_counter global total_tasks, call_completed_map, global_return_values_map global need_pool_or_done_cond, caller_id_finished_count, new_pool_needed global current_max_recursive_level, shared_vars_map, shared_vars_list_map global class_map, worker_checking_level_lock, failure_count, thread_stats global glob_status_queue, ui_controller, concurrent_compressed_upload_lock def InitializeMultiprocessingVariables(): """Initializes module-level variables that will be inherited by subprocesses. On Windows, a multiprocessing.Manager object should only be created within an "if __name__ == '__main__':" block. This function must be called, otherwise every command that calls Command.Apply will fail. While multiprocessing variables are initialized at the beginning of gsutil execution, new processes and threads are created only by calls to Command.Apply. When multiple processes and threads are used, the flow of startup/teardown looks like this: 1. __main__: initializes multiprocessing variables, including any necessary Manager processes (here and in gslib.utils.parallelism_framework_util). 2. __main__: Registers signal handlers for terminating signals responsible for cleaning up multiprocessing variables and manager processes upon exit. 3. Command.Apply registers signal handlers for the main process to kill itself after the cleanup handlers registered by __main__ have executed. 4. If worker processes have not been created for the current level of recursive calls, Command.Apply creates those processes. ---- Parallel operations start here, so steps are no longer numbered. ---- - Command.Apply in the main thread starts the ProducerThread. - The Producer thread adds task arguments to the global task queue. - It optionally starts the SeekAheadThread which estimates total work for the Apply call. - Command.Apply in the main thread starts the UIThread, which will consume messages from the global status queue, process them, and display them to the user. - Each worker process creates a thread pool to perform work. - The worker process registers signal handlers to kill itself in response to a terminating signal. - The main thread of the worker process moves items from the global task queue to the process-local task queue. - Worker threads retrieve items from the process-local task queue, perform the work, and post messages to the global status queue. - Worker threads may themselves call Command.Apply. - This creates a new pool of worker subprocesses with the same size as the main pool. This pool is shared amongst all Command.Apply calls at the given recursion depth. - This reuses the global UIThread, global status queue, and global task queue. - This starts a new ProducerThread. - A SeekAheadThread is not started at this level; only one such thread exists at the top level, and it provides estimates for top-level work only. - The ProducerThread runs out of tasks, or the user signals cancellation. - The ProducerThread cancels the SeekAheadThread (if it is running) via an event. - The ProducerThread enqueues special terminating messages on the global task queue and global status queue, signaling the UI Thread to shut down and the main thread to continue operation. - In the termination case, existing processes exit in response to terminating signals from the main process. ---- Parallel operations end here. ---- 5. Further top-level calls to Command.Apply can be made, which will repeat all of the steps made in #4, except that worker processes will be reused. """ # This list of global variables must exactly match the above list of # declarations. # pylint: disable=global-variable-undefined global manager, consumer_pools, task_queues, caller_id_lock, caller_id_counter global total_tasks, call_completed_map, global_return_values_map, thread_stats global need_pool_or_done_cond, caller_id_finished_count, new_pool_needed global current_max_recursive_level, shared_vars_map, shared_vars_list_map global class_map, worker_checking_level_lock, failure_count, glob_status_queue global concurrent_compressed_upload_lock manager = multiprocessing_context.Manager() # List of ConsumerPools - used during shutdown to clean up child processes. consumer_pools = [] # List of all existing task queues - used by all pools to find the queue # that's appropriate for the given recursive_apply_level. task_queues = [] # Used to assign a globally unique caller ID to each Apply call. caller_id_lock = manager.Lock() caller_id_counter = ProcessAndThreadSafeInt(True) # Map from caller_id to total number of tasks to be completed for that ID. total_tasks = AtomicDict(manager=manager) # Map from caller_id to a boolean which is True iff all its tasks are # finished. call_completed_map = AtomicDict(manager=manager) # Used to keep track of the set of return values for each caller ID. global_return_values_map = AtomicDict(manager=manager) # Condition used to notify any waiting threads that a task has finished or # that a call to Apply needs a new set of consumer processes. need_pool_or_done_cond = manager.Condition() # Lock used to prevent multiple worker processes from asking the main thread # to create a new consumer pool for the same level. worker_checking_level_lock = manager.Lock() # Map from caller_id to the current number of completed tasks for that ID. caller_id_finished_count = AtomicDict(manager=manager) # Used as a way for the main thread to distinguish between being woken up # by another call finishing and being woken up by a call that needs a new set # of consumer processes. new_pool_needed = ProcessAndThreadSafeInt(True) current_max_recursive_level = ProcessAndThreadSafeInt(True) # Map from (caller_id, name) to the value of that shared variable. shared_vars_map = AtomicDict(manager=manager) shared_vars_list_map = AtomicDict(manager=manager) # Map from (process id, thread id) to a _ThreadStat object (see WorkerThread). # Used to keep track of thread idle time and execution time. thread_stats = AtomicDict(manager=manager) # Map from caller_id to calling class. class_map = manager.dict() # Number of tasks that resulted in an exception in calls to Apply(). failure_count = ProcessAndThreadSafeInt(True) # Central queue for status reporting across multiple processes and threads. # It's possible that if many processes and threads are executing small file # writes or metadata changes quickly, performance may be bounded by lock # contention on the queue. Initial testing conducted with # 12 processes * 5 threads per process showed little difference. If this # becomes a performance bottleneck in the future, consider creating a queue # per-process and having the UI thread poll all of the queues; that approach # would need to address: # - Queue fairness if one queue grows to be disproportionately large # - Reasonable time correlation with events as they occur # # This queue must be torn down after worker processes/threads and the # UI thread have been torn down. Otherwise, these threads may have # undefined behavior when trying to interact with a non-existent queue. glob_status_queue = manager.Queue(MAX_QUEUE_SIZE) # Semaphore lock used to prevent resource exhaustion when running many # compressed uploads in parallel. concurrent_compressed_upload_lock = manager.BoundedSemaphore( GetMaxConcurrentCompressedUploads()) def TeardownMultiprocessingProcesses(): """Should be called by signal handlers prior to shut down.""" # Shut down all processes in consumer pools in preparation for exiting. ShutDownGsutil() # Shut down command and util's multiprocessing.Manager(). # pylint: disable=global-variable-not-assigned,global-variable-undefined global manager # pylint: enable=global-variable-not-assigned,global-variable-undefined manager.shutdown() gslib.utils.parallelism_framework_util.top_level_manager.shutdown() def InitializeThreadingVariables(): """Initializes module-level variables used when running multi-threaded. When multiprocessing is not available (or on Windows where only 1 process is used), thread-safe analogs to the multiprocessing global variables must be initialized. This function is the thread-safe analog to InitializeMultiprocessingVariables. """ # pylint: disable=global-variable-undefined global global_return_values_map, shared_vars_map, failure_count global caller_id_finished_count, shared_vars_list_map, total_tasks global need_pool_or_done_cond, call_completed_map, class_map, thread_stats global task_queues, caller_id_lock, caller_id_counter, glob_status_queue global worker_checking_level_lock, current_max_recursive_level global concurrent_compressed_upload_lock caller_id_counter = ProcessAndThreadSafeInt(False) caller_id_finished_count = AtomicDict() caller_id_lock = threading.Lock() call_completed_map = AtomicDict() class_map = AtomicDict() current_max_recursive_level = ProcessAndThreadSafeInt(False) failure_count = ProcessAndThreadSafeInt(False) glob_status_queue = Queue.Queue(MAX_QUEUE_SIZE) global_return_values_map = AtomicDict() need_pool_or_done_cond = threading.Condition() shared_vars_list_map = AtomicDict() shared_vars_map = AtomicDict() thread_stats = AtomicDict() task_queues = [] total_tasks = AtomicDict() worker_checking_level_lock = threading.Lock() concurrent_compressed_upload_lock = threading.BoundedSemaphore( GetMaxConcurrentCompressedUploads()) # Each subclass of Command must define a property named 'command_spec' that is # an instance of the following class. CommandSpec = namedtuple( 'CommandSpec', [ # Name of command. 'command_name', # Usage synopsis. 'usage_synopsis', # List of command name aliases. 'command_name_aliases', # Min number of args required by this command. 'min_args', # Max number of args required by this command, or NO_MAX. 'max_args', # Getopt-style string specifying acceptable sub args. 'supported_sub_args', # True if file URLs are acceptable for this command. 'file_url_ok', # True if provider-only URLs are acceptable for this command. 'provider_url_ok', # Index in args of first URL arg. 'urls_start_arg', # List of supported APIs 'gs_api_support', # Default API to use for this command 'gs_default_api', # Private arguments (for internal testing) 'supported_private_args', 'argparse_arguments', ]) class Command(HelpProvider, GcloudStorageCommandMixin): """Base class for all gsutil commands.""" # Each subclass must override this with an instance of CommandSpec. command_spec = None _commands_with_subcommands_and_subopts = ('acl', 'defacl', 'iam', 'kms', 'label', 'logging', 'notification', 'retention', 'web') # This keeps track of the recursive depth of the current call to Apply. recursive_apply_level = 0 # If the multiprocessing module isn't available, we'll use this to keep track # of the caller_id. sequential_caller_id = -1 @staticmethod def CreateCommandSpec(command_name, usage_synopsis=None, command_name_aliases=None, min_args=0, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=None, gs_default_api=None, supported_private_args=None, argparse_arguments=None): """Creates an instance of CommandSpec, with defaults.""" return CommandSpec(command_name=command_name, usage_synopsis=usage_synopsis, command_name_aliases=command_name_aliases or [], min_args=min_args, max_args=max_args, supported_sub_args=supported_sub_args, file_url_ok=file_url_ok, provider_url_ok=provider_url_ok, urls_start_arg=urls_start_arg, gs_api_support=gs_api_support or [ApiSelector.XML], gs_default_api=gs_default_api or ApiSelector.XML, supported_private_args=supported_private_args, argparse_arguments=argparse_arguments or []) # Define a convenience property for command name, since it's used many places. def _GetDefaultCommandName(self): return self.command_spec.command_name command_name = property(_GetDefaultCommandName) def _CalculateUrlsStartArg(self): """Calculate the index in args of the first URL arg. Returns: Index of the first URL arg (according to the command spec). """ return self.command_spec.urls_start_arg def _TranslateDeprecatedAliases(self, args): """Map deprecated aliases to the corresponding new command, and warn.""" new_command_args = OLD_ALIAS_MAP.get(self.command_alias_used, None) if new_command_args: # Prepend any subcommands for the new command. The command name itself # is not part of the args, so leave it out. args = new_command_args[1:] + args self.logger.warn('\n'.join( textwrap.wrap( ('You are using a deprecated alias, "%(used_alias)s", for the ' '"%(command_name)s" command. This will stop working on 9/9/2014. ' 'Please use "%(command_name)s" with the appropriate sub-command in ' 'the future. See "gsutil help %(command_name)s" for details.') % { 'used_alias': self.command_alias_used, 'command_name': self.command_name }))) return args def __init__(self, command_runner, args, headers, debug, trace_token, parallel_operations, bucket_storage_uri_class, gsutil_api_class_map_factory, logging_filters=None, command_alias_used=None, perf_trace_token=None, user_project=None): """Instantiates a Command. Args: command_runner: CommandRunner (for commands built atop other commands). args: Command-line args (arg0 = actual arg, not command name ala bash). headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). trace_token: Trace token to pass to the API implementation. parallel_operations: Should command operations be executed in parallel? bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. gsutil_api_class_map_factory: Creates map of cloud storage interfaces. Settable for testing/mocking. logging_filters: Optional list of logging. Filters to apply to this command's logger. command_alias_used: The alias that was actually used when running this command (as opposed to the "official" command name, which will always correspond to the file name). perf_trace_token: Performance measurement trace token to use when making API calls. user_project: Project to be billed for this request. Implementation note: subclasses shouldn't need to define an __init__ method, and instead depend on the shared initialization that happens here. If you do define an __init__ method in a subclass you'll need to explicitly call super().__init__(). But you're encouraged not to do this, because it will make changing the __init__ interface more painful. """ # Save class values from constructor params. super().__init__() self.command_runner = command_runner self.unparsed_args = args self.headers = headers self.debug = debug self.trace_token = trace_token self.perf_trace_token = perf_trace_token self.parallel_operations = parallel_operations self.user_project = user_project self.bucket_storage_uri_class = bucket_storage_uri_class self.gsutil_api_class_map_factory = gsutil_api_class_map_factory self.exclude_symlinks = False self.recursion_requested = False self.all_versions = False self.command_alias_used = command_alias_used self.seek_ahead_gsutil_api = None # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global ui_controller # pylint: enable=global-variable-undefined # pylint: enable=global-variable-not-assigned # Global instance of a threaded logger object. self.logger = CreateOrGetGsutilLogger(self.command_name) if logging_filters: for log_filter in logging_filters: self.logger.addFilter(log_filter) if self.headers is not None: self.non_metadata_headers = GetNonMetadataHeaders(self.headers) else: self.non_metadata_headers = None if self.command_spec is None: raise CommandException('"%s" command implementation is missing a ' 'command_spec definition.' % self.command_name) self.quiet_mode = not self.logger.isEnabledFor(logging.INFO) ui_controller = UIController(quiet_mode=self.quiet_mode, dump_status_messages_file=boto.config.get( 'GSUtil', 'dump_status_messages_file', None)) # Parse and validate args. self.args = self._TranslateDeprecatedAliases(args) self.ParseSubOpts() # Named tuple public functions start with _ # pylint: disable=protected-access self.command_spec = self.command_spec._replace( urls_start_arg=self._CalculateUrlsStartArg()) if (len(self.args) < self.command_spec.min_args or len(self.args) > self.command_spec.max_args): self.RaiseWrongNumberOfArgumentsException() if self.command_name not in self._commands_with_subcommands_and_subopts: self.CheckArguments() # Build the support and default maps from the command spec. support_map = { 'gs': self.command_spec.gs_api_support, 's3': [ApiSelector.XML] } default_map = { 'gs': self.command_spec.gs_default_api, 's3': ApiSelector.XML } self.gsutil_api_map = GsutilApiMapFactory.GetApiMap( self.gsutil_api_class_map_factory, support_map, default_map) self.project_id = None self.gsutil_api = CloudApiDelegator(self.bucket_storage_uri_class, self.gsutil_api_map, self.logger, MainThreadUIQueue( sys.stderr, ui_controller), debug=self.debug, http_headers=self.non_metadata_headers, trace_token=self.trace_token, perf_trace_token=self.perf_trace_token, user_project=self.user_project) # Cross-platform path to run gsutil binary. self.gsutil_cmd = '' # If running on Windows, invoke python interpreter explicitly. if IS_WINDOWS: self.gsutil_cmd += 'python ' # Add full path to gsutil to make sure we test the correct version. self.gsutil_path = gslib.GSUTIL_PATH self.gsutil_cmd += self.gsutil_path # We're treating recursion_requested like it's used by all commands, but # only some of the commands accept the -R option. if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-r' or o == '-R': self.recursion_requested = True break self.multiprocessing_is_available = ( CheckMultiprocessingAvailableAndInit().is_available) def RaiseWrongNumberOfArgumentsException(self): """Raises exception for wrong number of arguments supplied to command.""" if len(self.args) < self.command_spec.min_args: tail_str = 's' if self.command_spec.min_args > 1 else '' message = ('The %s command requires at least %d argument%s.' % (self.command_name, self.command_spec.min_args, tail_str)) else: message = ('The %s command accepts at most %d arguments.' % (self.command_name, self.command_spec.max_args)) message += ' Usage:\n%s\nFor additional help run:\n gsutil help %s' % ( self.command_spec.usage_synopsis, self.command_name) raise CommandException(message) def RaiseInvalidArgumentException(self): """Raises exception for specifying an invalid argument to command.""" message = ('Incorrect option(s) specified. Usage:\n%s\n' 'For additional help run:\n gsutil help %s' % (self.command_spec.usage_synopsis, self.command_name)) raise CommandException(message) def ParseSubOpts(self, check_args=False, args=None, should_update_sub_opts_and_args=True): """Parses sub-opt args. Args: check_args: True to have CheckArguments() called after parsing. args: List of args. If None, self.args will be used. should_update_sub_opts_and_args: True if self.sub_opts and self.args should be updated with the values returned after parsing. Else return a tuple of sub_opts, args returned by getopt.getopt. This is done to allow this method to be called from get_gcloud_storage_args in which case we do not want to update self.sub_opts and self.args. Raises: RaiseInvalidArgumentException: Invalid args specified. """ if args is None: unparsed_args = self.args else: unparsed_args = args try: parsed_sub_opts, parsed_args = getopt.getopt( unparsed_args, self.command_spec.supported_sub_args, self.command_spec.supported_private_args or []) except getopt.GetoptError: self.RaiseInvalidArgumentException() if should_update_sub_opts_and_args: self.sub_opts, self.args = parsed_sub_opts, parsed_args if check_args: self.CheckArguments() else: if check_args: # This is just for sanity check. Only get_gcloud_storage_args will # call this method with should_update_sub_opts_and_args=False, and it # does not set check_args to True. raise TypeError('Requested to check arguments' ' but sub_opts and args have not been updated.') return parsed_sub_opts, parsed_args def CheckArguments(self): """Checks that command line arguments match the command_spec. Any commands in self._commands_with_subcommands_and_subopts are responsible for calling this method after handling initial parsing of their arguments. This prevents commands with sub-commands as well as options from breaking the parsing of getopt. TODO: Provide a function to parse commands and sub-commands more intelligently once we stop allowing the deprecated command versions. Raises: CommandException if the arguments don't match. """ if (not self.command_spec.file_url_ok and HaveFileUrls(self.args[self.command_spec.urls_start_arg:])): raise CommandException('"%s" command does not support "file://" URLs. ' 'Did you mean to use a gs:// URL?' % self.command_name) if (not self.command_spec.provider_url_ok and HaveProviderUrls(self.args[self.command_spec.urls_start_arg:])): raise CommandException('"%s" command does not support provider-only ' 'URLs.' % self.command_name) def WildcardIterator(self, url_string, all_versions=False): """Helper to instantiate gslib.WildcardIterator. Args are same as gslib.WildcardIterator interface, but this method fills in most of the values from instance state. Args: url_string: URL string naming wildcard objects to iterate. all_versions: If true, the iterator yields all versions of objects matching the wildcard. If false, yields just the live object version. Returns: WildcardIterator for use by caller. """ return CreateWildcardIterator(url_string, self.gsutil_api, all_versions=all_versions, project_id=self.project_id, logger=self.logger) def GetSeekAheadGsutilApi(self): """Helper to instantiate a Cloud API instance for a seek-ahead iterator. This must be separate from the core command.gsutil_api instance for thread-safety, since other iterators typically use that instance and the SeekAheadIterator operates in parallel. Returns: Cloud API instance for use by the seek-ahead iterator. """ # This is initialized in Initialize(Multiprocessing|Threading)Variables # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global glob_status_queue # pylint: enable=global-variable-not-assigned # pylint: enable=global-variable-undefined if not self.seek_ahead_gsutil_api: self.seek_ahead_gsutil_api = CloudApiDelegator( self.bucket_storage_uri_class, self.gsutil_api_map, logging.getLogger('dummy'), glob_status_queue, debug=self.debug, http_headers=self.non_metadata_headers, trace_token=self.trace_token, perf_trace_token=self.perf_trace_token, user_project=self.user_project) return self.seek_ahead_gsutil_api def RunCommand(self): """Abstract function in base class. Subclasses must implement this. The return value of this function will be used as the exit status of the process, so subclass commands should return an integer exit code (0 for success, a value in [1,255] for failure). """ raise CommandException('Command %s is missing its RunCommand() ' 'implementation' % self.command_name) ############################################################ # Shared helper functions that depend on base class state. # ############################################################ # TODO: Refactor ACL functions to a different module and pass the # command object as state, as opposed to defining them as member functions # of the command class. def ApplyAclFunc(self, acl_func, acl_excep_handler, url_strs, object_fields=None): """Sets the standard or default object ACL depending on self.command_name. Args: acl_func: ACL function to be passed to Apply. acl_excep_handler: ACL exception handler to be passed to Apply. url_strs: URL strings on which to set ACL. object_fields: If present, list of object metadata fields to retrieve; if None, default name expansion iterator fields will be used. Raises: CommandException if an ACL could not be set. """ multi_threaded_url_args = [] urls = list(map(StorageUrlFromString, url_strs)) if (UrlsAreMixOfBucketsAndObjects(urls) and not self.recursion_requested): raise CommandException('Cannot operate on a mix of buckets and objects.') # Handle bucket ACL setting operations single-threaded, because # our threading machinery currently assumes it's working with objects # (name_expansion_iterator), and normally we wouldn't expect users to need # to set ACLs on huge numbers of buckets at once anyway. for url in urls: if url.IsCloudUrl() and url.IsBucket(): if self.recursion_requested: # If user specified -R option, convert any bucket args to bucket # wildcards (e.g., gs://bucket/*), to prevent the operation from # being applied to the buckets themselves. url.object_name = '*' multi_threaded_url_args.append(url.url_string) else: # Convert to a NameExpansionResult so we can re-use the threaded # function for the single-threaded implementation. RefType is unused. for blr in self.WildcardIterator( url.url_string).IterBuckets(bucket_fields=['id']): name_expansion_for_url = NameExpansionResult( source_storage_url=url, is_multi_source_request=False, is_multi_top_level_source_request=False, names_container=False, expanded_storage_url=blr.storage_url, expanded_result=None) acl_func(self, name_expansion_for_url) else: multi_threaded_url_args.append(url.url_string) if len(multi_threaded_url_args) >= 1: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, multi_threaded_url_args, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=object_fields) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), multi_threaded_url_args, self.recursion_requested, all_versions=self.all_versions) # Perform requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(acl_func, name_expansion_iterator, acl_excep_handler, fail_on_error=not self.continue_on_error, seek_ahead_iterator=seek_ahead_iterator) if not self.everything_set_okay and not self.continue_on_error: raise CommandException('ACLs for some objects could not be set.') def SetAclFunc(self, name_expansion_result, thread_state=None): """Sets the object ACL for the name_expansion_result provided. Args: name_expansion_result: NameExpansionResult describing the target object. thread_state: If present, use this gsutil Cloud API instance for the set. """ if thread_state: assert not self.def_acl gsutil_api = thread_state else: gsutil_api = self.gsutil_api op_string = 'default object ACL' if self.def_acl else 'ACL' url = name_expansion_result.expanded_storage_url self.logger.info('Setting %s on %s...', op_string, url) if (gsutil_api.GetApiSelector(url.scheme) == ApiSelector.XML and url.scheme != 'gs'): # If we are called with a non-google ACL model, we need to use the XML # passthrough. acl_arg should either be a canned ACL or an XML ACL. self._SetAclXmlPassthrough(url, gsutil_api) else: # Normal Cloud API path. acl_arg is a JSON ACL or a canned ACL. self._SetAclGsutilApi(url, gsutil_api) PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) def _SetAclXmlPassthrough(self, url, gsutil_api): """Sets the ACL for the URL provided using the XML passthrough functions. This function assumes that self.def_acl, self.canned, and self.continue_on_error are initialized, and that self.acl_arg is either an XML string or a canned ACL string. Args: url: CloudURL to set the ACL on. gsutil_api: gsutil Cloud API to use for the ACL set. Must support XML passthrough functions. """ orig_prefer_api = gsutil_api.prefer_api try: gsutil_api.prefer_api = ApiSelector.XML gsutil_api.XmlPassThroughSetAcl(self.acl_arg, url, canned=self.canned, def_obj_acl=self.def_acl, provider=url.scheme) except ServiceException as e: if self.continue_on_error: self.everything_set_okay = False self.logger.error(e) else: raise finally: gsutil_api.prefer_api = orig_prefer_api def _SetAclGsutilApi(self, url, gsutil_api): """Sets the ACL for the URL provided using the gsutil Cloud API. This function assumes that self.def_acl, self.canned, and self.continue_on_error are initialized, and that self.acl_arg is either a JSON string or a canned ACL string. Args: url: CloudURL to set the ACL on. gsutil_api: gsutil Cloud API to use for the ACL set. """ try: if url.IsBucket(): if self.def_acl: if self.canned: gsutil_api.PatchBucket(url.bucket_name, apitools_messages.Bucket(), canned_def_acl=self.acl_arg, provider=url.scheme, fields=['id']) else: def_obj_acl = AclTranslation.JsonToMessage( self.acl_arg, apitools_messages.ObjectAccessControl) if not def_obj_acl: # Use a sentinel value to indicate a private (no entries) default # object ACL. def_obj_acl.append(PRIVATE_DEFAULT_OBJ_ACL) bucket_metadata = apitools_messages.Bucket( defaultObjectAcl=def_obj_acl) gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) else: if self.canned: gsutil_api.PatchBucket(url.bucket_name, apitools_messages.Bucket(), canned_acl=self.acl_arg, provider=url.scheme, fields=['id']) else: bucket_acl = AclTranslation.JsonToMessage( self.acl_arg, apitools_messages.BucketAccessControl) bucket_metadata = apitools_messages.Bucket(acl=bucket_acl) gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) else: # url.IsObject() if self.canned: gsutil_api.PatchObjectMetadata(url.bucket_name, url.object_name, apitools_messages.Object(), provider=url.scheme, generation=url.generation, canned_acl=self.acl_arg) else: object_acl = AclTranslation.JsonToMessage( self.acl_arg, apitools_messages.ObjectAccessControl) object_metadata = apitools_messages.Object(acl=object_acl) gsutil_api.PatchObjectMetadata(url.bucket_name, url.object_name, object_metadata, provider=url.scheme, generation=url.generation) except ArgumentException as e: raise except ServiceException as e: if self.continue_on_error: self.everything_set_okay = False self.logger.error(e) else: raise def SetAclCommandHelper(self, acl_func, acl_excep_handler): """Sets ACLs on the self.args using the passed-in acl function. Args: acl_func: ACL function to be passed to Apply. acl_excep_handler: ACL exception handler to be passed to Apply. """ acl_arg = self.args[0] url_args = self.args[1:] # Disallow multi-provider setacl requests, because there are differences in # the ACL models. if not UrlsAreForSingleProvider(url_args): raise CommandException('"%s" command spanning providers not allowed.' % self.command_name) # Determine whether acl_arg names a file containing XML ACL text vs. the # string name of a canned ACL. if os.path.isfile(acl_arg): with codecs.open(acl_arg, 'r', UTF8) as f: acl_arg = f.read() self.canned = False else: # No file exists, so expect a canned ACL string. # validate=False because we allow wildcard urls. storage_uri = boto.storage_uri( url_args[0], debug=self.debug, validate=False, bucket_storage_uri_class=self.bucket_storage_uri_class) canned_acls = storage_uri.canned_acls() if acl_arg not in canned_acls: raise CommandException('Invalid canned ACL "%s".' % acl_arg) self.canned = True # Used to track if any ACLs failed to be set. self.everything_set_okay = True self.acl_arg = acl_arg self.ApplyAclFunc(acl_func, acl_excep_handler, url_args) if not self.everything_set_okay and not self.continue_on_error: raise CommandException('ACLs for some objects could not be set.') def _WarnServiceAccounts(self): """Warns service account users who have received an AccessDenied error. When one of the metadata-related commands fails due to AccessDenied, user must ensure that they are listed as an Owner in the API console. """ # Import this here so that the value will be set first in # gcs_oauth2_boto_plugin. # pylint: disable=g-import-not-at-top from gcs_oauth2_boto_plugin.oauth2_plugin import IS_SERVICE_ACCOUNT if IS_SERVICE_ACCOUNT: # This method is only called when canned ACLs are used, so the warning # definitely applies. self.logger.warning('\n'.join( textwrap.wrap( 'It appears that your service account has been denied access while ' 'attempting to perform a metadata operation. If you believe that you ' 'should have access to this metadata (i.e., if it is associated with ' 'your account), please make sure that your service account' 's email ' 'address is listed as an Owner in the Permissions tab of the API ' 'console. See "gsutil help creds" for further information.\n'))) def GetAndPrintAcl(self, url_str): """Prints the standard or default object ACL depending on self.command_name. Args: url_str: URL string to get ACL for. """ blr = self.GetAclCommandBucketListingReference(url_str) url = StorageUrlFromString(url_str) if (self.gsutil_api.GetApiSelector(url.scheme) == ApiSelector.XML and url.scheme != 'gs'): # Need to use XML passthrough. try: acl = self.gsutil_api.XmlPassThroughGetAcl(url, def_obj_acl=self.def_acl, provider=url.scheme) print(acl.to_xml()) except AccessDeniedException as _: self._WarnServiceAccounts() raise else: if self.command_name == 'defacl': acl = blr.root_object.defaultObjectAcl if not acl: self.logger.warn( 'No default object ACL present for %s. This could occur if ' 'the default object ACL is private, in which case objects ' 'created in this bucket will be readable only by their ' 'creators. It could also mean you do not have OWNER permission ' 'on %s and therefore do not have permission to read the ' 'default object ACL. It could also mean that %s has Bucket ' 'Policy Only enabled and therefore object ACLs and default ' 'object ACLs are disabled (see ' 'https://cloud.google.com/storage/docs/bucket-policy-only).', url_str, url_str, url_str) else: acl = blr.root_object.acl # Use the access controls api to check if the acl is actually empty or # if the user has 403 access denied or 400 invalid argument. if not acl: self._ListAccessControlsAcl(url) print(AclTranslation.JsonFromMessage(acl)) def _ListAccessControlsAcl(self, storage_url): """Returns either bucket or object access controls for a storage url. Args: storage_url: StorageUrl object representing the bucket or object. Returns: BucketAccessControls, ObjectAccessControls, or None if storage_url does not represent a cloud bucket or cloud object. Raises: ServiceException if there was an error in the request. """ if storage_url.IsBucket(): return self.gsutil_api.ListBucketAccessControls( storage_url.bucket_name, provider=storage_url.scheme) elif storage_url.IsObject(): return self.gsutil_api.ListObjectAccessControls( storage_url.bucket_name, storage_url.object_name, provider=storage_url.scheme) else: return None def GetAclCommandBucketListingReference(self, url_str): """Gets a single bucket listing reference for an acl get command. Args: url_str: URL string to get the bucket listing reference for. Returns: BucketListingReference for the URL string. Raises: CommandException if string did not result in exactly one reference. """ # We're guaranteed by caller that we have the appropriate type of url # string for the call (ex. we will never be called with an object string # by getdefacl) wildcard_url = StorageUrlFromString(url_str) if wildcard_url.IsObject(): plurality_iter = PluralityCheckableIterator( self.WildcardIterator(url_str).IterObjects( bucket_listing_fields=['acl'])) else: # Bucket or provider. We call IterBuckets explicitly here to ensure that # the root object is populated with the acl. if self.command_name == 'defacl': bucket_fields = ['defaultObjectAcl'] else: bucket_fields = ['acl'] plurality_iter = PluralityCheckableIterator( self.WildcardIterator(url_str).IterBuckets( bucket_fields=bucket_fields)) if plurality_iter.IsEmpty(): raise CommandException('No URLs matched') if plurality_iter.HasPlurality(): raise CommandException( '%s matched more than one URL, which is not allowed by the %s ' 'command' % (url_str, self.command_name)) return list(plurality_iter)[0] def GetSingleBucketUrlFromArg(self, arg, bucket_fields=None): """Gets a single bucket URL based on the command arguments. Args: arg: String argument to get bucket URL for. bucket_fields: Fields to populate for the bucket. Returns: (StorageUrl referring to a single bucket, Bucket metadata). Raises: CommandException if args did not match exactly one bucket. """ plurality_checkable_iterator = self.GetBucketUrlIterFromArg( arg, bucket_fields=bucket_fields) if plurality_checkable_iterator.HasPlurality(): raise CommandException('%s matched more than one URL, which is not\n' 'allowed by the %s command' % (arg, self.command_name)) blr = list(plurality_checkable_iterator)[0] return StorageUrlFromString(blr.url_string), blr.root_object def GetBucketUrlIterFromArg(self, arg, bucket_fields=None): """Gets a single bucket URL based on the command arguments. Args: arg: String argument to iterate over. bucket_fields: Fields to populate for the bucket. Returns: PluralityCheckableIterator over buckets. Raises: CommandException if iterator matched no buckets. """ arg_url = StorageUrlFromString(arg) if not arg_url.IsCloudUrl() or arg_url.IsObject(): raise CommandException('"%s" command must specify a bucket' % self.command_name) plurality_checkable_iterator = PluralityCheckableIterator( self.WildcardIterator(arg).IterBuckets(bucket_fields=bucket_fields)) if plurality_checkable_iterator.IsEmpty(): raise CommandException('No URLs matched') return plurality_checkable_iterator ###################### # Private functions. # ###################### def _ResetConnectionPool(self): # Each OS process needs to establish its own set of connections to # the server to avoid writes from different OS processes interleaving # onto the same socket (and garbling the underlying SSL session). # We ensure each process gets its own set of connections here by # reinitializing state that tracks connections. connection_pool = StorageUri.provider_pool if connection_pool: for i in connection_pool: connection_pool[i].connection.close() StorageUri.provider_pool = {} StorageUri.connection = None def _GetProcessAndThreadCount(self, process_count, thread_count, parallel_operations_override, print_macos_warning=True): """Determines the values of process_count and thread_count. These values are used for parallel operations. If we're not performing operations in parallel, then ignore existing values and use process_count = thread_count = 1. Args: process_count: A positive integer or None. In the latter case, we read the value from the .boto config file. thread_count: A positive integer or None. In the latter case, we read the value from the .boto config file. parallel_operations_override: Used to override self.parallel_operations. This allows the caller to safely override the top-level flag for a single call. print_macos_warning: Print a warning about parallel processing on MacOS if true. Returns: (process_count, thread_count): The number of processes and threads to use, respectively. """ # Set OS process and python thread count as a function of options # and config. if self.parallel_operations or parallel_operations_override: if not process_count: process_count = boto.config.getint( 'GSUtil', 'parallel_process_count', gslib.commands.config.DEFAULT_PARALLEL_PROCESS_COUNT) if process_count < 1: raise CommandException('Invalid parallel_process_count "%d".' % process_count) if not thread_count: thread_count = boto.config.getint( 'GSUtil', 'parallel_thread_count', gslib.commands.config.DEFAULT_PARALLEL_THREAD_COUNT) if thread_count < 1: raise CommandException('Invalid parallel_thread_count "%d".' % thread_count) else: # If -m not specified, then assume 1 OS process and 1 Python thread. process_count = 1 thread_count = 1 should_prohibit_multiprocessing, os_name = ShouldProhibitMultiprocessing() if should_prohibit_multiprocessing and process_count > 1: raise CommandException('\n'.join( textwrap.wrap( ('It is not possible to set process_count > 1 on %s. Please ' 'update your config file(s) (located at %s) and set ' '"parallel_process_count = 1".') % (os_name, ', '.join(GetFriendlyConfigFilePaths()))))) is_main_thread = self.recursive_apply_level == 0 if print_macos_warning and os_name == 'macOS' and process_count > 1 and is_main_thread: self.logger.info( 'If you experience problems with multiprocessing on MacOS, they ' 'might be related to https://bugs.python.org/issue33725. You can ' 'disable multiprocessing by editing your .boto config or by adding ' 'the following flag to your command: ' '`-o "GSUtil:parallel_process_count=1"`. Note that multithreading is ' 'still available even if you disable multiprocessing.\n') self.logger.debug('process count: %d', process_count) self.logger.debug('thread count: %d', thread_count) return (process_count, thread_count) def _SetUpPerCallerState(self): """Set up the state for a caller id, corresponding to one Apply call.""" # pylint: disable=global-variable-undefined,global-variable-not-assigned # These variables are initialized in InitializeMultiprocessingVariables or # InitializeThreadingVariables global global_return_values_map, shared_vars_map, failure_count global caller_id_finished_count, shared_vars_list_map, total_tasks global need_pool_or_done_cond, call_completed_map, class_map global task_queues, caller_id_lock, caller_id_counter # Get a new caller ID. with caller_id_lock: caller_id_counter.Increment() caller_id = caller_id_counter.GetValue() # Create a copy of self with an incremented recursive level. This allows # the class to report its level correctly if the function called from it # also needs to call Apply. cls = copy.copy(self) cls.recursive_apply_level += 1 # Thread-safe loggers can't be pickled, so we will remove it here and # recreate it later in the WorkerThread. This is not a problem since any # logger with the same name will be treated as a singleton. cls.logger = None # Likewise, the default API connection(s) can't be pickled, but are unused # anyway as each thread gets its own API delegator. cls.gsutil_api = None cls.seek_ahead_gsutil_api = None class_map[caller_id] = cls total_tasks[caller_id] = -1 # -1 => the producer hasn't finished yet. call_completed_map[caller_id] = False caller_id_finished_count[caller_id] = 0 global_return_values_map[caller_id] = [] return caller_id def _CreateNewConsumerPool(self, num_processes, num_threads, status_queue): """Create a new pool of processes that call _ApplyThreads.""" processes = [] task_queue = _NewMultiprocessingQueue() task_queues.append(task_queue) current_max_recursive_level.Increment() if current_max_recursive_level.GetValue() > MAX_RECURSIVE_DEPTH: raise CommandException('Recursion depth of Apply calls is too great.') for _ in range(num_processes): recursive_apply_level = len(consumer_pools) p = multiprocessing_context.Process(target=self._ApplyThreads, args=(num_threads, num_processes, recursive_apply_level, status_queue)) p.daemon = True processes.append(p) _CryptoRandomAtFork() p.start() consumer_pool = _ConsumerPool(processes, task_queue) consumer_pools.append(consumer_pool) class ParallelOverrideReason(object): """Enum class to describe purpose of overriding parallel operations.""" # For the case when we use slice parallelism. SLICE = 'slice' # For the case when we run a helper Apply call (such as in the _DiffIterator # of rsync) and override to make the command go faster. SPEED = 'speed' # For when we run Apply calls in perfdiag. PERFDIAG = 'perfdiag' def Apply(self, func, args_iterator, exception_handler, shared_attrs=None, arg_checker=_UrlArgChecker, parallel_operations_override=None, process_count=None, thread_count=None, should_return_results=False, fail_on_error=False, seek_ahead_iterator=None): """Calls _Parallel/SequentialApply based on multiprocessing availability. Args: func: Function to call to process each argument. args_iterator: Iterable collection of arguments to be put into the work queue. exception_handler: Exception handler for WorkerThread class. shared_attrs: List of attributes to manage across sub-processes. arg_checker: Used to determine whether we should process the current argument or simply skip it. Also handles any logging that is specific to a particular type of argument. parallel_operations_override: A string (see ParallelOverrideReason) describing the reason to override self.parallel_operations. This allows the caller to safely override the top-level flag for a single call. process_count: The number of processes to use. If not specified, then the configured default will be used. thread_count: The number of threads per process. If not specified, then the configured default will be used.. should_return_results: If true, then return the results of all successful calls to func in a list. fail_on_error: If true, then raise any exceptions encountered when executing func. This is only applicable in the case of process_count == thread_count == 1. seek_ahead_iterator: If present, a seek-ahead iterator that will provide an approximation of the total number of tasks and bytes that will be iterated by the ProducerThread. Used only if multiple processes and/or threads are used. Returns: Results from spawned threads. """ # This is initialized in Initialize(Multiprocessing|Threading)Variables # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global thread_stats # pylint: enable=global-variable-not-assigned # pylint: enable=global-variable-undefined if shared_attrs: original_shared_vars_values = {} # We'll add these back in at the end. for name in shared_attrs: original_shared_vars_values[name] = getattr(self, name) # By setting this to 0, we simplify the logic for computing deltas. # We'll add it back after all of the tasks have been performed. setattr(self, name, 0) (process_count, thread_count) = self._GetProcessAndThreadCount( process_count, thread_count, parallel_operations_override) is_main_thread = (self.recursive_apply_level == 0 and self.sequential_caller_id == -1) if is_main_thread: # This initializes the initial performance summary parameters. LogPerformanceSummaryParams(num_processes=process_count, num_threads=thread_count) # We don't honor the fail_on_error flag in the case of multiple threads # or processes. fail_on_error = fail_on_error and (process_count * thread_count == 1) # Only check this from the first call in the main thread. Apart from the # fact that it's wasteful to try this multiple times in general, it also # will never work when called from a subprocess since we use daemon # processes, and daemons can't create other processes. if (is_main_thread and not self.multiprocessing_is_available and process_count > 1): # Run the check again and log the appropriate warnings. This was run # before, when the Command object was created, in order to calculate # self.multiprocessing_is_available, but we don't want to print the # warning until we're sure the user actually tried to use multiple # threads or processes. CheckMultiprocessingAvailableAndInit(logger=self.logger) caller_id = self._SetUpPerCallerState() # If any shared attributes passed by caller, create a dictionary of # shared memory variables for every element in the list of shared # attributes. if shared_attrs: shared_vars_list_map[caller_id] = shared_attrs for name in shared_attrs: shared_vars_map[(caller_id, name)] = 0 # Make all of the requested function calls. usable_processes_count = (process_count if self.multiprocessing_is_available else 1) if thread_count * usable_processes_count > 1: self._ParallelApply( func, args_iterator, exception_handler, caller_id, arg_checker, usable_processes_count, thread_count, should_return_results, fail_on_error, seek_ahead_iterator=seek_ahead_iterator, parallel_operations_override=parallel_operations_override) if is_main_thread: _AggregateThreadStats() else: self._SequentialApply(func, args_iterator, exception_handler, caller_id, arg_checker, should_return_results, fail_on_error) if shared_attrs: for name in shared_attrs: # This allows us to retain the original value of the shared variable, # and simply apply the delta after what was done during the call to # apply. final_value = (original_shared_vars_values[name] + shared_vars_map.get( (caller_id, name))) setattr(self, name, final_value) if should_return_results: return global_return_values_map.get(caller_id) def _MaybeSuggestGsutilDashM(self): """Outputs a suggestion to the user to use gsutil -m.""" if not (boto.config.getint('GSUtil', 'parallel_process_count', 0) == 1 and boto.config.getint('GSUtil', 'parallel_thread_count', 0) == 1): self.logger.info('\n' + textwrap.fill( '==> NOTE: You are performing a sequence of gsutil operations that ' 'may run significantly faster if you instead use gsutil -m %s ...\n' 'Please see the -m section under "gsutil help options" for further ' 'information about when gsutil -m can be advantageous.' % self.command_spec.command_name) + '\n') # pylint: disable=g-doc-args def _SequentialApply(self, func, args_iterator, exception_handler, caller_id, arg_checker, should_return_results, fail_on_error): """Performs all function calls sequentially in the current thread. No other threads or processes will be spawned. This degraded functionality is used when the multiprocessing module is not available or the user requests only one thread and one process. """ # Create a WorkerThread to handle all of the logic needed to actually call # the function. Note that this thread will never be started, and all work # is done in the current thread. worker_thread = WorkerThread(None, False, headers=self.non_metadata_headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) args_iterator = iter(args_iterator) # Count of sequential calls that have been made. Used for producing # suggestion to use gsutil -m. sequential_call_count = 0 while True: # Try to get the next argument, handling any exceptions that arise. try: args = next(args_iterator) except StopIteration as e: break except Exception as e: # pylint: disable=broad-except _IncrementFailureCount() if fail_on_error: raise else: try: exception_handler(self, e) except Exception as _: # pylint: disable=broad-except self.logger.debug( 'Caught exception while handling exception for %s:\n%s', func, traceback.format_exc()) continue sequential_call_count += 1 if (sequential_call_count == OFFER_GSUTIL_M_SUGGESTION_THRESHOLD or sequential_call_count % OFFER_GSUTIL_M_SUGGESTION_FREQUENCY == 0): # Output suggestion near beginning of run, so user sees it early, and # every so often while the command is executing, so they can ^C and try # gsutil -m. self._MaybeSuggestGsutilDashM() if arg_checker(self, args): # Now that we actually have the next argument, perform the task. task = Task(func, args, caller_id, exception_handler, should_return_results, arg_checker, fail_on_error) worker_thread.PerformTask(task, self) lines_since_suggestion_last_printed = (sequential_call_count % OFFER_GSUTIL_M_SUGGESTION_FREQUENCY) if lines_since_suggestion_last_printed >= GetTermLines(): # Output suggestion at end of long run, in case user missed it at the # start and it scrolled off-screen. self._MaybeSuggestGsutilDashM() PutToQueueWithTimeout(self.gsutil_api.status_queue, FinalMessage(time.time())) # If the final iterated argument results in an exception, and that # exception modifies shared_attrs, we need to publish the results. worker_thread.shared_vars_updater.Update(caller_id, self) # Now that all the work is done, log the types of source URLs encountered. self._ProcessSourceUrlTypes(args_iterator) # pylint: disable=g-doc-args def _ParallelApply(self, func, args_iterator, exception_handler, caller_id, arg_checker, process_count, thread_count, should_return_results, fail_on_error, seek_ahead_iterator=None, parallel_operations_override=None): r"""Dispatches input arguments across a thread/process pool. Pools are composed of parallel OS processes and/or Python threads, based on options (-m or not) and settings in the user's config file. If only one OS process is requested/available, dispatch requests across threads in the current OS process. In the multi-process case, we will create one pool of worker processes for each level of the tree of recursive calls to Apply. E.g., if A calls Apply(B), and B ultimately calls Apply(C) followed by Apply(D), then we will only create two sets of worker processes - B will execute in the first, and C and D will execute in the second. If C is then changed to call Apply(E) and D is changed to call Apply(F), then we will automatically create a third set of processes (lazily, when needed) that will be used to execute calls to E and F. This might look something like: Pool1 Executes: B / \ Pool2 Executes: C D / \ Pool3 Executes: E F Apply's parallelism is generally broken up into 4 cases: - If process_count == thread_count == 1, then all tasks will be executed by _SequentialApply. - If process_count > 1 and thread_count == 1, then the main thread will create a new pool of processes (if they don't already exist) and each of those processes will execute the tasks in a single thread. - If process_count == 1 and thread_count > 1, then this process will create a new pool of threads to execute the tasks. - If process_count > 1 and thread_count > 1, then the main thread will create a new pool of processes (if they don't already exist) and each of those processes will, upon creation, create a pool of threads to execute the tasks. Args: caller_id: The caller ID unique to this call to command.Apply. See command.Apply for description of other arguments. """ # This is initialized in Initialize(Multiprocessing|Threading)Variables # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global glob_status_queue, ui_controller # pylint: enable=global-variable-not-assigned # pylint: enable=global-variable-undefined is_main_thread = self.recursive_apply_level == 0 if (parallel_operations_override == self.ParallelOverrideReason.SLICE and self.recursive_apply_level <= 1): # The operation uses slice parallelism if the recursive apply level > 0 or # if we're executing _ParallelApply without the -m option. glob_status_queue.put(PerformanceSummaryMessage(time.time(), True)) if not IS_WINDOWS and is_main_thread: # For multi-thread or multi-process scenarios, the main process must # kill itself on a terminating signal, because sys.exit(1) only exits # the currently executing thread, leaving orphaned processes. The main # thread is responsible for cleaning up multiprocessing variables such # as manager processes. Therefore, the main thread's signal handling # chain is: # 1: __main__._CleanupSignalHandler (clean up processes) # 2: MultithreadedSignalHandler (kill self) for signal_num in (signal.SIGINT, signal.SIGTERM): RegisterSignalHandler(signal_num, MultithreadedMainSignalHandler, is_final_handler=True) if not task_queues: # The process we create will need to access the next recursive level # of task queues if it makes a call to Apply, so we always keep around # one more queue than we know we need. OTOH, if we don't create a new # process, the existing process still needs a task queue to use. if process_count > 1: task_queues.append(_NewMultiprocessingQueue()) else: task_queue = _NewThreadsafeQueue() task_queues.append(task_queue) # Create a top-level worker pool since this is the first execution # of ParallelApply on the main thread. WorkerPool(thread_count, self.logger, task_queue=task_queue, bucket_storage_uri_class=self.bucket_storage_uri_class, gsutil_api_map=self.gsutil_api_map, debug=self.debug, status_queue=glob_status_queue, headers=self.non_metadata_headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) if process_count > 1: # Handle process pool creation. # Check whether this call will need a new set of workers. # Each worker must acquire a shared lock before notifying the main thread # that it needs a new worker pool, so that at most one worker asks for # a new worker pool at once. try: if not is_main_thread: worker_checking_level_lock.acquire() if self.recursive_apply_level >= current_max_recursive_level.GetValue(): with need_pool_or_done_cond: # Only the main thread is allowed to create new processes - # otherwise, we will run into some Python bugs. if is_main_thread: self._CreateNewConsumerPool(process_count, thread_count, glob_status_queue) else: # Notify the main thread that we need a new consumer pool. new_pool_needed.Reset(reset_value=1) need_pool_or_done_cond.notify_all() # The main thread will notify us when it finishes. need_pool_or_done_cond.wait() finally: if not is_main_thread: worker_checking_level_lock.release() else: # Handle new worker thread pool creation. if not is_main_thread: try: worker_checking_level_lock.acquire() if self.recursive_apply_level > _GetCurrentMaxRecursiveLevel(): # We don't have a thread pool for this level of recursive apply # calls, so create a pool and corresponding task queue. _IncrementCurrentMaxRecursiveLevel() task_queue = _NewThreadsafeQueue() task_queues.append(task_queue) WorkerPool(thread_count, self.logger, task_queue=task_queue, bucket_storage_uri_class=self.bucket_storage_uri_class, gsutil_api_map=self.gsutil_api_map, debug=self.debug, status_queue=glob_status_queue, headers=self.non_metadata_headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) finally: worker_checking_level_lock.release() task_queue = task_queues[self.recursive_apply_level] # Only use the seek-ahead iterator in the main thread to provide an # overall estimate of operations. if seek_ahead_iterator and not is_main_thread: seek_ahead_iterator = None # Kick off a producer thread to throw tasks in the global task queue. We # do this asynchronously so that the main thread can be free to create new # consumer pools when needed (otherwise, any thread with a task that needs # a new consumer pool must block until we're completely done producing; in # the worst case, every worker blocks on such a call and the producer fills # up the task queue before it finishes, so we block forever). producer_thread = ProducerThread( copy.copy(self), args_iterator, caller_id, func, task_queue, should_return_results, exception_handler, arg_checker, fail_on_error, seek_ahead_iterator=seek_ahead_iterator, status_queue=(glob_status_queue if is_main_thread else None)) # Start the UI thread that is responsible for displaying operation status # (aggregated across processes and threads) to the user. ui_thread = None if is_main_thread: ui_thread = UIThread(glob_status_queue, sys.stderr, ui_controller) # Wait here until either: # 1. We're the main thread in the multi-process case, and someone needs # a new consumer pool - in which case we create one and continue # waiting. # 2. Someone notifies us that all of the work we requested is done, in # which case we retrieve the results (if applicable) and stop # waiting. # At most one of these can be true, because the main thread is blocked on # its call to Apply, and a thread will not ask for a new consumer pool # unless it had more work to do. while True: with need_pool_or_done_cond: if call_completed_map[caller_id]: break elif (process_count > 1 and is_main_thread and new_pool_needed.GetValue()): new_pool_needed.Reset() self._CreateNewConsumerPool(process_count, thread_count, glob_status_queue) need_pool_or_done_cond.notify_all() # Note that we must check the above conditions before the wait() call; # otherwise, the notification can happen before we start waiting, in # which case we'll block forever. need_pool_or_done_cond.wait() # We've completed all tasks (or excepted), so signal the UI thread to # terminate. if is_main_thread: PutToQueueWithTimeout(glob_status_queue, ZERO_TASKS_TO_DO_ARGUMENT) ui_thread.join(timeout=UI_THREAD_JOIN_TIMEOUT) # Now that all the work is done, log the types of source URLs encountered. self._ProcessSourceUrlTypes(producer_thread.args_iterator) # We encountered an exception from the producer thread before any arguments # were enqueued, but it wouldn't have been propagated, so we'll now # explicitly raise it here. if producer_thread.unknown_exception: # pylint: disable=raising-bad-type raise producer_thread.unknown_exception # We encountered an exception from the producer thread while iterating over # the arguments, so raise it here if we're meant to fail on error. if producer_thread.iterator_exception and fail_on_error: # pylint: disable=raising-bad-type raise producer_thread.iterator_exception if is_main_thread and not parallel_operations_override: PutToQueueWithTimeout(glob_status_queue, FinalMessage(time.time())) def _ProcessSourceUrlTypes(self, args_iterator): """Logs the URL type information to analytics collection.""" if not isinstance(args_iterator, CopyObjectsIterator): return LogPerformanceSummaryParams(is_daisy_chain=args_iterator.is_daisy_chain, has_file_src=args_iterator.has_file_src, has_cloud_src=args_iterator.has_cloud_src, provider_types=args_iterator.provider_types) def _ApplyThreads(self, thread_count, process_count, recursive_apply_level, status_queue): """Assigns the work from the multi-process global task queue. Work is assigned to an individual process for later consumption either by the WorkerThreads or (if thread_count == 1) this thread. Args: thread_count: The number of threads used to perform the work. If 1, then perform all work in this thread. process_count: The number of processes used to perform the work. recursive_apply_level: The depth in the tree of recursive calls to Apply of this thread. status_queue: Multiprocessing/threading queue for progress reporting and performance aggregation. """ assert process_count > 1, ( 'Invalid state, calling command._ApplyThreads with only one process.') _CryptoRandomAtFork() # Separate processes should exit on a terminating signal, # but to avoid race conditions only the main process should handle # multiprocessing cleanup. Override child processes to use a single signal # handler. for catch_signal in GetCaughtSignals(): signal.signal(catch_signal, ChildProcessSignalHandler) self._ResetConnectionPool() self.recursive_apply_level = recursive_apply_level task_queue = task_queues[recursive_apply_level] # Ensure fairness across processes by filling our WorkerPool # only with as many tasks as it has WorkerThreads. This semaphore is # acquired each time that a task is retrieved from the queue and released # each time a task is completed by a WorkerThread. worker_semaphore = threading.BoundedSemaphore(thread_count) # TODO: Presently, this pool gets recreated with each call to Apply. We # should be able to do it just once, at process creation time. worker_pool = WorkerPool( thread_count, self.logger, worker_semaphore=worker_semaphore, bucket_storage_uri_class=self.bucket_storage_uri_class, gsutil_api_map=self.gsutil_api_map, debug=self.debug, status_queue=status_queue, headers=self.non_metadata_headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) num_enqueued = 0 while True: while not worker_semaphore.acquire(blocking=False): # Because Python signal handlers are only called in between atomic # instructions, if we block the main thread on an available worker # thread, we won't be able to respond to signals such as a # user-initiated CTRL-C until a worker thread completes a task. # We poll the semaphore periodically as a compromise between # efficiency and user responsiveness. time.sleep(0.01) task = task_queue.get() if task.args != ZERO_TASKS_TO_DO_ARGUMENT: # If we have no tasks to do and we're performing a blocking call, we # need a special signal to tell us to stop - otherwise, we block on # the call to task_queue.get() forever. worker_pool.AddTask(task) num_enqueued += 1 else: # No tasks remain; since no work was dispatched to a thread, don't # block the semaphore on a WorkerThread completion. worker_semaphore.release() # Below here lie classes and functions related to controlling the flow of tasks # between various threads and processes. class _ConsumerPool(object): def __init__(self, processes, task_queue): self.processes = processes self.task_queue = task_queue def ShutDown(self): for process in self.processes: KillProcess(process.pid) class Task( namedtuple('Task', ( 'func args caller_id exception_handler should_return_results arg_checker ' 'fail_on_error'))): """Task class representing work to be completed. Args: func: The function to be executed. args: The arguments to func. caller_id: The globally-unique caller ID corresponding to the Apply call. exception_handler: The exception handler to use if the call to func fails. should_return_results: True iff the results of this function should be returned from the Apply call. arg_checker: Used to determine whether we should process the current argument or simply skip it. Also handles any logging that is specific to a particular type of argument. fail_on_error: If true, then raise any exceptions encountered when executing func. This is only applicable in the case of process_count == thread_count == 1. """ pass # TODO: Refactor the various threading code that doesn't need to depend on # command.py globals (ProducerThread, UIThread) to different files to aid # readability and reduce the size of command.py. def _StartSeekAheadThread(seek_ahead_iterator, seek_ahead_thread_cancel_event): """Initializes and runs the seek-ahead thread. We defer starting this thread until it is needed, since it is only useful when the ProducerThread iterates more results than it can store on the global task queue. Args: seek_ahead_iterator: Iterator that yields SeekAheadResults. seek_ahead_thread_cancel_event: threading.Event for signaling the seek-ahead thread to terminate. Returns: The thread object for the initialized thread. """ # This is initialized in Initialize(Multiprocessing|Threading)Variables # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global glob_status_queue # pylint: enable=global-variable-not-assigned # pylint: enable=global-variable-undefined return SeekAheadThread(seek_ahead_iterator, seek_ahead_thread_cancel_event, glob_status_queue) class ProducerThread(threading.Thread): """Thread used to enqueue work for other processes and threads.""" def __init__(self, cls, args_iterator, caller_id, func, task_queue, should_return_results, exception_handler, arg_checker, fail_on_error, seek_ahead_iterator=None, status_queue=None): """Initializes the producer thread. Args: cls: Instance of Command for which this ProducerThread was created. args_iterator: Iterable collection of arguments to be put into the work queue. caller_id: Globally-unique caller ID corresponding to this call to Apply. func: The function to be called on each element of args_iterator. task_queue: The queue into which tasks will be put, to later be consumed by Command._ApplyThreads. should_return_results: True iff the results for this call to command.Apply were requested. exception_handler: The exception handler to use when errors are encountered during calls to func. arg_checker: Used to determine whether we should process the current argument or simply skip it. Also handles any logging that is specific to a particular type of argument. fail_on_error: If true, then raise any exceptions encountered when executing func. This is only applicable in the case of process_count == thread_count == 1. seek_ahead_iterator: If present, a seek-ahead iterator that will provide an approximation of the total number of tasks and bytes that will be iterated by the ProducerThread. status_queue: status_queue to inform task_queue estimation. Only valid when calling from the main thread, else None. Even if this is the main thread, the status_queue will only properly work if args is a collection of NameExpansionResults, which is the type that gives us initial information about files to be processed. Otherwise, nothing will be added to the queue. """ super(ProducerThread, self).__init__() self.func = func self.cls = cls self.args_iterator = args_iterator self.caller_id = caller_id self.task_queue = task_queue self.arg_checker = arg_checker self.exception_handler = exception_handler self.should_return_results = should_return_results self.fail_on_error = fail_on_error self.shared_variables_updater = _SharedVariablesUpdater() self.daemon = True self.unknown_exception = None self.iterator_exception = None self.seek_ahead_iterator = seek_ahead_iterator self.status_queue = status_queue self.start() def run(self): num_tasks = 0 cur_task = None last_task = None task_estimation_threshold = None seek_ahead_thread = None seek_ahead_thread_cancel_event = None seek_ahead_thread_considered = False args = None try: total_size = 0 self.args_iterator = iter(self.args_iterator) while True: try: args = next(self.args_iterator) except StopIteration as e: break except Exception as e: # pylint: disable=broad-except _IncrementFailureCount() if self.fail_on_error: self.iterator_exception = e raise else: try: self.exception_handler(self.cls, e) except Exception as _: # pylint: disable=broad-except self.cls.logger.debug( 'Caught exception while handling exception for %s:\n%s', self.func, traceback.format_exc()) self.shared_variables_updater.Update(self.caller_id, self.cls) continue if self.arg_checker(self.cls, args): num_tasks += 1 if self.status_queue: if not num_tasks % 100: # Time to update the total number of tasks. if (isinstance(args, NameExpansionResult) or isinstance(args, CopyObjectInfo) or isinstance(args, RsyncDiffToApply)): PutToQueueWithTimeout( self.status_queue, ProducerThreadMessage(num_tasks, total_size, time.time())) if (isinstance(args, NameExpansionResult) or isinstance(args, CopyObjectInfo)): if args.expanded_result: json_expanded_result = json.loads(args.expanded_result) if 'size' in json_expanded_result: total_size += int(json_expanded_result['size']) elif isinstance(args, RsyncDiffToApply): if args.copy_size: total_size += int(args.copy_size) if not seek_ahead_thread_considered: if task_estimation_threshold is None: task_estimation_threshold = _GetTaskEstimationThreshold() if task_estimation_threshold <= 0: # Disable the seek-ahead thread (never start it). seek_ahead_thread_considered = True elif num_tasks >= task_estimation_threshold: if self.seek_ahead_iterator: seek_ahead_thread_cancel_event = threading.Event() seek_ahead_thread = _StartSeekAheadThread( self.seek_ahead_iterator, seek_ahead_thread_cancel_event) # For integration testing only, force estimation to complete # prior to producing further results. if boto.config.get('GSUtil', 'task_estimation_force', None): seek_ahead_thread.join(timeout=SEEK_AHEAD_JOIN_TIMEOUT) seek_ahead_thread_considered = True last_task = cur_task cur_task = Task(self.func, args, self.caller_id, self.exception_handler, self.should_return_results, self.arg_checker, self.fail_on_error) if last_task: self.task_queue.put(last_task) except Exception as e: # pylint: disable=broad-except # This will also catch any exception raised due to an error in the # iterator when fail_on_error is set, so check that we failed for some # other reason before claiming that we had an unknown exception. if not self.iterator_exception: self.unknown_exception = e finally: # We need to make sure to update total_tasks[caller_id] before we enqueue # the last task. Otherwise, a worker can retrieve the last task and # complete it, then check total_tasks and determine that we're not done # producing all before we update total_tasks. This approach forces workers # to wait on the last task until after we've updated total_tasks. total_tasks[self.caller_id] = num_tasks if not cur_task: # This happens if there were zero arguments to be put in the queue. cur_task = Task(None, ZERO_TASKS_TO_DO_ARGUMENT, self.caller_id, None, None, None, None) self.task_queue.put(cur_task) # If the seek ahead thread is still running, cancel it and wait for it # to exit since we've enumerated all of the tasks already. We don't want # to delay command completion on an estimate that has become meaningless. if seek_ahead_thread is not None: seek_ahead_thread_cancel_event.set() # It's possible that the seek-ahead-thread may attempt to put to the # status queue after it has been torn down, for example if the system # is overloaded. Because the put uses a timeout, it should never block # command termination or signal handling. seek_ahead_thread.join(timeout=SEEK_AHEAD_JOIN_TIMEOUT) # Send a final ProducerThread message that definitively states # the amount of actual work performed. if (self.status_queue and (isinstance(args, NameExpansionResult) or isinstance( args, CopyObjectInfo) or isinstance(args, RsyncDiffToApply))): PutToQueueWithTimeout( self.status_queue, ProducerThreadMessage(num_tasks, total_size, time.time(), finished=True)) # It's possible that the workers finished before we updated total_tasks, # so we need to check here as well. _NotifyIfDone(self.caller_id, caller_id_finished_count.get(self.caller_id)) class WorkerPool(object): """Pool of worker threads to which tasks can be added.""" def __init__(self, thread_count, logger, worker_semaphore=None, task_queue=None, bucket_storage_uri_class=None, gsutil_api_map=None, debug=0, status_queue=None, headers=None, perf_trace_token=None, trace_token=None, user_project=None): # In the multi-process case, a worker sempahore is required to ensure # even work distribution. # # In the single process case, the input task queue directly feeds worker # threads from the ProducerThread. Since worker threads will consume only # one task at a time from the queue, there is no need for a semaphore to # ensure even work distribution. # # Thus, exactly one of task_queue or worker_semaphore must be provided. assert (worker_semaphore is None) != (task_queue is None) self.headers = headers self.perf_trace_token = perf_trace_token self.trace_token = trace_token self.user_project = user_project self.task_queue = task_queue or _NewThreadsafeQueue() self.threads = [] for _ in range(thread_count): worker_thread = WorkerThread( self.task_queue, logger, worker_semaphore=worker_semaphore, bucket_storage_uri_class=bucket_storage_uri_class, gsutil_api_map=gsutil_api_map, debug=debug, status_queue=status_queue, headers=self.headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) self.threads.append(worker_thread) worker_thread.start() def AddTask(self, task): """Adds a task to the task queue; used only in the multi-process case.""" self.task_queue.put(task) class WorkerThread(threading.Thread): """Thread where all the work will be performed. This makes the function calls for Apply and takes care of all error handling, return value propagation, and shared_vars. Note that this thread is NOT started upon instantiation because the function- calling logic is also used in the single-threaded case. """ # This is initialized in Initialize(Multiprocessing|Threading)Variables # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global thread_stats # pylint: enable=global-variable-not-assigned # pylint: enable=global-variable-undefined def __init__(self, task_queue, logger, worker_semaphore=None, bucket_storage_uri_class=None, gsutil_api_map=None, debug=0, status_queue=None, headers=None, perf_trace_token=None, trace_token=None, user_project=None): """Initializes the worker thread. Args: task_queue: The thread-safe queue from which this thread should obtain its work. logger: Logger to use for this thread. worker_semaphore: threading.BoundedSemaphore to be released each time a task is completed, or None for single-threaded execution. bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. gsutil_api_map: Map of providers and API selector tuples to api classes which can be used to communicate with those providers. Used for the instantiating CloudApiDelegator class. debug: debug level for the CloudApiDelegator class. status_queue: Queue for reporting status updates. user_project: Project to be billed for this request. """ super(WorkerThread, self).__init__() self.pid = os.getpid() self.init_time = time.time() self.task_queue = task_queue self.worker_semaphore = worker_semaphore self.daemon = True self.cached_classes = {} self.shared_vars_updater = _SharedVariablesUpdater() self.headers = headers self.perf_trace_token = perf_trace_token self.trace_token = trace_token self.user_project = user_project # Note that thread_gsutil_api is not initialized in the sequential # case; task functions should use utils.cloud_api_helper.GetCloudApiInstance # to retrieve the main thread's CloudApiDelegator in that case. self.thread_gsutil_api = None if bucket_storage_uri_class and gsutil_api_map: self.thread_gsutil_api = CloudApiDelegator( bucket_storage_uri_class, gsutil_api_map, logger, status_queue, debug=debug, http_headers=self.headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) @CaptureThreadStatException def _StartBlockedTime(self): """Update the thread_stats AtomicDict before task_queue.get() is called.""" if thread_stats.get((self.pid, self.ident)) is None: thread_stats[(self.pid, self.ident)] = _ThreadStat(self.init_time) # While this read/modify/write is not an atomic operation on the dict, # we are protected since the (process ID, thread ID) tuple is unique # to this thread, making this thread the only reader/writer for this key. thread_stat = thread_stats[(self.pid, self.ident)] thread_stat.StartBlockedTime() thread_stats[(self.pid, self.ident)] = thread_stat @CaptureThreadStatException def _EndBlockedTime(self): """Update the thread_stats AtomicDict after task_queue.get() is called.""" thread_stat = thread_stats[(self.pid, self.ident)] thread_stat.EndBlockedTime() thread_stats[(self.pid, self.ident)] = thread_stat def PerformTask(self, task, cls): """Makes the function call for a task. Args: task: The Task to perform. cls: The instance of a class which gives context to the functions called by the Task's function. E.g., see SetAclFuncWrapper. """ caller_id = task.caller_id try: results = task.func(cls, task.args, thread_state=self.thread_gsutil_api) if task.should_return_results: global_return_values_map.Increment(caller_id, [results], default_value=[]) except Exception as e: # pylint: disable=broad-except _IncrementFailureCount() if task.fail_on_error: raise # Only happens for single thread and process case. else: try: task.exception_handler(cls, e) except Exception as _: # pylint: disable=broad-except # Don't allow callers to raise exceptions here and kill the worker # threads. cls.logger.debug( 'Caught exception while handling exception for %s:\n%s', task, traceback.format_exc()) finally: if self.worker_semaphore: self.worker_semaphore.release() self.shared_vars_updater.Update(caller_id, cls) # Even if we encounter an exception, we still need to claim that that # the function finished executing. Otherwise, we won't know when to # stop waiting and return results. num_done = caller_id_finished_count.Increment(caller_id, 1) _NotifyIfDone(caller_id, num_done) def run(self): while True: self._StartBlockedTime() task = self.task_queue.get() self._EndBlockedTime() if task.args == ZERO_TASKS_TO_DO_ARGUMENT: # This can happen in the single-process case because worker threads # consume ProducerThread tasks directly. continue caller_id = task.caller_id # Get the instance of the command with the appropriate context. cls = self.cached_classes.get(caller_id, None) if not cls: cls = copy.copy(class_map[caller_id]) cls.logger = CreateOrGetGsutilLogger(cls.command_name) self.cached_classes[caller_id] = cls self.PerformTask(task, cls) class _ThreadStat(object): """Stores thread idle and execution time statistics.""" def __init__(self, init_time): self.total_idle_time = 0 # The last time EndBlockedTime was called, which is the last time a # task_queue.get() completed or when we initialized the thread. self.end_block_time = init_time # The last time StartBlockedTime was called, which is the last time a # task_queue.get() call started. self.start_block_time = time.time() # Between now and thread initialization, we were not blocked. self.total_execution_time = 0 def StartBlockedTime(self): self.start_block_time = time.time() exec_time = self.start_block_time - self.end_block_time self.total_execution_time += exec_time def EndBlockedTime(self): self.end_block_time = time.time() idle_time = self.end_block_time - self.start_block_time self.total_idle_time += idle_time def AggregateStat(self, end_time): """Decide final stats upon Apply completion.""" if self.end_block_time > self.start_block_time: # Apply ended before we blocked on task_queue.get(), or there was an # exception during StartBlockedTime. In both of these cases, we were not # blocked on task_queue.get() and so can add this time to execution time. self.total_execution_time += end_time - self.end_block_time else: # Apply ended while we were blocked on task_queue.get(), or there was an # exception during EndBlockedTime. In both of these cases, we were in the # midst of or just finishing a task_queue.get() call, and so can add this # time to idle time. self.total_idle_time += end_time - self.start_block_time def _AggregateThreadStats(): """At the end of the top-level Apply call, aggregate the thread stats dict. This should only be called in the main process and thread because it logs to the MetricsCollector. """ cur_time = time.time() total_idle_time = total_execution_time = 0 for thread_stat in thread_stats.values(): thread_stat.AggregateStat(cur_time) total_idle_time += thread_stat.total_idle_time total_execution_time += thread_stat.total_execution_time LogPerformanceSummaryParams(thread_idle_time=total_idle_time, thread_execution_time=total_execution_time) class _SharedVariablesUpdater(object): """Used to update shared variable for a class in the global map. Note that each thread will have its own instance of the calling class for context, and it will also have its own instance of a _SharedVariablesUpdater. This is used in the following way: 1. Before any tasks are performed, each thread will get a copy of the calling class, and the globally-consistent value of this shared variable will be initialized to whatever it was before the call to Apply began. 2. After each time a thread performs a task, it will look at the current values of the shared variables in its instance of the calling class. 2.A. For each such variable, it computes the delta of this variable between the last known value for this class (which is stored in a dict local to this class) and the current value of the variable in the class. 2.B. Using this delta, we update the last known value locally as well as the globally-consistent value shared across all classes (the globally consistent value is simply increased by the computed delta). """ def __init__(self): self.last_shared_var_values = {} def Update(self, caller_id, cls): """Update any shared variables with their deltas.""" shared_vars = shared_vars_list_map.get(caller_id, None) if shared_vars: for name in shared_vars: key = (caller_id, name) last_value = self.last_shared_var_values.get(key, 0) # Compute the change made since the last time we updated here. This is # calculated by simply subtracting the last known value from the current # value in the class instance. delta = getattr(cls, name) - last_value self.last_shared_var_values[key] = delta + last_value # Update the globally-consistent value by simply increasing it by the # computed delta. shared_vars_map.Increment(key, delta) def _NotifyIfDone(caller_id, num_done): """Notify any threads waiting for results that something has finished. Each waiting thread will then need to check the call_completed_map to see if its work is done. Note that num_done could be calculated here, but it is passed in as an optimization so that we have one less call to a globally-locked data structure. Args: caller_id: The caller_id of the function whose progress we're checking. num_done: The number of tasks currently completed for that caller_id. """ num_to_do = total_tasks[caller_id] if num_to_do == num_done and num_to_do >= 0: # Notify the Apply call that's sleeping that it's ready to return. with need_pool_or_done_cond: call_completed_map[caller_id] = True need_pool_or_done_cond.notify_all() # pylint: disable=global-variable-not-assigned,global-variable-undefined def ShutDownGsutil(): """Shut down all processes in consumer pools in preparation for exiting.""" global glob_status_queue for q in queues: try: q.cancel_join_thread() except: # pylint: disable=bare-except pass for consumer_pool in consumer_pools: consumer_pool.ShutDown() try: glob_status_queue.cancel_join_thread() except: # pylint: disable=bare-except pass def _GetCurrentMaxRecursiveLevel(): global current_max_recursive_level return current_max_recursive_level.GetValue() def _IncrementCurrentMaxRecursiveLevel(): global current_max_recursive_level current_max_recursive_level.Increment() def _IncrementFailureCount(): global failure_count failure_count.Increment() def DecrementFailureCount(): global failure_count failure_count.Decrement() def GetFailureCount(): """Returns the number of failures processed during calls to Apply().""" global failure_count return failure_count.GetValue() def ResetFailureCount(): """Resets the failure_count variable to 0 - useful if error is expected.""" global failure_count failure_count.Reset() no_op_credentials.py 0000644 00000002043 15025013616 0010603 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """No-op implementation of credentials for JSON HTTP requests.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class NoOpCredentials(object): def __init__(self): pass def authorize(self, http_obj): # pylint: disable=invalid-name return http_obj def set_store(self, store): # pylint: disable=invalid-name pass metrics_tuple.py 0000644 00000002677 15025013616 0010010 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Defines a Metric object used for sending information to Google Analytics.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import namedtuple # A Metric contains all of the information required to post a GA event. It is # not a custom metric, which is a GA term for a specific number to track on the # Analytics dashboard. This is not nested within MetricsCollector to allow # pickle to dump a list of Metrics. Metric = namedtuple( 'Metric', [ # The URL of the request endpoint. 'endpoint', # The HTTP method of request. 'method', # The URL-encoded body to send with the request. 'body', # The user-agent string to send as a header. 'user_agent', ]) wildcard_iterator.py 0000644 00000113347 15025013616 0010630 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2010 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Wildcard iterator class and supporting functions.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import fnmatch import glob import logging import os import re import textwrap import six from gslib.bucket_listing_ref import BucketListingBucket from gslib.bucket_listing_ref import BucketListingObject from gslib.bucket_listing_ref import BucketListingPrefix from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import CloudApi from gslib.cloud_api import NotFoundException from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import GenerationFromUrlAndString from gslib.storage_url import StorageUrlFromString from gslib.storage_url import StripOneSlash from gslib.storage_url import WILDCARD_REGEX from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import UTF8 from gslib.utils.text_util import FixWindowsEncodingIfNeeded from gslib.utils.text_util import PrintableStr if six.PY3: # StandardError was removed, so use the base exception type instead StandardError = Exception FLAT_LIST_REGEX = re.compile(r'(?P<before>.*?)\*\*(?P<after>.*)') _UNICODE_EXCEPTION_TEXT = ( 'Invalid Unicode path encountered (%s). gsutil cannot proceed ' 'with such files present. Please remove or rename this file and ' 'try again. NOTE: the path printed above replaces the ' 'problematic characters with a hex-encoded printable ' 'representation. For more details (including how to convert to a ' 'gsutil-compatible encoding) see `gsutil help encoding`.') class WildcardIterator(object): """Class for iterating over Google Cloud Storage strings containing wildcards. The base class is abstract; you should instantiate using the wildcard_iterator() static factory method, which chooses the right implementation depending on the base string. """ # TODO: Standardize on __str__ and __repr__ here and elsewhere. Define both # and make one return the other. def __repr__(self): """Returns string representation of WildcardIterator.""" return 'WildcardIterator(%s)' % self.wildcard_url.url_string class CloudWildcardIterator(WildcardIterator): """WildcardIterator subclass for buckets, bucket subdirs and objects. Iterates over BucketListingRef matching the Url string wildcard. It's much more efficient to first get metadata that's available in the Bucket (for example to get the name and size of each object), because that information is available in the object list results. """ def __init__(self, wildcard_url, gsutil_api, all_versions=False, project_id=None, logger=None): """Instantiates an iterator that matches the wildcard URL. Args: wildcard_url: CloudUrl that contains the wildcard to iterate. gsutil_api: Cloud storage interface. Passed in for thread safety, also settable for testing/mocking. all_versions: If true, the iterator yields all versions of objects matching the wildcard. If false, yields just the live object version. project_id: Project ID to use for bucket listings. logger: logging.Logger used for outputting debug messages during iteration. If None, the root logger will be used. """ self.wildcard_url = wildcard_url self.all_versions = all_versions self.gsutil_api = gsutil_api self.project_id = project_id self.logger = logger or logging.getLogger() def __iter__(self, bucket_listing_fields=None, expand_top_level_buckets=False): """Iterator that gets called when iterating over the cloud wildcard. In the case where no wildcard is present, returns a single matching object, single matching prefix, or one of each if both exist. Args: bucket_listing_fields: Iterable fields to include in bucket listings. Ex. ['name', 'acl']. Iterator is responsible for converting these to list-style format ['items/name', 'items/acl'] as well as adding any fields necessary for listing such as prefixes. API implementation is responsible for adding pagination fields. If this is None, all fields are returned. expand_top_level_buckets: If true, yield no BUCKET references. Instead, expand buckets into top-level objects and prefixes. Yields: BucketListingRef of type BUCKET, OBJECT or PREFIX. """ single_version_request = self.wildcard_url.HasGeneration() # For wildcard expansion purposes, we need at a minimum the name of # each object and prefix. If we're not using the default of requesting # all fields, make sure at least these are requested. The Cloud API # tolerates specifying the same field twice. get_fields = None if bucket_listing_fields: get_fields = set() for field in bucket_listing_fields: get_fields.add(field) bucket_listing_fields = self._GetToListFields( get_fields=bucket_listing_fields) bucket_listing_fields.update(['items/name', 'prefixes']) get_fields.update(['name']) # If we're making versioned requests, ensure generation and # metageneration are also included. if single_version_request or self.all_versions: bucket_listing_fields.update( ['items/generation', 'items/metageneration']) get_fields.update(['generation', 'metageneration']) # Handle bucket wildcarding, if any, in _ExpandBucketWildcards. Then # iterate over the expanded bucket strings and handle any object # wildcarding. for bucket_listing_ref in self._ExpandBucketWildcards(bucket_fields=['id']): bucket_url_string = bucket_listing_ref.url_string if self.wildcard_url.IsBucket(): # IsBucket() guarantees there are no prefix or object wildcards, and # thus this is a top-level listing of buckets. if expand_top_level_buckets: url = StorageUrlFromString(bucket_url_string) for obj_or_prefix in self.gsutil_api.ListObjects( url.bucket_name, delimiter='/', all_versions=self.all_versions, provider=self.wildcard_url.scheme, fields=bucket_listing_fields): if obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT: yield self._GetObjectRef(bucket_url_string, obj_or_prefix.data, with_version=self.all_versions) else: # CloudApi.CsObjectOrPrefixType.PREFIX: yield self._GetPrefixRef(bucket_url_string, obj_or_prefix.data) else: yield bucket_listing_ref else: # By default, assume a non-wildcarded URL is an object, not a prefix. # This prevents unnecessary listings (which are slower, more expensive, # and also subject to eventual consistency). if (not ContainsWildcard(self.wildcard_url.url_string) and self.wildcard_url.IsObject() and not self.all_versions): try: get_object = self.gsutil_api.GetObjectMetadata( self.wildcard_url.bucket_name, self.wildcard_url.object_name, generation=self.wildcard_url.generation, provider=self.wildcard_url.scheme, fields=get_fields) yield self._GetObjectRef(self.wildcard_url.bucket_url_string, get_object, with_version=(self.all_versions or single_version_request)) return except (NotFoundException, AccessDeniedException): # It's possible this is a prefix - try to list instead. pass # Expand iteratively by building prefix/delimiter bucket listing # request, filtering the results per the current level's wildcard # (if present), and continuing with the next component of the # wildcard. See _BuildBucketFilterStrings() documentation for details. if single_version_request: url_string = '%s%s#%s' % (bucket_url_string, self.wildcard_url.object_name, self.wildcard_url.generation) else: # Rstrip any prefixes to correspond with rstripped prefix wildcard # from _BuildBucketFilterStrings(). url_string = '%s%s' % ( bucket_url_string, StripOneSlash(self.wildcard_url.object_name) or '/') # Cover root object named '/' case. urls_needing_expansion = [url_string] while urls_needing_expansion: url = StorageUrlFromString(urls_needing_expansion.pop(0)) (prefix, delimiter, prefix_wildcard, suffix_wildcard) = (self._BuildBucketFilterStrings(url.object_name)) regex_patterns = self._GetRegexPatterns(prefix_wildcard) # If we have a suffix wildcard, we only care about listing prefixes. listing_fields = (set(['prefixes']) if suffix_wildcard else bucket_listing_fields) # List bucket for objects matching prefix up to delimiter. for obj_or_prefix in self.gsutil_api.ListObjects( url.bucket_name, prefix=prefix, delimiter=delimiter, all_versions=self.all_versions or single_version_request, provider=self.wildcard_url.scheme, fields=listing_fields): for pattern in regex_patterns: if obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT: gcs_object = obj_or_prefix.data if pattern.match(gcs_object.name): if not suffix_wildcard or (StripOneSlash(gcs_object.name) == suffix_wildcard): if not single_version_request or ( self._SingleVersionMatches(gcs_object.generation)): yield self._GetObjectRef( bucket_url_string, gcs_object, with_version=(self.all_versions or single_version_request)) break else: # CloudApi.CsObjectOrPrefixType.PREFIX prefix = obj_or_prefix.data if ContainsWildcard(prefix): # TODO: Disambiguate user-supplied strings from iterated # prefix and object names so that we can better reason # about wildcards and handle this case without raising # an error. raise CommandException( 'Cloud folder %s%s contains a wildcard; gsutil does ' 'not currently support objects with wildcards in their ' 'name.' % (bucket_url_string, prefix)) # If the prefix ends with a slash, remove it. Note that we only # remove one slash so that we can successfully enumerate dirs # containing multiple slashes. rstripped_prefix = StripOneSlash(prefix) if pattern.match(rstripped_prefix): if suffix_wildcard and rstripped_prefix != suffix_wildcard: # There's more wildcard left to expand. url_append_string = '%s%s' % (bucket_url_string, rstripped_prefix + '/' + suffix_wildcard) urls_needing_expansion.append(url_append_string) else: # No wildcard to expand, just yield the prefix. yield self._GetPrefixRef(bucket_url_string, prefix) break def _GetRegexPatterns(self, wildcard_pattern): """Returns list of regex patterns derived from the wildcard patterns. Args: wildcard_pattern (str): A wilcard_pattern to filter the resources. Returns: List of compiled regex patterns. This translates the wildcard_pattern and also creates some additional patterns so that we can treat ** in a/b/c/**/d.txt as zero or more folders. This means, a/b/c/d.txt will also be returned along with a/b/c/e/f/d.txt. """ # Case 1: The original pattern should always be present. wildcard_patterns = [wildcard_pattern] if '/**/' in wildcard_pattern: # Case 2: Will fetch object gs://bucket/dir1/a.txt if pattern is # gs://bucket/dir1/**/a.txt updated_pattern = wildcard_pattern.replace('/**/', '/') wildcard_patterns.append(updated_pattern) else: updated_pattern = wildcard_pattern for pattern in (wildcard_pattern, updated_pattern): if pattern.startswith('**/'): # Case 3 (using wildcard_pattern): Will fetch object gs://bucket/a.txt # if pattern is gs://bucket/**/a.txt. Note that '/**/' will match # '/a.txt' not 'a.txt'. # Case 4:(using updated_pattern) Will fetch gs://bucket/dir1/dir2/a.txt # if the pattern is gs://bucket/**/dir1/**/a.txt wildcard_patterns.append(pattern[3:]) return [re.compile(fnmatch.translate(p)) for p in wildcard_patterns] def _BuildBucketFilterStrings(self, wildcard): """Builds strings needed for querying a bucket and filtering results. This implements wildcard object name matching. Args: wildcard: The wildcard string to match to objects. Returns: (prefix, delimiter, prefix_wildcard, suffix_wildcard) where: prefix is the prefix to be sent in bucket GET request. delimiter is the delimiter to be sent in bucket GET request. prefix_wildcard is the wildcard to be used to filter bucket GET results. suffix_wildcard is wildcard to be appended to filtered bucket GET results for next wildcard expansion iteration. For example, given the wildcard gs://bucket/abc/d*e/f*.txt we would build prefix= abc/d, delimiter=/, prefix_wildcard=d*e, and suffix_wildcard=f*.txt. Using this prefix and delimiter for a bucket listing request will then produce a listing result set that can be filtered using this prefix_wildcard; and we'd use this suffix_wildcard to feed into the next call(s) to _BuildBucketFilterStrings(), for the next iteration of listing/filtering. Raises: AssertionError if wildcard doesn't contain any wildcard chars. """ # Generate a request prefix if the object name part of the wildcard starts # with a non-wildcard string (e.g., that's true for 'gs://bucket/abc*xyz'). match = WILDCARD_REGEX.search(wildcard) if not match: # Input "wildcard" has no wildcard chars, so just return tuple that will # cause a bucket listing to match the given input wildcard. Example: if # previous iteration yielded gs://bucket/dir/ with suffix_wildcard abc, # the next iteration will call _BuildBucketFilterStrings() with # gs://bucket/dir/abc, and we will return prefix ='dir/abc', # delimiter='/', prefix_wildcard='dir/abc', and suffix_wildcard=''. prefix = wildcard delimiter = '/' prefix_wildcard = wildcard suffix_wildcard = '' else: if match.start() > 0: # Wildcard does not occur at beginning of object name, so construct a # prefix string to send to server. prefix = wildcard[:match.start()] wildcard_part = wildcard[match.start():] else: prefix = None wildcard_part = wildcard end = wildcard_part.find('/') if end != -1: wildcard_part = wildcard_part[:end + 1] prefix_wildcard = (prefix or '') + wildcard_part if not prefix_wildcard.endswith('**/'): # Remove trailing '/' so we will match gs://bucket/abc* as well as # gs://bucket/abc*/ with the same wildcard regex. Don't do this for # double wildcards because those won't pass a delimiter to the API. prefix_wildcard = StripOneSlash(prefix_wildcard) suffix_wildcard = wildcard[match.end():] end = suffix_wildcard.find('/') if end == -1: suffix_wildcard = '' else: suffix_wildcard = suffix_wildcard[end + 1:] # To implement recursive (**) wildcarding, if prefix_wildcard # suffix_wildcard starts with '**' don't send a delimiter, and combine # suffix_wildcard at end of prefix_wildcard. if prefix_wildcard.find('**') != -1: delimiter = None prefix_wildcard += suffix_wildcard suffix_wildcard = '' else: delimiter = '/' # The following debug output is useful for tracing how the algorithm # walks through a multi-part wildcard like gs://bucket/abc/d*e/f*.txt self.logger.debug( 'wildcard=%s, prefix=%s, delimiter=%s, ' 'prefix_wildcard=%s, suffix_wildcard=%s\n', PrintableStr(wildcard), PrintableStr(prefix), PrintableStr(delimiter), PrintableStr(prefix_wildcard), PrintableStr(suffix_wildcard)) return (prefix, delimiter, prefix_wildcard, suffix_wildcard) def _SingleVersionMatches(self, listed_generation): decoded_generation = GenerationFromUrlAndString(self.wildcard_url, listed_generation) return str(self.wildcard_url.generation) == str(decoded_generation) def _ExpandBucketWildcards(self, bucket_fields=None): """Expands bucket and provider wildcards. Builds a list of bucket url strings that can be iterated on. Args: bucket_fields: If present, populate only these metadata fields for buckets. Example value: ['acl', 'defaultObjectAcl'] Yields: BucketListingRefereneces of type BUCKET. """ bucket_url = StorageUrlFromString(self.wildcard_url.bucket_url_string) if (bucket_fields and set(bucket_fields) == set(['id']) and not ContainsWildcard(self.wildcard_url.bucket_name)): # If we just want the name of a non-wildcarded bucket URL, # don't make an RPC. yield BucketListingBucket(bucket_url) elif (self.wildcard_url.IsBucket() and not ContainsWildcard(self.wildcard_url.bucket_name)): # If we have a non-wildcarded bucket URL, get just that bucket. yield BucketListingBucket(bucket_url, root_object=self.gsutil_api.GetBucket( self.wildcard_url.bucket_name, provider=self.wildcard_url.scheme, fields=bucket_fields)) else: regex = fnmatch.translate(self.wildcard_url.bucket_name) prog = re.compile(regex) fields = self._GetToListFields(bucket_fields) if fields: fields.add('items/id') for bucket in self.gsutil_api.ListBuckets( fields=fields, project_id=self.project_id, provider=self.wildcard_url.scheme): if prog.match(bucket.id): url = StorageUrlFromString('%s://%s/' % (self.wildcard_url.scheme, bucket.id)) yield BucketListingBucket(url, root_object=bucket) def _GetToListFields(self, get_fields=None): """Prepends 'items/' to the input fields and converts it to a set. This way field sets requested for GetBucket can be used in ListBucket calls. Note that the input set must contain only bucket or object fields; listing fields such as prefixes or nextPageToken should be added after calling this function. Args: get_fields: Iterable fields usable in GetBucket/GetObject calls. Returns: Set of fields usable in ListBuckets/ListObjects calls. """ if get_fields: list_fields = set() for field in get_fields: list_fields.add('items/' + field) return list_fields def _GetObjectRef(self, bucket_url_string, gcs_object, with_version=False): """Creates a BucketListingRef of type OBJECT from the arguments. Args: bucket_url_string: Wildcardless string describing the containing bucket. gcs_object: gsutil_api root Object for populating the BucketListingRef. with_version: If true, return a reference with a versioned string. Returns: BucketListingRef of type OBJECT. """ # Generation can be None in test mocks, so just return the # live object for simplicity. if with_version and gcs_object.generation is not None: generation_str = GenerationFromUrlAndString(self.wildcard_url, gcs_object.generation) object_string = '%s%s#%s' % (bucket_url_string, gcs_object.name, generation_str) else: object_string = '%s%s' % (bucket_url_string, gcs_object.name) object_url = StorageUrlFromString(object_string) return BucketListingObject(object_url, root_object=gcs_object) def _GetPrefixRef(self, bucket_url_string, prefix): """Creates a BucketListingRef of type PREFIX from the arguments. Args: bucket_url_string: Wildcardless string describing the containing bucket. prefix: gsutil_api Prefix for populating the BucketListingRef Returns: BucketListingRef of type PREFIX. """ prefix_url = StorageUrlFromString('%s%s' % (bucket_url_string, prefix)) return BucketListingPrefix(prefix_url, root_object=prefix) def IterBuckets(self, bucket_fields=None): """Iterates over the wildcard, returning refs for each expanded bucket. This ignores the object part of the URL entirely and expands only the the bucket portion. It will yield BucketListingRefs of type BUCKET only. Args: bucket_fields: Iterable fields to include in bucket listings. Ex. ['defaultObjectAcl', 'logging']. This function is responsible for converting these to listing-style format ['items/defaultObjectAcl', 'items/logging'], as well as adding any fields necessary for listing such as 'items/id'. API implemenation is responsible for adding pagination fields. If this is None, all fields are returned. Yields: BucketListingRef of type BUCKET, or empty iterator if no matches. """ for blr in self._ExpandBucketWildcards(bucket_fields=bucket_fields): yield blr def IterAll(self, bucket_listing_fields=None, expand_top_level_buckets=False): """Iterates over the wildcard, yielding bucket, prefix or object refs. Args: bucket_listing_fields: If present, populate only these metadata fields for listed objects. expand_top_level_buckets: If true and the wildcard expands only to Bucket(s), yields the expansion of each bucket into a top-level listing of prefixes and objects in that bucket instead of a BucketListingRef to that bucket. Yields: BucketListingRef, or empty iterator if no matches. """ for blr in self.__iter__(bucket_listing_fields=bucket_listing_fields, expand_top_level_buckets=expand_top_level_buckets): yield blr def IterObjects(self, bucket_listing_fields=None): """Iterates over the wildcard, yielding only object BucketListingRefs. Args: bucket_listing_fields: If present, populate only these metadata fields for listed objects. Yields: BucketListingRefs of type OBJECT or empty iterator if no matches. """ for blr in self.__iter__(bucket_listing_fields=bucket_listing_fields, expand_top_level_buckets=True): if blr.IsObject(): yield blr def _GetFileObject(filepath): """Returns an apitools Object class with supported file attributes. To provide size estimates for local to cloud file copies, we need to retrieve expose the local file's size. Args: filepath: Path to the file. Returns: apitools Object that with file name and size attributes filled-in. """ # TODO: If we are preserving POSIX attributes, we could instead call # os.stat() here. return apitools_messages.Object(size=os.path.getsize(filepath)) class FileWildcardIterator(WildcardIterator): """WildcardIterator subclass for files and directories. If you use recursive wildcards ('**') only a single such wildcard is supported. For example you could use the wildcard '**/*.txt' to list all .txt files in any subdirectory of the current directory, but you couldn't use a wildcard like '**/abc/**/*.txt' (which would, if supported, let you find .txt files in any subdirectory named 'abc'). """ def __init__(self, wildcard_url, exclude_tuple=None, ignore_symlinks=False, logger=None): """Instantiates an iterator over BucketListingRefs matching wildcard URL. Args: wildcard_url: FileUrl that contains the wildcard to iterate. exclude_tuple: (base_url, exclude_pattern), where base_url is top-level URL to list; exclude_pattern is a regex of paths to ignore during iteration. ignore_symlinks: If True, ignore symlinks during iteration. logger: logging.Logger used for outputting debug messages during iteration. If None, the root logger will be used. """ self.wildcard_url = wildcard_url self.exclude_tuple = exclude_tuple self.ignore_symlinks = ignore_symlinks self.logger = logger or logging.getLogger() def __iter__(self, bucket_listing_fields=None): """Iterator that gets called when iterating over the file wildcard. In the case where no wildcard is present, returns a single matching file or directory. Args: bucket_listing_fields: Iterable fields to include in listings. Ex. ['size']. Currently only 'size' is supported. If present, will populate yielded BucketListingObject.root_object with the file name and size. Raises: WildcardException: if invalid wildcard found. Yields: BucketListingRef of type OBJECT (for files) or PREFIX (for directories) """ include_size = (bucket_listing_fields and 'size' in set(bucket_listing_fields)) wildcard = self.wildcard_url.object_name match = FLAT_LIST_REGEX.match(wildcard) if match: # Recursive wildcarding request ('.../**/...'). # Example input: wildcard = '/tmp/tmp2pQJAX/**/*' base_dir = match.group('before')[:-1] remaining_wildcard = match.group('after') # At this point for the above example base_dir = '/tmp/tmp2pQJAX' and # remaining_wildcard = '/*' if remaining_wildcard.startswith('*'): raise WildcardException('Invalid wildcard with more than 2 consecutive ' '*s (%s)' % wildcard) # If there was no remaining wildcard past the recursive wildcard, # treat it as if it were a '*'. For example, file://tmp/** is equivalent # to file://tmp/**/* if not remaining_wildcard: remaining_wildcard = '*' # Skip slash(es). remaining_wildcard = remaining_wildcard.lstrip(os.sep) filepaths = self._IterDir(base_dir, remaining_wildcard) else: # Not a recursive wildcarding request. filepaths = glob.iglob(wildcard) for filepath in filepaths: expanded_url = StorageUrlFromString(filepath) try: if self.ignore_symlinks and os.path.islink(filepath): if self.logger: self.logger.info('Skipping symbolic link %s...', filepath) continue if os.path.isdir(filepath): yield BucketListingPrefix(expanded_url) else: blr_object = _GetFileObject(filepath) if include_size else None yield BucketListingObject(expanded_url, root_object=blr_object) except UnicodeEncodeError: raise CommandException('\n'.join( textwrap.wrap(_UNICODE_EXCEPTION_TEXT % repr(filepath)))) def _IterDir(self, directory, wildcard): """An iterator over the specified dir and wildcard. Args: directory (unicode): The path of the directory to iterate over. wildcard (str): The wildcard characters used for filename pattern matching. Yields: (str) A string containing the path to a file somewhere under the directory hierarchy of `directory`. Raises: ComandException: If this method encounters a file path that it cannot decode as UTF-8. """ if os.path.splitdrive(directory)[0] == directory: # For Windows-style paths that consist of a drive letter followed by a # colon, os.path.join behaves in an odd manner. It intentionally will not # join ['c:' and 'foo'] as 'c:\\foo', but rather as 'c:foo'. The latter # format is not handled correctly by gsutil, so we check if the path # specifies the root of a volume, and if so, append a backslash so that # the resulting joined path looks like 'c:\\foo'. directory += '\\' # UTF8-encode directory before passing it to os.walk() so if there are # non-valid UTF8 chars in the file name (e.g., that can happen if the file # originated on Windows) os.walk() will not attempt to decode and then die # with a "codec can't decode byte" error, and instead we can catch the error # at yield time and print a more informative error message. for dirpath, dirnames, filenames in os.walk(six.ensure_text(directory), topdown=True): filtered_dirnames = [] for dirname in dirnames: full_dir_path = os.path.join(dirpath, dirname) # Removes directories in place to prevent them and their children from # being iterated. See https://docs.python.org/3/library/os.html#os.walk if not self._ExcludeDir(full_dir_path): filtered_dirnames.append(dirname) else: # If a symlink is excluded above we don't want to print 2 messages. continue # This only prints a log message as os.walk() will not, by default, # walk down into symbolic links that resolve to directories. if self.logger and os.path.islink(full_dir_path): self.logger.info('Skipping symlink directory "%s"', full_dir_path) dirnames[:] = filtered_dirnames for f in fnmatch.filter(filenames, wildcard): try: yield os.path.join(dirpath, FixWindowsEncodingIfNeeded(f)) except UnicodeDecodeError: # Note: We considered several ways to deal with this, but each had # problems: # 1. Raise an exception and try to catch in a higher layer (the # gsutil cp command), so we can properly support the gsutil cp -c # option. That doesn't work because raising an exception during # iteration terminates the generator. # 2. Accumulate a list of bad filenames and skip processing each # during iteration, then raise at the end, with exception text # printing the bad paths. That doesn't work because iteration is # wrapped in PluralityCheckableIterator, so it's possible there # are not-yet-performed copy operations at the time we reach the # end of the iteration and raise the exception - which would cause # us to skip copying validly named files. Moreover, the gsutil # cp command loops over argv, so if you run the command gsutil cp # -rc dir1 dir2 gs://bucket, an invalid unicode name inside dir1 # would cause dir2 never to be visited. # 3. Print the invalid pathname and skip it during iteration. That # would work but would mean gsutil cp could exit with status 0 # even though some files weren't copied. # 4. Change the WildcardIterator to include an error status along with # the result. That would solve the problem but would be a # substantial change (WildcardIterator is used in many parts of # gsutil), and we didn't feel that magnitude of change was # warranted by this relatively uncommon corner case. # Instead we chose to abort when one such file is encountered, and # require the user to remove or rename the files and try again. raise CommandException('\n'.join( textwrap.wrap(_UNICODE_EXCEPTION_TEXT % repr(os.path.join(dirpath, f))))) def _ExcludeDir(self, dir): """Check a directory to see if it should be excluded from os.walk. Args: dir: String representing the directory to check. Returns: True if the directory should be excluded. """ if self.exclude_tuple is None: return False (base_url, exclude_dirs, exclude_pattern) = self.exclude_tuple if not exclude_dirs: return False str_to_check = StorageUrlFromString( dir).url_string[len(base_url.url_string):] if str_to_check.startswith(self.wildcard_url.delim): str_to_check = str_to_check[1:] if exclude_pattern.match(str_to_check): if self.logger: self.logger.info('Skipping excluded directory %s...', dir) return True # pylint: disable=unused-argument def IterObjects(self, bucket_listing_fields=None): """Iterates over the wildcard, yielding only object (file) refs. Args: bucket_listing_fields: Iterable fields to include in listings. Ex. ['size']. Currently only 'size' is supported. If present, will populate yielded BucketListingObject.root_object with the file name and size. Yields: BucketListingRefs of type OBJECT or empty iterator if no matches. """ for bucket_listing_ref in self.IterAll( bucket_listing_fields=bucket_listing_fields): if bucket_listing_ref.IsObject(): yield bucket_listing_ref # pylint: disable=unused-argument def IterAll(self, bucket_listing_fields=None, expand_top_level_buckets=False): """Iterates over the wildcard, yielding BucketListingRefs. Args: bucket_listing_fields: Iterable fields to include in listings. Ex. ['size']. Currently only 'size' is supported. If present, will populate yielded BucketListingObject.root_object with the file name and size. expand_top_level_buckets: Ignored; filesystems don't have buckets. Yields: BucketListingRefs of type OBJECT (file) or PREFIX (directory), or empty iterator if no matches. """ for bucket_listing_ref in self.__iter__( bucket_listing_fields=bucket_listing_fields): yield bucket_listing_ref def IterBuckets(self, unused_bucket_fields=None): """Placeholder to allow polymorphic use of WildcardIterator. Args: unused_bucket_fields: Ignored; filesystems don't have buckets. Raises: WildcardException: in all cases. """ raise WildcardException( 'Iterating over Buckets not possible for file wildcards') class WildcardException(StandardError): """Exception raised for invalid wildcard URLs.""" def __init__(self, reason): StandardError.__init__(self) self.reason = reason def __repr__(self): return 'WildcardException: %s' % self.reason def __str__(self): return 'WildcardException: %s' % self.reason def CreateWildcardIterator(url_str, gsutil_api, all_versions=False, project_id=None, exclude_tuple=None, ignore_symlinks=False, logger=None): """Instantiate a WildcardIterator for the given URL string. Args: url_str: URL string naming wildcard object(s) to iterate. gsutil_api: Cloud storage interface. Passed in for thread safety, also settable for testing/mocking. all_versions: If true, the iterator yields all versions of objects matching the wildcard. If false, yields just the live object version. project_id: Project id to use for bucket listings. exclude_tuple: (base_url, exclude_pattern), where base_url is top-level URL to list; exclude_pattern is a regex of paths to ignore during iteration. ignore_symlinks: For FileUrls, ignore symlinks during iteration if true. logger: logging.Logger used for outputting debug messages during iteration. If None, the root logger will be used. Returns: A WildcardIterator that handles the requested iteration. """ url = StorageUrlFromString(url_str) logger = logger or logging.getLogger() if url.IsFileUrl(): return FileWildcardIterator(url, exclude_tuple=exclude_tuple, ignore_symlinks=ignore_symlinks, logger=logger) else: # Cloud URL return CloudWildcardIterator(url, gsutil_api, all_versions=all_versions, project_id=project_id) sig_handling.py 0000644 00000013214 15025013616 0007544 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Signal handling functions.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import re import signal import sys import traceback from gslib import metrics from gslib.exception import ControlCException from gslib.utils.constants import UTF8 from gslib.utils.system_util import IS_WINDOWS if IS_WINDOWS: import ctypes # pylint: disable=g-import-not-at-top # Maps from signal_num to list of signal handlers to call. _non_final_signal_handlers = {} # Maps from signal_num to the final signal handler (if any) that should be # called for that signal. _final_signal_handlers = {} def RegisterSignalHandler(signal_num, handler, is_final_handler=False): """Registers a handler for signal signal_num. Unlike calling signal.signal(): - This function can be called from any thread (and will cause the handler to be run by the main thread when the signal is received). - Handlers are cumulative: When a given signal is received, all registered handlers will be executed (with the exception that only the last handler to register with is_final_handler=True will be called). Handlers should make no ordering assumptions, other than that the last handler to register with is_final_handler=True will be called after all the other handlers. Args: signal_num: The signal number with which to associate handler. handler: The handler. is_final_handler: Bool indicator whether handler should be called last among all the handlers for this signal_num. The last handler to register this way survives; other handlers registered with is_final_handler=True will not be called when the signal is received. Raises: RuntimeError: if attempt is made to register a signal_num not in GetCaughtSignals. """ if signal_num not in GetCaughtSignals(): raise RuntimeError('Attempt to register handler (%s) for signal %d, which ' 'is not in GetCaughtSignals' % (handler, signal_num)) if is_final_handler: _final_signal_handlers[signal_num] = handler else: _non_final_signal_handlers[signal_num].append(handler) def _SignalHandler(signal_num, cur_stack_frame): """Global signal handler. When a signal is caught we execute each registered handler for that signal. Args: signal_num: Signal that was caught. cur_stack_frame: Unused. """ if signal_num in _non_final_signal_handlers: for handler in _non_final_signal_handlers[signal_num]: handler(signal_num, cur_stack_frame) if signal_num in _final_signal_handlers: _final_signal_handlers[signal_num](signal_num, cur_stack_frame) def InitializeSignalHandling(): """Initializes global signal handling. Sets up global signal handler for each signal we handle. """ for signal_num in GetCaughtSignals(): _non_final_signal_handlers[signal_num] = [] # Make main signal handler catch the signal. signal.signal(signal_num, _SignalHandler) def GetCaughtSignals(): """Returns terminating signals that can be caught on this OS platform.""" signals = [signal.SIGINT, signal.SIGTERM] if not IS_WINDOWS: # Windows doesn't have SIGQUIT. signals.append(signal.SIGQUIT) return signals def KillProcess(pid): """Make best effort to kill the given process. We ignore all exceptions so a caller looping through a list of processes will continue attempting to kill each, even if one encounters a problem. Args: pid: The process ID. """ try: # os.kill doesn't work in Python3 versions before 3.2. if IS_WINDOWS and ((3, 0) <= sys.version_info[:3] < (3, 2)): kernel32 = ctypes.windll.kernel32 handle = kernel32.OpenProcess(1, 0, pid) kernel32.TerminateProcess(handle, 0) else: os.kill(pid, signal.SIGKILL) except: # pylint: disable=bare-except pass # pylint: disable=unused-argument def MultithreadedMainSignalHandler(signal_num, cur_stack_frame): """Final signal handler for multi-threaded main process.""" if signal_num == signal.SIGINT: if logging.getLogger().isEnabledFor(logging.DEBUG): stack_trace = ''.join(traceback.format_list(traceback.extract_stack())) err = ('DEBUG: Caught CTRL-C (signal %d) - Exception stack trace:\n' ' %s' % (signal_num, re.sub('\\n', '\n ', stack_trace))) try: sys.stderr.write(err.encode(UTF8)) except (UnicodeDecodeError, TypeError) as e: # Can happen when outputting invalid Unicode filenames. # or in python3, where stderr only accepts arguments of type # str, not bytes. sys.stderr.write(err) else: sys.stderr.write('Caught CTRL-C (signal %d) - exiting\n' % signal_num) metrics.LogFatalError(exception=ControlCException()) metrics.Shutdown() KillProcess(os.getpid()) def ChildProcessSignalHandler(signal_num, cur_stack_frame): """Final signal handler for child processes.""" KillProcess(os.getpid()) # pylint: enable=unused-argument resumable_streaming_upload.py 0000644 00000021032 15025013616 0012507 0 ustar 00 # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper class for streaming resumable uploads.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import os import six from gslib.exception import CommandException from gslib.utils.boto_util import GetJsonResumableChunkSize from gslib.utils.constants import UTF8 class ResumableStreamingJsonUploadWrapper(object): """Wraps an input stream in a buffer for resumable uploads. This class takes a non-seekable input stream, buffers it, and exposes it as a stream with limited seek capabilities such that it can be used in a resumable JSON API upload. max_buffer_size bytes of buffering is supported. """ def __init__(self, stream, max_buffer_size, test_small_buffer=False): """Initializes the wrapper. Args: stream: Input stream. max_buffer_size: Maximum size of internal buffer; should be >= the chunk size of the resumable upload API to ensure that at least one full chunk write can be replayed in the event of a server error. test_small_buffer: Skip check for buffer size vs. chunk size, for testing. """ self._orig_fp = stream if not test_small_buffer and max_buffer_size < GetJsonResumableChunkSize(): raise CommandException('Resumable streaming upload created with buffer ' 'size %s, JSON resumable upload chunk size %s. ' 'Buffer size must be >= JSON resumable upload ' 'chunk size to ensure that uploads can be ' 'resumed.' % (max_buffer_size, GetJsonResumableChunkSize())) self._max_buffer_size = max_buffer_size self._buffer = collections.deque() self._buffer_start = 0 self._buffer_end = 0 self._position = 0 @property def mode(self): """Returns the mode of the underlying file descriptor, or None.""" return getattr(self._orig_fp, 'mode', None) def read(self, size=-1): # pylint: disable=invalid-name """"Reads from the wrapped stream. Args: size: The amount of bytes to read. If omitted or negative, the entire contents of the stream will be read and returned. Returns: Bytes from the wrapped stream. """ read_all_bytes = size is None or size < 0 if read_all_bytes: bytes_remaining = self._max_buffer_size else: bytes_remaining = size data = b'' buffered_data = [] if self._position < self._buffer_end: # There was a backwards seek, so read from the buffer first. # TODO: Performance test to validate if it is worth re-aligning # the buffers in this case. Also, seeking through the buffer for # each read on a long catch-up is probably not performant, but we'd # need a more complex data structure than a deque to get around this. pos_in_buffer = self._buffer_start buffer_index = 0 # First, find the start position in the buffer. while pos_in_buffer + len(self._buffer[buffer_index]) < self._position: # When this loop exits, buffer_index will refer to a buffer that # has at least some overlap with self._position, and # pos_in_buffer will be >= self._position pos_in_buffer += len(self._buffer[buffer_index]) buffer_index += 1 # Read until we've read enough or we're out of buffer. while pos_in_buffer < self._buffer_end and bytes_remaining > 0: buffer_len = len(self._buffer[buffer_index]) # This describes how far into the current buffer self._position is. offset_from_position = self._position - pos_in_buffer bytes_available_this_buffer = buffer_len - offset_from_position read_size = min(bytes_available_this_buffer, bytes_remaining) buffered_data.append(self._buffer[buffer_index] [offset_from_position:offset_from_position + read_size]) bytes_remaining -= read_size pos_in_buffer += buffer_len buffer_index += 1 self._position += read_size # At this point we're guaranteed that if there are any bytes left to read, # then self._position == self._buffer_end, and we can read from the # wrapped stream if needed. if read_all_bytes: # TODO: The user is requesting reading until the end of an # arbitrary length stream, which is bad we'll need to return data # with no size limits; if the stream is sufficiently long, we could run # out of memory. We could break this down into smaller reads and # buffer it as we go, but we're still left returning the data all at # once to the caller. We could raise, but for now trust the caller to # be sane and have enough memory to hold the remaining stream contents. new_data = self._orig_fp.read(size) data_len = len(new_data) if not buffered_data: data = new_data else: buffered_data.append(new_data) data = b''.join(buffered_data) self._position += data_len elif bytes_remaining: new_data = self._orig_fp.read(bytes_remaining) if not buffered_data: data = new_data else: buffered_data.append(new_data) data = b''.join(buffered_data) data_len = len(new_data) if data_len: self._position += data_len self._buffer.append(new_data) self._buffer_end += data_len oldest_data = None while self._buffer_end - self._buffer_start > self._max_buffer_size: oldest_data = self._buffer.popleft() self._buffer_start += len(oldest_data) if oldest_data: refill_amount = self._max_buffer_size - (self._buffer_end - self._buffer_start) if refill_amount: self._buffer.appendleft(oldest_data[-refill_amount:]) self._buffer_start -= refill_amount else: if six.PY3: if buffered_data: buffered_data = [ bd.encode(UTF8) if isinstance(bd, str) else bd for bd in buffered_data ] data = b''.join(buffered_data) if buffered_data else b'' return data def tell(self): # pylint: disable=invalid-name """Returns the current stream position.""" return self._position def seekable(self): # pylint: disable=invalid-name """Returns true since limited seek support exists.""" return True def seek(self, offset, whence=os.SEEK_SET): # pylint: disable=invalid-name """Seeks on the buffered stream. Args: offset: The offset to seek to; must be within the buffer bounds. whence: Must be os.SEEK_SET. Raises: CommandException if an unsupported seek mode or position is used. """ if whence == os.SEEK_SET: if offset < self._buffer_start or offset > self._buffer_end: raise CommandException('Unable to resume upload because of limited ' 'buffering available for streaming uploads. ' 'Offset %s was requested, but only data from ' '%s to %s is buffered.' % (offset, self._buffer_start, self._buffer_end)) # Move to a position within the buffer. self._position = offset elif whence == os.SEEK_END: if offset > self._max_buffer_size: raise CommandException('Invalid SEEK_END offset %s on streaming ' 'upload. Only %s can be buffered.' % (offset, self._max_buffer_size)) # Read to the end and rely on buffering to handle the offset. while self.read(self._max_buffer_size): pass # Now we're at the end. self._position -= offset else: raise CommandException('Invalid seek mode on streaming upload. ' '(mode %s, offset %s)' % (whence, offset)) def close(self): # pylint: disable=invalid-name return self._orig_fp.close() cred_types.py 0000644 00000002234 15025013616 0007257 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Defines gsutil-supported credential types.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class CredTypes(object): HMAC = "HMAC" IMPERSONATION = "Service Account Impersonation" OAUTH2_SERVICE_ACCOUNT = "OAuth 2.0 Service Account" OAUTH2_USER_ACCOUNT = "Oauth 2.0 User Account" GCE = "GCE" DEVSHELL = "Google Developer Shell" EXTERNAL_ACCOUNT = "External Account" EXTERNAL_ACCOUNT_AUTHORIZED_USER = "External Account Authorized User" iamcredentials_api.py 0000644 00000027346 15025013616 0010746 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """JSON gsutil Cloud API implementation for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import logging import traceback from apitools.base.py import exceptions as apitools_exceptions from boto import config from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import ServiceException from gslib.no_op_credentials import NoOpCredentials from gslib.third_party.iamcredentials_apitools import iamcredentials_v1_client as apitools_client from gslib.third_party.iamcredentials_apitools import iamcredentials_v1_messages as apitools_messages from gslib.utils import system_util from gslib.utils.boto_util import GetCertsFile from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNewHttp from gslib.utils.boto_util import GetNumRetries TRANSLATABLE_APITOOLS_EXCEPTIONS = (apitools_exceptions.HttpError) if system_util.InvokedViaCloudSdk(): _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gcloud auth login`') else: _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gsutil config`') class IamcredentailsApi(object): """Wraps calls to the Cloud IAM Credentials v1 interface via apitools.""" def __init__(self, logger=None, credentials=None, debug=0): """Performs necessary setup for interacting with Google Cloud IAM Credentials. Args: logger: logging.logger for outputting log messages. credentials: Credentials to be used for interacting with Cloud IAM debug: Debug level for the API implementation (0..3). """ super(IamcredentailsApi, self).__init__() self.logger = logger self.credentials = credentials self.certs_file = GetCertsFile() self.http = GetNewHttp() self.http_base = 'https://' # The original config option had a typo; supporting it as a fall-back # for legacy reasons. legacy_typo = 'gs_iamcredentails_host' self.host_base = config.get( 'Credentials', 'gs_iamcredentials_host', config.get('Credentials', legacy_typo, 'iamcredentials.googleapis.com')) gs_iamcred_port = config.get('Credentials', 'gs_iamcredentails_port', None) self.host_port = (':' + gs_iamcred_port) if gs_iamcred_port else '' self.url_base = (self.http_base + self.host_base + self.host_port) log_request = (debug >= 3) log_response = (debug >= 3) self.api_client = apitools_client.IamcredentialsV1( url=self.url_base, http=self.http, log_request=log_request, log_response=log_response, credentials=self.credentials) self.num_retries = GetNumRetries() self.api_client.num_retries = self.num_retries self.max_retry_wait = GetMaxRetryDelay() self.api_client.max_retry_wait = self.max_retry_wait if isinstance(self.credentials, NoOpCredentials): # This API key is not secret and is used to identify gsutil during # anonymous requests. self.api_client.AddGlobalParam( 'key', u'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM') def SignBlob(self, service_account_id, message): """Sign the blob using iamcredentials.SignBlob API.""" name = 'projects/-/serviceAccounts/%s' % service_account_id sign_blob_request = apitools_messages.SignBlobRequest(payload=message) request = ( apitools_messages.IamcredentialsProjectsServiceAccountsSignBlobRequest( name=name, signBlobRequest=sign_blob_request)) try: return self.api_client.projects_serviceAccounts.SignBlob(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, service_account_id) def GenerateAccessToken(self, service_account_id, scopes): """Generates an access token for the given service account.""" name = 'projects/-/serviceAccounts/%s' % service_account_id generate_access_token_request = apitools_messages.GenerateAccessTokenRequest( scope=scopes) request = (apitools_messages. IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequest( name=name, generateAccessTokenRequest=generate_access_token_request)) try: return self.api_client.projects_serviceAccounts.GenerateAccessToken( request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, service_account_id) def _TranslateExceptionAndRaise(self, e, service_account_id=None): """Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. service_account_id: Optional service account in request that caused the exception. Raises: Translated CloudApi exception, or the original exception if it was not translatable. """ if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('TranslateExceptionAndRaise: %s', traceback.format_exc()) translated_exception = self._TranslateApitoolsException( e, service_account_id=service_account_id) if translated_exception: raise translated_exception else: raise def _TranslateApitoolsException(self, e, service_account_id=None): """Translates apitools exceptions into their gsutil equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. service_account_id: Optional service account ID that caused the exception. Returns: CloudStorageApiServiceException for translatable exceptions, None otherwise. """ if isinstance(e, apitools_exceptions.HttpError): message = self._GetMessageFromHttpError(e) if e.status_code == 400: # It is possible that the Project ID is incorrect. Unfortunately the # JSON API does not give us much information about what part of the # request was bad. return BadRequestException(message or 'Bad Request', status=e.status_code) elif e.status_code == 401: if 'Login Required' in str(e): return AccessDeniedException(message or 'Access denied: login required.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) elif e.status_code == 403: # Messaging for when the the originating credentials don't have access # to impersonate a service account. if 'The caller does not have permission' in str(e): return AccessDeniedException( 'Service account impersonation failed. Please go to the Google ' 'Cloud Platform Console (https://cloud.google.com/console), ' 'select IAM & admin, then Service Accounts, and grant your ' 'originating account the Service Account Token Creator role on ' 'the target service account.') # The server's errors message when IAM Credentials API aren't enabled # are pretty great so we just display them. if 'IAM Service Account Credentials API has not been used' in str(e): return AccessDeniedException(message) if 'The account for the specified project has been disabled' in str(e): return AccessDeniedException(message or 'Account disabled.', status=e.status_code) elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e): return AccessDeniedException(message or 'Access denied: quota exceeded. ' 'Is your project ID valid?', status=e.status_code) elif 'User Rate Limit Exceeded' in str(e): return AccessDeniedException( 'Rate limit exceeded. Please retry this ' 'request later.', status=e.status_code) elif 'Access Not Configured' in str(e): return AccessDeniedException( 'Access Not Configured. Please go to the Google Cloud Platform ' 'Console (https://cloud.google.com/console#/project) for your ' 'project, select APIs & services, and enable the Google Cloud ' 'IAM Credentials API.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) else: return AccessDeniedException(message or e.message or service_account_id, status=e.status_code) elif e.status_code == 404: return NotFoundException(message or e.message, status=e.status_code) elif e.status_code == 409 and service_account_id: return ServiceException('The key %s already exists.' % service_account_id, status=e.status_code) elif e.status_code == 412: return PreconditionException(message, status=e.status_code) return ServiceException(message, status=e.status_code) def _GetMessageFromHttpError(self, http_error): if isinstance(http_error, apitools_exceptions.HttpError): if getattr(http_error, 'content', None): try: json_obj = json.loads(http_error.content) if 'error' in json_obj and 'message' in json_obj['error']: return json_obj['error']['message'] except Exception: # pylint: disable=broad-except # If we couldn't decode anything, just leave the message as None. pass def _GetAcceptableScopesFromHttpError(self, http_error): try: www_authenticate = http_error.response['www-authenticate'] # In the event of a scope error, the www-authenticate field of the HTTP # response should contain text of the form # # 'Bearer realm="https://oauth2.googleapis.com/", # error=insufficient_scope, # scope="${space separated list of acceptable scopes}"' # # Here we use a quick string search to find the scope list, just looking # for a substring with the form 'scope="${scopes}"'. scope_idx = www_authenticate.find('scope="') if scope_idx >= 0: scopes = www_authenticate[scope_idx:].split('"')[1] return 'Acceptable scopes: %s' % scopes except Exception: # pylint: disable=broad-except # Return None if we have any trouble parsing out the acceptable scopes. pass tz_utc.py 0000644 00000004316 15025013616 0006431 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Provides a tzinfo subclass to represent the UTC timezone.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from datetime import timedelta from datetime import tzinfo class UTC(tzinfo): """Timezone information class used to convert datetime timestamps to UTC. This class is necessary to convert timestamps to UTC. By default Python datetime objects are timezone unaware. This created problems when interacting with cloud object timestamps which are timezone-aware. This issue appeared when handling the timeCreated metadata attribute; the values returned by the service were placed in RFC 3339 format in the storage_v1_messages module. RFC 3339 requires a timezone in any timestamp. This caused problems as the datetime object elsewhere in the code was timezone unaware and was different by exactly one hour. The main problem was that the local system used daylight savings time which adjusted the timestamp ahead by one hour. """ def utcoffset(self, _): """An offset of the number of minutes away from UTC this tzinfo object is. Returns: A time duration of zero. UTC is zero minutes away from UTC. """ return timedelta(0) def tzname(self, _): """A method to retrieve the name of this timezone object. Returns: The name of the timezone (i.e. 'UTC'). """ return 'UTC' def dst(self, _): """A fixed offset to handle daylight savings time (DST). Returns: A time duration of zero as UTC does not use DST. """ return timedelta(0) exception.py 0000644 00000012265 15025013616 0007121 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2010 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """gsutil exceptions. The exceptions in this module are for use across multiple different classes. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import six NO_URLS_MATCHED_PREFIX = 'No URLs matched' NO_URLS_MATCHED_GENERIC = (NO_URLS_MATCHED_PREFIX + '. Do the files you\'re operating on exist?') NO_URLS_MATCHED_TARGET = NO_URLS_MATCHED_PREFIX + ': %s' if six.PY3: # StandardError was removed, so use the base exception type instead StandardError = Exception class AbortException(StandardError): """Exception raised when a user aborts a command that needs to do cleanup.""" def __init__(self, reason): StandardError.__init__(self) self.reason = reason def __repr__(self): return 'AbortException: %s' % self.reason def __str__(self): return 'AbortException: %s' % self.reason class CommandException(StandardError): """Exception raised when a problem is encountered running a gsutil command. This exception should be used to signal user errors or system failures (like timeouts), not bugs (like an incorrect param value). For the latter you should raise Exception so we can see where/how it happened via gsutil -D (which will include a stack trace for raised Exceptions). """ def __init__(self, reason, informational=False): """Instantiate a CommandException. Args: reason: Text describing the problem. informational: Indicates reason should be printed as FYI, not a failure. """ StandardError.__init__(self) self.reason = reason self.informational = informational def __repr__(self): return str(self) def __str__(self): return 'CommandException: %s' % self.reason class ControlCException(Exception): """Exception to report to analytics when the user exits via ctrl-C. This exception is never actually raised, but is used by analytics collection to provide a more descriptive name for user exit. """ pass class GcloudStorageTranslationError(Exception): """Exception raised when a gsutil command can't be translated to gcloud.""" pass class HashMismatchException(Exception): """Exception raised when data integrity validation fails.""" pass class IamChOnResourceWithConditionsException(Exception): """Raised when trying to use "iam ch" on an IAM policy with conditions. Because the syntax for conditions is fairly complex, it doesn't make sense to specify them on the command line using a colon-delimited set of values in the same way you'd specify simple bindings - it would be a complex and potentially surprising interface, which isn't what you want when dealing with permissions. Additionally, providing partial functionality -- e.g. if a policy contains bindings with conditions, still allow users to interact with bindings that don't contain conditions -- might sound tempting, but results in a bad user experience. Bindings can be thought of as a mapping from (role, condition) -> [members]. Thus, a user might think they're editing the binding for (role1, condition1), but they'd really be editing the binding for (role1, None). Thus, we just raise an error if we encounter a binding with conditions present, and encourage users to use "iam {get,set}" instead. """ def __init__(self, message): Exception.__init__(self, message) self.message = message def __repr__(self): return str(self) def __str__(self): return 'IamChOnResourceWithConditionsException: %s' % self.message class InvalidUrlError(Exception): """Exception raised when URL is invalid.""" def __init__(self, message): Exception.__init__(self, message) self.message = message def __repr__(self): return str(self) def __str__(self): return 'InvalidUrlError: %s' % self.message class ExternalBinaryError(Exception): """Exception raised when gsutil runs an external binary, and it fails.""" def __init__(self, message): Exception.__init__(self, message) self.message = message def __repr__(self): return 'ExternalBinaryError: %s' % self.message command_runner.py 0000644 00000056324 15025013616 0010136 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Class that runs a named gsutil command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import difflib import logging import os import pkgutil import sys import textwrap import time import six from six.moves import input import boto from boto import config from boto.storage_uri import BucketStorageUri import gslib from gslib import metrics from gslib.cloud_api_delegator import CloudApiDelegator from gslib.command import Command from gslib.command import CreateOrGetGsutilLogger from gslib.command import GetFailureCount from gslib.command import OLD_ALIAS_MAP from gslib.command import ShutDownGsutil import gslib.commands from gslib.cs_api_map import ApiSelector from gslib.cs_api_map import GsutilApiClassMapFactory from gslib.cs_api_map import GsutilApiMapFactory from gslib.discard_messages_queue import DiscardMessagesQueue from gslib.exception import CommandException from gslib.gcs_json_api import GcsJsonApi from gslib.no_op_credentials import NoOpCredentials from gslib.tab_complete import MakeCompleter from gslib.utils import boto_util from gslib.utils import shim_util from gslib.utils import system_util from gslib.utils.constants import RELEASE_NOTES_URL from gslib.utils.constants import UTF8 from gslib.utils.metadata_util import IsCustomMetadataHeader from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit from gslib.utils.text_util import CompareVersions from gslib.utils.text_util import InsistAsciiHeader from gslib.utils.text_util import InsistAsciiHeaderValue from gslib.utils.text_util import print_to_fd from gslib.utils.unit_util import SECONDS_PER_DAY from gslib.utils.update_util import LookUpGsutilVersion from gslib.utils.update_util import GsutilPubTarball def HandleHeaderCoding(headers): """Handles coding of headers and their values. Alters the dict in-place. Converts a dict of headers and their values to their appropriate types. We ensure that all headers and their values will contain only ASCII characters, with the exception of custom metadata header values; these values may contain Unicode characters, and thus if they are not already unicode-type objects, we attempt to decode them to Unicode using UTF-8 encoding. Args: headers: A dict mapping headers to their values. All keys and values must be either str or unicode objects. Raises: CommandException: If a header or its value cannot be encoded in the required encoding. """ if not headers: return for key in headers: InsistAsciiHeader(key) if IsCustomMetadataHeader(key): if not isinstance(headers[key], six.text_type): try: headers[key] = headers[key].decode(UTF8) except UnicodeDecodeError: raise CommandException('\n'.join( textwrap.wrap( 'Invalid encoding for header value (%s: %s). Values must be ' 'decodable as Unicode. NOTE: the value printed above ' 'replaces the problematic characters with a hex-encoded ' 'printable representation. For more details (including how to ' 'convert to a gsutil-compatible encoding) see `gsutil help ' 'encoding`.' % (repr(key), repr(headers[key]))))) else: # Non-custom-metadata headers and their values must be ASCII characters. InsistAsciiHeaderValue(key, headers[key]) def HandleArgCoding(args): """Handles coding of command-line args. Alters the list in-place. Args: args: A list of command-line args. Raises: CommandException: if errors encountered. """ # Python passes arguments from the command line as byte strings. To # correctly interpret them, we decode them as utf-8. for i in range(len(args)): arg = args[i] if not isinstance(arg, six.text_type): try: args[i] = arg.decode(UTF8) except UnicodeDecodeError: raise CommandException('\n'.join( textwrap.wrap( 'Invalid encoding for argument (%s). Arguments must be decodable ' 'as Unicode. NOTE: the argument printed above replaces the ' 'problematic characters with a hex-encoded printable ' 'representation. For more details (including how to convert to a ' 'gsutil-compatible encoding) see `gsutil help encoding`.' % repr(arg)))) def _StringToSysArgType(unicode_str): """Converts a string literal (unicode) to the same type as sys.argv[0].""" # TODO(PY3-ONLY): If we remove the PY2 code branch, this method becomes # a no-op, so we can just remove the whole method when we move to PY3. if six.PY2: return unicode_str.encode(UTF8) return unicode_str class CommandRunner(object): """Runs gsutil commands and does some top-level argument handling.""" def __init__(self, bucket_storage_uri_class=BucketStorageUri, gsutil_api_class_map_factory=GsutilApiClassMapFactory, command_map=None): """Instantiates a CommandRunner. Args: bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. gsutil_api_class_map_factory: Creates map of cloud storage interfaces. Settable for testing/mocking. command_map: Map of command names to their implementations for testing/mocking. If not set, the map is built dynamically. """ self.bucket_storage_uri_class = bucket_storage_uri_class self.gsutil_api_class_map_factory = gsutil_api_class_map_factory if command_map: self.command_map = command_map else: self.command_map = self._LoadCommandMap() def _LoadCommandMap(self): """Returns dict mapping each command_name to implementing class.""" # Import all gslib.commands submodules. for _, module_name, _ in pkgutil.iter_modules(gslib.commands.__path__): __import__('gslib.commands.%s' % module_name) command_map = {} # Only include Command subclasses in the dict. for command in Command.__subclasses__(): command_map[command.command_spec.command_name] = command for command_name_aliases in command.command_spec.command_name_aliases: command_map[command_name_aliases] = command return command_map def _GetTabCompleteLogger(self): """Returns a logger for tab completion.""" return CreateOrGetGsutilLogger('tab_complete') def _ConfigureCommandArgumentParserArguments(self, parser, subcommands_or_arguments, gsutil_api): """Creates parsers recursively for potentially nested subcommands. Args: parser: argparse parser object. subcommands_or_arguments: list of CommandArgument objects, or recursive dict with subcommand names as keys. gsutil_api: gsutil Cloud API instance to use. Raises: RuntimeError: if argument is configured with unsupported completer TypeError: if subcommands_or_arguments is not a dict or list """ logger = self._GetTabCompleteLogger() def HandleList(): for command_argument in subcommands_or_arguments: action = parser.add_argument(*command_argument.args, **command_argument.kwargs) if command_argument.completer: action.completer = MakeCompleter(command_argument.completer, gsutil_api) def HandleDict(): subparsers = parser.add_subparsers() for subcommand_name, subcommand_value in subcommands_or_arguments.items(): cur_subcommand_parser = subparsers.add_parser(subcommand_name, add_help=False) logger.info( 'Constructing argument parsers for {}'.format(subcommand_name)) self._ConfigureCommandArgumentParserArguments(cur_subcommand_parser, subcommand_value, gsutil_api) if isinstance(subcommands_or_arguments, list): HandleList() elif isinstance(subcommands_or_arguments, dict): HandleDict() else: error_format = ('subcommands_or_arguments {} should be list or dict, ' 'found type {}') raise TypeError( error_format.format(subcommands_or_arguments, type(subcommands_or_arguments))) def GetGsutilApiForTabComplete(self): """Builds and returns a gsutil_api based off gsutil_api_class_map_factory. Returns: the gsutil_api instance """ # This should match the support map for the "ls" command. support_map = { 'gs': [ApiSelector.XML, ApiSelector.JSON], 's3': [ApiSelector.XML] } default_map = {'gs': ApiSelector.JSON, 's3': ApiSelector.XML} gsutil_api_map = GsutilApiMapFactory.GetApiMap( self.gsutil_api_class_map_factory, support_map, default_map) gsutil_api = CloudApiDelegator(self.bucket_storage_uri_class, gsutil_api_map, self._GetTabCompleteLogger(), DiscardMessagesQueue(), debug=0) return gsutil_api def ConfigureCommandArgumentParsers(self, main_parser): """Configures argparse arguments and argcomplete completers for commands. Args: main_parser: argparse object that can be called to get subparsers to add subcommands (called just 'commands' in gsutil) """ gsutil_api = self.GetGsutilApiForTabComplete() # build a dict mapping from command name to the argparse arguments. # this dict has values with either a recursive dictionary or a list of # CommandArgument objects. command_to_argparse_arguments = { command.command_spec.command_name: command.command_spec.argparse_arguments for command in self.command_map.values() } # At this point command_to_argparse_arguments looks like # { # 'retention': { # 'set' : `set arguments array` # 'get' : `set arguments array` # ... # 'event': { # 'set': `event set arguments array` # 'release': `event release arguments array` # } # }, # ... other commands here # } # # Which will be passed into the helper and called recursively on the items # in the dict, with the base case being the arguments arrays, where the # arguments are added to the subparser for the lowest level command. self._ConfigureCommandArgumentParserArguments( main_parser, command_to_argparse_arguments, gsutil_api) def RunNamedCommand(self, command_name, args=None, headers=None, debug=0, trace_token=None, parallel_operations=False, skip_update_check=False, logging_filters=None, do_shutdown=True, perf_trace_token=None, user_project=None, collect_analytics=False): """Runs the named command. Used by gsutil main, commands built atop other commands, and tests. Args: command_name: The name of the command being run. args: Command-line args (arg0 = actual arg, not command name ala bash). headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). trace_token: Trace token to pass to the underlying API. parallel_operations: Should command operations be executed in parallel? skip_update_check: Set to True to disable checking for gsutil updates. logging_filters: Optional list of logging.Filters to apply to this command's logger. do_shutdown: Stop all parallelism framework workers iff this is True. perf_trace_token: Performance measurement trace token to pass to the underlying API. user_project: The project to bill this request to. collect_analytics: Set to True to collect an analytics metric logging this command. Raises: CommandException: if errors encountered. Returns: Return value(s) from Command that was run. """ command_changed_to_update = False if (not skip_update_check and self.MaybeCheckForAndOfferSoftwareUpdate(command_name, debug)): command_name = 'update' command_changed_to_update = True args = [_StringToSysArgType('-n')] # Check for opt-in analytics. if system_util.IsRunningInteractively() and collect_analytics: metrics.CheckAndMaybePromptForAnalyticsEnabling() self.MaybePromptForPythonUpdate(command_name) if not args: args = [] # Include api_version header in all commands. api_version = boto.config.get_value('GSUtil', 'default_api_version', '1') if not headers: headers = {} headers['x-goog-api-version'] = api_version if command_name not in self.command_map: close_matches = difflib.get_close_matches(command_name, self.command_map.keys(), n=1) if close_matches: # Instead of suggesting a deprecated command alias, suggest the new # name for that command. translated_command_name = (OLD_ALIAS_MAP.get(close_matches[0], close_matches)[0]) print('Did you mean this?', file=sys.stderr) print('\t%s' % translated_command_name, file=sys.stderr) elif command_name == 'update' and gslib.IS_PACKAGE_INSTALL: sys.stderr.write( 'Update command is not supported for package installs; ' 'please instead update using your package manager.') raise CommandException('Invalid command "%s".' % command_name) # Call str() on this string because the type of objects in `args` differ # on Python 2 vs 3 (bytes vs unicode), and we want to compare using the # same as whatever is in `args`. if _StringToSysArgType('--help') in args: new_args = [command_name] original_command_class = self.command_map[command_name] subcommands = original_command_class.help_spec.subcommand_help_text.keys() for arg in args: if arg in subcommands: new_args.append(arg) break # Take the first match and throw away the rest. args = new_args command_name = 'help' HandleArgCoding(args) HandleHeaderCoding(headers) command_class = self.command_map[command_name] command_inst = command_class(self, args, headers, debug, trace_token, parallel_operations, self.bucket_storage_uri_class, self.gsutil_api_class_map_factory, logging_filters, command_alias_used=command_name, perf_trace_token=perf_trace_token, user_project=user_project) # Log the command name, command alias, and sub-options after being parsed by # RunCommand and the command constructor. For commands with subcommands and # suboptions, we need to log the suboptions again within the command itself # because the command constructor will not parse the suboptions fully. if collect_analytics: metrics.LogCommandParams(command_name=command_inst.command_name, sub_opts=command_inst.sub_opts, command_alias=command_name) if command_inst.translate_to_gcloud_storage_if_requested(): # This does not mean that the gcloud storage command worked. # It only means that we succesfully attempted running gcloud storage. # The command itself might have failed. return_code = command_inst.run_gcloud_storage() else: # Run gsutil. return_code = command_inst.RunCommand() if CheckMultiprocessingAvailableAndInit().is_available and do_shutdown: ShutDownGsutil() if GetFailureCount() > 0: return_code = 1 if command_changed_to_update: # If the command changed to update, the user's original command was # not executed. return_code = 1 print('\n'.join( textwrap.wrap( 'Update was successful. Exiting with code 1 as the original command ' 'issued prior to the update was not executed and should be re-run.' ))) return return_code def SkipUpdateCheck(self): """Helper function that will determine if update checks should be skipped. Args: command_name: The name of the command being run. Returns: True if: - gsutil is not connected to a tty (e.g., if being run from cron); - user is running gsutil -q - user specified gs_host (which could be a non-production different service instance, in which case credentials won't work for checking gsutil tarball).""" logger = logging.getLogger() if (not system_util.IsRunningInteractively() or not logger.isEnabledFor(logging.INFO) or boto_util.HasUserSpecifiedGsHost()): return True return False def MaybePromptForPythonUpdate(self, command_name): """Alert the user that they should install Python 3. Args: command_name: The name of the command being run. Returns: True if a prompt was output. """ if (sys.version_info.major != 2 or self.SkipUpdateCheck() or command_name not in ('update', 'ver', 'version') or boto.config.getbool('GSUtil', 'skip_python_update_prompt', False)): return False # Notify the user about Python 2 deprecation. print_to_fd( 'Gsutil 5 drops Python 2 support. Please install Python 3 to update ' 'to the latest version of gsutil. https://goo.gle/py3\n') return True def MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug): """Checks the last time we checked for an update and offers one if needed. Offer is made if the time since the last update check is longer than the configured threshold offers the user to update gsutil. Args: command_name: The name of the command being run. debug: Debug level to pass in to boto connection (range 0..3). Returns: True if the user decides to update. """ # Don't try to interact with user if: # - SkipUpdateChecks returns True. # - user is running the config command (which could otherwise attempt to # check for an update for a user running behind a proxy, who has not yet # configured gsutil to go through the proxy; for such users we need the # first connection attempt to be made by the gsutil config command). # - user is running the version command (which gets run when using # gsutil -D, which would prevent users with proxy config problems from # sending us gsutil -D output). # - user is running the update command (which could otherwise cause an # additional note that an update is available when user is already trying # to perform an update); # - user is using a Cloud SDK install (which should only be updated via # gcloud components update) logger = logging.getLogger() if (self.SkipUpdateCheck() or command_name in ('config', 'update', 'ver', 'version') or system_util.InvokedViaCloudSdk()): return False software_update_check_period = boto.config.getint( 'GSUtil', 'software_update_check_period', 30) # Setting software_update_check_period to 0 means periodic software # update checking is disabled. if software_update_check_period == 0: return False last_checked_for_gsutil_update_timestamp_file = ( boto_util.GetLastCheckedForGsutilUpdateTimestampFile()) cur_ts = int(time.time()) if not os.path.isfile(last_checked_for_gsutil_update_timestamp_file): # Set last_checked_ts from date of VERSION file, so if the user installed # an old copy of gsutil it will get noticed (and an update offered) the # first time they try to run it. last_checked_ts = gslib.GetGsutilVersionModifiedTime() with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f: f.write(str(last_checked_ts)) else: try: with open(last_checked_for_gsutil_update_timestamp_file, 'r') as f: last_checked_ts = int(f.readline()) except (TypeError, ValueError): return False if (cur_ts - last_checked_ts > software_update_check_period * SECONDS_PER_DAY): # Create a credential-less gsutil API to check for the public # update tarball. gsutil_api = GcsJsonApi(self.bucket_storage_uri_class, logger, DiscardMessagesQueue(), credentials=NoOpCredentials(), debug=debug) cur_ver = gslib.VERSION try: cur_ver = LookUpGsutilVersion(gsutil_api, GsutilPubTarball()) except Exception: return False with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f: f.write(str(cur_ts)) (g, m) = CompareVersions(cur_ver, gslib.VERSION) if m: print_to_fd('\n'.join( textwrap.wrap( 'A newer version of gsutil (%s) is available than the version you ' 'are running (%s). NOTE: This is a major new version, so it is ' 'strongly recommended that you review the release note details at ' '%s before updating to this version, especially if you use gsutil ' 'in scripts.' % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL)))) if gslib.IS_PACKAGE_INSTALL: return False print_to_fd('\n') answer = input('Would you like to update [y/N]? ') return answer and answer.lower()[0] == 'y' elif g: print_to_fd('\n'.join( textwrap.wrap( 'A newer version of gsutil (%s) is available than the version you ' 'are running (%s). A detailed log of gsutil release changes is ' 'available at %s if you would like to read them before updating.' % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL)))) if gslib.IS_PACKAGE_INSTALL: return False print_to_fd('\n') answer = input('Would you like to update [Y/n]? ') return not answer or answer.lower()[0] != 'n' return False kms_api.py 0000644 00000032366 15025013616 0006552 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """JSON gsutil Cloud API implementation for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import logging import traceback from apitools.base.py import exceptions as apitools_exceptions from boto import config from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import ServiceException from gslib.gcs_json_credentials import SetUpJsonCredentialsAndCache from gslib.no_op_credentials import NoOpCredentials from gslib.third_party.kms_apitools import cloudkms_v1_client as apitools_client from gslib.third_party.kms_apitools import cloudkms_v1_messages as apitools_messages from gslib.utils import system_util from gslib.utils.boto_util import GetCertsFile from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNewHttp from gslib.utils.boto_util import GetNumRetries TRANSLATABLE_APITOOLS_EXCEPTIONS = (apitools_exceptions.HttpError) if system_util.InvokedViaCloudSdk(): _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gcloud auth login`') else: _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gsutil config`') class KmsApi(object): """Wraps calls to the Cloud KMS v1 interface via apitools.""" def __init__(self, logger=None, credentials=None, debug=0): """Performs necessary setup for interacting with Google Cloud KMS. Args: logger: logging.logger for outputting log messages. credentials: Credentials to be used for interacting with Cloud KMS debug: Debug level for the API implementation (0..3). """ super(KmsApi, self).__init__() self.logger = logger self.certs_file = GetCertsFile() self.http = GetNewHttp() self.http_base = 'https://' self.host_base = config.get('Credentials', 'gs_kms_host', 'cloudkms.googleapis.com') gs_kms_port = config.get('Credentials', 'gs_kms_port', None) self.host_port = (':' + gs_kms_port) if gs_kms_port else '' self.url_base = (self.http_base + self.host_base + self.host_port) SetUpJsonCredentialsAndCache(self, logger, credentials=credentials) log_request = (debug >= 3) log_response = (debug >= 3) self.api_client = apitools_client.CloudkmsV1(url=self.url_base, http=self.http, log_request=log_request, log_response=log_response, credentials=self.credentials) self.num_retries = GetNumRetries() self.api_client.num_retries = self.num_retries self.max_retry_wait = GetMaxRetryDelay() self.api_client.max_retry_wait = self.max_retry_wait if isinstance(self.credentials, NoOpCredentials): # This API key is not secret and is used to identify gsutil during # anonymous requests. self.api_client.AddGlobalParam( 'key', u'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM') def GetKeyIamPolicy(self, key_name): request = (apitools_messages. CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequest( resource=key_name)) try: return (self.api_client.projects_locations_keyRings_cryptoKeys. GetIamPolicy(request)) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, key_name=key_name) def SetKeyIamPolicy(self, key_name, policy): policy_request = apitools_messages.SetIamPolicyRequest(policy=policy) request = (apitools_messages. CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequest( resource=key_name, setIamPolicyRequest=policy_request)) try: return (self.api_client.projects_locations_keyRings_cryptoKeys. SetIamPolicy(request)) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, key_name=key_name) def CreateKeyRing(self, project, keyring_name, location='global'): """Attempts to create the specified keyRing. Args: project: (str) The project id in which to create the keyRing and key. keyring_name: (str) The name of the keyRing, e.g. my-keyring. Note that this must be unique within the location. location: (str) The location in which to create the keyRing. Defaults to 'global'. Returns: (str) The fully-qualified name of the keyRing, e.g.: projects/my-project/locations/global/keyRings/my-keyring Raises: Translated CloudApi exception if we were unable to create the keyRing. Note that in the event of a 409 status code (resource already exists) when attempting creation, we continue and treat this as a success. """ keyring_msg = apitools_messages.KeyRing( name='projects/%s/locations/%s/keyRings/%s' % (project, location, keyring_name)) keyring_create_request = ( apitools_messages.CloudkmsProjectsLocationsKeyRingsCreateRequest( keyRing=keyring_msg, keyRingId=keyring_name, parent='projects/%s/locations/%s' % (project, location))) try: self.api_client.projects_locations_keyRings.Create(keyring_create_request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: if e.status_code != 409: raise return 'projects/%s/locations/%s/keyRings/%s' % (project, location, keyring_name) def CreateCryptoKey(self, keyring_fqn, key_name): """Attempts to create the specified cryptoKey. Args: keyring_fqn: (str) The fully-qualified name of the keyRing, e.g. projects/my-project/locations/global/keyRings/my-keyring. key_name: (str) The name of the desired key, e.g. my-key. Note that this must be unique within the keyRing. Returns: (str) The fully-qualified name of the cryptoKey, e.g.: projects/my-project/locations/global/keyRings/my-keyring/cryptoKeys/my-key Raises: Translated CloudApi exception if we were unable to create the cryptoKey. Note that in the event of a 409 status code (resource already exists) when attempting creation, we continue and treat this as a success. """ cryptokey_msg = apitools_messages.CryptoKey(purpose=( apitools_messages.CryptoKey.PurposeValueValuesEnum.ENCRYPT_DECRYPT)) cryptokey_create_request = ( apitools_messages. CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequest( cryptoKey=cryptokey_msg, cryptoKeyId=key_name, parent=keyring_fqn)) try: self.api_client.projects_locations_keyRings_cryptoKeys.Create( cryptokey_create_request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: if e.status_code != 409: raise return '%s/cryptoKeys/%s' % (keyring_fqn.rstrip('/'), key_name) def _TranslateExceptionAndRaise(self, e, key_name=None): """Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. key_name: Optional key name in request that caused the exception. Raises: Translated CloudApi exception, or the original exception if it was not translatable. """ if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('TranslateExceptionAndRaise: %s', traceback.format_exc()) translated_exception = self._TranslateApitoolsException(e, key_name=key_name) if translated_exception: raise translated_exception else: raise def _GetMessageFromHttpError(self, http_error): if isinstance(http_error, apitools_exceptions.HttpError): if getattr(http_error, 'content', None): try: json_obj = json.loads(http_error.content) if 'error' in json_obj and 'message' in json_obj['error']: return json_obj['error']['message'] except Exception: # pylint: disable=broad-except # If we couldn't decode anything, just leave the message as None. pass def _GetAcceptableScopesFromHttpError(self, http_error): try: www_authenticate = http_error.response['www-authenticate'] # In the event of a scope error, the www-authenticate field of the HTTP # response should contain text of the form # # 'Bearer realm="https://oauth2.googleapis.com/", # error=insufficient_scope, # scope="${space separated list of acceptable scopes}"' # # Here we use a quick string search to find the scope list, just looking # for a substring with the form 'scope="${scopes}"'. scope_idx = www_authenticate.find('scope="') if scope_idx >= 0: scopes = www_authenticate[scope_idx:].split('"')[1] return 'Acceptable scopes: %s' % scopes except Exception: # pylint: disable=broad-except # Return None if we have any trouble parsing out the acceptable scopes. pass def _TranslateApitoolsException(self, e, key_name=None): """Translates apitools exceptions into their gsutil equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. key_name: Optional key name in request that caused the exception. Returns: CloudStorageApiServiceException for translatable exceptions, None otherwise. """ if isinstance(e, apitools_exceptions.HttpError): message = self._GetMessageFromHttpError(e) if e.status_code == 400: # It is possible that the Project ID is incorrect. Unfortunately the # JSON API does not give us much information about what part of the # request was bad. return BadRequestException(message or 'Bad Request', status=e.status_code) elif e.status_code == 401: if 'Login Required' in str(e): return AccessDeniedException(message or 'Access denied: login required.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) elif e.status_code == 403: if 'The account for the specified project has been disabled' in str(e): return AccessDeniedException(message or 'Account disabled.', status=e.status_code) elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e): return AccessDeniedException(message or 'Access denied: quota exceeded. ' 'Is your project ID valid?', status=e.status_code) elif 'User Rate Limit Exceeded' in str(e): return AccessDeniedException( 'Rate limit exceeded. Please retry this ' 'request later.', status=e.status_code) elif 'Access Not Configured' in str(e): return AccessDeniedException( 'Access Not Configured. Please go to the Google Cloud Platform ' 'Console (https://cloud.google.com/console#/project) for your ' 'project, select APIs & services, and enable the Google Cloud ' 'KMS API.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) else: return AccessDeniedException(message or e.message or key_name, status=e.status_code) elif e.status_code == 404: return NotFoundException(message or e.message, status=e.status_code) elif e.status_code == 409 and key_name: return ServiceException('The key %s already exists.' % key_name, status=e.status_code) elif e.status_code == 412: return PreconditionException(message, status=e.status_code) return ServiceException(message, status=e.status_code) __main__.py 0000644 00000103131 15025013616 0006634 0 ustar 00 #!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Main module for Google Cloud Storage command line tool.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import datetime import errno import getopt import logging import os import re import signal import socket import sys import textwrap import traceback import six from six.moves import configparser from six.moves import range from google.auth import exceptions as google_auth_exceptions import gslib.exception from gslib.exception import CommandException from gslib.exception import ControlCException from gslib.utils.version_check import check_python_version_support from gslib.utils.arg_helper import GetArgumentsAndOptions from gslib.utils.user_agent_helper import GetUserAgent # Load the gsutil version number and append it to boto.UserAgent so the value is # set before anything instantiates boto. This has to run after THIRD_PARTY_DIR # is modified (done in gsutil.py) but before any calls are made that would cause # boto.s3.Connection to be loaded - otherwise the Connection class would end up # with a static reference to the pre-modified version of the UserAgent field, # so boto requests would not include gsutil/version# in the UserAgent string. import boto import gslib from gslib.utils import system_util, text_util # pylint: disable=g-import-not-at-top # This module also imports boto, and will override the UserAgent global variable # if imported above. from gslib import metrics # We parse the options and arguments here so we can pass the results to the user # agent helper. try: opts, args = GetArgumentsAndOptions() except CommandException as e: reason = e.reason if e.informational else 'CommandException: %s' % e.reason err = '%s\n' % reason try: text_util.print_to_fd(err, end='', file=sys.stderr) except UnicodeDecodeError: # Can happen when outputting invalid Unicode filenames. sys.stderr.write(err) if e: metrics.LogFatalError(e) sys.exit(1) # This calculated user agent can be stored for use in StorageV1. gslib.USER_AGENT = GetUserAgent(args, metrics.MetricsCollector.IsDisabled()) boto.UserAgent += gslib.USER_AGENT # pylint: disable=g-bad-import-order import httplib2 import oauth2client from google_reauth import reauth_creds from google_reauth import errors as reauth_errors from gslib import context_config from gslib import wildcard_iterator from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ArgumentException from gslib.cloud_api import BadRequestException from gslib.cloud_api import ProjectIdException from gslib.cloud_api import ServiceException from gslib.command_runner import CommandRunner import apitools.base.py.exceptions as apitools_exceptions from gslib.utils import boto_util from gslib.utils import constants from gslib.utils import system_util from gslib.sig_handling import GetCaughtSignals from gslib.sig_handling import InitializeSignalHandling from gslib.sig_handling import RegisterSignalHandler CONFIG_KEYS_TO_REDACT = ['proxy', 'proxy_port', 'proxy_user', 'proxy_pass'] # We don't use the oauth2 authentication plugin directly; importing it here # ensures that it's loaded and available by default when an operation requiring # authentication is performed. try: # pylint: disable=unused-import,g-import-not-at-top import gcs_oauth2_boto_plugin except ImportError: pass DEBUG_WARNING = """ ***************************** WARNING ***************************** *** You are running gsutil with debug output enabled. *** Be aware that debug output includes authentication credentials. *** Make sure to remove the value of the Authorization header for *** each HTTP request printed to the console prior to posting to *** a public medium such as a forum post or Stack Overflow. ***************************** WARNING ***************************** """.lstrip() TRACE_WARNING = """ ***************************** WARNING ***************************** *** You are running gsutil with trace output enabled. *** Be aware that trace output includes authentication credentials *** and may include the contents of any files accessed during the trace. ***************************** WARNING ***************************** """.lstrip() HTTP_WARNING = """ ***************************** WARNING ***************************** *** You are running gsutil with the "https_validate_certificates" config *** variable set to False. This option should always be set to True in *** production environments to protect against man-in-the-middle attacks, *** and leaking of user data. ***************************** WARNING ***************************** """.lstrip() debug_level = 0 test_exception_traces = False # pylint: disable=unused-argument def _CleanupSignalHandler(signal_num, cur_stack_frame): """Cleans up if process is killed with SIGINT, SIGQUIT or SIGTERM. Note that this method is called after main() has been called, so it has access to all the modules imported at the start of main(). Args: signal_num: Unused, but required in the method signature. cur_stack_frame: Unused, but required in the method signature. """ _Cleanup() if (gslib.utils.parallelism_framework_util. CheckMultiprocessingAvailableAndInit().is_available): gslib.command.TeardownMultiprocessingProcesses() def _Cleanup(): for fname in boto_util.GetCleanupFiles(): try: os.unlink(fname) except: # pylint: disable=bare-except pass def _OutputAndExit(message, exception=None): """Outputs message to stderr and exits gsutil with code 1. This function should only be called in single-process, single-threaded mode. Args: message: Message to print to stderr. exception: The exception that caused gsutil to fail. """ if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS or test_exception_traces: stack_trace = traceback.format_exc() err = ('DEBUG: Exception stack trace:\n %s\n%s\n' % (re.sub('\\n', '\n ', stack_trace), message)) else: err = '%s\n' % message try: text_util.print_to_fd(err, end='', file=sys.stderr) except UnicodeDecodeError: # Can happen when outputting invalid Unicode filenames. sys.stderr.write(err) if exception: metrics.LogFatalError(exception) sys.exit(1) def _OutputUsageAndExit(command_runner): command_runner.RunNamedCommand('help') sys.exit(1) class GsutilFormatter(logging.Formatter): """A logging.Formatter that supports logging microseconds (%f).""" def formatTime(self, record, datefmt=None): if datefmt: return datetime.datetime.fromtimestamp(record.created).strftime(datefmt) # Use default implementation if datefmt is not specified. return super(GsutilFormatter, self).formatTime(record, datefmt=datefmt) def _ConfigureRootLogger(level=logging.INFO): """Similar to logging.basicConfig() except it always adds a handler.""" log_format = '%(levelname)s %(asctime)s %(filename)s] %(message)s' date_format = '%m%d %H:%M:%S.%f' formatter = GsutilFormatter(fmt=log_format, datefmt=date_format) handler = logging.StreamHandler() handler.setFormatter(formatter) root_logger = logging.getLogger() root_logger.addHandler(handler) root_logger.setLevel(level) def main(): InitializeSignalHandling() # Any modules used in initializing multiprocessing variables must be # imported after importing gslib.__main__. # pylint: disable=redefined-outer-name,g-import-not-at-top import gslib.boto_translation import gslib.command import gslib.utils.parallelism_framework_util # pylint: disable=unused-variable from gcs_oauth2_boto_plugin import oauth2_client from apitools.base.py import credentials_lib # pylint: enable=unused-variable if (gslib.utils.parallelism_framework_util. CheckMultiprocessingAvailableAndInit().is_available): # These setup methods must be called, and, on Windows, they can only be # called from within an "if __name__ == '__main__':" block. gslib.command.InitializeMultiprocessingVariables() gslib.boto_translation.InitializeMultiprocessingVariables() else: gslib.command.InitializeThreadingVariables() # This needs to be done after InitializeMultiprocessingVariables(), since # otherwise we can't call CreateLock. try: # pylint: disable=unused-import,g-import-not-at-top import gcs_oauth2_boto_plugin gsutil_client_id, gsutil_client_secret = ( system_util.GetGsutilClientIdAndSecret()) gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret( gsutil_client_id, gsutil_client_secret) gcs_oauth2_boto_plugin.oauth2_helper.SetLock( gslib.utils.parallelism_framework_util.CreateLock()) credentials_lib.SetCredentialsCacheFileLock( gslib.utils.parallelism_framework_util.CreateLock()) except ImportError: pass global debug_level global test_exception_traces supported, err = check_python_version_support() if not supported: raise CommandException(err) sys.exit(1) boto_util.MonkeyPatchBoto() system_util.MonkeyPatchHttp() # In gsutil 4.0 and beyond, we don't use the boto library for the JSON # API. However, we still store gsutil configuration data in the .boto # config file for compatibility with previous versions and user convenience. # Many users have a .boto configuration file from previous versions, and it # is useful to have all of the configuration for gsutil stored in one place. command_runner = CommandRunner() if not boto_util.BOTO_IS_SECURE: raise CommandException('\n'.join( textwrap.wrap( 'Your boto configuration has is_secure = False. Gsutil cannot be ' 'run this way, for security reasons.'))) headers = {} parallel_operations = False quiet = False version = False debug_level = 0 trace_token = None perf_trace_token = None test_exception_traces = False user_project = None # If user enters no commands just print the usage info. if len(sys.argv) == 1: sys.argv.append('help') # Change the default of the 'https_validate_certificates' boto option to # True (it is currently False in boto). if not boto.config.has_option('Boto', 'https_validate_certificates'): if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.setbool('Boto', 'https_validate_certificates', True) for signal_num in GetCaughtSignals(): RegisterSignalHandler(signal_num, _CleanupSignalHandler) try: for o, a in opts: if o in ('-d', '--debug'): # Also causes boto to include httplib header output. debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS elif o in ('-D', '--detailedDebug'): # We use debug level 3 to ask gsutil code to output more detailed # debug output. This is a bit of a hack since it overloads the same # flag that was originally implemented for boto use. And we use -DD # to ask for really detailed debugging (i.e., including HTTP payload). if debug_level == constants.DEBUGLEVEL_DUMP_REQUESTS: debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS else: debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS elif o in ('-?', '--help'): _OutputUsageAndExit(command_runner) elif o in ('-h', '--header'): (hdr_name, _, hdr_val) = a.partition(':') if not hdr_name: _OutputUsageAndExit(command_runner) headers[hdr_name.lower()] = hdr_val elif o in ('-m', '--multithreaded'): parallel_operations = True elif o in ('-q', '--quiet'): quiet = True elif o == '-u': user_project = a elif o in ('-v', '--version'): version = True elif o in ('-i', '--impersonate-service-account'): constants.IMPERSONATE_SERVICE_ACCOUNT = a elif o == '--perf-trace-token': perf_trace_token = a elif o == '--trace-token': trace_token = a elif o == '--testexceptiontraces': # Hidden flag for integration tests. test_exception_traces = True # Avoid printing extra warnings to stderr regarding long retries by # setting the threshold very high. constants.LONG_RETRY_WARN_SEC = 3600 elif o in ('-o', '--option'): (opt_section_name, _, opt_value) = a.partition('=') if not opt_section_name: _OutputUsageAndExit(command_runner) (opt_section, _, opt_name) = opt_section_name.partition(':') if not opt_section or not opt_name: _OutputUsageAndExit(command_runner) if not boto.config.has_section(opt_section): boto.config.add_section(opt_section) boto.config.set(opt_section, opt_name, opt_value) # Now that any Boto option overrides (via `-o` args) have been parsed, # perform initialization that depends on those options. boto_util.configured_certs_file = (boto_util.ConfigureCertsFile()) metrics.LogCommandParams(global_opts=opts) httplib2.debuglevel = debug_level if trace_token: sys.stderr.write(TRACE_WARNING) if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS: sys.stderr.write(DEBUG_WARNING) _ConfigureRootLogger(level=logging.DEBUG) command_runner.RunNamedCommand('ver', ['-l']) config_items = [] for config_section in ('Boto', 'GSUtil'): try: config_items.extend(boto.config.items(config_section)) except configparser.NoSectionError: pass for i in range(len(config_items)): config_item_key = config_items[i][0] if config_item_key in CONFIG_KEYS_TO_REDACT: config_items[i] = (config_item_key, 'REDACTED') sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv)) sys.stderr.write('config_file_list: %s\n' % boto_util.GetFriendlyConfigFilePaths()) sys.stderr.write('config: %s\n' % str(config_items)) else: # Non-debug log level. root_logger_level = logging.WARNING if quiet else logging.INFO # oauth2client uses INFO and WARNING logging in places that would better # correspond to gsutil's debug logging (e.g., when refreshing # access tokens), so we bump the threshold one level higher where # appropriate. These log levels work for regular- and quiet-level logging. oa2c_logger_level = logging.WARNING oa2c_multiprocess_file_storage_logger_level = logging.ERROR _ConfigureRootLogger(level=root_logger_level) oauth2client.client.logger.setLevel(oa2c_logger_level) oauth2client.contrib.multiprocess_file_storage.logger.setLevel( oa2c_multiprocess_file_storage_logger_level) # pylint: disable=protected-access oauth2client.transport._LOGGER.setLevel(oa2c_logger_level) reauth_creds._LOGGER.setLevel(oa2c_logger_level) # pylint: enable=protected-access # TODO(reauth): Fix once reauth pins to pyu2f version newer than 0.1.3. # Fixes pyu2f v0.1.3 bug. import six # pylint: disable=g-import-not-at-top six.input = six.moves.input if not boto_util.CERTIFICATE_VALIDATION_ENABLED: sys.stderr.write(HTTP_WARNING) if version: command_name = 'version' elif not args: command_name = 'help' else: command_name = args[0] if command_name != 'test': # Don't initialize mTLS authentication because # tests that need it will do this initialization themselves. context_config.create_context_config(logging.getLogger()) _CheckAndWarnForProxyDifferences() # Both 1 and 2 are valid _ARGCOMPLETE values; this var tells argcomplete at # what argv[] index the command to match starts. We want it to start at the # value for the path to gsutil, so: # $ gsutil <command> # Should be the 1st argument, so '1' # $ python gsutil <command> # Should be the 2nd argument, so '2' # Both are valid; most users invoke gsutil in the first style, but our # integration and prerelease tests invoke it in the second style, as we need # to specify the Python interpreter used to run gsutil. if os.environ.get('_ARGCOMPLETE', '0') in ('1', '2'): return _PerformTabCompletion(command_runner) return _RunNamedCommandAndHandleExceptions( command_runner, command_name, args=args[1:], headers=headers, debug_level=debug_level, trace_token=trace_token, parallel_operations=parallel_operations, perf_trace_token=perf_trace_token, user_project=user_project) finally: _Cleanup() def _CheckAndWarnForProxyDifferences(): # If there are both boto config and environment variable config present for # proxies, unset the environment variable and warn if it differs. boto_port = boto.config.getint('Boto', 'proxy_port', 0) if boto.config.get('Boto', 'proxy', None) or boto_port: for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: if proxy_env_var in os.environ and os.environ[proxy_env_var]: differing_values = [] proxy_info = boto_util.ProxyInfoFromEnvironmentVar(proxy_env_var) if proxy_info.proxy_host != boto.config.get('Boto', 'proxy', None): differing_values.append( 'Boto proxy host: "%s" differs from %s proxy host: "%s"' % (boto.config.get('Boto', 'proxy', None), proxy_env_var, proxy_info.proxy_host)) if (proxy_info.proxy_user != boto.config.get('Boto', 'proxy_user', None)): differing_values.append( 'Boto proxy user: "%s" differs from %s proxy user: "%s"' % (boto.config.get('Boto', 'proxy_user', None), proxy_env_var, proxy_info.proxy_user)) if (proxy_info.proxy_pass != boto.config.get('Boto', 'proxy_pass', None)): differing_values.append( 'Boto proxy password differs from %s proxy password' % proxy_env_var) # Only compare ports if at least one is present, since the # boto logic for selecting default ports has not yet executed. if ((proxy_info.proxy_port or boto_port) and proxy_info.proxy_port != boto_port): differing_values.append( 'Boto proxy port: "%s" differs from %s proxy port: "%s"' % (boto_port, proxy_env_var, proxy_info.proxy_port)) if differing_values: sys.stderr.write('\n'.join( textwrap.wrap( 'WARNING: Proxy configuration is present in both the %s ' 'environment variable and boto configuration, but ' 'configuration differs. boto configuration proxy values will ' 'be used. Differences detected:' % proxy_env_var))) sys.stderr.write('\n%s\n' % '\n'.join(differing_values)) # Regardless of whether the proxy configuration values matched, # delete the environment variable so as not to confuse boto. del os.environ[proxy_env_var] def _HandleUnknownFailure(e): # Called if we fall through all known/handled exceptions. raise _OutputAndExit(message='Failure: %s.' % e, exception=e) def _HandleCommandException(e): if e.informational: _OutputAndExit(message=e.reason, exception=e) else: _OutputAndExit(message='CommandException: %s' % e.reason, exception=e) # pylint: disable=unused-argument def _HandleControlC(signal_num, cur_stack_frame): """Called when user hits ^C. This function prints a brief message instead of the normal Python stack trace (unless -D option is used). Args: signal_num: Signal that was caught. cur_stack_frame: Unused. """ if debug_level >= 2: stack_trace = ''.join(traceback.format_list(traceback.extract_stack())) _OutputAndExit('DEBUG: Caught CTRL-C (signal %d) - Exception stack trace:\n' ' %s' % (signal_num, re.sub('\\n', '\n ', stack_trace)), exception=ControlCException()) else: _OutputAndExit('Caught CTRL-C (signal %d) - exiting' % signal_num, exception=ControlCException()) def _HandleSigQuit(signal_num, cur_stack_frame): r"""Called when user hits ^\, so we can force breakpoint a running gsutil.""" import pdb # pylint: disable=g-import-not-at-top pdb.set_trace() def _ConstructAccountProblemHelp(reason): """Constructs a help string for an access control error. Args: reason: e.reason string from caught exception. Returns: Contructed help text. """ default_project_id = boto.config.get_value('GSUtil', 'default_project_id') # pylint: disable=line-too-long, g-inconsistent-quotes acct_help = ( "Your request resulted in an AccountProblem (403) error. Usually this " "happens if you attempt to create a bucket without first having " "enabled billing for the project you are using. Please ensure billing is " "enabled for your project by following the instructions at " "`Google Cloud Platform Console<https://support.google.com/cloud/answer/6158867>`. " ) if default_project_id: acct_help += ( "In the project overview, ensure that the Project Number listed for " "your project matches the project ID (%s) from your boto config file. " % default_project_id) acct_help += ( "If the above doesn't resolve your AccountProblem, please send mail to " "buganizer-system+187143@google.com requesting assistance, noting the " "exact command you ran, the fact that you received a 403 AccountProblem " "error, and your project ID. Please do not post your project ID on " "StackOverflow. " "Note: It's possible to use Google Cloud Storage without enabling " "billing if you're only listing or reading objects for which you're " "authorized, or if you're uploading objects to a bucket billed to a " "project that has billing enabled. But if you're attempting to create " "buckets or upload objects to a bucket owned by your own project, you " "must first enable billing for that project.") return acct_help def _CheckAndHandleCredentialException(e, args): # Provide detail to users who have no boto config file (who might previously # have been using gsutil only for accessing publicly readable buckets and # objects). if (not boto_util.HasConfiguredCredentials() and not boto.config.get_value( 'Tests', 'bypass_anonymous_access_warning', False)): # The check above allows tests to assert that we get a particular, # expected failure, rather than always encountering this error message # when there are no configured credentials. This allows tests to # simulate a second user without permissions, without actually requiring # two separate configured users. if system_util.InvokedViaCloudSdk(): message = '\n'.join( textwrap.wrap( 'You are attempting to access protected data with no configured ' 'credentials. Please visit ' 'https://cloud.google.com/console#/project and sign up for an ' 'account, and then run the "gcloud auth login" command to ' 'configure gsutil to use these credentials.')) else: message = '\n'.join( textwrap.wrap( 'You are attempting to access protected data with no configured ' 'credentials. Please visit ' 'https://cloud.google.com/console#/project and sign up for an ' 'account, and then run the "gsutil config" command to configure ' 'gsutil to use these credentials.')) _OutputAndExit(message=message, exception=e) elif (e.reason and (e.reason == 'AccountProblem' or e.reason == 'Account disabled.' or 'account for the specified project has been disabled' in e.reason) and ','.join(args).find('gs://') != -1): _OutputAndExit('\n'.join( textwrap.wrap(_ConstructAccountProblemHelp(e.reason))), exception=e) def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None, headers=None, debug_level=0, trace_token=None, parallel_operations=False, perf_trace_token=None, user_project=None): """Runs the command and handles common exceptions.""" # Note that this method is run at the end of main() and thus has access to # all of the modules imported there. # pylint: disable=g-import-not-at-top try: # Catch ^C so we can print a brief message instead of the normal Python # stack trace. Register as a final signal handler because this handler kills # the main gsutil process (so it must run last). RegisterSignalHandler(signal.SIGINT, _HandleControlC, is_final_handler=True) # Catch ^\ so we can force a breakpoint in a running gsutil. if not system_util.IS_WINDOWS: RegisterSignalHandler(signal.SIGQUIT, _HandleSigQuit) return command_runner.RunNamedCommand(command_name, args, headers, debug_level, trace_token, parallel_operations, perf_trace_token=perf_trace_token, collect_analytics=True, user_project=user_project) except AttributeError as e: if str(e).find('secret_access_key') != -1: _OutputAndExit( 'Missing credentials for the given URI(s). Does your ' 'boto config file contain all needed credentials?', exception=e) else: _OutputAndExit(message=str(e), exception=e) except CommandException as e: _HandleCommandException(e) except getopt.GetoptError as e: _HandleCommandException(CommandException(e.msg)) except boto.exception.InvalidUriError as e: _OutputAndExit(message='InvalidUriError: %s.' % e.message, exception=e) except gslib.exception.InvalidUrlError as e: _OutputAndExit(message='InvalidUrlError: %s.' % e.message, exception=e) except boto.auth_handler.NotReadyToAuthenticate as e: _OutputAndExit(message='NotReadyToAuthenticate', exception=e) except gslib.exception.ExternalBinaryError as e: _OutputAndExit(message=str(e), exception=e) except OSError as e: # In Python 3, IOError (next except) is an alias for OSError # Sooo... we need the same logic here if (e.errno == errno.EPIPE or (system_util.IS_WINDOWS and e.errno == errno.EINVAL) and not system_util.IsRunningInteractively()): # If we get a pipe error, this just means that the pipe to stdout or # stderr is broken. This can happen if the user pipes gsutil to a command # that doesn't use the entire output stream. Instead of raising an error, # just swallow it up and exit cleanly. sys.exit(0) else: _OutputAndExit(message='OSError: %s.' % e.strerror, exception=e) except IOError as e: if (e.errno == errno.EPIPE or (system_util.IS_WINDOWS and e.errno == errno.EINVAL) and not system_util.IsRunningInteractively()): # If we get a pipe error, this just means that the pipe to stdout or # stderr is broken. This can happen if the user pipes gsutil to a command # that doesn't use the entire output stream. Instead of raising an error, # just swallow it up and exit cleanly. sys.exit(0) else: raise except wildcard_iterator.WildcardException as e: _OutputAndExit(message=e.reason, exception=e) except ProjectIdException as e: _OutputAndExit( 'You are attempting to perform an operation that requires a ' 'project id, with none configured. Please re-run ' 'gsutil config and make sure to follow the instructions for ' 'finding and entering your default project id.', exception=e) except BadRequestException as e: if e.reason == 'MissingSecurityHeader': _CheckAndHandleCredentialException(e, args) _OutputAndExit(message=e, exception=e) except AccessDeniedException as e: _CheckAndHandleCredentialException(e, args) _OutputAndExit(message=e, exception=e) except ArgumentException as e: _OutputAndExit(message=e, exception=e) except ServiceException as e: _OutputAndExit(message=e, exception=e) except (oauth2client.client.HttpAccessTokenRefreshError, google_auth_exceptions.OAuthError, google_auth_exceptions.RefreshError) as e: if system_util.InvokedViaCloudSdk(): _OutputAndExit( 'Your credentials are invalid. ' 'Please run\n$ gcloud auth login', exception=e) else: _OutputAndExit( 'Your credentials are invalid. For more help, see ' '"gsutil help creds", or re-run the gsutil config command (see ' '"gsutil help config").', exception=e) except apitools_exceptions.HttpError as e: # These should usually be retried by the underlying implementation or # wrapped by CloudApi ServiceExceptions, but if we do get them, # print something useful. _OutputAndExit('HttpError: %s, %s' % (getattr(e.response, 'status', ''), e.content or ''), exception=e) except socket.error as e: if e.args[0] == errno.EPIPE: # Retrying with a smaller file (per suggestion below) works because # the library code send loop (in boto/s3/key.py) can get through the # entire file and then request the HTTP response before the socket # gets closed and the response lost. _OutputAndExit( 'Got a "Broken pipe" error. This can happen to clients using Python ' '2.x, when the server sends an error response and then closes the ' 'socket (see http://bugs.python.org/issue5542). If you are trying to ' 'upload a large object you might retry with a small (say 200k) ' 'object, and see if you get a more specific error code.', exception=e) elif e.args[0] == errno.ECONNRESET and ' '.join(args).contains('s3://'): _OutputAndExit('\n'.join( textwrap.wrap( 'Got a "Connection reset by peer" error. One way this can happen is ' 'when copying data to/from an S3 regional bucket. If you are using a ' 'regional S3 bucket you could try re-running this command using the ' 'regional S3 endpoint, for example ' 's3://s3-<region>.amazonaws.com/your-bucket. For details about this ' 'problem see https://github.com/boto/boto/issues/2207')), exception=e) else: _HandleUnknownFailure(e) except oauth2client.client.FlowExchangeError as e: _OutputAndExit('\n%s\n\n' % '\n'.join( textwrap.wrap( 'Failed to retrieve valid credentials (%s). Make sure you selected and ' 'pasted the ENTIRE authorization code (including any numeric prefix ' "e.g. '4/')." % e)), exception=e) except reauth_errors.ReauthSamlLoginRequiredError: if system_util.InvokedViaCloudSdk(): _OutputAndExit('You must re-authenticate with your SAML IdP. ' 'Please run\n$ gcloud auth login') else: _OutputAndExit('You must re-authenticate with your SAML IdP. ' 'Please run\n$ gsutil config') except Exception as e: # pylint: disable=broad-except config_paths = ', '.join(boto_util.GetFriendlyConfigFilePaths()) # Check for two types of errors related to service accounts. These errors # appear to be the same except for their messages, but they are caused by # different problems and both have unhelpful error messages. Moreover, # the error type belongs to PyOpenSSL, which is not necessarily installed. if 'mac verify failure' in str(e): _OutputAndExit( 'Encountered an error while refreshing access token. ' 'If you are using a service account,\nplease verify that the ' 'gs_service_key_file_password field in your config file(s),' '\n%s, is correct.' % config_paths, exception=e) elif 'asn1 encoding routines' in str(e): _OutputAndExit( 'Encountered an error while refreshing access token. ' 'If you are using a service account,\nplease verify that the ' 'gs_service_key_file field in your config file(s),\n%s, is correct.' % config_paths, exception=e) _HandleUnknownFailure(e) def _PerformTabCompletion(command_runner): """Performs gsutil-specific tab completion for the shell.""" # argparse and argcomplete are bundled with the Google Cloud SDK. # When gsutil is invoked from the Google Cloud SDK, both should be available. try: import argcomplete import argparse except ImportError as e: _OutputAndExit('A library required for performing tab completion was' ' not found.\nCause: %s' % e, exception=e) parser = argparse.ArgumentParser(add_help=False) command_runner.ConfigureCommandArgumentParsers(parser) argcomplete.autocomplete(parser, exit_method=sys.exit) return 0 if __name__ == '__main__': sys.exit(main()) commands/__pycache__/defacl.cpython-39.pyc 0000644 00000026404 15025013616 0014431 0 ustar 00 a +(Wg : � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlm Z dd l m Z dd l m Z dd l m Z dd l mZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddlm Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl'm)Z) dd l*m+Z+ d Z,d Z-d Z.d Z/d!Z0d"Z1e,e-�2d#� e.�2d#� d$ Z3d%d#�4e/e0 e1 g� Z5ee3e5�Z6ee-e0�Z7ee,e/�Z8ee.e1�Z9G d&d'� d'e�Z:dS )(zFImplementation of default object acl command for Google Cloud Storage.� )�absolute_import)�print_function)�division)�unicode_literalsN)� gcs_json_api)�metrics)�AccessDeniedException)�BadRequestException)� Preconditions)�ServiceException)�Command)�SetAclExceptionHandler)�SetAclFuncWrapper)�CommandArgument)� ApiSelector)�CommandException)�CreateHelpText)�StorageUrlFromString)�UrlsAreForSingleProvider)�storage_v1_messages)� acl_helper)�NO_MAX)�Retry)�GcloudStorageFlag)�GcloudStorageMap)�PRIVATE_DEFAULT_OBJ_ACLzJ gsutil defacl set (<file-path>|<predefined-acl>) gs://<bucket_name>... z( gsutil defacl get gs://<bucket_name> zF gsutil defacl ch [-f] -u|-g|-d|-p <grant>... gs://<bucket_name>... a[ <B>SET</B> The ``defacl set`` command sets default object ACLs for the specified buckets. If you specify a default object ACL for a certain bucket, Cloud Storage applies the default object ACL to all new objects uploaded to that bucket, unless an ACL for that object is separately specified during upload. Similar to the ``acl set`` command, the ``defacl set`` command specifies either a predefined ACL or the path to a file that contains ACL text. See "gsutil help acl" for examples of editing and setting ACLs via the acl command. See `Predefined ACLs <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ for a list of predefined ACLs. Setting a default object ACL on a bucket provides a convenient way to ensure newly uploaded objects have a specific ACL. If you don't set the bucket's default object ACL, it will default to project-private. If you then upload objects that need a different ACL, you will need to perform a separate ACL update operation for each object. Depending on how many objects require updates, this could be very time-consuming. z{ <B>GET</B> Gets the default ACL text for a bucket, which you can save and edit for use with the "defacl set" command. a� <B>CH</B> The "defacl ch" (or "defacl change") command updates the default object access control list for a bucket. The syntax is shared with the "acl ch" command, so see the "CH" section of "gsutil help acl" for the full help description. <B>CH EXAMPLES</B> Grant anyone on the internet READ access by default to any object created in the bucket example-bucket: gsutil defacl ch -u AllUsers:R gs://example-bucket NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see "gsutil help setmeta". Add the user john.doe@example.com to the default object ACL on bucket example-bucket with READ access: gsutil defacl ch -u john.doe@example.com:READ gs://example-bucket Add the group admins@example.com to the default object ACL on bucket example-bucket with OWNER access: gsutil defacl ch -g admins@example.com:O gs://example-bucket Remove the group admins@example.com from the default object ACL on bucket example-bucket: gsutil defacl ch -d admins@example.com gs://example-bucket Add the owners of project example-project-123 to the default object ACL on bucket example-bucket with READ access: gsutil defacl ch -p owners-example-project-123:R gs://example-bucket NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access to a project's viewers/editors respectively. <B>CH OPTIONS</B> The "ch" sub-command has the following options -d Remove all roles associated with the matching entity. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. With this option the gsutil exit status will be 0 even if some ACLs couldn't be changed. -g Add or modify a group entity's role. -p Add or modify a project viewers/editors/owners role. -u Add or modify a user entity's role. � z z. The defacl command has three sub-commands: c s� e Zd ZdZejdg d�ededdddej ej gej e � � e � � ge �d�ge � � gd�d � Zejdg d �d d eeeed �d�Z� fdd�Zdd� Zdd� Zdd� Zdd� Zee ddd�dd � �Z d d � Z d d � Z � Z S )!� DefAclCommandz(Implementation of gsutil defacl command.�defacl)� setdefacl� getdefacl�chdefacl� z fg:u:d:p:F� )�set�get�ch) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)z default aclr r r! � command_helpz*Get, set, or change default ACL on buckets)r% r$ r& )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s� | j �d�}|dkr&tg d�i d�}n�|dkr�| j �d�}tj�|�r`tdddd | gi d�}q�|tjv rvtj| }n|}tdddd | gi d�}nN|d kr�| �� t � | j �| _ tg d �t d �t d �t d �t d�t d�d�d�}t � �|�S )Nr r% )�storage�buckets�describez,--format=multi(defaultObjectAcl:format=json)z--raw)�gcloud_command�flag_mapr$ r9 r: �updatez--default-object-acl-file=z --predefined-default-object-acl=r& )r9 r: r> z --add-default-object-acl-grantz!--remove-default-object-acl-grantz--continue-on-error)�-g�-p�-u�-dz-f)�args�popr �os�path�isfiler �+FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION� ParseSubOptsr �translate_sub_opts_for_shim�sub_optsr �super�get_gcloud_storage_args)�self� sub_command�gcloud_storage_map�acl_file_or_predefined_acl�predefined_acl�� __class__� �$/opt/gsutil/gslib/commands/defacl.pyrM � sT � ��������� z%DefAclCommand.get_gcloud_storage_argsc C s6 | j s| �� | j d �� dks*| jdkr.dS dS d S )Nr r$ r r# )rC �$RaiseWrongNumberOfArgumentsException�lower�command_alias_used�rN rU rU rV �_CalculateUrlsStartArg� s �z$DefAclCommand._CalculateUrlsStartArgc C sR t | jd ��� s td| j ��z| �tt� W n tyL | � � � Y n0 d S )N����)URL must name a bucket for the %s command) r rC �IsBucketr � command_name�SetAclCommandHelperr r r �_WarnServiceAccountsrZ rU rU rV � _SetDefAcl s � zDefAclCommand._SetDefAclc C s4 t | jd ��� s td| j ��| �| jd � d S )Nr r] )r rC r^ r r_ �GetAndPrintAclrZ rU rU rV � _GetDefAcl s �zDefAclCommand._GetDefAclc C s: d| _ g | _| jr�| jD ]�\}}|dkrB| j�tj|tjjd�� |dkrd| j�tj|tjjd�� |dkr�| j�tj|tjj d�� |dkr| j�t� |�� q| js�t d��t | j �r�t| j d �jd kr�t d �| j���t� }| j D ]4}| �|�D ]$}|j�� �st d ��|�|j� q�q�|D ]}| �|� �q$d S ) zDParses options and changes default object ACLs on specified buckets.Tr? )� scope_typerA r@ rB zFPlease specify at least one access change with the -g, -u, or -d flagsr �gsz2The "{0}" command can only be used with gs:// URLsz5The defacl ch command can only be applied to buckets.N)�parse_versions�changesrK �appendr � AclChange� ChangeType�GROUP�USER�PROJECT�AclDelr r rC r �scheme�formatr_ r$ �WildcardIterator� storage_urlr^ �add�ApplyAclChanges)rN �o�aZ bucket_urlsZurl_arg�resultrs rU rU rV � _ChDefAcl sL ��� ��� �zDefAclCommand._ChDefAcl� )�tries� timeout_secsc C s� | j j|j|jddgd�}|j}| �||�dkrB| j�d|� dS |sP|�t � zFt |j d�}t j |d�}| j j|j|||jd gd � | j�d |� W nX ty� } z td t|� ��W Y d}~n.d}~0 ty� | �� td | ��Y n0 dS )z8Applies the changes in self.changes to the provided URL.�defaultObjectAcl�metageneration)�provider�fieldsr zNo changes to %sN)�meta_gen_match)r} �id)� preconditionsr r� zUpdated default ACL on %sz$Received bad request from server: %szTFailed to set acl for %s. Please ensure you have OWNER-role access to this resource.)� gsutil_api� GetBucket� bucket_namerp r} �$_ApplyAclChangesAndReturnChangeCount�logger�infori r r r~ �apitools_messages�Bucket� PatchBucketr r �strr ra )rN �url�bucket� current_aclr� �bucket_metadata�erU rU rV ru = s8 � �$ �z DefAclCommand.ApplyAclChangesc C s* d}| j D ]}||�||d| j�7 }q |S )Nr r )rh �Executer� )rN rs Zdefacl_message�modification_count�changerU rU rV r� e s �z2DefAclCommand._ApplyAclChangesAndReturnChangeCountc C s� | j �d�}| jdd� d| _d| _|dkr4| j}n2|dkrD| j}n"|dv rT| j}ntd|| j f ��t j |g| j d � |� dS ) z+Command entry point for the defacl command.r T)� check_argsFr% r$ )r& r� zEInvalid subcommand "%s" for the %s command. See "gsutil help defacl".)� subcommandsrK ) rC rD rI �def_acl�continue_on_errorrd rb ry r r_ r �LogCommandParamsrK )rN �action_subcommand�funcrU rU rV � RunCommandl s$ ��zDefAclCommand.RunCommand)!�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr � MakeFileURLOrCannedACLArgument�%MakeZeroOrMoreCloudBucketURLsArgument� MakeNCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� _ch_help_text� help_specrM r[ rb rd ry r r ru r� r� � __classcell__rU rU rS rV r � sP � ���� 5 ( 'r );r� � __future__r r r r rE �gslibr r �gslib.cloud_apir r r r � gslib.commandr r r �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.help_providerr �gslib.storage_urlr r �"gslib.third_party.storage_apitoolsr r� Z gslib.utilsr �gslib.utils.constantsr �gslib.utils.retry_utilr �gslib.utils.shim_utilr r � gslib.utils.translation_helperr � _SET_SYNOPSIS� _GET_SYNOPSIS� _CH_SYNOPSIS�_SET_DESCRIPTION�_GET_DESCRIPTION�_CH_DESCRIPTION�lstripr� �join� _DESCRIPTIONr� r� r� r� r rU rU rU rV �<module> s\ ; ��� commands/__pycache__/kms.cpython-39.pyc 0000644 00000035606 15025013616 0014011 0 ustar 00 a +(Wg�R � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl)m+Z+ dZ,dZ-d Z.e,e-�/d � e.�/d � d Z0d Z1d Z2d Z3d!e1e2 e3 Z4ee0e4�Z5ee,e1�Z6ee-e2�Z7ee.e3�Z8e+d"d#ge*d$�e*d%�d&�d'�Z9e+d"d#gd(e*d$�id'�Z:G d)d*� d*e�Z;dS )+z/This module provides the kms command to gsutil.� )�absolute_import)�print_function)�division)�unicode_literalsN)�metrics)�AccessDeniedException)�ServiceException)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�KmsApi)�PopulateProjectId)�Binding)�storage_v1_messages)� text_util)�NO_MAX)� ValidateCMEK)�Retry)�GcloudStorageFlag)�GcloudStorageMapz4 gsutil kms authorize [-p <proj_id>] -k <kms_key> zJ gsutil kms encryption [(-d|[-k <kms_key>])] [-w] gs://<bucket_name>... z, gsutil kms serviceaccount [-p <proj_id>] � a� <B>AUTHORIZE</B> The authorize sub-command checks that the default (or supplied) project has a Cloud Storage service agent created for it, and if not, it creates one. It then adds appropriate encrypt/decrypt permissions to Cloud KMS resources such that the service agent can write and read Cloud KMS-encrypted objects in buckets associated with the service agent's project. <B>AUTHORIZE EXAMPLES</B> Authorize "my-project" to use a Cloud KMS key: gsutil kms authorize -p my-project \ -k projects/key-project/locations/us-east1/keyRings/key-ring/cryptoKeys/my-key <B>AUTHORIZE OPTIONS</B> -k <key> The path to the KMS key to use. The path has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -p <project> The ID or number of the project being authorized to use the Cloud KMS key. If this flag is not included, your default project is authorized. a� <B>ENCRYPTION</B> The encryption sub-command is used to set, display, or clear a bucket's default KMS key, which is used to encrypt newly-written objects if no other key is specified. <B>ENCRYPTION EXAMPLES</B> Set the default KMS key for my-bucket: gsutil kms encryption \ -k projects/key-project/locations/us-east1/keyRings/key-ring/cryptoKeys/my-key \ gs://my-bucket Show the default KMS key for my-bucket, if one is set: gsutil kms encryption gs://my-bucket Clear the default KMS key so newly-written objects are not encrypted using it: gsutil kms encryption -d gs://my-bucket Once you clear the default KMS key, newly-written objects are encrypted with Google-managed encryption keys by default. <B>ENCRYPTION OPTIONS</B> -k <key> Set the default KMS key for my-bucket using the full path to the key, which has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -w (used with -k key) Display a warning rather than failing if gsutil is unable to verify that the specified key contains the correct IAM bindings for encryption/decryption. This is useful for users that do not have getIamPolicy permission but know that the key has the correct IAM policy for encryption in the user's project. -d Clear the default KMS key. a" <B>SERVICEACCOUNT</B> The serviceaccount sub-command displays the Cloud Storage service agent that is used to perform Cloud KMS operations against your default project (or a supplied project). <B>SERVICEACCOUNT EXAMPLES</B> Show the service account for my-project: gsutil kms serviceaccount -p my-project <B>SERVICEACCOUNT OPTIONS</B> -p <project> The ID or number of the project whose Cloud Storage service agent is being requested. If this flag is not included, your default project is used. aS The kms command is used to configure Google Cloud Storage and Cloud KMS resources to support encryption of Cloud Storage objects with `Cloud KMS keys <https://cloud.google.com/storage/docs/encryption/customer-managed-keys>`_. The kms command has three sub-commands that deal with configuring Cloud Storage's integration with Cloud KMS: ``authorize``, ``encryption``, and ``serviceaccount``. Before using this command, read the `prerequisites <https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys#prereqs>`_. for using Cloud KMS with Cloud Storage. �storagez service-agentz --projectz--authorize-cmek)�-p�-k��gcloud_command�flag_mapr c s� e Zd ZdZejdededdddej gej g e � d�gg d�d� Z ej dg dd eeeed�d �Zeeed �i d �Z� fd d�Zdd� Zeeddd�dd� �Zdd� Z dd� Z dd� Z dd � Z d d � Z d d � Z!d!d"� Z"d#d"� Z"� Z#S )$� KmsCommandz!Implements of gsutil kms command.�kms� zdk:p:wF�� authorize� encryption�serviceaccount) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz Configure Cloud KMS encryption)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)r$ r& r c s� | j d dkr�tdtddgtd�td�td�d�d �ii d �}d | j v sRd | j v rj|jd jd g7 _q�|jd jg d �7 _ntj}t� �|�S )Nr r% r �bucketsz --clear-default-encryption-keyz--default-encryption-key� )�-dr �-wr r: r �update)�describezi--format=value[separator=": "](name, encryption.defaultKmsKeyName.yesno(no="No default encryption key."))z--raw)�argsr r r r �gcloud_storage_map�super�get_gcloud_storage_args)�selfr? �� __class__� �!/opt/gsutil/gslib/commands/kms.pyrA � s( ��� � � z"KmsCommand.get_gcloud_storage_argsc C s� | � � d| _d | _d| _| jrx| jD ]P\}}|dkr>|| _q&|dkrX|| _t| j� q&|dkrhd| _q&|dkr&d| _q&| jr�| jdks�| js�td� t � d ����| js�t d �| _d S ) NFr r r: Tr; r% r zoThe "-w" option should only be specified for the "encryption" subcommand and must be used with the "-k" option.) �CheckArguments� clear_kms_key�kms_key� warn_on_key_authorize_failure�sub_opts� project_idr �subcommand_namer �join�textwrap�wrapr )rB rM �o�arE rE rF �_GatherSubOptions s6 ����z KmsCommand._GatherSubOptions� )�tries� timeout_secsc C s� | j j|dd�j}t| jd�}| j�d|� z^|�|�}| j�d|� d}tdd| gd �}||jvr�|j� |� |� ||� d }||fW S t y� | j r�t �d �t�d ��� |df Y S � Y n0 d S )a� Authorizes a project's service account to be used with a KMS key. Authorizes the Cloud Storage-owned service account for project_id to be used with kms_key. Args: project_id: (str) Project id string (not number). kms_key: (str) Fully qualified resource name for the KMS key. Returns: (str, bool) A 2-tuple consisting of: 1) The email address for the service account associated with the project, which is authorized to encrypt/decrypt with the specified key. 2) A bool value - True if we had to grant the service account permission to encrypt/decrypt with the given key; False if the required permission was already present. �gs��provider)�loggerzGetting IAM policy for %szCurrent policy is %sFz*roles/cloudkms.cryptoKeyEncrypterDecrypterzserviceAccount:%s)�role�membersTr a Warning: Check that your Cloud Platform project's service account has the "cloudkms.cryptoKeyEncrypterDecrypter" role for the specified key. Without this role, you may not be able to encrypt or decrypt objects using the key which will prevent you from uploading or downloading objects.N)� gsutil_api�GetProjectServiceAccount� email_addressr rZ �debugZGetKeyIamPolicyr �bindings�appendZSetKeyIamPolicyr rJ r � print_to_fdrN rO rP )rB rL rI �service_accountZkms_api�policyZadded_new_binding�bindingrE rE rF �_AuthorizeProject s4 � � �� z KmsCommand._AuthorizeProjectc C sj | � d� | js$td| j| jf ��| �| j| j�\}}|rRtd| j| jf � ntd| j| jf � dS )Nr$ z,%s %s requires a key to be specified with -kz9Authorized project %s to encrypt and decrypt with key: %szFProject %s was already authorized to encrypt and decrypt with key: %s.r )rS rI r � command_namerM rg rL �print)rB �_�newly_authorizedrE rE rF � _AuthorizeS s � � �zKmsCommand._Authorizec C s@ t j�� |_tdt|��d� � | jj|j |dg|j d� dS )z�Clears the defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. z)Clearing default encryption key for %s...�/r% ��fieldsrY N) �apitools_messages�Bucket�EncryptionValuer% ri �str�rstripr] � PatchBucket� bucket_name�scheme)rB �bucket_metadata� bucket_urlrE rE rF �_EncryptionClearKeyb s � �z KmsCommand._EncryptionClearKeyc C s� |j }z|| d }}W n, tyD | �|| j�\}}|||< Y n0 |r^t�d|| jf � tjj| jd�|_ t dt |�� d� � | j j|j|dg|jd� dS ) aa Sets defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. svc_acct_for_project_num: (Dict[int, str]) Mapping of project numbers to their corresponding service account. Fz,Authorized service account %s to use key: %s)�defaultKmsKeyNamez(Setting default KMS key for bucket %s...rm r% rn N)� projectNumber�KeyErrorrg rI r rc rp rq rr r% ri rs rt r] ru rv rw )rB rx ry �svc_acct_for_project_numZbucket_project_numberrd rk rE rE rF �_EncryptionSetKeyq s0 � ��� � �z KmsCommand._EncryptionSetKeyc st � � d� i �� �fdd�}d}� j}|s2� �� |D ]$}� �|�}|D ]}d}||� qHq6|spttt|� ��dS )Nr% c s� | j }|jdkr td� j ��� jj|jddg|jd�}� jrN� �||� dS � j rf� � ||�� dS t |�� d�}|j r�|j jr�td||j jf � n td | � dS ) z6Set, clear, or get the defaultKmsKeyName for a bucket.rW z7The %s command can only be used with gs:// bucket URLs.r% r| rn r rm z!Default encryption key for %s: %sz'Bucket %s has no default encryption key)� storage_urlrw r rh r] � GetBucketrv rH rz rI r rs rt r% r{ ri )�blrry rx �bucket_url_string�rB r~ rE rF �_EncryptionForBucket� s: ��� �� � z4KmsCommand._Encryption.<locals>._EncryptionForBucketFTr )rS r> �$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr r �list)rB r� � some_matched�url_args�url_str� bucket_iter�bucket_listing_refrE r� rF � _Encryption� s % zKmsCommand._Encryptionc C sz | � � | jsdg| _| jr:| jD ]\}}|dkr"|| _q"| jsJtd �| _| j�d| j� | jj| jdd�j }t |� dS )Nzgs://r z'Checking service account for project %srW rX r ) rG r> rK rL r rZ r` r] r^ r_ ri )rB rQ rR rd rE rE rF �_ServiceAccount� s" ��zKmsCommand._ServiceAccountc C sT z2t � | j| jj�\| _| _tj| jd� || �W S t jyN | �� Y n0 d S )N)rK ) �getoptr> � command_specr* rK r �LogCommandParams� GetoptError� RaiseInvalidArgumentException)rB �funcrE rE rF �_RunSubCommand� s � zKmsCommand._RunSubCommandc C s2 | j jdd�tjkr.td�t�d| j ����dS )�(Command entry point for the kms command.rW rX r z�The "%s" command can only be used with the GCS JSON API. If you have only supplied hmac credentials in your boto file, please instead supply a credential type that can be used with the JSON API.N) r] �GetApiSelectorr �JSONr rN rO rP rh )rB rE rE rF � RunCommand� s ���zKmsCommand.RunCommandc C s� | j jdd�tjkr.td�t�d| j ����t j t j t j d�}| j �d�| _| j|v rxtj| jgd� | �|| j �S td| j| jf ��d S ) r� rW rX r z�The "%s" command can only be used with the GCS JSON API, which cannot use HMAC credentials. Please supply a credential type that is compatible with the JSON API (e.g. OAuth2) in your boto config file.r# r )� subcommandsz+Invalid subcommand "%s" for the %s command.N)r] r� r r� r rN rO rP rh r rl r� r� r> �poprM r r� r� )rB Zmethod_for_subcommandrE rE rF r� � s& ���� �)$�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r r� r � MakeNCloudBucketURLsArgumentr� �HelpSpec�_DETAILED_HELP_TEXT�_authorize_help_text�_encryption_help_text�_serviceaccount_help_text� help_specr �_AUTHORIZE_COMMAND�_SERVICEACCOUNT_COMMANDr? rA rS r r rg rl rz r r� r� r� r� � __classcell__rE rE rC rF r � s\ ���� �� 4!; r )<r� � __future__r r r r r� rO �gslibr �gslib.cloud_apir r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr Z gslib.kms_apir �gslib.project_idr Z3gslib.third_party.kms_apitools.cloudkms_v1_messagesr �"gslib.third_party.storage_apitoolsr rp Z gslib.utilsr �gslib.utils.constantsr � gslib.utils.encryption_helperr �gslib.utils.retry_utilr �gslib.utils.shim_utilr r Z_AUTHORIZE_SYNOPSISZ_ENCRYPTION_SYNOPSISZ_SERVICEACCOUNT_SYNOPSIS�lstripr� Z_AUTHORIZE_DESCRIPTIONZ_ENCRYPTION_DESCRIPTIONZ_SERVICEACCOUNT_DESCRIPTION� _DESCRIPTIONr� r� r� r� r� r� r rE rE rE rF �<module> s~ ��+ �� ������� commands/__pycache__/compose.cpython-39.pyc 0000644 00000011111 15025013616 0014645 0 ustar 00 a +(Wg" � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dZ dZ de de Z!G dd� de �Z"dS )z;Implementation of compose command for Google Cloud Storage.� )�absolute_import)�print_function)�division)�unicode_literals)�config)�BucketListingObject)�Command)�CommandArgument)� ApiSelector)�CommandException)�ContainsWildcard)�StorageUrlFromString)�storage_v1_messages)�GetEncryptionKeyWrapper)�GcloudStorageFlag)�GcloudStorageMap)�PreconditionsFromHeaders� zb gsutil compose gs://bucket/source_obj1 [gs://bucket/source_obj2 ...] gs://bucket/composite_obj z <B>SYNOPSIS</B> aH <B>DESCRIPTION</B> The compose command creates a new object whose content is the concatenation of a given sequence of source objects under the same bucket. gsutil uses the content type of the first source object to determine the destination object's content type and does not modify or delete the source objects as part of the compose command. For more information, see the `composite objects topic <https://cloud.google.com/storage/docs/composite-objects>`_. There is a limit (currently %d) to the number of components that can be composed in a single operation. c @ s� e Zd ZdZejddgeded ddddej ej gej e � � gd� Z ejddgdd ei d �Zeg d �i d �Zd d� Zdd� ZdS )�ComposeCommandz)Implementation of gsutil compose command.�compose�concat� � F) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz>Concatenate a sequence of objects into a new composite object.)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�objectsr )�gcloud_command�flag_mapc C s |j dkrtdt|� ��d S )N�gsz7"compose" called on URL with unsupported provider (%s).)�schemer �str)�self�url� r4 �%/opt/gsutil/gslib/commands/compose.py� CheckProvider_ s �z ComposeCommand.CheckProviderc C s� | j d }| j dd� | _ t|�}| �|� |�� r@td| ��tj|j|jd�}g }d}| j D ]�}t |�r|| � |�� � }nt t|��g}|D ]v}|j } | �| � | j|jkr�td��|s�| }tjj| jd�} | �� r�t| j�| _|�| � t|�tkr�tdt ��q�q`|�std��| jj|j|j|jd d gd �} | j|_| j|_t| j�pPi �} | j � d |t|�� | jj ||| |jt t �d � dS )z,Command entry point for the compose command.���NzQA version-specific URL (%s) cannot be the destination for gsutil compose - abort.)�name�bucketz,GCS does not support inter-bucket composing.)r8 z>"compose" called with too many component objects. Limit is %d.z/"compose" requires at least 1 component object.�contentEncoding� contentType)�provider�fieldsz)Composing %s from %d component object(s).)� preconditionsr< �encryption_tuple)!�argsr r6 � HasGenerationr �apitools_messages�Object� object_name� bucket_namer �WildcardIterator� IterObjectsr � storage_url�ComposeRequest�SourceObjectsValueListEntry�int� generation�append�len�MAX_COMPOSE_ARITY� gsutil_api�GetObjectMetadatar0 r; r: r �headers�logger�info� ComposeObjectr r ) r2 Ztarget_url_strZ target_url�dst_obj_metadata� componentsZ first_src_url� src_url_strZ src_url_iter�blr�src_url�src_obj_metadataZfirst_src_obj_metadatar> r4 r4 r5 � RunCommande sp �� �� � � ��zComposeCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISrO r �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr �gcloud_storage_mapr6 r\ r4 r4 r4 r5 r = s: �� �r N)#r` � __future__r r r r �botor �gslib.bucket_listing_refr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.storage_urlr r �"gslib.third_party.storage_apitoolsr rB � gslib.utils.encryption_helperr �gslib.utils.shim_utilr r � gslib.utils.translation_helperr rO rb rh r r4 r4 r4 r5 �<module> s6 � �� commands/__pycache__/rpo.cpython-39.pyc 0000644 00000013722 15025013616 0014012 0 ustar 00 a +(Wgc � @ s2 d Z ddlmZ ddlmZ ddlZddlmZ ddlmZ ddl m Z ddl m Z dd l mZ dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ dZd�d�e��Z d�e �Z dZ e e � d� Z dZ!dZ"de" e! Z#ee e#�Z$ee e!�Z%ee e"�Z&eg d�i d�Z'eg d�i d�Z(G dd � d e�Z)dS ) z/This module provides the rpo command to gsutil.� )�absolute_import)�print_functionN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)Z ASYNC_TURBO�DEFAULTz({})�|z+ gsutil rpo set {} gs://<bucket_name>... z( gsutil rpo get gs://<bucket_name>... � am <B>SET</B> The ``rpo set`` command configures turbo replication for dual-region Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use turbo replication: gsutil rpo set ASYNC_TURBO gs://redbucket gs://bluebucket Configure your buckets to NOT use turbo replication: gsutil rpo set DEFAULT gs://redbucket gs://bluebucket z� <B>GET</B> The ``rpo get`` command returns the replication setting for the specified Cloud Storage buckets. <B>GET EXAMPLES</B> Check if your buckets are using turbo replication: gsutil rpo get gs://redbucket gs://bluebucket a The ``rpo`` command is used to retrieve or configure the `replication setting <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_ of dual-region Cloud Storage buckets. This command has two sub-commands: ``get`` and ``set``. )�storage�buckets�listzL--format=value[separator=": "](format("gs://{}", name),rpo.yesno(no="None"))z--raw��gcloud_command�flag_map)r r �updatez--recovery-point-objectivec @ s� e Zd ZdZejdg ededdddej gej e � d�ge de e �d�e �� gd �d � Zejdg d d eeed �d �Zeeed �i d�Zdd� Zdd� Zdd� Zdd� Z dd� Z dS )� RpoCommandz"Implements the gsutil rpo command.�rpo� � F� �mode)�choices��get�set) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzConfigure replication)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textr c C s |j jdkrtd| j ��d S )N�gsz7The %s command can only be used with gs:// bucket URLs.)� storage_url�schemer � command_name)�self�blr� r: �!/opt/gsutil/gslib/commands/rpo.py�,_ValidateBucketListingRefAndReturnBucketName� s ��z7RpoCommand._ValidateBucketListingRefAndReturnBucketNamec C sF |j }| jj|jdg|jd�}|j}t|��d�}td||f � dS )z"Gets the rpo setting for a bucket.r ��fields�provider�/z%s: %sN) r5 � gsutil_api� GetBucket� bucket_namer6 r �str�rstrip�print)r8 r9 � bucket_url�bucket_metadatar �bucketr: r: r; �_GetRpo� s �zRpoCommand._GetRpoc C sh |j }|}|tvr td�t���tj|d�}| j�d|t |�� d�f � | j j |j |dg|jd� dS )z"Sets the rpo setting for a bucket.z.Invalid value for rpo set. Should be one of {})r zSetting rpo %s for %sr@ r r= r )r5 �VALID_RPO_VALUESr �format�VALID_RPO_VALUES_STRING�apitools_messages�Bucket�logger�inforD rE rA � PatchBucketrC r6 )r8 r9 Z rpo_valuerG Zformatted_rpo_valuerH r: r: r; �_SetRpo� s$ �� � �zRpoCommand._SetRpoc C s | j �d�}|dvr td��d}g }|dkr4| j}n$|dkrX| j}| j �d�}|�|� | j�d�tj kr�td� t � d | j ����d }| j }|s�| �� |D ]\}| �|�}|D ]H} | j�| jj�tj kr�td� t � d | j ����d }|| g|�R � q�q�|�sttt|� ��dS ) z-Handles rpo command on Cloud Storage buckets.r r zrpo only supports get|setNr r! r4 r z=The "%s" command can only be with the Cloud Storage JSON API.Fz2The "%s" command can only be used for GCS buckets.T)�args�popr rJ rS �appendrA �GetApiSelectorr �JSON�join�textwrap�wrapr7 �$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr5 r6 r r ) r8 � subcommand�subcommand_func�subcommand_args� setting_arg� some_matched�url_args�url_str� bucket_iter�bucket_listing_refr: r: r; �_Rpo� sN �� ����zRpoCommand._Rpoc C sZ | j d }| jdd� |dks&|dkrJtj| jd� tj|gd� | �� S td| ��d S ) z(Command entry point for the rpo command.r T)� check_argsr r! )�sub_opts)� subcommandsz-Invalid subcommand "%s", use get|set instead.N)rT � ParseSubOptsr �LogCommandParamsri rg r )r8 �action_subcommandr: r: r; � RunCommand� s �zRpoCommand.RunCommandN) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rX r �MakeNCloudURLsArgumentr rK �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr � GET_COMMAND� SET_COMMAND�gcloud_storage_mapr< rJ rS rg rn r: r: r: r; r c sP ����� �� .r )*rr � __future__r r rZ �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rN �gslib.utils.constantsr �gslib.utils.shim_utilr rK rL rY rM � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprt �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONry r{ rz r} r~ r r: r: r: r; �<module> sL � �� �� commands/__pycache__/perfdiag.cpython-39.pyc 0000644 00000174265 15025013616 0015005 0 ustar 00 a +(Wg�u � @ sB d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlm Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZdd lm Z dd lm Z dd lm Z dd lm Z ddl Z dd l m!Z! dd l m"Z" ddl#m$Z$ ddl#m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z4 ddl5m6Z6 ddl7m8Z8 ddl7m9Z9 ddl:m;Z; ddl<m=Z= dd l<m>Z> dd l?m@Z@ dd lAmBZB dd lAmCZC dd lAmDZD dd!lAmEZE dd"lAmFZF dd#lGmHZH dd$lGmIZI dd%lGmJZJ dd&lGmKZK dd'lGmLZL d(ZMd)eM d* ZNe d+d,�ZOe d-d.�ZPe d/d0�ZQe d1d2�ZRe d3d4�ZSi ZTG d5d6� d6eU�ZVG d7d8� d8eV�ZWdQd9d:�ZXdRd;d<�ZYdSd=d>�ZZdTd?d@�Z[dUdAdB�Z\dCdD� Z]dEdF� Z^G dGdH� dHe_�Z`dVdKdL�ZaG dMdN� dNe$�ZbdOdP� ZcdS )Wz%Contains the perfdiag gsutil command.� )�absolute_import)�print_function)�division)�unicode_literalsN)� defaultdict)� namedtuple)� cStringIO)� http_client)�range)�NotFoundException)�ServiceException)�Command)�DummyArgChecker)�CommandArgument)�config)� ApiSelector)�CommandException)�FilePart)�StorageUrlFromString)�storage_v1_messages)� text_util)�GetMaxRetryDelay)�ResumableThreshold)�UTF8)�GetCloudApiInstance)� GetDownloadSerializationData)�"CalculateB64EncodedMd5FromContents)�CheckFreeSpace)�GetDiskCounters)� GetFileSize)�IS_LINUX)�IsRunningInCiEnvironment)� DivideAndCeil)�HumanReadableToBytes)�MakeBitsHumanReadable)�MakeHumanReadable)� Percentilez� gsutil perfdiag [-i in.json] gsutil perfdiag [-o out.json] [-n objects] [-c processes] [-k threads] [-p parallelism type] [-y slices] [-s size] [-d directory] [-t tests] [-j ratio] gs://<bucket_name>... z <B>SYNOPSIS</B> a�! <B>DESCRIPTION</B> The ``perfdiag`` command runs a suite of diagnostic tests for a given Cloud Storage bucket. The ``bucket_name`` parameter must name an existing bucket to which the user has write permission. Several test files will be uploaded to and downloaded from this bucket. All test files will be deleted at the completion of the diagnostic if it finishes successfully. For a list of relevant permissions, see `Cloud IAM permissions for gsutil commands <https://cloud.google.com/storage/docs/access-control/iam-gsutil>`_. gsutil performance can be influenced by a number of factors originating at the client, server, or network level. Some examples include the following: + CPU speed + Available memory + The access path to the local disk + Network bandwidth + Contention and error rates along the path between gsutil and Google servers + Operating system buffering configuration + Firewalls and other network elements The `perfdiag` command is provided so that customers can run a known measurement suite when troubleshooting performance problems. <B>PROVIDING DIAGNOSTIC OUTPUT TO THE CLOUD STORAGE TEAM</B> If the Cloud Storage team asks you to run a performance diagnostic please use the following command, and email the output file (output.json) to the @google.com address provided by the Cloud Storage team: gsutil perfdiag -o output.json gs://your-bucket Additional resources for discussing ``perfdiag`` results include the `Stack Overflow tag for Cloud Storage <https://stackoverflow.com/questions/tagged/google-cloud-storage>`_ and the `gsutil GitHub repository <https://github.com/GoogleCloudPlatform/gsutil/issues>`_. <B>OPTIONS</B> -n Sets the number of objects to use when downloading and uploading files during tests. Defaults to 5. -c Sets the number of `processes <https://en.wikipedia.org/wiki/Process_(computing)>`_ to use while running throughput experiments. The default value is 1. -k Sets the number of `threads <https://en.wikipedia.org/wiki/Thread_(computing)>`_ per process to use while running throughput experiments. Each process will receive an equal number of threads. The default value is 1. NOTE: All specified threads and processes will be created, but may not by saturated with work if too few objects (specified with ``-n``) and too few components (specified with ``-y``) are specified. -p Sets the type of parallelism to be used (only applicable when threads or processes are specified and threads * processes > 1). The default is to use ``fan``. Must be one of the following: fan Use one thread per object. This is akin to using gsutil ``-m cp``, with sliced object download / parallel composite upload disabled. slice Use Y (specified with ``-y``) threads for each object, transferring one object at a time. This is akin to using parallel object download / parallel composite upload, without ``-m``. Sliced uploads not supported for s3. both Use Y (specified with ``-y``) threads for each object, transferring multiple objects at a time. This is akin to simultaneously using sliced object download / parallel composite upload and ``gsutil -m cp``. Parallel composite uploads not supported for s3. -y Sets the number of slices to divide each file/object into while transferring data. Only applicable with the slice (or both) parallelism type. The default is 4 slices. -s Sets the size (in bytes) for each of the N (set with ``-n``) objects used in the read and write throughput tests. The default is 1 MiB. This can also be specified using byte suffixes such as 500K or 1M. NOTE: these values are interpreted as multiples of 1024 (K=1024, M=1024*1024, etc.) NOTE: If ``rthru_file`` or ``wthru_file`` are performed, N (set with ``-n``) times as much disk space as specified will be required for the operation. -d Sets the directory to store temporary local files in. If not specified, a default temporary directory will be used. -t Sets the list of diagnostic tests to perform. The default is to run the ``lat``, ``rthru``, and ``wthru`` diagnostic tests. Must be a comma-separated list containing one or more of the following: lat For N (set with ``-n``) objects, write the object, retrieve its metadata, read the object, and finally delete the object. Record the latency of each operation. list Write N (set with ``-n``) objects to the bucket, record how long it takes for the eventually consistent listing call to return the N objects in its result, delete the N objects, then record how long it takes listing to stop returning the N objects. rthru Runs N (set with ``-n``) read operations, with at most C (set with -c) reads outstanding at any given time. rthru_file The same as ``rthru``, but simultaneously writes data to the disk, to gauge the performance impact of the local disk on downloads. wthru Runs N (set with ``-n``) write operations, with at most C (set with ``-c``) writes outstanding at any given time. wthru_file The same as wthru, but simultaneously reads data from the disk, to gauge the performance impact of the local disk on uploads. -m Adds metadata to the result JSON file. Multiple ``-m`` values can be specified. Example: gsutil perfdiag -m "key1:val1" -m "key2:val2" gs://bucketname Each metadata key will be added to the top-level "metadata" dictionary in the output JSON file. -o Writes the results of the diagnostic to an output file. The output is a JSON file containing system information and performance diagnostic results. The file can be read and reported later using the ``-i`` option. -i Reads the JSON output file created using the ``-o`` command and prints a formatted description of the results. -j Applies gzip transport encoding and sets the target compression ratio for the generated test files. This ratio can be an integer between 0 and 100 (inclusive), with 0 generating a file with uniform data, and 100 generating random data. When you specify the ``-j`` option, files being uploaded are compressed in-memory and on-the-wire only. See `cp -j <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_ for specific semantics. <B>MEASURING AVAILABILITY</B> The ``perfdiag`` command ignores the boto num_retries configuration parameter. Instead, it always retries on HTTP errors in the 500 range and keeps track of how many 500 errors were encountered during the test. The availability measurement is reported at the end of the test. Note that HTTP responses are only recorded when the request was made in a single process. When using multiple processes or threads, read and write throughput measurements are performed in an external process, so the availability numbers reported won't include the throughput measurements. <B>NOTE</B> The ``perfdiag`` command runs a series of tests that collects system information, such as the following: + Retrieves requester's IP address. + Executes DNS queries to Google servers and collects the results. + Collects network statistics information from the output of ``netstat -s`` and evaluates the BIOS product name string. + If a proxy server is configured, attempts to connect to it to retrieve the location and storage class of the bucket being used for performance testing. None of this information will be sent to Google unless you proactively choose to send it. � FileDataTuplez size md5 data�FanDownloadTuplez6need_to_slice object_name file_name serialization_data�SliceDownloadTuplez<object_name file_name serialization_data start_byte end_byte�FanUploadTuplez9need_to_slice file_name object_name use_file gzip_encoded�SliceUploadTuplez@file_name object_name use_file file_start file_size gzip_encodedc @ s e Zd ZdZdS )�Errorz%Base exception class for this module.N��__name__� __module__� __qualname__�__doc__� r2 r2 �&/opt/gsutil/gslib/commands/perfdiag.pyr, 2 s r, c @ s e Zd ZdZdS )�InvalidArgumentz)Raised on invalid arguments to functions.Nr- r2 r2 r2 r3 r4 7 s r4 c C s@ t | |�}|jr&| �|j|j|j� n| �|j||j|j� dS )a Function argument to apply for performing fanned parallel downloads. Args: cls: The calling PerfDiagCommand class instance. args: A FanDownloadTuple object describing this download. thread_state: gsutil Cloud API instance to use for the operation. N)r � need_to_slice�PerformSlicedDownload� object_name� file_name�serialization_data�Download��cls�args� thread_state� gsutil_apir2 r2 r3 �_DownloadObject< s ��r@ c C s, t | |�}| �|j||j|j|j|j� dS )a Function argument to apply for performing sliced downloads. Args: cls: The calling PerfDiagCommand class instance. args: A SliceDownloadTuple object describing this download. thread_state: gsutil Cloud API instance to use for the operation. N)r r: r7 r8 r9 � start_byte�end_byter; r2 r2 r3 �_DownloadSliceM s �rC c C sN t | |�}|jr.| j|j|j|j||jd� n | j|j|j||j|jd� dS )a Function argument to apply for performing fanned parallel uploads. Args: cls: The calling PerfDiagCommand class instance. args: A FanUploadTuple object describing this upload. thread_state: gsutil Cloud API instance to use for the operation. �� gzip_encodedN)r r5 �PerformSlicedUploadr8 r7 �use_filerE �Uploadr; r2 r2 r3 � _UploadObjectZ s ��rI c C s2 t | |�}| j|j|j||j|j|j|jd� dS )a Function argument to apply for performing sliced parallel uploads. Args: cls: The calling PerfDiagCommand class instance. args: A SliceUploadTuple object describing this upload. thread_state: gsutil Cloud API instance to use for the operation. rD N)r rH r8 r7 rG � file_start� file_sizerE r; r2 r2 r3 � _UploadSliceq s �rL c C s t | |�}| �||� dS )a Function argument to apply for performing parallel object deletions. Args: cls: The calling PerfDiagCommand class instance. object_name: The object name to delete from the test bucket. thread_state: gsutil Cloud API instance to use for the operation. N)r �Delete)r< r7 r> r? r2 r2 r3 �_DeleteWrapper� s rN c C s | j �t|�� dS �z9Simple exception handler to allow post-completion status.N��logger�error�str)r< �er2 r2 r3 �_PerfdiagExceptionHandler� s rU c C s d S �Nr2 )�_r2 r2 r3 �_DummyTrackerCallback� s rX c @ s$ e Zd ZdZdZdd� Zdd� ZdS )� DummyFilezDA dummy, file-like object that throws away everything written to it.Zbwc O s d S rV r2 )�selfr= �kwargsr2 r2 r3 �write� s zDummyFile.writec C s d S rV r2 �rZ r2 r2 r3 �close� s zDummyFile.closeN)r. r/ r0 r1 �moder\ r^ r2 r2 r2 r3 rY � s rY �d � P c C sx |d }t �t||��}d}||k rtt||| �}t|d| �}|| }| �|d|� � | �d| � ||7 }q dS )am Writes data into a file like object. Args: fp: A file like object to write the data to. file_size: The amount of data to write to the file. random_ratio: The percentage of randomly generated data to write. This can be any number between 0 and 100 (inclusive), with 0 producing uniform data, and 100 producing random data. max_unique_random_bytes: The maximum number of bytes to generate pseudo-randomly before beginning to repeat bytes. The default was chosen as the next prime larger than 5 MiB. g Y@r � N� x)�os�urandom�min�intr\ ) �fprK � random_ratioZmax_unique_random_bytesZ random_bytesZtotal_bytes_written� num_bytesZ num_bytes_seqZnum_bytes_randomr2 r2 r3 �_GenerateFileData� s rk c @ s� e Zd ZdZejdg d�eddddddejej gej e � d�gd� Z ej dg d d ei d �Zd Zd ZdZdZdZdZdZdZdZdZeeeeeefZeeefZeeefZ e j � dde j j!j"j#�Z$e j �%ddd�Z&dZ'dZ(d Z)d Z*dZd d �Z+d!d"� Z,d[d%d&�Z-d'd(� Z.d)d*� Z/e0j1d+d,� �Z2d-d.� Z3d/d0� Z4G d1d2� d2e5j6�Z7d3d4� Z8d5d6� Z9d7d8� Z:d\d9d:�Z;d]d;d<�Z<d^d=d>�Z=d_d?d@�Z>dAdB� Z?d`dDdE�Z@dadFdG�ZAdHdI� ZBdJdK� ZCdLdM� ZDdNdO� ZEdPdQ� ZFdRdS� ZGdTdU� ZHdVdW� ZIdXdY� ZJdCS )b�PerfDiagCommandz*Implementation of gsutil perfdiag command.�perfdiag)Zdiag� diagnosticZperfZ performancer rb zn:c:k:p:y:s:d:t:m:i:o:j:F) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzRun performance diagnostic)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)r i i � � ZrthruZ rthru_fileZwthruZ wthru_file�lat�list�fan�slice�both� Credentials�gs_host�gs_port�P � � i @ g N@Tc C s� | j �d|� |rtjnd}tj|tj|d�}|�� \}}tjrTt|t �rT|� t �}|rv|j rvt d|j d�|�f ��|r~|S |j S )a� Executes a command in a subprocess. Args: cmd: List containing the command to execute. raise_on_error: Whether or not to raise an exception when a process exits with a non-zero return code. return_output: If set to True, the return value of the function is the stdout of the process. mute_stderr: If set to True, the stderr of the process is not printed to the console. Returns: The return code of the process or the stdout if return_output is set. Raises: Exception: If raise_on_error is set to True and any process exits with a non-zero return code. zRunning command: %sN)�stdout�stderrz8Received non-zero return code (%d) from subprocess '%s'.� )rQ �debug� subprocess�PIPE�Popen� communicate�six�PY3� isinstance�bytes�decoder � returncoder �join) rZ �cmd�raise_on_error� return_output� mute_stderrr� �pZ stdoutdatarW r2 r2 r3 �_Exec s �zPerfDiagCommand._Execc C s$ | j | j td�kr | j�d� dS )zBOutputs a warning message if a large amount of data is being used.�2GiBz2This is a large operation, and could take a while.N)� num_objects� thru_filesizer# rQ �infor] r2 r2 r3 �_WarnIfLargeData< s z PerfDiagCommand._WarnIfLargeData�gsutil_test_filer` c C s� t jd|| jdd�\}}t�|d�� }t|||� W d � n1 sF0 Y |sX|r�t|d��P}|rpt|�nd}|r�t|�nd} |r�|� � nd} t || | �t |< W d � n1 s�0 Y | j � |� |S )a� Creates a temporary file of the given size and returns its path. Args: file_size: The size of the temporary file to create. mem_metadata: If true, store md5 and file size in memory at temp_file_dict[fpath].md5, tempfile_data[fpath].file_size. mem_data: If true, store the file data in memory at temp_file_dict[fpath].data prefix: The prefix to use for the temporary file. Defaults to gsutil_test_file. random_ratio: The percentage of randomly generated data to write. This can be any number between 0 and 100 (inclusive), with 0 producing uniform data, and 100 producing random data. Returns: The file path of the created temporary file. z.binF)�suffix�prefix�dir�text�wbN�rb)�tempfile�mkstemp� directoryrd �fdopenrk �openr r �readr' �temp_file_dict�temporary_files�add) rZ rK � mem_metadata�mem_datar� ri �fd�fpathrh �md5�datar2 r2 r3 � _MakeTempFileA s � * . z PerfDiagCommand._MakeTempFilec C s� i | _ t� | _t� | _d| _d| _tt�| _d| _ d| _ | j | j v rrg | _ | jD ] }| j|ddd�}| j �|� qP| j �| j| j| j| jf��r�| jdddd�| _| j �| j| jf�r�| j| jdd| jd�| _tj� | j�| _ | j �| j| jf��r�g | _ g | _ t | j!�}|| j| j" k�r�| j#�$d| j"t%| j�f � | �&� t'| j"�D ]:}| j| jd| jd�}| j �|� | j �tj� |�� �qJn t(d t%| j| j" �t%|�f ��t)� | _*| j#�+| �,� � d S ) z?Performs setup operations needed before diagnostics can be run.r FT)r� r� i P )r� r� ri z) Creating %d local files each of size %s.)r� ri zbNot enough free disk space for throughput files: %s of disk space required, but only %s available.N)-�results�setr� �temporary_objects�total_requests�request_errorsr rg �error_responses_by_code�connection_breaks�teardown_completed�LAT� diag_tests� latency_files�test_lat_file_sizesr� �append� intersection�RTHRU�WTHRU� RTHRU_FILE� WTHRU_FILE�tcp_warmup_filer� �gzip_compression_ratio�mem_thru_file_namerd �path�basename�mem_thru_object_name�thru_file_names�thru_object_namesr r� r� rQ r� r% r� r r rY � discard_sink� addFilter�_PerfdiagFilter)rZ rK r� Zfree_disk_spacerW r8 r2 r2 r3 �_SetUpi sp ��� ��� ���zPerfDiagCommand._SetUpc C sr | j sht�� z,| jD ]}t�|� q| jr8t�| j� W n t yL Y n0 | j D ]}| � || j � qTd| _ dS )zDPerforms operations to clean things up after performing diagnostics.TN) r� r� �clearr� rd �remove�delete_directory�rmdirr� �OSErrorr� rM r? )rZ r� r7 r2 r2 r3 � _TearDown� s zPerfDiagCommand._TearDownc c sH | j �d|� t�� }dV t�� }|| �|| � | j �d|� dS )a� A context manager that measures time. A context manager that prints a status message before and after executing the inner command and times how long the inner command takes. Keeps track of the timing, aggregated by the given key. Args: key: The key to insert the timing value into a dictionary bucket. bucket: A dictionary to place the timing value in. Yields: For the context manager. z%s starting...Nz%s done.)rQ r� �timer� )rZ �key�bucket�t0�t1r2 r2 r3 �_Time� s zPerfDiagCommand._Timec C sD d}d}d}d}d}|�s@t t�� d| d t� �}z |� }| jd7 _d}W q t| j��y< } z�|d7 }|| jkr�| j�d� W Y d}~�q@t |t ��r|j dkr�| j |j d7 < | jd7 _| j d7 _ |d7 }t�|� n� || jk�r(| j�d � W Y d}~�q@n| jd7 _W Y d}~qd}~0 0 q|S ) z�Runs an operation with retry logic. Args: func: The function to run. Returns: True if the operation succeeds, False if aborted. Fr N� rb Tz,Reached maximum total retries. Not retrying.i� z3Reached maximum server error retries. Not retrying.)rf �randomr r� �tuple� exceptions�MAX_TOTAL_RETRIESrQ r� r� r �statusr� r� r� �sleep�MAX_SERVER_ERROR_RETRIESr� ) rZ �func�successZserver_error_retriedZ total_retried�iZ return_val� next_sleeprT r2 r2 r3 � _RunOperation� s> �&z PerfDiagCommand._RunOperationc s� t t��jd< t�j�D ]�}�j�d|d � �jD ]�}t| � �j � � �t j � |��_� j�t��}�j�d||�� t���� ���fdd�}��|� ���fdd�}��|�}t|������fd d �}��|� ���fd d �}��|� q4qd S )zRuns latency tests.�latencyz Running latency iteration %d...rb zN File of size %s located on disk at '%s' being diagnosed in the cloud at '%s'.c s` t �� j�} ��d� �jd ��* �jj| ���jdgd� W d � n1 sR0 Y d S )Nz UPLOAD_%dr� �name)�size�provider�fields)r� �BytesIOr� r� r� r? � UploadObjectr� )Zio_fp)� file_datarK rZ � upload_targetr2 r3 �_Upload; s �z1PerfDiagCommand._RunLatencyTests.<locals>._Uploadc sX �� d� �jd ��. �jj�j�j�jg d�d�W d � S 1 sJ0 Y d S )Nz METADATA_%dr� )r� � contentType� mediaLinkr� �r� r� )r� r� r? �GetObjectMetadata� bucket_namer7 r� r2 �rK rZ �urlr2 r3 � _MetadataF s �z3PerfDiagCommand._RunLatencyTests.<locals>._Metadatac sX �� d� �jd ��. �jj�j�j�j�j�d� W d � n1 sJ0 Y d S )Nz DOWNLOAD_%dr� )r� r9 )r� r� r? �GetObjectMediar r7 r� r� r2 )rK rZ r9 r r2 r3 � _DownloadR s �z3PerfDiagCommand._RunLatencyTests.<locals>._Downloadc sR �� d� �jd ��( �jj�j�j�jd� W d � n1 sD0 Y d S )Nz DELETE_%dr� �r� )r� r� r? � DeleteObjectr r7 r� r2 r r2 r3 �_Delete] s �z1PerfDiagCommand._RunLatencyTests.<locals>._DeleteN)r r� r� r r� rQ r� r� r� � bucket_url�Clonerd r� r� r7 r� r% � StorageUrlToUploadObjectMetadatar� r ) rZ r� r� Zreadable_file_sizer� r Zdownload_metadatar r r2 )r� rK rZ r9 r� r r3 �_RunLatencyTests' s. � z PerfDiagCommand._RunLatencyTestsc @ s e Zd Zdd� ZdS )z PerfDiagCommand._PerfdiagFilterc C s* |� � }d|v p&d|v p&d|v p&d|v S )NzCopying file:///z Copying gs://z Computing CRCzgsutil -m perfdiag)� getMessage)rZ �record�msgr2 r2 r3 �filterg s ��z&PerfDiagCommand._PerfdiagFilter.filterN)r. r/ r0 r r2 r2 r2 r3 r� e s r� c C s | j �t|�� dS rO rP )rZ rT r2 r2 r3 rU m s z)PerfDiagCommand._PerfdiagExceptionHandlerc C sh g }t t|��D ]0}|r || nd}|�t||| ||| �� q| jt|tdt| jj | j | j d� dS )ay Performs a parallel download of multiple objects using the fan strategy. Args: need_to_slice: If True, additionally apply the slice strategy to each object in object_names. object_names: A list of object names to be downloaded. Each object must already exist in the test bucket. file_names: A list, corresponding by index to object_names, of file names for downloaded data. If None, discard downloaded data. serialization_data: A list, corresponding by index to object_names, of serialization data for each object. N�r� r� �� arg_checker� parallel_operations_override� process_count� thread_count) r �lenr� r( �Applyr@ rU r �ParallelOverrideReason�PERFDIAG� processes�threads)rZ r5 � object_names� file_namesr9 r= r� r8 r2 r2 r3 �PerformFannedDownloadq s" ���z%PerfDiagCommand.PerformFannedDownloadc C s� |r:t |d�� }|�| j� W d � n1 s00 Y t| j| j�}g }t| j�D ]>}|| }t|d | d | jd �} |�t||||| �� qV| j t |t dt | j j| j| jd� dS )a3 Performs a download of an object using the slice strategy. Args: object_name: The name of the object to download. file_name: The name of the file to download data to, or None if data should be discarded. serialization_data: The serialization data for the object. �abNrb r r )r� �truncater� r" � num_slicesr rf r� r) r rC rU r r r r r ) rZ r7 r8 r9 rh �component_sizer= r� rA rB r2 r2 r3 r6 � s, * ���z%PerfDiagCommand.PerformSlicedDownloadc C sZ g }t t|��D ]"}|�t||| || ||�� q| jt|tdt| jj | j | j d� dS )aB Performs a parallel upload of multiple files using the fan strategy. The metadata for file_name should be present in temp_file_dict prior to calling. Also, the data for file_name should be present in temp_file_dict if use_file is specified as False. Args: need_to_slice: If True, additionally apply the slice strategy to each file in file_names. file_names: A list of file names to be uploaded. object_names: A list, corresponding by by index to file_names, of object names to upload data to. use_file: If true, use disk I/O, otherwise read upload data from memory. gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. r r N) r r r� r* r rI rU r r r r r )rZ r5 r! r rG rE r= r� r2 r2 r3 �PerformFannedUpload� s ���z#PerfDiagCommand.PerformFannedUploadc s0 t �j�j�}�fdd�t�j�D �}g }t�j�D ]<} | | } t|t| j| �}|�t||| || ||�� q4z��j t |t dt �j j�j�jd� g �t�j�D ] } tjj|| d�} ��| � q�� ���fdd�} ��| � W �j t|t dt �j j�j�jd� n$�j t|t dt �j j�j�jd� 0 dS ) a� Performs a parallel upload of a file using the slice strategy. The metadata for file_name should be present in temp_file_dict prior to calling. Also, the data from for file_name should be present in temp_file_dict if use_file is specified as False. Args: file_name: The name of the file to upload. object_name: The name of the object to upload to. use_file: If true, use disk I/O, otherwise read upload data from memory. gsutil_api: CloudApi instance to use for operations in this thread. gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a semaphore is used to limit resource usage. c s g | ]}� t |� �qS r2 )rS ��.0r� )r7 r2 r3 � <listcomp>� s z7PerfDiagCommand.PerformSlicedUpload.<locals>.<listcomp>r r )r� c s. t �� } �| _�jj| _� j�| �jd� d S �Nr )�apitools_messages�Objectr� r r r� � ComposeObjectr� )�dst_obj_metadata�r? r7 �request_componentsrZ r2 r3 �_Compose s �z5PerfDiagCommand.PerformSlicedUpload.<locals>._ComposeN)r" r� r% r rf r� r� r� r+ r rL rU r r r r r r, �ComposeRequest�SourceObjectsValueListEntryr� rN ) rZ r8 r7 rG r? rE r& Zcomponent_object_namesr= r� Zcomponent_start�src_obj_metadatar2 r2 r0 r3 rF � sf � � ��� �� � �z#PerfDiagCommand.PerformSlicedUploadc C s� |rdnd}|rdnd}| j �d|| jt| j�f � | �� | j| j| j| jd�| j |< |�r| j }| j }g }t | j�D ]�}| j �| j | � | j| jv r�| jj| jj| j | ddg| jjd �}n | �| j | | j | | j|�}t�| j | � t| j | d ��� |�t |�� qznF| j �| j � | �| j | j | j|�}d }| j g| j }t |�g| j }tj � | j!�} | j �| � | �| j!| | j� | �"| | j� t#�#� } | jd k�r�| jd k�r�t | j�D ]2}|�r�|| nd } | �"|| | j| || � �q�nz| j| j$| j%fv �r*| j| j%k} | �&| |||� nH| j| j'k�rrt | j�D ].}|�rT|| nd } | �(|| | || � �qBt#�#� } | | }| j| j }|| }|| j | d < || j | d< || j | d< d S )zRuns read throughput tests.�read_throughput_file�read_throughput� with file I/O� z9 Running read throughput tests %s (%s objects of size %s))rK r r � parallelismr� r �r� r� r# Nrb � time_took�total_bytes_copied�bytes_per_second))rQ r� r� r% r� r� r r �parallel_strategyr� r� r� r r� r� r� r� r? r r r �schemerH rd �unlinkr� r^ r� r r� r� r� r� r� r: r� �FAN�BOTHr" �SLICEr6 )rZ rG � test_name�file_io_stringr! r r9 r� � obj_metadata�warmup_obj_namer� r8 r5 r� r< r= r> r2 r2 r3 �_RunReadThruTests s� ���� � ��� � � � z!PerfDiagCommand._RunReadThruTestsc s� |rdnd}|rdnd}� j �d|� jt� j�f � � �� � j� j� j� j� j � j d�� j |< t j �� j�}� j�|� � �� j|� j� |r�� j}� j}n&� jg� j }� fdd�t� j�D �}|D ]}� j�|� q�t�� }� jd k�r0� jd k�r0t� j�D ]&} � j|| || � j|� j d � �qnx� j� j� jfv �rh� j� jk} � j| |||� j d � n@� j� j k�r�t� j�D ]&} � j || || |� j� j d � �q�t�� } | | } � j� j } | | }| � j | d < | � j | d < |� j | d < dS )z Runs write throughput tests.�write_throughput_file�write_throughputr8 r9 z: Running write throughput tests %s (%s objects of size %s))rK r r r: �gzip_encoded_writesr� c s g | ]}� j t|� �qS r2 )r� rS r( r] r2 r3 r* � s z6PerfDiagCommand._RunWriteThruTests.<locals>.<listcomp>rb rD r< r= r> N) rQ r� r� r% r� r� r r r? rL r� r� rd r� r� r� r� r� rH r? r� r� r� r r� rB rC r' rD rF )rZ rG rE rF rH r! r r7 r� r� r5 r� r< r= r> r2 r] r3 �_RunWriteThruTestss sz ���� � � � � z"PerfDiagCommand._RunWriteThruTestsc sT d�j i�jd< g }g }d�dd� td�D ��}d| d �t�j �D ]N}�jd d d �d �}tj�|�}|�|� |�t d ||d d �� �j � |� qF�j � d �j � �jt|ttd� g �g �t�� }t|��t� ����fdd�� � ���fdd�}�j � d�jj�j � �� �rT��|� �� �rt�� | �jk�r�j �d� �qT�qt�� } t����| | d��jd d< dd� |D �}�j � d�j � �jt|ttd� �j � d�jj�j � g �g �t�� }t|����r(� ��fdd�} ��| � ��r�t�� | �jk�rڈj �d� �q(�q�t�� } t����| | d��jd d < d S ) z0Runs eventual consistency listing latency tests.� num_files�listingr9 c S s g | ]}t �tj��qS r2 )r� �choice�string�ascii_lowercase)r) rW r2 r2 r3 r* � � z1PerfDiagCommand._RunListTests.<locals>.<listcomp>r� zgsutil-perfdiag-list-�-r T)r� r� r� Fz' Writing %s objects for listing test...)r c sp t � � } t�jj�jj�d�jdgd��}t|��jkrH�j � d�j� t � � }� � || � t dd� |D ��S )z>Lists and returns objects in the bucket. Also records latency.�/z items/name)r� � delimiterr� r� z5Listing produced more than the expected %d object(s).c S s g | ] }|j j�qS r2 )r� r� )r) �objr2 r2 r3 r* � rS z@PerfDiagCommand._RunListTests.<locals>._List.<locals>.<listcomp>) r� r� r? � ListObjectsr r r� r r� rQ �warningr� r� )r� �objectsr� )�list_latencies� list_prefixrZ r2 r3 �_List� s" ���z,PerfDiagCommand._RunListTests.<locals>._Listc s& � � } �� | �@ � ��t��� d S rV )�updater� r ��names)r] �expected_objects� files_seen� found_objectsr2 r3 �_ListAfterUpload� s z7PerfDiagCommand._RunListTests.<locals>._ListAfterUploadz5Listing bucket %s waiting for %s objects to appear...z)Maximum time reached waiting for listing.)�num_listing_callsr[ �files_seen_after_listingr< �insertc S s g | ]}|�qS r2 r2 )r) r7 r2 r2 r3 r* rS z'Deleting %s objects for listing test...z8Listing bucket %s waiting for %s objects to disappear...c s" � � } �� | � ��t��� d S rV )�intersection_updater� r r_ )r] rb rc r2 r3 �_ListAfterDelete s z7PerfDiagCommand._RunListTests.<locals>._ListAfterDelete�deleteN)r� r� r� r r� rd r� r� r� r* r� r� rQ r� r rI rU r r� r� r r r� �MAX_LISTING_WAIT_TIMErY r rN ) rZ Z list_objectsr= Z random_idrW r� r7 Ztotal_start_timerd Ztotal_end_timeri r2 )r] ra rb rc r[ r\ rZ r3 � _RunListTests� s� � � �� � ��� � �z PerfDiagCommand._RunListTestsNc s� d��du rt | j��j�� }||_t|��zz|rBt||���n t | j||� � } t� | ��������fdd�� � fdd�} �r�| n� } �� | �W �r��� � S n�r��� � 0 dS )a� Performs an upload to the test bucket. The file is uploaded to the bucket referred to by self.bucket_url, and has name object_name. Args: file_name: The path to the local file, and the key to its entry in temp_file_dict. object_name: The name of the remote object. gsutil_api: Cloud API instance to use for this upload. use_file: If true, use disk I/O, otherwise read everything from memory. file_start: The first byte in the file to upload to the object. (only should be specified for sliced uploads) file_size: The size of the file to upload. (only should be specified for sliced uploads) gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. Returns: Uploaded Object Metadata. Nc sH � t � k r&�j���j� g d��d�S �j���j� g d�t�d�S d S )N)r� r r� )r� r� r� rE )r� r� r� �tracker_callbackrE )r r� r� �UploadObjectResumablerX r2 )rK rh r? rE rZ r� r2 r3 � _InnerUploadX s" ��z,PerfDiagCommand.Upload.<locals>._InnerUploadc s2 t jj� � � W d � S 1 s$0 Y d S rV )�gslib�command�!concurrent_compressed_upload_lockr2 )ro r2 r3 �_InnerUploadLockm s z0PerfDiagCommand.Upload.<locals>._InnerUploadLock) r� r� r r r7 r r r� r� r� r� r^ ) rZ r8 r7 r? rG rJ rK rE � upload_urlr� rs Z upload_funcr2 )ro rK rh r? rE rZ r� r3 rH , s, � �zPerfDiagCommand.Uploadc sn d�zV|dur$t |d������ n�j�� ������fdd�}��|� W �rj��� n�rh��� 0 dS )a� Downloads an object from the test bucket. Args: object_name: The name of the object (in the test bucket) to download. gsutil_api: CloudApi instance to use for the download. file_name: Optional file name to write downloaded data to. If None, downloaded data is discarded immediately. serialization_data: Optional serialization data, used so that we don't have to get the metadata before downloading. start_byte: The first byte in the object to download. (only should be specified for sliced downloads) end_byte: The last byte in the object to download. (only should be specified for sliced downloads) Nzr+bc s" �j �jj���j�� �d� d S )N)r� rA rB r9 )r r r r� r2 �rB rh r? r7 rZ r9 rA r2 r3 �_InnerDownload� s �z0PerfDiagCommand.Download.<locals>._InnerDownload)r� �seekr� r� r^ )rZ r7 r? r8 r9 rA rB rv r2 ru r3 r: w s �zPerfDiagCommand.Downloadc s6 z � ��fdd�}�� |� W n ty0 Y n0 dS )z�Deletes an object from the test bucket. Args: object_name: The name of the object to delete. gsutil_api: Cloud API instance to use for this delete. c s � j �jj��jd� d S r+ )r r r r� r2 �r? r7 rZ r2 r3 � _InnerDelete� s �z,PerfDiagCommand.Delete.<locals>._InnerDeleteN)r� r )rZ r7 r? ry r2 rx r3 rM � s zPerfDiagCommand.Deletec C s. d}g }t dd��P}|�� dd� }|D ]*}|�� \}}}}|d �� r(|�|� q(W d � n1 sh0 Y i }t dd���}|D ]�}|�� dd� } | \ }}}} }} } } }}}||v r�t| �| } t|�| }t| �} t| �} t| �} t|�}| | | || |f||< q�W d � n1 �s 0 Y |S ) a� Retrieves disk I/O statistics for all disks. Adapted from the psutil module's psutil._pslinux.disk_io_counters: http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py Originally distributed under under a BSD license. Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola. Returns: A dictionary containing disk names mapped to the disk counters from /disk/diskstats. i z/proc/partitions�rr� N���z/proc/diskstats� )r� � readlines�split�isdigitr� rg )rZ � sector_size� partitions�f�lines�linerW r� �retdict�values�reads�rbytes�rtime�writes�wbytes�wtimer2 r2 r3 �_GetDiskCounters� s, * 6z PerfDiagCommand._GetDiskCountersc C s� z| j ddgddd�}W n ty8 | j�d� Y dS 0 |�� �� }d}d}d}d}|�d�D ]�}d |v std |v rxd}|r�|du r�d |v s�d |v s�d |v r�d�dd� |D ��}|r�|du r�d|v s�d|v r�d�dd� |D ��}|r`|du r`d|v �sd|v �sd|v r`d�dd� |D ��}q`i }z(t|�|d< t|�|d< t|�|d< W n0 t t f�yz d|d< d|d< d|d< Y n0 |S )z�Tries to parse out TCP packet information from netstat output. Returns: A dictionary containing TCP information, or None if netstat is not available. �netstat�-sTF)r� r� zGnetstat not found on your system; some measurement data will be missingN� ztcp:ztcp statisticszsegments retransmitedzretransmit timeoutszsegments retransmittedr9 c s s | ]}|t jv r|V qd S rV �rQ �digits�r) �cr2 r2 r3 � <genexpr>� rS z/PerfDiagCommand._GetTcpStats.<locals>.<genexpr>zsegments receivedzpackets receivedc s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� rS zsegments send outz packets sentz segments sentc s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� rS �tcp_retransmit� tcp_received�tcp_sent) r� r� rQ rY �strip�lowerr~ r� rg � ValueError� TypeError)rZ �netstat_outputZ found_tcpr� r� r� r� �resultr2 r2 r3 � _GetTcpStats� s` � ��� �� ��� z PerfDiagCommand._GetTcpStatsc C s� i }t jt jt jt jf}tj�ddd�|d< tj�ddd�}tj�ddd�}t |�|d < tj�dd |rjdnd �r|| j � d � zt � t � � �|d < W n |y� d|d < Y n0 | j|d< t�dt�� �|d< dd| jg}z8| j|ddd�}t�d|�}|�r|�d�nd|d< W n ttf�y0 d|d< Y n0 t�r�z@tdd�� }|�� dk|d< W d � n1 �sl0 Y W n t�y� Y n0 |�dd ��r�t � � } d d d d d | g}z t � } | j|d| d�|d!< W n ttf�y� d|d!< Y n0 | j j | j j d"d#g| j j!d$�} | j"|d%< | j#|d&< z,t�� } t � | j� t�� } | | |d'< W n |�yf Y n0 z t �$| j�\} }}||d(< W n |�y� g }g |d(< Y n0 g |d)< |D ]>}z"t �%|�\} }}|d) �&| � W n |�y� Y n0 �q�z@g d*�}| j|ddd�}t�d+|�}|�r*|�d,�nd|d-< W n ttf�yT d|d-< Y n0 i |d.< |D ]X}z<t � � }t�� } |�'|| j(f� t�� } | | |d. |< W n |�y� Y n0 �qb|�rXd}z*t�� } t � |�}t�� } | | |d/< W n |�y Y n0 z<t � � }t�� } |�'|�p$||f� t�� } | | |d0< W n |�yV Y n0 zt)�*� |d1< W n t+�y� d|d1< Y n0 zt,t-�.� �|d2< W n t/tf�y� d|d2< Y n0 d}d}d}d}z�td3d���}|D ]�}|�0d4��r t1d�2d5d6� |D ���d7 }n||�0d8��r6t1d�2d9d6� |D ���d7 }nR|�0d:��r`t1d�2d;d6� |D ���d7 }n(|�0d<��r�t1d�2d=d6� |D ���d7 }�q�W d � n1 �s�0 Y W n t3t4f�y� Y n0 ||||d>�|d?< i |d@< t5t�D ]>}t6t|�}|�7� �r�t8|t9j:��rdA|v �s�||d@ |< �q�i |dB< g dC�} | D ]l} zPt| d��}|�;� } W d � n1 �sn0 Y | �<� |dB t-j=�>| �< W n t3�y� Y n0 �q>|| j?dD< dS )Ez Collects system information.�Boto� is_secureT�boto_https_enabled�proxyN� proxy_portr � using_proxy� proxy_rdnsFz�DNS lookups are disallowed in this environment, so some information is not included in this perfdiag run. To allow local DNS lookups while using a proxy, set proxy_rdns to False in your boto file.� ip_addressr9 �tempdir�%a, %d %b %Y %H:%M:%S +0000� gmt_timestamp�nslookupz -type=CNAME)r� r� z = (?P<googserv>[^.]+)\.Zgoogserv�googserv_routez /sys/class/dmi/id/product_namerz zGoogle Compute Engine �on_gce�gcloudZcomputeZ instancesr� z --filter=�gce_instance_info�location� storageClassr; �bucket_location�bucket_storageClass�google_host_dns_latency� googserv_ips�googserv_hostnames)r� z -type=TXTzo-o.myaddr.google.com.z text\s+=\s+"(?P<dnsip>[\.\d]+)"Zdnsip� dns_o-o_ip� google_host_connect_latencies�proxy_dns_latency�proxy_host_connect_latency� cpu_count�load_avgz /proc/meminfoZMemTotalc s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� � rS z2PerfDiagCommand._CollectSysInfo.<locals>.<genexpr>�� ZMemFreec s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� � rS ZBuffersc s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� � rS ZCachedc s s | ]}|t jv r|V qd S rV r� r� r2 r2 r3 r� � rS )� mem_total�mem_free� mem_buffers� mem_cached�meminfoZ gsutil_configr� �tcp_proc_values)z /proc/sys/net/core/rmem_defaultz/proc/sys/net/core/rmem_maxz /proc/sys/net/core/wmem_defaultz/proc/sys/net/core/wmem_maxz!/proc/sys/net/ipv4/tcp_timestampsz/proc/sys/net/ipv4/tcp_sackz%/proc/sys/net/ipv4/tcp_window_scaling�sysinfo)@�socketrR �herror�gaierror�timeout�botor �get�getint�boolrQ r� � gethostbyname� gethostnamer� r� �strftime�gmtime� XML_API_HOSTr� �re�search�groupr r� r r� �readliner! r? � GetBucketr r r@ r� r� �gethostbyname_ex� gethostbyaddrr� �connect� XML_API_PORT�multiprocessingr� �NotImplementedErrorr� rd � getloadavg�AttributeError� startswithrg r� �IOErrorr� r� �getattr�isupperr� r� � string_typesr� r� r� r� r� ) rZ r� Z socket_errors� proxy_hostr� r� Znslookup_cname_output�mrh �hostnamer� Z bucket_infor� r� rW Z ipaddrlistZ googserv_ipZnslookup_txt_output�sock�proxy_ipr� r� r� r� r� r� �attr� attr_valueZstats_to_check�fname�valuer2 r2 r3 �_CollectSysInfo sJ � � � � 4� � � � � � �,� � ( z PerfDiagCommand._CollectSysInfoc s� t |�}tt|��| � t�t� fdd�|D ��| �}tjt|��d�ddd� tjd� d �d �ddd� tjd|d �d �ddd� tjdt |d �d �d �ddd� t�dt |d �d �d �d� dS )zAPrints out mean, standard deviation, median, and 90th percentile.c 3 s | ]}|� d V qdS )r� Nr2 )r) �x��meanr2 r3 r� � rS z0PerfDiagCommand._DisplayStats.<locals>.<genexpr>� r9 r� ��endz%.1fr� � � g �?r| g�������?N) r �float�sum�math�sqrtr � print_to_fdrS �rjustr& )rZ �trials�n�stdevr2 r� r3 � _DisplayStats� s ��z PerfDiagCommand._DisplayStatsc 1 C s t �� t �d� t �d�d�� t �d� d| jv �r�t �� t �d� t �d�d�� t �d� t �d� t �d� t| jd �D �]8}t| jd | �}|�d �\}}t|�}|d kr�t jd �d �d dd� t jt|��d �d dd� | � |� |dk�r:t jd�d �d dd� t jt|��d �d dd� | � |� |dk�r~t jd�d �d dd� t jt|��d �d dd� | � |� |dkr�t jd�d �d dd� t jt|��d �d dd� | � |� q�d| jv �r`t �� t �d� t �d�d�� t �d� | jd }t �d| j t|d �t|d �f � t �dt |d d � � d |v �r`t �d |d � d | jv �r�t �� t �d� t �d!�d�� t �d� | jd }t �d| j t|d �t|d �f � t �dt |d d � � d |v �r�t �d |d � d"| jv �r�t �� t �d� t �d#�d�� t �d� | jd" }t �d| j t|d �t|d �f � t �d$t |d d � � d |v �r�t �d |d � d%| jv �r:t �� t �d� t �d&�d�� t �d� | jd% }t �d| j t|d �t|d �f � t �d$t |d d � � d |v �r:t �d |d � d'| jv �rxt �� t �d� t �d(�d�� t �d� | jd' } | d) } | d* } t �d+| d, � t �d-| d. � t �d/| d0 � t �d1d2� d3d4� | d5 D �� � t �d6d2� t t| d7 �� � t �d8| d, � t �d-| d. � t �d/| d0 � t �d1d2� d9d4� | d5 D �� � t �d6d2� t t| d7 �� � d:| jv � r�t �� t �d� t �d;�d�� t �d� | jd: } t �d<| d= � t �d>| d? � t �d@| jdA � t �dB| j�dCdD� � t �dE| j�dFdD� � dG| v �r�| dG } dH}zt�| dI�}W n t�y\ Y n0 |�r�t�|�}tj�|�}t �dJ|�dK� � dL| v �r�t �dM| dL � | dL �r�t �dN| dO �dPdQ� � t �dR| dS � t �dT| dU � t �dV| dW � t �dXdY� | dZ � � t �d[dY� | d\ � � t �d]| d^ � t �d_| d` � t �da| db � zPt �dct| dd de � � t �dft| dd dg | dd dh | dd di � � W n t�y� Y n0 dj| v �rFdk| v �rF| dj }| dk }dlD ]J}z.|dm| |dm| }t �dn||f � W n t�y> Y n0 �q�n t �do� dp| v � rjdq| v � rjt jdrdd� | dp }| dq }t jd ds�dt�dd� duD ]}t j|�d �dd� �q�t �� t|�D ]�}|| }|| }|\}} } } } } |\}!}"}#}$}%}&t jd |�dt�dd� |!| |"| |#| |$| |%| |&| g}'|'D ] }t jt|��d �dd� � q>t �� �q�dv| v � r�t jdwdd� t�| dv �D ]}(t �dx|( � � q�dy| v � r�t �dz| dy � d{| v � r�t �d|| d{ � d}| v � r�t �d~| d} d � d�| v � rBt �d�� t�| d� �D ] \})}*t �d�|)|*d f � � q d�| v � rbt �d�| d� d � d�| v � r�t �d�| d� d � d�| jv � r�d�| jv � r�t �� t �d� t �d��d�� t �d� t| jd� �}+t| jd� �},t| jd� �}-|+d�k� r|+|, t |+� d� nd�}.t �d�|+ � t �d�|, � t �d�|- � t �d�|. � d�| jv � r�tt�| jd� ��}/|/� r�t �d�� t �dP� d�d4� |/D ��� | j � r�t | j d���"}0t j | j|0d�d�� W dH � n1 � s�0 Y t �� t �d�| j � t �� dHS )�z/Displays results collected from diagnostic run.zN==============================================================================zDIAGNOSTIC RESULTS�N r� zN------------------------------------------------------------------------------ZLatencyzOOperation Size Trials Mean (ms) Std Dev (ms) Median (ms) 90th % (ms)zO========= ========= ====== ========= ============ =========== ===========rW ZMETADATA�Metadatar� r9 r� r� �DOWNLOADr: �UPLOADrH �DELETErM rK zWrite Throughputz5Copied %s %s file(s) for a total transfer size of %s.rK r= zWrite throughput: %s/s.r> � r: zParallelism strategy: %srJ z Write Throughput With File I/Or7 zRead ThroughputzRead throughput: %s/s.r6 z Read Throughput With File I/OrO ZListingrg rj zAfter inserting %s objects:rN z0 Total time for objects to appear: %.2g secondsr< z" Number of listing calls made: %sre z) Individual listing call latencies: [%s]�, c s s | ]}d | V qdS �z%.2gsNr2 �r) r� r2 r2 r3 r� � rS z2PerfDiagCommand._DisplayResults.<locals>.<genexpr>r[ z' Files reflected after each call: [%s]rf zAfter deleting %s objects:c s s | ]}d | V qdS r r2 r r2 r2 r3 r� � rS r� zSystem InformationzIP Address: %sr� zTemporary Directory: %sr� zBucket URI: %s� bucket_urizgsutil Version: %s�gsutil_version�Unknownzboto Version: %s� boto_versionr� Nr� zMeasurement time: %sz%Y-%m-%d %I:%M:%S %p %Zr� zRunning on GCE: %szGCE Instance: %sr� r� z zBucket location: %sr� zBucket storage class: %sr� zGoogle Server: %sr� z!Google Server IP Addresses: %sz r� z Google Server Hostnames: %sr� z#Google DNS thinks your IP is: %sr� zCPU Count: %sr� zCPU Load Average: %sr� zTotal Memory: %sr� r� zFree Memory: %sr� r� r� � netstat_end� netstat_start)�sent�receivedZ retransmitztcp_%sz!TCP segments %s during test: %dzQTCP segment counts not available because "netstat" was not found during test runs�disk_counters_end�disk_counters_startzDisk Counter Deltas: Zdiskr� )r� r� r� r� r� r� r� zTCP /proc values: z %s = %sr� zBoto HTTPS Enabled: %sr� z$Requests routed through proxy: %sr� zALatency of the DNS lookup for Google Storage server (ms): %.1fg @�@r� z7Latencies connecting to Google Storage server IPs (ms):z %s = %.1fr� z@Latency of the DNS lookup for the configured proxy (ms): %.1fr� z8Latency connecting to the configured proxy (ms): %.1fr� r� zIn-Process HTTP Statisticsr� r r` z Total HTTP requests made: %dzHTTP 5xx errors: %dzHTTP connections broken: %dzAvailability: %.7g%%r� zError responses by code:c s s | ]}d | V qdS )z %s: %sNr2 r� r2 r2 r3 r� 2 rS �wr� )�indentz Output file written to '%s'.)!r r� �centerr� �sortedr~ rg r� r% r� r� r$ r� �maprS r� r� �strptimer� �calendar�timegm�datetime� fromtimestampr� �replacer� r� � iteritemsr� � output_filer� �json�dump)1rZ r� r� �op�numbytesZ write_thruZwrite_thru_fileZ read_thruZread_thru_filerO rg rj r� Z ts_string� timetuple� localtimeZlocaldtZ netstat_afterZnetstat_beforeZtcp_type�deltaZ disk_afterZ disk_beforeZcolnameZdiskname�before�afterZreads1Zwrites1Zrbytes1Zwbytes1Zrtime1Zwtime1Zreads2Zwrites2Zrbytes2Zwbytes2Zrtime2Zwtime2Zdeltas�item�ipr� �totalZ numerrorsZ numbreaksZ availabilityZ sorted_codesr� r2 r2 r3 �_DisplayResults s� ��� � � � ����� � ����� � ����� � ����� � ��������������� � � � �� � ��� � �� �� � � � � �� � �� �� ��� � 2z PerfDiagCommand._DisplayResultsc C s> z t |�}|dk rt|��|W S ty8 t|��Y n0 dS )a^ Tries to convert val argument to a positive integer. Args: val: The value (as a string) to convert to a positive integer. msg: The error message to place in the CommandException on an error. Returns: A valid positive integer. Raises: CommandException: If the supplied value is not a valid positive integer. rb N)rg r r� )rZ �valr r2 r2 r3 �_ParsePositiveInteger< s z%PerfDiagCommand._ParsePositiveIntegerc C sh d| _ d| _d| _d| _d| _d| _t�� | _d| _ t | j �| _ d| _ d| _i | _d| _d| _| j�r| jD �]�\}}|dkr�| �|d �| _ |d kr�| �|d �| _|d kr�| �|d �| _|dkr�|�� | jv r�|�� | _n td| ��|dkr�| �|d�| _|dk�r2zt|�| _W n t�y0 td��Y n0 |dk�rd|| _tj�| j��sdd| _ t� | j� |dk�r�t � | _ |� � � d�D ].}|�� | j v�r�td| ��| j � |� �q�|dk�r�|� d�}t!|�dk�r�td | ��|\}}|| j|< |d k�r tj�"|�| _ |d k�r�tj�"|�| _tj�#| j��s@td | ��zNt$| jd ��,}t%�&|�| _'| j(�)d!| j� W d � n1 �s�0 Y W n t�y� td"| ��Y n0 dS |d#krld| _zt*|�| _W n t�y� d$| _Y n0 | jd%k �s| jdkrltd&��ql| jdk�s&| jdk�r8| j�s8| j+| _n(| jdk�r`| jdk�r`| j�r`td'��| j,�sp| �-� t.| j,d% �| _/| j/j0| _1| j/�2� �s�| j/�3� �r�td(| j,d% ��| jtd)�k�r�| j4| j v �s�| j5| j v �r�td*��| j| j6| j7fv }| j1d+k�o| j �8| j5| j9�} |�r,| �r,td,��| j:j;| j/j<| j/j0d-gd.� t=j>t?j@t?jAt?jBt=jCtDg| _EdS )/z&Parses arguments for perfdiag command.r� rb N� r� Fr` z-nz,The -n parameter must be a positive integer.z-cz,The -c parameter must be a positive integer.z-kz,The -k parameter must be a positive integer.z-pz)'%s' is not a valid parallelism strategy.z-yz,The -y parameter must be a positive integer.r� zInvalid -s parameter.z-dTz-t�,z8List of test names (-t) contains invalid test name '%s'.z-m�:r� z,Invalid metadata key-value combination '%s'.z-oz-iz Invalid input file (-i): '%s'.rz zRead input file: '%s'.z'Could not decode input file (-i): '%s'.z-jr{ r z7The -j parameter must be between 0 and 100 (inclusive).zrCannot specify parallelism strategy (-p) without also specifying multiple threads and/or processes (-c and/or -k).zOThe perfdiag command requires a URL that specifies a bucket. "%s" is not valid.r� z~For in-memory tests maximum file size is 2GiB. For larger file sizes, specify rthru_file and/or wthru_file with the -t option.�s3zQSliced uploads are not available for s3. Use -p fan or sequential uploads for s3.�idr )Fr� r r r? r% r� r� � gettempdirr� r� r� �DEFAULT_DIAG_TESTSr� r � input_file� metadata_keysrL r� �sub_optsr. r� �PARALLEL_STRATEGIESr r# r� rd r� �exists�makedirsr� r~ �ALL_DIAG_TESTSr� r �abspath�isfiler� r �loadr� rQ r� rg rB r= �$RaiseWrongNumberOfArgumentsExceptionr r r@ r� � IsCloudUrl�IsBucketr� r� rD rC r� r� r? r� r r � HTTPExceptionr� rR r� r� � BadStatusLiner r� ) rZ �o�arE �piecesr� r� r� Z perform_sliceZslice_not_availabler2 r2 r3 � _ParseArgsQ s ��� �� � � 4� �� � � � �� �� � �zPerfDiagCommand._ParseArgsc C s� | � � | jr| �� dS tj�ddd� | j�d| j| j | j | j | j t | j�d�| j�| j| j� �z�| �� | �� | �� }|r�|| jd d< tr�t� | jd d < t| j �| jd < d | jd < | j| jd < | j| jv r�| � � | j | jv r�| � � | j | jv �r| j dd� | j!| jv �r(| j dd� | j"| jv �r>| � � | j#| jv �rT| �$� | �� }|�rp|| jd d< t�r�t� | jd d< | j%| jd< | j&| jd< | j'| jd< | j(| jd< t)j*| jd< tj+| jd< | �,� | �� W | �,� n | �,� 0 dS )z3Called by gsutil when the command is being invoked.r r� � num_retries�0z�Number of iterations to run: %d Base bucket URI: %s Number of processes: %d Number of threads: %d Parallelism strategy: %s Throughput file size: %s Diagnostics to run: %s Gzip compression ratio: %s Gzip transport encoding writes: %sr r� r r r rm Z json_format�metadataT)rG r r r� r� r� r� r r )-rH r6 r, r� r r� rQ r� r� r r r r? r% r� r� r� r� rL r� r� r� r� r r rS r7 r� r r� rI r� rM r� r� �LISTrl r� r� r� r� rp �VERSION� __version__r� )rZ r� r2 r2 r3 � RunCommand� sd � zPerfDiagCommand.RunCommand)TFF)r FFr� r` )F)F)F)F)Fr NF)NNr N)Kr. r/ r0 r1 r �CreateCommandSpec� _SYNOPSISr �XML�JSONr � MakeNCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr� r� r� r� r� r� rL rB rD rC r< r5 r9 r� r r� �gs� connection� GSConnection� DefaultHostr� r� r� r� r� ZKEY_BUFFER_SIZErk r� r� r� r� r� � contextlib�contextmanagerr� r� r �logging�Filterr� rU r" r6 r' rF rI rM rl rH r: rM r� r� r� r� r, r. rH rO r2 r2 r2 r3 rl � s� �� � � # � (P* .> " � * � K X Hu � N � +)9 ` : rl c C s< | � � r,| �� r,t�� }| j|_| j|_|S td| ��d S )NzENon-cloud URL upload target %s was created in perfdiag implemenation.) rA �IsObjectr, r- r7 r� r r� r )� storage_urlr� r2 r2 r3 r 0 s �r )N)N)N)N)N)r r` ra )dr1 � __future__r r r r r � collectionsr r r] r r r_ r� r� rd r� r� r� rQ r� r� r� r� �boto.gs.connectionr� � six.movesr r r rp �gslib.cloud_apir r � gslib.commandr r �gslib.command_argumentr �gslib.commandsr �gslib.cs_api_mapr �gslib.exceptionr �gslib.file_partr �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr r, Z gslib.utilsr Zgslib.utils.boto_utilr r �gslib.utils.constantsr � gslib.utils.cloud_api_helperr r �gslib.utils.hashing_helperr Zgslib.utils.system_utilr r r r r! �gslib.utils.unit_utilr" r# r$ r% r& rQ rW r' r( r) r* r+ r� � Exceptionr, r4 r@ rC rI rL rN rU rX �objectrY rk rl r r2 r2 r2 r3 �<module> s� �� < ���� � $ t commands/__pycache__/du.cpython-39.pyc 0000644 00000020735 15025013616 0013624 0 ustar 00 a +(Wg�) � @ s0 d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m Z ddl!m"Z" ddlm#Z# dZ$de$ d Z%G dd� de �Z&dS ) zCImplementation of Unix-like du command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�BucketListingObject)�Command)�CommandArgument)� ApiSelector)�CommandException)�ContainsWildcard)�StorageUrlFromString)� ls_helper)�NO_MAX)�S3_DELETE_MARKER_GUID)�UTF8)�GcloudStorageFlag)�GcloudStorageMap)� print_to_fd)�MakeHumanReadable)� text_utilz gsutil du url... z <B>SYNOPSIS</B> ak <B>DESCRIPTION</B> The du command displays the amount of space in bytes used up by the objects in a bucket, subdirectory, or project. The syntax emulates the Linux ``du -b`` command, which reports the disk usage of files and subdirectories. For example, the following command reports the total space used by all objects and subdirectories under gs://your-bucket/dir: gsutil du -s -a gs://your-bucket/dir <B>OPTIONS</B> -0 Ends each output line with a 0 byte rather than a newline. You can use this to make the output machine-readable. -a Includes both live and noncurrent object versions. Also prints the generation and metageneration number for each listed object. If this flag is not specified, only live object versions are included. -c Includes a total size at the end of the output. -e Exclude a pattern from the report. Example: -e "*.o" excludes any object that ends in ".o". Can be specified multiple times. -h Prints object sizes in human-readable format. For example, ``1 KiB``, ``234 MiB``, or ``2GiB``. -s Displays only the total size for each argument, omitting the list of individual objects. -X Similar to ``-e``, but excludes patterns from the given file. The patterns to exclude should be listed one per line. <B>EXAMPLES</B> To list the size of each object in a bucket: gsutil du gs://bucketname To list the size of each object in the ``prefix`` subdirectory: gsutil du gs://bucketname/prefix/* To include the total number of bytes in human-readable form: gsutil du -ch gs://bucketname To see only the summary of the total number of (live) bytes in two given buckets: gsutil du -s gs://bucket1 gs://bucket2 To list the size of each object in a bucket with `Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_ enabled, including noncurrent objects: gsutil du -a gs://bucketname To list the size of each object in a bucket, except objects that end in ".bak", with each object printed ending in a null byte: gsutil du -e "*.bak" -0 gs://bucketname To list the size of each bucket in a project and the total size of the project: gsutil -o GSUtil:default_project_id=project-name du -shc c @ s� e Zd ZdZejdg ededdddej ej gej e � � gd� Z ejdg dd ei d �Zed dged �ed �ed�ed�ed�ed�ed�d�d�Zdd� Zdd� Zdd� ZdS ) � DuCommandz$Implementation of gsutil du command.Zdur z 0ace:hsX:FT) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzDisplay object size usage)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storagez--zero-terminatorz--all-versionsz--totalz--exclude-name-patternz--readable-sizesz --summarizez--exclude-name-pattern-file)�-0�-a�-c�-e�-h�-s�-X)�gcloud_command�flag_mapc C s< | j rt|�nt�|�}tjdj|t�|�d�| jd� d S )Nz{size:<11} {name})�size�name)�end) �human_readabler �six� text_typer r �format� ensure_text� line_ending)�self� num_bytesr4 � size_string� r? � /opt/gsutil/gslib/commands/du.py�_PrintSummaryLine� s � � ��zDuCommand._PrintSummaryLinec C s� |j }|j}|jr4t|jjv r4d}d}d}|d7 }n$| jrDt|j�nt|j�}|j}d}| j s�dj |t � |�t � | j �d�}t|tjdd� ||fS ) z�Print listing info for given bucket_listing_ref. Args: bucket_listing_ref: BucketListing being listed. Returns: Tuple (number of objects, object size) Raises: Exception: if calling bug encountered. �0r z<DeleteMarker>� z{size:<11} {url}{ending})r3 �urlZending� )�filer5 )� root_object� url_string�metadatar �additionalPropertiesr6 r r3 �str� summary_onlyr9 r7 r: r; r �sys�stdout)r< �bucket_listing_ref�obj�url_strr> r= Znum_objsZ url_detailr? r? r@ � _PrintInfoAboutBucketListingRef� s. � � � �z)DuCommand._PrintInfoAboutBucketListingRefc s� d� _ d� _d� _d� _d� _g � _� jr�� jD ]�\}}|dkrHd� _ q0|dkrXd� _q0|dkrhd� _q0|dkr~� j�|� q0|d kr�d� _q0|d kr�d� _q0|d kr0|d kr�tj }d}n"t j r�t |d �n t |d t d�}d}dd� |D �� _|r0|�� q0� j�s dg� _d}d}� fdd�}d&dd�}� fdd�} � jD �]H} t| �} | �� �r`td� j ��dg} tj� j� j|||| |� jd� j| d � } | �� �r�� �d | j �jd gd �}n6| �� �r� �d | j| j f �jd gd �}n t | �g}|D ]�}|j }|�� �r� j�rt|j d!d"��}| � |�\}}}|�!� �rT|dk�rTt"| ��rT� j�sTd}||7 }� j�r� �#||j$�%d#��&t �� �q�q:� j�r�� �#|d$� |�r�td%��dS )'z'Command entry point for the du command.� Fr* � r+ Tr, r- r. r/ r0 �-�r)�encodingc S s g | ]}t �|�� ��qS r? )r7 r: �strip)�.0�liner? r? r@ � <listcomp>� � z(DuCommand.RunCommand.<locals>.<listcomp>zgs://r c s � � | �S �N)rR )�blr�r< r? r@ �_PrintObjectLong� s z.DuCommand.RunCommand.<locals>._PrintObjectLongNc S s d S r] r? )Z unused_blrr? r? r@ � _PrintNothing� s z+DuCommand.RunCommand.<locals>._PrintNothingc s � j s� �| |j�t�� d S r] )rL rA rH �encoder )r= r^ r_ r? r@ �_PrintDirectory� s z-DuCommand.RunCommand.<locals>._PrintDirectoryz$Only cloud URLs are supported for %sr3 ) Zprint_object_funcZprint_dir_funcZprint_dir_header_funcZprint_dir_summary_funcZprint_newline_func� all_versionsZshould_recurse�exclude_patterns�fieldsz%s://*�id)� bucket_fieldsz%s://%sz**)�wildcard_suffix�/�totalz$One or more URLs matched no objects.)N)'r; rd Z produce_totalr6 rL re �sub_opts�appendrM �stdinr7 �PY2�openr �close�argsr � IsFileUrlr � command_namer ZLsHelper�WildcardIterator�logger� IsProvider�scheme� IterBuckets�IsBucket� bucket_namer � storage_url�CreatePrefixUrlZExpandUrlAndPrint�IsObjectr rA rH �rstriprb )r< �o�a�fZf_close� total_bytesZgot_nomatch_errorsr` ra rc �url_argZtop_level_storage_url�bucket_listing_fieldsZlisting_helperZtop_level_iterr^ r| �_Zexp_objsZ exp_bytesr? r_ r@ � RunCommand� s� �� ��� ��� ���� zDuCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r �gcloud_storage_maprA rR r� r? r? r? r@ r z sL ��� �� #r )'r� � __future__r r r r �localerM r7 �gslib.bucket_listing_refr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.storage_urlr r Z gslib.utilsr �gslib.utils.constantsr r r �gslib.utils.shim_utilr r Zgslib.utils.text_utilr �gslib.utils.unit_utilr r r� r� r r? r? r? r@ �<module> s< ��K commands/__pycache__/mv.cpython-39.pyc 0000644 00000012076 15025013616 0013635 0 ustar 00 a +(Wgm � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd l m Z dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ dZde d ZG dd� de�ZdS )zCImplementation of Unix-like mv command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literals)�Command)�CommandArgument)�CP_AND_MV_SHIM_FLAG_MAP)� CP_SUB_ARGS)�'ShimTranslatePredefinedAclSubOptForCopy)� ApiSelector)�CommandException)�StorageUrlFromString)�NO_MAX)�GcloudStorageMapzb gsutil mv [-p] src_url dst_url gsutil mv [-p] src_url... dst_url gsutil mv [-p] -I dst_url z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> The ``gsutil mv`` command allows you to move data between your local file system and the cloud, move data within the cloud, and move data between cloud storage providers. For example, to move all objects from a bucket to a local directory you could use: gsutil mv gs://my_bucket/* dir Similarly, to move all objects from a local directory to a bucket you could use: gsutil mv ./dir gs://my_bucket <B>RENAMING GROUPS OF OBJECTS</B> You can use the ``gsutil mv`` command to rename all objects with a given prefix to have a new prefix. For example, the following command renames all objects under gs://my_bucket/oldprefix to be under gs://my_bucket/newprefix, otherwise preserving the naming structure: gsutil mv gs://my_bucket/oldprefix gs://my_bucket/newprefix If you do a rename as specified above and you want to preserve ACLs, you should use the ``-p`` option (see OPTIONS). If you have a large number of files to move you might want to use the ``gsutil -m`` option, to perform a multi-threaded/multi-processing move: gsutil -m mv gs://my_bucket/oldprefix gs://my_bucket/newprefix <B>NON-ATOMIC OPERATION</B> Unlike the case with many file systems, the gsutil mv command does not perform a single atomic operation. Rather, it performs a copy from source to destination followed by removing the source for each object. A consequence of this is that, in addition to normal network and operation charges, if you move a Nearline Storage, Coldline Storage, or Archive Storage object, deletion and data retrieval charges apply. See the `documentation <https://cloud.google.com/storage/pricing>`_ for pricing details. <B>OPTIONS</B> All options that are available for the gsutil cp command are also available for the gsutil mv command (except for the -R flag, which is implied by the ``gsutil mv`` command). Please see the `Options section for cp <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_ for more information. c sx e Zd ZdZejdg d�edeeddde j e j ge j e � � gd� Zejdd d gd d ei d �Z� fdd�Zdd� Z� ZS )� MvCommandz�Implementation of gsutil mv command. Note that there is no atomic rename operation - this command is simply a shorthand for 'cp' followed by 'rm'. �mv)�moveZren�rename� TFr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_argumentsr r � command_helpzMove/rename objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s& t | j� tddgtd�}t� �|�S )N�storager )�gcloud_command�flag_map)r �sub_optsr r �super�get_gcloud_storage_args)�self�gcloud_storage_map�� __class__� � /opt/gsutil/gslib/commands/mv.pyr, � s �z!MvCommand.get_gcloud_storage_argsc C s� | j dd� D ]:}t|�}|�� r|�� s2|�� rtd|| j d f ��qdg}| jr`|�d� |�| j � | j j d|| j | j | j| j| jd� dS )z'Command entry point for the mv command.r ���z�You cannot move a source bucket using the mv command. If you meant to move all objects in the bucket, you can use a command like: gsutil mv %s/* %sz-Mz-R�cp)�args�headers�debug� trace_token� user_project�parallel_operations)r5 r � IsCloudUrl�IsBucket� IsProviderr �recursion_requested�append�extend� unparsed_args�command_runner�RunNamedCommandr6 r7 r8 r9 r: )r- Z arg_to_check�urlrA r1 r1 r2 � RunCommand� s( � � zMvCommand.RunCommand)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r r �XML�JSONr �%MakeZeroOrMoreCloudOrFileURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr, rE � __classcell__r1 r1 r/ r2 r ` s4 ��� r N)rI � __future__r r r r � gslib.commandr �gslib.command_argumentr Zgslib.commands.cpr r r �gslib.cs_api_mapr �gslib.exceptionr �gslib.storage_urlr �gslib.utils.constantsr �gslib.utils.shim_utilr rK rQ r r1 r1 r1 r2 �<module> s* ��9 commands/__pycache__/rm.cpython-39.pyc 0000644 00000025561 15025013616 0013634 0 ustar 00 a +(Wg@ � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlm Z dd lm Z dd lm Z dd l m Z dd l mZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl%m'Z' dd l(m)Z) dd l*m+Z+ e j,Z-d Z.d e. d Z/d!d"� Z0d#d$� Z1d+d%d&�Z2d'd(� Z3G d)d*� d*e �Z4dS ),zCImplementation of Unix-like rm command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�BucketNotFoundException)�NotEmptyException)�NotFoundException)�ServiceException)�Command)�DecrementFailureCount)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_PREFIX)�NO_URLS_MATCHED_TARGET)�NameExpansionIterator)� SeekAheadNameExpansionIterator)�StorageUrlFromString)�MetadataMessage)� constants)�parallelism_framework_util)�GetCloudApiInstance)�Retry)�GcloudStorageFlag)�GcloudStorageMap)� StdinIterator)�PreconditionsFromHeadersz7 gsutil rm [-f] [-r] url... gsutil rm [-f] [-r] -I z <B>SYNOPSIS</B> aU <B>DESCRIPTION</B> NOTE: As part of verifying the existence of objects prior to deletion, ``gsutil rm`` makes ``GET`` requests to Cloud Storage for object metadata. These requests incur `network and operations charges <https://cloud.google.com/storage/pricing>`_. The gsutil rm command removes objects and/or buckets. For example, the following command removes the object ``kitten.png``: gsutil rm gs://bucket/kitten.png Use the -r option to specify recursive object deletion. For example, the following command removes gs://bucket/subdir and all objects and subdirectories under it: gsutil rm -r gs://bucket/subdir When working with versioning-enabled buckets, note that the -r option removes all object versions in the subdirectory. To remove only the live version of each object in the subdirectory, use the `** wildcard <https://cloud.google.com/storage/docs/wildcards>`_. The following command removes all versions of all objects in a bucket, and then deletes the bucket: gsutil rm -r gs://bucket To remove all objects and their versions from a bucket without deleting the bucket, use the ``-a`` option: gsutil rm -a gs://bucket/** If you have a large number of objects to remove, use the ``gsutil -m`` option, which enables multi-threading/multi-processing: gsutil -m rm -r gs://my_bucket/subdir You can pass a list of URLs (one per line) to remove on stdin instead of as command line arguments by using the -I option. This allows you to use gsutil in a pipeline to remove objects identified by a program, such as: some_program | gsutil -m rm -I The contents of stdin can name cloud URLs and wildcards of cloud URLs. Note that ``gsutil rm`` refuses to remove files from the local file system. For example, this fails: gsutil rm *.txt WARNING: Object removal cannot be undone. Cloud Storage is designed to give developers a high amount of flexibility and control over their data, and Google maintains strict controls over the processing and purging of deleted data. If you have concerns that your application software or your users may at some point erroneously delete or replace data, see `Options for controlling data lifecycles <https://cloud.google.com/storage/docs/control-data-lifecycles>`_ for ways to protect your data from accidental data deletion. <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when removing multiple objects. If some of the objects could not be removed, gsutil's exit status will be non-zero even if this flag is set. Execution will still halt if an inaccessible bucket is encountered. This option is implicitly set when running "gsutil -m rm ...". -I Causes gsutil to read the list of objects to remove from stdin. This allows you to run a program that generates the list of objects to remove. -R, -r The -R and -r options are synonymous. Causes bucket or bucket subdirectory contents (all objects and subdirectories that it contains) to be removed recursively. If used with a bucket-only URL (like gs://bucket), after deleting objects and subdirectories gsutil deletes the bucket. This option implies the -a option and deletes all object versions. If you only want to delete live object versions, use the `** wildcard <https://cloud.google.com/storage/docs/wildcards>`_ instead of -r. -a Delete all versions of an object. c C sf | j s| j�t|�� t|t�r@| jd7 _| j�t|�� n"t| j|�rTt � n| j d7 _ dS )z9Simple exception handler to allow post-completion status.� N) �continue_on_error�logger�error�str� isinstancer �bucket_not_found_count� _ExceptionMatchesBucketToDelete�bucket_strings_to_deleter �op_failure_count��cls�e� r* � /opt/gsutil/gslib/commands/rm.py�_RemoveExceptionHandler� s r, c C s. t |t�rt|jv s t |t�r&t� n|�dS )z3When removing folders, we don't mind if none exist.N)r" r r �reasonr r r' r* r* r+ � _RemoveFoldersExceptionHandler� s �r. c C s | j ||d� d S )N�� thread_state)� RemoveFunc)r( �name_expansion_resultr0 r* r* r+ �_RemoveFuncWrapper� s r3 c C sB | r>t d }|t|�v r>t|��|�}t|�dko<|d | v S dS )a� Returns True if the exception matches a bucket slated for deletion. A recursive delete call on an empty bucket will raise an exception when listing its objects, but if we plan to delete the bucket that shouldn't result in a user-visible error. Args: bucket_strings_to_delete: Buckets slated for recursive deletion. e: Exception to check. Returns: True if the exception was a no-URLs-matched exception and it matched one of bucket_strings_to_delete, None otherwise. � � r N)r r! �split�len)r% r) �msg�partsr* r* r+ r$ � s r$ c @ s� e Zd ZdZejdg d�edejdddde j e j ge j e � � gd� Zejdg d�dd ei d �Zed dged �ed �ed �ed�ed�d�d�Zdd� Zddd�ZdS )� RmCommandz$Implementation of gsutil rm command.�rm)�del�delete�remover ZafIrRF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzRemove objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�-r�-a�-Iz--continue-on-error)rR �-RrS rT �-f)�gcloud_command�flag_mapc sr � j � _d� _d� _� jrv� jD ]T\}}|dkr8d� _q |dkrHd� _q |dkrXd� _q |dksh|dkr d� _d� _q � jr�� jr�td��t� }n� js�td ��� j}d � _ d � _ g }g � _ � j�r d g}|D ]L}t |���� � s��� rΈ �|�j|d �D ] }|�|j� � j �|� q�q�t� j�p(i �� _z�t� j� j� j� j|� j� j� j� j�p\� j d � }d} � j�s�t � j� j� � � |� j� j� jd�} � j t |t � j ddg| d� W nl t�y� } z$t!� j | ��r�t"� n� W Y d} ~ n8d} ~ 0 t#�y } z� j�s � W Y d} ~ n d} ~ 0 0 � j �r0td��� j �rH� j�sHtd��� j�rg } |D ],}t |����$� �rX| �|�%d�d � �qX| �rd� _z<t� j� j� j� j| � j� j� jd�}� j t |t&dd� W n6 t�y } z | j'�(t)��s� W Y d} ~ n d} ~ 0 0 |D ]6�� j�*d�� t+t,ddd �� �fd d ��} | � �q � j �rn� j �rVd nd } td!� j | | f ��d S )"z'Command entry point for the rm command.FrS TrV rT rR rU z&No arguments allowed with the -I flag.z5The rm command (without -I) expects at least one URL.r �id)� bucket_fields)� project_id� all_versionsr N)r\ r[ r& r# )� fail_on_error� shared_attrs�seek_ahead_iteratorz.Encountered non-existent bucket during listingz Some files could not be removed.�*z *_$folder$)r[ r\ )r] �Removing %s...� r )�tries� timeout_secsc s � j j�j�jd� d S )N)�provider)� gsutil_api� DeleteBucket� bucket_name�schemer* ��self�urlr* r+ �BucketDeleteWithRetry~ s z3RmCommand.RunCommand.<locals>.BucketDeleteWithRetry�sr4 z(%d file%s/object%s could not be removed.)-�parallel_operationsr �read_args_from_stdinr\ �sub_opts�recursion_requested�argsr r r& r# r% r �IsBucket� IsProvider�WildcardIterator� IterBuckets�append� storage_urlr �headers� preconditionsr � command_name�debugr rf r[ r �GetSeekAheadGsutilApi�Applyr3 r, r$ r r �IsObject�rstripr. r- � startswithr �infor r )rk �o�unused_a�url_strsZbucket_urls_to_deleterZ �url_str�blr�name_expansion_iteratorr_ r) Zfolder_object_wildcardsrm � plural_strr* rj r+ � RunCommand� s� � � � � � � �zRmCommand.RunCommandNc C s� t | |d�}|j}| j�d|� z"|j|j|j| j|j|j d� W n8 t yz } z | j�d|� t � W Y d }~n d }~0 0 t |j tt�� d�� d S )Nr/ ra )r{ � generationre zCannot find %s)� message_time)r �expanded_storage_urlr r� � DeleteObjectrh � object_namer{ r� ri r r �_PutToQueueWithTimeout� status_queuer �time)rk r2 r0 rf � exp_src_urlr) r* r* r+ r1 � s � �zRmCommand.RemoveFunc)N)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �NO_MAXr �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r �gcloud_storage_mapr� r1 r* r* r* r+ r: � sF �� �� r: )N)5r� � __future__r r r r r� �gslib.cloud_apir r r r � gslib.commandr r �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r r �gslib.name_expansionr r �gslib.storage_urlr �gslib.thread_messager Z gslib.utilsr r � gslib.utils.cloud_api_helperr �gslib.utils.retry_utilr �gslib.utils.shim_utilr r Zgslib.utils.system_utilr � gslib.utils.translation_helperr �PutToQueueWithTimeoutr� r� r� r, r. r3 r$ r: r* r* r* r+ �<module> sP ��[ commands/__pycache__/iam.cpython-39.pyc 0000644 00000061147 15025013616 0013764 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl mZ ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl'm)Z) ddl*m+Z+ ddl,m-Z- ddl,m.Z. ddl,m/Z/ dd l,m0Z0 dd l1m2Z3 dd l4m5Z5 dd l6m7Z7 dd l8m9Z9 dd!l8m:Z: dd"l4m;Z; dd#l<m=Z= dd$l<m>Z> dd%l<m?Z? dd&l<m@Z@ dd'l<mAZA dd(l<mBZB dd)lCmDZD dd*lEmFZF dd+lEmGZG d,ZHd-ZId.ZJd/ZKd0ZLd1ZMeHeI�Nd2� eJ�Nd2� d3 ZOd4d2�PeKeLeMg� ZQe$eOeQ�ZRe$eIeK�ZSe$eHeL�ZTe$eJeM�ZUe �Vd5�ZWd6ZXd7d8� ZYd9d:� ZZd;d<� Z[d=d>� Z\d?d@� Z]G dAdB� dBe�Z^dS )Cz8Implementation of IAM policy management command for GCS.� )�absolute_import)�print_function)�division)�unicode_literalsN)�zip)� protojson)� DecodeError)�config)�ArgumentException)�PreconditionException)�ServiceException)�Command)�GetFailureCount)�CommandArgument)� ApiSelector)�CommandException)�&IamChOnResourceWithConditionsException)�CreateHelpText)�LogCommandParams)�NameExpansionIterator)� SeekAheadNameExpansionIterator)�PluralityCheckableIterator)�GetSchemeFromUrlString)�IsKnownUrlScheme)�StorageUrlFromString)� UrlsAreMixOfBucketsAndObjects)�storage_v1_messages)� shim_util)�GetCloudApiInstance)�IAM_POLICY_VERSION)�NO_MAX)� iam_helper)�BindingStringToTuple)� BindingsTuple��DeserializeBindingsTuple)�IsEqualBindings)� PatchBindings��SerializeBindingsTuple)�Retry)�GcloudStorageMap)�GcloudStorageFlagz3 gsutil iam set [-afRr] [-e <etag>] file url ... z gsutil iam get url a� gsutil iam ch [-fRr] binding ... url where each binding is of the form: [-d] ("user"|"serviceAccount"|"domain"|"group"):id:role[,...] [-d] ("allUsers"|"allAuthenticatedUsers"):role[,...] -d ("user"|"serviceAccount"|"domain"|"group"):id -d ("allUsers"|"allAuthenticatedUsers") NOTE: The "iam ch" command does not support changing Cloud IAM policies with bindings that contain conditions. As such, "iam ch" cannot be used to add conditions to a policy or to change the policy of a resource that already contains conditions. See additional details below. NOTE: The "gsutil iam" command does not allow you to add convenience values (projectOwner, projectEditor, projectViewer), but you can remove existing ones. a <B>GET</B> The ``iam get`` command gets the Cloud IAM policy for a bucket or object, which you can save and edit for use with the ``iam set`` command. The following examples save the bucket or object's Cloud IAM policy to a text file: gsutil iam get gs://example > bucket_iam.txt gsutil iam get gs://example/important.txt > object_iam.txt The Cloud IAM policy returned by ``iam get`` includes an etag. The etag is used in the precondition check for ``iam set`` unless you override it using ``iam set -e``. a� <B>SET</B> The ``iam set`` command sets a Cloud IAM policy on one or more buckets or objects, replacing the existing policy on those buckets or objects. For an example of the correct formatting for a Cloud IAM policy, see the output of the ``iam get`` command. You can use the ``iam ch`` command to edit an existing policy, even in the presence of concurrent updates. You can also edit the policy concurrently using the ``-e`` flag to override the Cloud IAM policy's etag. Specifying ``-e`` with an empty string (i.e. ``gsutil iam set -e '' ...``) instructs gsutil to skip the precondition check when setting the Cloud IAM policy. When you set a Cloud IAM policy on a large number of objects, you should use the gsutil ``-m`` option for concurrent processing. The following command applies ``iam.txt`` to all objects in the ``dogs`` bucket: gsutil -m iam set -r iam.txt gs://dogs Note that only object-level operations are parallelized; setting a Cloud IAM policy on a large number of buckets with the ``-m`` flag does not improve performance. <B>SET OPTIONS</B> The ``set`` sub-command has the following options: -R, -r Performs ``iam set`` recursively on all objects under the specified bucket. This flag can only be set if the policy exclusively uses ``roles/storage.legacyObjectReader`` or ``roles/storage.legacyObjectOwner``. This flag cannot be used if the bucket is configured for uniform bucket-level access. -a Performs ``iam set`` on all object versions. -e <etag> Performs the precondition check on each object with the specified etag before setting the policy. You can retrieve the policy's etag using ``iam get``. -f The default gsutil error-handling mode is fail-fast. This flag changes the request to fail-silent mode. This option is implicitly set when you use the gsutil ``-m`` option. av <B>CH</B> The ``iam ch`` command incrementally updates Cloud IAM policies. You can specify multiple access grants or removals in a single command. The access changes are applied as a batch to each url in the order in which they appear in the command line arguments. Each access change specifies a principal and a role that is either granted or revoked. You can use gsutil ``-m`` to handle object-level operations in parallel. NOTE: The ``iam ch`` command cannot be used to change the Cloud IAM policy of a resource that contains conditions in its policy bindings. Attempts to do so result in an error. To change the Cloud IAM policy of such a resource, you can perform a read-modify-write operation by saving the policy to a file using ``iam get``, editing the file, and setting the updated policy using ``iam set``. <B>CH EXAMPLES</B> Examples for the ``ch`` sub-command: To grant a single role to a single principal for some targets: gsutil iam ch user:john.doe@example.com:objectCreator gs://ex-bucket To make a bucket's objects publicly readable: gsutil iam ch allUsers:objectViewer gs://ex-bucket To grant multiple bindings to a bucket: gsutil iam ch user:john.doe@example.com:objectCreator \ domain:www.my-domain.org:objectViewer gs://ex-bucket To specify more than one role for a particular principal: gsutil iam ch user:john.doe@example.com:objectCreator,objectViewer \ gs://ex-bucket To specify a custom role for a particular principal: gsutil iam ch user:john.doe@example.com:roles/customRoleName gs://ex-bucket To apply a grant and simultaneously remove a binding to a bucket: gsutil iam ch -d group:readers@example.com:legacyBucketReader \ group:viewers@example.com:objectViewer gs://ex-bucket To remove a user from all roles on a bucket: gsutil iam ch -d user:john.doe@example.com gs://ex-bucket <B>CH OPTIONS</B> The ``ch`` sub-command has the following options: -d Removes roles granted to the specified principal. -R, -r Performs ``iam ch`` recursively to all objects under the specified bucket. This flag can only be set if the policy exclusively uses ``roles/storage.legacyObjectReader`` or ``roles/storage.legacyObjectOwner``. This flag cannot be used if the bucket is configured for uniform bucket-level access. -f The default gsutil error-handling mode is fail-fast. This flag changes the request to fail-silent mode. This is implicitly set when you invoke the gsutil ``-m`` option. � z a7 Cloud Identity and Access Management (Cloud IAM) allows you to control who has access to the resources in your Google Cloud project. For more information, see `Cloud Identity and Access Management <https://cloud.google.com/storage/docs/access-control/iam>`_. The iam command has three sub-commands: z [a-z]+://.+z�To change the IAM policy of a resource that has bindings containing conditions, perform a read-modify-write operation using "iam get" and "iam set".c C s t | �r|std��d S )Nz/Cannot operate on a mix of buckets and objects.)r r )�urls�recursion_requested� r0 �!/opt/gsutil/gslib/commands/iam.py�*_RaiseErrorIfUrlsAreMixOfBucketsAndObjects� s r2 c C s$ |\}}| j |jdd� |D �|d�S )Nc S s g | ] }t |��qS r0 r$ ��.0�tr0 r0 r1 � <listcomp> � z$_PatchIamWrapper.<locals>.<listcomp>�� thread_state)�PatchIamHelper�expanded_storage_url)�cls� iter_resultr9 Zserialized_bindings_tuples�expansion_resultr0 r0 r1 �_PatchIamWrapper s �r? c C s$ |\}}| j |jt�tj|�|d�S )Nr8 )� SetIamHelperr; r �decode_message�apitools_messages�Policy)r< r= r9 Zserialized_policyr> r0 r0 r1 �_SetIamWrapper s �rD c C s | j �t|�� d S �N��logger�error�str�r< �er0 r0 r1 �_SetIamExceptionHandler s rL c C s | j �t|�� d S rE rF rJ r0 r0 r1 �_PatchIamExceptionHandler s rM c s e Zd ZdZejdededdddej gej e � d�ge � d�e � � ge �� e � � gd�d � Zejdg d d eeeed�d �Zd d� Z� fdd�Zdd� Zdd� Zd/dd�Z� fdd�Zd0dd�Z d1d d �Z d2d d �Z d3d d!�Z d4d"d#�Z e!e"d$d%d&�d5d'd(��Z#d)d*� Z$d+d,� Z%d-d.� Z&� Z'S )6� IamCommandz%Implementation of gsutil iam command.�iam� zafRrd:e:TF� )�get�set�ch) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz9Get, set, or change bucket and/or object IAM permissions.)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s | j s|d �� rdS dS d S )Nr �objects�buckets)r/ �IsObject)�self� url_patternsr0 r0 r1 �_get_resource_typeD s z IamCommand._get_resource_typec s� | j �d�| _| jdkrLt| j d ��� r0d}nd}td|ddgi d�}�n.| jd k�r@tg d �td �td �td �td�td�td�d�d�}| �� | j dd � }tt t|��}d}t | j �D ]4\}\}}|dv r�d}q�|dkr�|dkr�d| j |< q�t ||� |�s|d �� �r d|j d< n d|j d< || j d d� | _ n:| jdk�rzt�t�ddd��tjju �rv| j�d � g S t� �|�S ) Nr rR rf rg �storage�get-iam-policy� --format=json)�gcloud_command�flag_maprS )rl N�set-iam-policyrn z--all-versionsz--etagz--etag=z--continue-on-errorz --recursive)�-a�-e� _empty_etag�-f�-R�-rrQ F�rw rv Trs � )rt ry rT �GSUtil�hidden_shim_mode�nonez|The shim maps iam ch commands to several gcloud storage commands, which cannot be determined without running gcloud storage.)�args�pop� sub_commandr rh r+ r, � ParseSubOpts�list�map� enumerate�sub_optsr2 ro r �HIDDEN_SHIM_MODEr rR �DRY_RUNrG �warning�super�get_gcloud_storage_args) ri � command_group�gcloud_storage_map� url_strings� url_objects�recurse�i�flag_key� flag_value�� __class__r0 r1 r� J s` �� �� ����z"IamCommand.get_gcloud_storage_argsc C s: |� � r6d|j }t|j��td��r.|d7 }t|��d S )NzInvalid Cloud URL "%s".z-Rrfzz This resource handle looks like a flag, which must appear before all bindings. See "gsutil help iam ch" for more details.)� IsFileUrl� object_namerS �issubsetr )ri �url� error_msgr0 r0 r1 �_RaiseIfInvalidUrl� s �z IamCommand._RaiseIfInvalidUrlc C s* d| _ d| _g }| jr^| jD ]@\}}|dv r4d| _q |dkrDd| _ q |dkr |�td|�� q g }t| j�}|D ]v}t�|�r�t t |��r�|�|� q�|dkr�z|�tdt |��� W q� t y� t d��Y q�0 qp|�td|�� qp|s�t d��|D ]}|�|� q�ttt|��}t|| j� ||fS )NFrx Tru z-dz?A -d flag is missing an argument specifying bindings to remove.z"Must specify at least one binding.)�continue_on_errorr/ r� �appendr" �iterr} �STORAGE_URI_REGEX�matchr r �next� StopIterationr r� r� r r2 )ri �patch_bindings_tuples�o�aZurl_pattern_strings�it�tokenr� r0 r0 r1 �_GetSettingsAndDiffs� sH � � �z IamCommand._GetSettingsAndDiffsNc C sZ | � |�\}}tj|| �� |tjtjdd�}|jdkrV| jrL| j�|j � n t |j ��|S )NT)�env�input�stderr�stdout�textr ) �._get_full_gcloud_storage_execution_information� subprocess�run�'_get_shim_command_environment_variables�PIPE� returncoder� rG rH r� r )ri r} �stdin�_�command�processr0 r0 r1 �_run_ch_subprocess� s � z IamCommand._run_ch_subprocessc sj | j dkrt� �� S | �� | �� \}}| �|�}g }| jrF|�d� d}|D �]}| �|� |dkr�| � g d�| t |�g �}|j r�d}qNt � |j�}dd� |D �} n t |�g} | D ]�} | � d |d | d g�} | j r�d}q�t � | j�} t�| d �} |D ] \}}t�|�}t| ||�} q�d d� tt�| ��D �| d < | j d |d| dgt j| dd�d�}|j r�d}q�qN|S )NrT rw r rf )rl �lsz--jsonrQ c S s g | ]}|d dkr|d �qS )�type� cloud_objectr� r0 )r4 �resourcer0 r0 r1 r6 � s �z1IamCommand.run_gcloud_storage.<locals>.<listcomp>rl rm rn �bindingsc S s" g | ]\}}|t t|��d ��qS �)�role�members)�sortedr� �r4 �r�mr0 r0 r1 r6 � s � �rq �-T)� sort_keys)r� )r r� �run_gcloud_storager� r� rk r/ r� r� r� rI r� �json�loadsr� r! ZBindingsDictToUpdateDictr' r� �six� iteritems�dumps)ri �bindings_tuples�patternsZ resource_typeZ list_settings� return_codeZ url_patternZ ls_processZ ls_outputr. r� Z get_process�policyr� �is_grant�diff� diff_dictZ set_processr� r0 r1 r� � s^ �� � � � �z IamCommand.run_gcloud_storagec C sR t | |d�}|�� r.|j|j|jddgd�}n |j|j|j|j|jddgd�}|S )a� Gets an IAM policy for a single, resolved bucket / object URL. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the global -m flag is specified. Will use self.gsutil_api if thread_state is set to None. Returns: Policy instance. r8 r� �etag)�provider�fields)� generationr� r� )r �IsBucket�GetBucketIamPolicy� bucket_name�scheme�GetObjectIamPolicyr� r� )ri � storage_urlr9 � gsutil_apir� r0 r0 r1 � GetIamHelper s ��zIamCommand.GetIamHelperc C s� | j d }t| �|�jdgd��}|�� r6td| ��|�� rPtd|| jf ��tt |�d j �}| j ||d�}t � t�|��}t j|ddd d �}t|� d S ) z,Gets IAM policy for single bucket or object.r �name)�bucket_listing_fieldsz%s matched no URLszD%s matched more than one URL, which is not allowed by the %s commandr8 T)�,z: rP )r� � separators�indentN)r} r �WildcardIterator�IterAll�IsEmptyr � HasPlurality� command_namer r� � url_stringr� r� r� r �encode_messager� �print)ri r9 �pattern�matchesr� r� � policy_jsonZ policy_strr0 r0 r1 �_GetIam) s, � ���zIamCommand._GetIamc C sJ t | |d�}|�� r*|j|j||jd� n |j|j|j||j|jd� dS )aP Sets IAM policy for a single, resolved bucket / object URL. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. policy: A Policy object to set on the bucket / object. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the -m flag is specified. Will use self.gsutil_api if thread_state is set to None. Raises: ServiceException passed from the API call if an HTTP error was returned. r8 )r� )r� r� N)r r� �SetBucketIamPolicyr� r� �SetObjectIamPolicyr� r� )ri r� r� r9 r� r0 r0 r1 �_SetIamHelperInternalB s ��z IamCommand._SetIamHelperInternalc C s< z| j |||d� W n" ty6 | jr0d| _n� Y n0 dS )zDHandles the potential exception raised by the internal set function.r8 FN)r� r r� �everything_set_okay)ri r� r� r9 r0 r0 r1 r@ c s zIamCommand.SetIamHelperc C s� z| j |||d� W np ty6 | jr0d| _n� Y nP ty� } z8| jrfd| _d| _| j�|j� n t |j��W Y d}~n d}~0 0 dS )a� Patches an IAM policy for a single, resolved bucket / object URL. The patch is applied by altering the policy from an IAM get request, and setting the new IAM with the specified etag. Because concurrent IAM set requests may alter the etag, we may need to retry this operation several times before success. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. bindings_tuples: A list of BindingsTuple instances. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the -m flag is specified. Will use self.gsutil_api if thread_state is set to None. r8 FTN) �_PatchIamHelperInternalr r� r� r �$tried_ch_on_resource_with_conditionsrG �debug�messager )ri r� r� r9 rK r0 r0 r1 r: m s � zIamCommand.PatchIamHelper� g �?)�tries� timeout_secsc C s� | j ||d�}|j|j }}|D ]:}|jr d| }|d7 }|d�t�dt ��7 }t|��q t |�} |D ]<\} } t � |�} t � | �} t | | | �}dd� t �|�D �}qht|| �r�| j�d|� d S tj||d�}| j|||d� d S ) Nr8 z"Could not patch IAM policy for %s.r- zfThe resource had conditions present in its IAM policy bindings, which is not supported by "iam ch". %sc S s$ g | ] \}}t jj|t|�d ��qS r� )rB rC �BindingsValueListEntryr� r� r0 r0 r1 r6 � s ��z6IamCommand._PatchIamHelperInternal.<locals>.<listcomp>zNo changes made to %s)r� r� )r� r� r� � condition�join�textwrap�wrap� IAM_CH_CONDITIONS_WORKAROUND_MSGr r� r! ZBindingsMessageToUpdateDictr' r� r� r& rG �inforB rC r� )ri r� r� r9 r� r� r� �bindingr� Z orig_bindingsr� r� Z bindings_dictr� Znew_bindings_dictr0 r0 r1 r� � s6 ��� � z"IamCommand._PatchIamHelperInternalc C s� | � � \}}g }|D ]J\}}g }|D ]&}tjj|d |d d�}|�|� q$|�t||d�� qd| _d| _g } |D ]b} z@| �� r�| j r�d| _ | �| j � q�| � | |� n | �| j � W qt t y� | �| � Y qt0 qt| �rxt| j| j| j| j| | j | j| j�p| jdgd � } t| j| j| �� | | j | jd �} t�d d � |D ��} | jt t | | �t | j | d � | jt � dk M _| j�s�d}| j�r�|d7 }|d� t!�"dt# ��7 }t$|��d S )Nr� r� )r� r� )r� r� TF�*r� �� all_versionsr� r� �r c S s g | ] }t |��qS r0 r( r3 r0 r0 r1 r6 � r7 z(IamCommand._PatchIam.<locals>.<listcomp>�� fail_on_error�seek_ahead_iteratorr z'Some IAM policies could not be patched.r- zjSome resources had conditions present in their IAM policy bindings, which is not supported by "iam ch". %s)%r� rB rC r� r� r# r� r� r� r/ �objectr� r: �AttributeErrorr� r r� r� rG r� r r� �parallel_operationsr �GetSeekAheadGsutilApi� itertools�repeat�Applyr? r rM r r� r� r� r� r )ri Zraw_bindings_tuplesrj r� r� r� Zbindings_messagesr Zbindings_message�threaded_wildcards�surl�name_expansion_iteratorr Z serialized_bindings_tuples_it�msgr0 r0 r1 � _PatchIam� s� � � � � �����zIamCommand._PatchIamc C s� d| _ d| _d| _d}d}| jr~| jD ]V\}}|dv r>d| _q&|dkrNd| _ q&|dkr^d| _q&|dkrtt|�}d}q&| �� q&| jd }| jd d � }z<t|d �� }t� |� � �}W d � n1 s�0 Y W n\ t y� t d | ��Y n@ t �y. } z&| j�d | � t d| ��W Y d } ~ n d } ~ 0 0 |�dg �} |�sN|�dd�}t�| |td��} zt�tj| �}W n$ t�y� t d||f ��Y n0 d| _g } ttt|��} t | | j� | D ]P}t |j � |� � �r| j�r�d|_ | �!|j � n | �"||� n | �!|j � �q�| �r�t#| j$| j| j| j%| | j| j| j �p@| j&dgd� }t'| j$| j| �(� | | j| jd�}t)�*t�+|��}| j,t-t.||�t/| j |d� | jt0� dk M _| j�s�t1d��d S )z7Set IAM policy for given wildcards on the command line.Fry rx Tru rr rs r rQ Nr� z.Specified IAM policy file "%s" does not exist.z'Invalid IAM policy file, ValueError: %sz Invalid IAM policy file "%s".r� r� )r� r� �versionz*Invalid IAM policy file "%s" or etag "%s".r r� r r r z#Some IAM policies could not be set.)2r� r/ r r� rI � RaiseInvalidArgumentExceptionr} �openr� r� �read�IOErrorr � ValueErrorrG r� rR r� r r rA rB rC r r� r� r� r r2 r� r� r� r� r� r@ r r� r� r r r r r r� r rD r rL r r )ri Z force_etagr� r� �argZfile_urlr� �fpr� rK r� r� r Zsurlsr r r Z policy_itr0 r0 r1 �_SetIam s� 0 � " �� � ��zIamCommand._SetIamc C s� | j �d�}| jdd� t| jd� d| _|dkrHt|gd� | �� nN|dkrft|gd� | �� n0|d kr�t|gd� | �� nt d || j f ��dS ) z(Command entry point for the acl command.r T)� check_args)r� FrR )� subcommandsrS rT zBInvalid subcommand "%s" for the %s command. See "gsutil help iam".) r} r~ r� r r� �def_aclr� r r r r� )ri �action_subcommandr0 r0 r1 � RunCommandv s" �zIamCommand.RunCommand)N)N)N)N)N)N)N)(�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �JSONr �MakeNCloudURLsArgument�MakeNFileURLsArgument� MakeZeroOrMoreCloudURLsArgument� MakeOneOrMoreBindingsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� _ch_help_text� help_specrk r� r� r� r� r� r� r� r� r@ r: r* r r� r r r" � __classcell__r0 r0 r� r1 rN sb ������ 8 1 : ! ! " � /InrN )_r& � __future__r r r r r r� �os�rer� r� r� � six.movesr �apitools.base.protorpcliter Z#apitools.base.protorpclite.messagesr �botor �gslib.cloud_apir r r � gslib.commandr r �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr Z gslib.metricsr �gslib.name_expansionr r �"gslib.plurality_checkable_iteratorr �gslib.storage_urlr r r r �"gslib.third_party.storage_apitoolsr rB Z gslib.utilsr � gslib.utils.cloud_api_helperr �gslib.utils.constantsr r r! Zgslib.utils.iam_helperr" r# r% r& r' r) �gslib.utils.retry_utilr* �gslib.utils.shim_utilr+ r, � _SET_SYNOPSIS� _GET_SYNOPSIS� _CH_SYNOPSIS�_GET_DESCRIPTION�_SET_DESCRIPTION�_CH_DESCRIPTION�lstripr( r� � _DESCRIPTIONr0 r1 r2 r3 �compiler� r� r2 r? rD rL rM rN r0 r0 r0 r1 �<module> s� +E ��� � commands/__pycache__/cp.cpython-39.pyc 0000644 00000124322 15025013616 0013613 0 ustar 00 a +(Wg�� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlm Z ddlm Z ddlm Z ddlm Z ddl m!Z! ddl m"Z" ddl m#Z# ddl$m%Z& ddl'm(Z( ddl'm)Z) ddl'm*Z* ddl+m,Z, ddl-m.Z. dd l-m/Z/ dd l0m1Z1 dd l0m2Z2 dd l0m3Z3 dd l0m4Z4 dd!l0m5Z5 dd"l0m6Z6 dd#l7m8Z8 dd$l7m9Z9 dd%l7m:Z: dd&l7m;Z; dd'l7m<Z< dd(l7m=Z= dd)l>m?Z? dd*l>m@Z@ dd+lAmBZB dd,lAmCZC dd-lAmDZD dd.lEmFZF dd/lEmGZG dd0lHmIZI dd1lHmJZJ d2ZKd3eK ZLd4ZMd5ZNd6ZOd7ZPd8ZQd9ZRd:ZSd;ZTd<ZUd=ZVd>ZWd?ZXd@ZYdA�ZeLeMeNeOePeQeReSeTeUeVeWeXeYg�Z[dBZ\e?dC�e?dD�e?dE�e?dF�e?dG�e?dH�e?dI�e?dJ�e?dK�e?dL�e?dM�e?dN�e?dO�e?dP�e?dQ�e?dR�dS�Z]dTdU� e^e]�_� �dVe?dV�fdWe?dV�fg D �Z`dXdY� ZadbdZd[�Zbd\d]� Zcd^d_� ZdG d`da� dae�ZedS )czCImplementation of Unix-like cp command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)� gcs_json_api)�Command)�CommandArgument)� ApiSelector)�CommandException)�LogPerformanceSummaryParams)�CopyObjectsIterator)�DestinationInfo)�NameExpansionIterator)�%NameExpansionIteratorDestinationTuple)� SeekAheadNameExpansionIterator)�ContainsWildcard)�IsCloudSubdirPlaceholder)�StorageUrlFromString)�storage_v1_messages)� cat_helper)� copy_helper)�parallelism_framework_util)�GetCloudApiInstance)�DEBUGLEVEL_DUMP_REQUESTS)�NO_MAX)�CreateCopyHelperOpts)� GetSourceFieldsNeededForCopy)�GZIP_ALL_FILES)�ItemExistsError)�Manifest)�SkipUnsupportedObjectError)�ConvertModeToBase8)�+DeserializeFileAttributesFromObjectMetadata)�InitializePreservePosixData)�POSIXAttributes)�'SerializeFileAttributesToObjectMetadata)� ValidateFilePermissionAccess)�GcloudStorageFlag)�GcloudStorageMap)�GetStreamFromFileUrl)� StdinIterator)�StdinIteratorCls)�NormalizeStorageClass)�RemoveCRLFFromString)�CalculateThroughput)�MakeHumanReadablezw gsutil cp [OPTION]... src_url dst_url gsutil cp [OPTION]... src_url... dst_url gsutil cp [OPTION]... -I dst_url z <B>SYNOPSIS</B> an <B>DESCRIPTION</B> The ``gsutil cp`` command allows you to copy data between your local file system and the cloud, within the cloud, and between cloud storage providers. For example, to upload all text files from the local directory to a bucket, you can run: gsutil cp *.txt gs://my-bucket You can also download data from a bucket. The following command downloads all text files from the top-level of a bucket to your current directory: gsutil cp gs://my-bucket/*.txt . You can use the ``-n`` option to prevent overwriting the content of existing files. The following example downloads text files from a bucket without clobbering the data in your directory: gsutil cp -n gs://my-bucket/*.txt . Use the ``-r`` option to copy an entire directory tree. For example, to upload the directory tree ``dir``: gsutil cp -r dir gs://my-bucket If you have a large number of files to transfer, you can perform a parallel multi-threaded/multi-processing copy using the top-level gsutil ``-m`` option (see "gsutil help options"): gsutil -m cp -r dir gs://my-bucket You can use the ``-I`` option with ``stdin`` to specify a list of URLs to copy, one per line. This allows you to use gsutil in a pipeline to upload or download objects as generated by a program: cat filelist | gsutil -m cp -I gs://my-bucket or: cat filelist | gsutil -m cp -I ./download_dir where the output of ``cat filelist`` is a list of files, cloud URLs, and wildcards of files and cloud URLs. NOTE: Shells like ``bash`` and ``zsh`` sometimes attempt to expand wildcards in ways that can be surprising. You may also encounter issues when attempting to copy files whose names contain wildcard characters. For more details about these issues, see `Wildcard behavior considerations <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. ah <B>HOW NAMES ARE CONSTRUCTED</B> The ``gsutil cp`` command attempts to name objects in ways that are consistent with the Linux ``cp`` command. This means that names are constructed depending on whether you're performing a recursive directory copy or copying individually-named objects, or whether you're copying to an existing or non-existent directory. When you perform recursive directory copies, object names are constructed to mirror the source directory structure starting at the point of recursive processing. For example, if ``dir1/dir2`` contains the file ``a/b/c``, then the following command creates the object ``gs://my-bucket/dir2/a/b/c``: gsutil cp -r dir1/dir2 gs://my-bucket In contrast, copying individually-named files results in objects named by the final path component of the source files. For example, assuming again that ``dir1/dir2`` contains ``a/b/c``, the following command creates the object ``gs://my-bucket/c``: gsutil cp dir1/dir2/** gs://my-bucket Note that in the above example, the '**' wildcard matches all names anywhere under ``dir``. The wildcard '*' matches names just one level deep. For more details, see `URI wildcards <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. The same rules apply for uploads and downloads: recursive copies of buckets and bucket subdirectories produce a mirrored filename structure, while copying individually or wildcard-named objects produce flatly-named files. In addition, the resulting names depend on whether the destination subdirectory exists. For example, if ``gs://my-bucket/subdir`` exists as a subdirectory, the following command creates the object ``gs://my-bucket/subdir/dir2/a/b/c``: gsutil cp -r dir1/dir2 gs://my-bucket/subdir In contrast, if ``gs://my-bucket/subdir`` does not exist, this same ``gsutil cp`` command creates the object ``gs://my-bucket/subdir/a/b/c``. NOTE: The `Google Cloud Platform Console <https://console.cloud.google.com>`_ creates folders by creating "placeholder" objects that end with a "/" character. gsutil skips these objects when downloading from the cloud to the local file system, because creating a file that ends with a "/" is not allowed on Linux and macOS. We recommend that you only create objects that end with "/" if you don't intend to download such objects using gsutil. a� <B>COPYING TO/FROM SUBDIRECTORIES; DISTRIBUTING TRANSFERS ACROSS MACHINES</B> You can use gsutil to copy to and from subdirectories by using a command like this: gsutil cp -r dir gs://my-bucket/data This causes ``dir`` and all of its files and nested subdirectories to be copied under the specified destination, resulting in objects with names like ``gs://my-bucket/data/dir/a/b/c``. Similarly, you can download from bucket subdirectories using the following command: gsutil cp -r gs://my-bucket/data dir This causes everything nested under ``gs://my-bucket/data`` to be downloaded into ``dir``, resulting in files with names like ``dir/data/a/b/c``. Copying subdirectories is useful if you want to add data to an existing bucket directory structure over time. It's also useful if you want to parallelize uploads and downloads across multiple machines (potentially reducing overall transfer time compared with running ``gsutil -m cp`` on one machine). For example, if your bucket contains this structure: gs://my-bucket/data/result_set_01/ gs://my-bucket/data/result_set_02/ ... gs://my-bucket/data/result_set_99/ you can perform concurrent downloads across 3 machines by running these commands on each machine, respectively: gsutil -m cp -r gs://my-bucket/data/result_set_[0-3]* dir gsutil -m cp -r gs://my-bucket/data/result_set_[4-6]* dir gsutil -m cp -r gs://my-bucket/data/result_set_[7-9]* dir Note that ``dir`` could be a local directory on each machine, or a directory mounted off of a shared file server. The performance of the latter depends on several factors, so we recommend experimenting to find out what works best for your computing environment. a� <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> If both the source and destination URL are cloud URLs from the same provider, gsutil copies data "in the cloud" (without downloading to and uploading from the machine where you run gsutil). In addition to the performance and cost advantages of doing this, copying in the cloud preserves metadata such as ``Content-Type`` and ``Cache-Control``. In contrast, when you download data from the cloud, it ends up in a file with no associated metadata, unless you have some way to keep or re-create that metadata. Copies spanning locations and/or storage classes cause data to be rewritten in the cloud, which may take some time (but is still faster than downloading and re-uploading). Such operations can be resumed with the same command if they are interrupted, so long as the command parameters are identical. Note that by default, the gsutil ``cp`` command does not copy the object ACL to the new object, and instead uses the default bucket ACL (see "gsutil help defacl"). You can override this behavior with the ``-p`` option. When copying in the cloud, if the destination bucket has Object Versioning enabled, by default ``gsutil cp`` copies only live versions of the source object. For example, the following command causes only the single live version of ``gs://bucket1/obj`` to be copied to ``gs://bucket2``, even if there are noncurrent versions of ``gs://bucket1/obj``: gsutil cp gs://bucket1/obj gs://bucket2 To also copy noncurrent versions, use the ``-A`` flag: gsutil cp -A gs://bucket1/obj gs://bucket2 The top-level gsutil ``-m`` flag is not allowed when using the ``cp -A`` flag. z� <B>CHECKSUM VALIDATION</B> gsutil automatically performs checksum validation for copies to and from Cloud Storage. For more information, see `Hashes and ETags <https://cloud.google.com/storage/docs/hashes-etags#cli>`_. a <B>RETRY HANDLING</B> The ``cp`` command retries when failures occur, but if enough failures happen during a particular copy or delete operation, or if a failure isn't retryable, the ``cp`` command skips that object and moves on. If any failures were not successfully retried by the end of the copy run, the ``cp`` command reports the number of failures and exits with a non-zero status. For details about gsutil's overall retry handling, see `Retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_. a� <B>RESUMABLE TRANSFERS</B> gsutil automatically resumes interrupted downloads and interrupted `resumable uploads <https://cloud.google.com/storage/docs/resumable-uploads#gsutil>`_, except when performing streaming transfers. In the case of an interrupted download, a partially downloaded temporary file is visible in the destination directory with the suffix ``_.gstmp`` in its name. Upon completion, the original file is deleted and replaced with the downloaded contents. Resumable transfers store state information in files under ~/.gsutil, named by the destination object or file. See "gsutil help prod" for details on using resumable transfers in production. a� <B>STREAMING TRANSFERS</B> Use '-' in place of src_url or dst_url to perform a `streaming transfer <https://cloud.google.com/storage/docs/streaming>`_. Streaming uploads using the `JSON API <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ are buffered in memory part-way back into the file and can thus sometimes resume in the event of network or service problems. gsutil does not support resuming streaming uploads using the XML API or resuming streaming downloads for either JSON or XML. If you have a large amount of data to transfer in these cases, we recommend that you write the data to a local file and copy that file rather than streaming it. a< <B>SLICED OBJECT DOWNLOADS</B> gsutil can automatically use ranged ``GET`` requests to perform downloads in parallel for large files being downloaded from Cloud Storage. See `sliced object download documentation <https://cloud.google.com/storage/docs/sliced-object-downloads>`_ for a complete discussion. a� <B>PARALLEL COMPOSITE UPLOADS</B> gsutil can automatically use `object composition <https://cloud.google.com/storage/docs/composite-objects>`_ to perform uploads in parallel for large, local files being uploaded to Cloud Storage. See the `parallel composite uploads documentation <https://cloud.google.com/storage/docs/parallel-composite-uploads>`_ for a complete discussion. am <B>CHANGING TEMP DIRECTORIES</B> gsutil writes data to a temporary directory in several cases: - when compressing data to be uploaded (see the ``-z`` and ``-Z`` options) - when decompressing data being downloaded (for example, when the data has ``Content-Encoding:gzip`` as a result of being uploaded using gsutil cp -z or gsutil cp -Z) - when running integration tests using the gsutil test command In these cases, it's possible the temporary file location on your system that gsutil selects by default may not have enough space. If gsutil runs out of space during one of these operations (for example, raising "CommandException: Inadequate temp space available to compress <your file>" during a ``gsutil cp -z`` operation), you can change where it writes these temp files by setting the TMPDIR environment variable. On Linux and macOS, you can set the variable as follows: TMPDIR=/some/directory gsutil cp ... You can also add this line to your ~/.bashrc file and restart the shell before running gsutil: export TMPDIR=/some/directory On Windows 7, you can change the TMPDIR environment variable from Start -> Computer -> System -> Advanced System Settings -> Environment Variables. You need to reboot after making this change for it to take effect. Rebooting is not necessary after running the export command on Linux and macOS. a <B>SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SUCH AS SYMLINKS AND DEVICES)</B> Please see the section about OS-specific file types in "gsutil help rsync". While that section refers to the ``rsync`` command, analogous points apply to the ``cp`` command. aT2 <B>OPTIONS</B> -a predef_acl Applies the specific predefined ACL to uploaded objects. See "gsutil help acls" for further details. -A Copy all source versions from a source bucket or folder. If not set, only the live version of each source object is copied. NOTE: This option is only useful when the destination bucket has Object Versioning enabled. Additionally, the generation numbers of copied versions do not necessarily match the order of the original generation numbers. -c If an error occurs, continue attempting to copy the remaining files. If any copies are unsuccessful, gsutil's exit status is non-zero, even if this flag is set. This option is implicitly set when running ``gsutil -m cp...``. NOTE: ``-c`` only applies to the actual copying operation. If an error, such as ``invalid Unicode file name``, occurs while iterating over the files in the local directory, gsutil prints an error message and aborts. -D Copy in "daisy chain" mode, which means copying between two buckets by first downloading to the machine where gsutil is run, then uploading to the destination bucket. The default mode is a "copy in the cloud," where data is copied between two buckets without uploading or downloading. During a "copy in the cloud," a source composite object remains composite at its destination. However, you can use "daisy chain" mode to change a composite object into a non-composite object. For example: gsutil cp -D gs://bucket/obj gs://bucket/obj_tmp gsutil mv gs://bucket/obj_tmp gs://bucket/obj NOTE: "Daisy chain" mode is automatically used when copying between providers: for example, when copying data from Cloud Storage to another provider. -e Exclude symlinks. When specified, symbolic links are not copied. -I Use ``stdin`` to specify a list of files or objects to copy. You can use gsutil in a pipeline to upload or download objects as generated by a program. For example: cat filelist | gsutil -m cp -I gs://my-bucket where the output of ``cat filelist`` is a one-per-line list of files, cloud URLs, and wildcards of files and cloud URLs. -j <ext,...> Applies gzip transport encoding to any file upload whose extension matches the ``-j`` extension list. This is useful when uploading files with compressible content such as .js, .css, or .html files. This also saves network bandwidth while leaving the data uncompressed in Cloud Storage. When you specify the ``-j`` option, files being uploaded are compressed in-memory and on-the-wire only. Both the local files and Cloud Storage objects remain uncompressed. The uploaded objects retain the ``Content-Type`` and name of the original files. Note that if you want to use the ``-m`` `top-level option <https://cloud.google.com/storage/docs/gsutil/addlhelp/GlobalCommandLineOptions>`_ to parallelize copies along with the ``-j/-J`` options, your performance may be bottlenecked by the "max_upload_compression_buffer_size" boto config option, which is set to 2 GiB by default. You can change this compression buffer size to a higher limit. For example: gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \ -m cp -j html,txt -r /local/source/dir gs://bucket/path -J Applies gzip transport encoding to file uploads. This option works like the ``-j`` option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If some of the source files don't compress well, such as binary data, using this option may result in longer uploads. -L <file> Outputs a manifest log file with detailed information about each item that was copied. This manifest contains the following information for each item: - Source path. - Destination path. - Source size. - Bytes transferred. - MD5 hash. - Transfer start time and date in UTC and ISO 8601 format. - Transfer completion time and date in UTC and ISO 8601 format. - Upload id, if a resumable upload was performed. - Final result of the attempted transfer, either success or failure. - Failure details, if any. If the log file already exists, gsutil uses the file as an input to the copy process, and appends log items to the existing file. Objects that are marked in the existing log file as having been successfully copied or skipped are ignored. Objects without entries are copied and ones previously marked as unsuccessful are retried. This option can be used in conjunction with the ``-c`` option to build a script that copies a large number of objects reliably, using a bash script like the following: until gsutil cp -c -L cp.log -r ./dir gs://bucket; do sleep 1 done The -c option enables copying to continue after failures occur, and the -L option allows gsutil to pick up where it left off without duplicating work. The loop continues running as long as gsutil exits with a non-zero status. A non-zero status indicates there was at least one failure during the copy operation. NOTE: If you are synchronizing the contents of a directory and a bucket, or the contents of two buckets, see "gsutil help rsync". -n No-clobber. When specified, existing files or objects at the destination are not replaced. Any items that are skipped by this option are reported as skipped. gsutil performs an additional GET request to check if an item exists before attempting to upload the data. This saves gsutil from retransmitting data, but the additional HTTP requests may make small object transfers slower and more expensive. -p Preserves ACLs when copying in the cloud. Note that this option has performance and cost implications only when using the XML API, as the XML API requires separate HTTP calls for interacting with ACLs. You can mitigate this performance issue using ``gsutil -m cp`` to perform parallel copying. Note that this option only works if you have OWNER access to all objects that are copied. If you want all objects in the destination bucket to end up with the same ACL, you can avoid these performance issues by setting a default object ACL on that bucket instead of using ``cp -p``. See "gsutil help defacl". Note that it's not valid to specify both the ``-a`` and ``-p`` options together. -P Enables POSIX attributes to be preserved when objects are copied. ``gsutil cp`` copies fields provided by ``stat``. These fields are the user ID of the owner, the group ID of the owning group, the mode or permissions of the file, and the access and modification time of the file. For downloads, these attributes are only set if the source objects were uploaded with this flag enabled. On Windows, this flag only sets and restores access time and modification time. This is because Windows doesn't support POSIX uid/gid/mode. -R, -r The ``-R`` and ``-r`` options are synonymous. They enable directories, buckets, and bucket subdirectories to be copied recursively. If you don't use this option for an upload, gsutil copies objects it finds and skips directories. Similarly, if you don't specify this option for a download, gsutil copies objects at the current bucket directory level and skips subdirectories. -s <class> Specifies the storage class of the destination object. If not specified, the default storage class of the destination bucket is used. This option is not valid for copying to non-cloud destinations. -U Skips objects with unsupported object types instead of failing. Unsupported object types include Amazon S3 objects in the GLACIER storage class. -v Prints the version-specific URL for each uploaded object. You can use these URLs to safely make concurrent upload requests, because Cloud Storage refuses to perform an update if the current object version doesn't match the version-specific URL. See `generation numbers <https://cloud.google.com/storage/docs/metadata#generation-number>`_ for more details. -z <ext,...> Applies gzip content-encoding to any file upload whose extension matches the ``-z`` extension list. This is useful when uploading files with compressible content such as .js, .css, or .html files, because it reduces network bandwidth and storage sizes. This can both improve performance and reduce costs. When you specify the ``-z`` option, the data from your files is compressed before it is uploaded, but your actual files are left uncompressed on the local disk. The uploaded objects retain the ``Content-Type`` and name of the original files, but have their ``Content-Encoding`` metadata set to ``gzip`` to indicate that the object data stored are compressed on the Cloud Storage servers and have their ``Cache-Control`` metadata set to ``no-transform``. For example, the following command: gsutil cp -z html \ cattypes.html tabby.jpeg gs://mycats does the following: - The ``cp`` command uploads the files ``cattypes.html`` and ``tabby.jpeg`` to the bucket ``gs://mycats``. - Based on the file extensions, gsutil sets the ``Content-Type`` of ``cattypes.html`` to ``text/html`` and ``tabby.jpeg`` to ``image/jpeg``. - The ``-z`` option compresses the data in the file ``cattypes.html``. - The ``-z`` option also sets the ``Content-Encoding`` for ``cattypes.html`` to ``gzip`` and the ``Cache-Control`` for ``cattypes.html`` to ``no-transform``. Because the ``-z/-Z`` options compress data prior to upload, they are not subject to the same compression buffer bottleneck that can affect the ``-j/-J`` options. Note that if you download an object with ``Content-Encoding:gzip``, gsutil decompresses the content before writing the local file. -Z Applies gzip content-encoding to file uploads. This option works like the ``-z`` option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If some of the source files don't compress well, such as binary data, using this option may result in files taking up more space in the cloud than they would if left uncompressed. --stet If the STET binary can be found in boto or PATH, cp will use the split-trust encryption tool for end-to-end encryption. z za:AcDeIL:MNnpPrRs:tUvz:Zj:Jz--all-versionsz--predefined-aclz--continue-on-errorz --daisy-chainz--ignore-symlinksz--read-paths-from-stdinz--gzip-in-flight-allz--gzip-in-flightz--manifest-pathz --no-clobberz--preserve-posixz--preserve-aclz--storage-classz--print-created-messagez--gzip-local-allz --gzip-local)�-A�-a�-c�-D�-e�-I�-J�-j�-L�-n�-P�-p�-s�-v�-Z�-zc C s i | ]\}}||�qS � rA )�.0�k�vrA rA � /opt/gsutil/gslib/commands/cp.py� <dictcomp>� s rF �-r�-Rc C s` d}t | �D ] \}\}}|dkr |} q*q |dur\| | d }| | d tj�||�f| |< dS )aZ Gcloud uses camel-case predefined/canned ACLs, and gsutil uses snake-case. The camel-case-snake-case difference is related to gcloud primarily using JSON API rather than the XML API. Predefined ACLs are also called "canned ACLs". Args: sub_opts: List of pairs representing flag keys and values, e.g. [('a', 'public-read')] Nr2 � r )� enumerater �+FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION�get)�sub_optsZpredefined_acl_idx�irC �_Zold_predefined_aclrA rA rE �'ShimTranslatePredefinedAclSubOptForCopy� s ��rP c C s | j ||| jd� d S )N)� thread_state�preserve_posix)�CopyFunc�preserve_posix_attrs)�cls�argsrQ rA rA rE �_CopyFuncWrapper� s �rW c C s4 | j �t|�� | jd7 _| j �dt�� � dS )�9Simple exception handler to allow post-completion status.rI z* Encountered exception while copying: %s N)�logger�error�str�op_failure_count�debug� traceback� format_exc�rU �erA rA rE �_CopyExceptionHandler� s �rb c C s | j �t|�� dS )rX N)rY rZ r[ r` rA rA rE �_RmExceptionHandler� s rc c s� e Zd ZdZejddgedeeddde j e j ge j dd ge � � gd � Zejddgd d ei d �Z� fdd�Zddd�Zdd� Zdd� Zdd� Z� ZS )� CpCommandaR Implementation of gsutil cp command. Note that CpCommand is run for both gsutil cp and gsutil mv. The latter happens by MvCommand calling CpCommand and passing the hidden (undocumented) -M option. This allows the copy and remove needed for each mv to run together (rather than first running all the cp's and then all the rm's, as we originally had implemented), which in turn avoids the following problem with removing the wrong objects: starting with a bucket containing only the object gs://bucket/obj, say the user does: gsutil mv gs://bucket/* gs://bucket/d.txt If we ran all the cp's and then all the rm's and we didn't expand the wildcard first, the cp command would first copy gs://bucket/obj to gs://bucket/d.txt, and the rm command would then remove that object. In the implementation prior to gsutil release 3.12 we avoided this by building a list of objects to process and then running the copies and then the removes; but building the list up front limits scalability (compared with the current approach of processing the bucket listing iterator on the fly). �cp�copyrI TFr Zstetztestcallbackfile=) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�supported_private_args�argparse_arguments� command_helpzCopy files and objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s2 | j �d� t| j� tddgtd�}t� �|�S )Nz�Unlike pure gsutil, this shim won't run composite uploads and sliced downloads in parallel by default. Use the -m flag to enable parallelism (i.e. "gsutil -m cp ...").�storagere )�gcloud_command�flag_map)rY �warnrP rM r) �CP_SHIM_FLAG_MAP�super�get_gcloud_storage_args)�self�gcloud_storage_map�� __class__rA rE r� � s � �z!CpCommand.get_gcloud_storage_argsNc C sV t | |d�}t�� }|jr d}n| j}|j}|j}|j} |j} |� � r^|� � r^t d||f ��|rz|� � rz|� � rzt d��| jr�|� � r�|� � r�t d��| r�t�|j|j|� t|�r�dS |jr�| j�|j�r�dS |jr�|jr�d| _|j� � �rRtj�|jj��sR| �rRzt� |jj� W n6 t �yP } z | j t j k�r<� W Y d} ~ n d} ~ 0 0 tj ||| | |j!|j|j| j|d� } t�"|| �} t�#|| � t�$|| ��r�t d ||| f ��| � � �r�| �%� �r�t d || f ��| � � �s�|j&�r�t d | ��d} |j'�r t(�)t*j+|j'�} |� � �r�|�r�| �s*t*�+� } t�,|j�\ }}}}}}}}}}t-|�}t.|||||d �}t*j+j/g d �}t0|||d� || _1| �r�| � � �r�t2| |j�}|j3j4}t5|j|j6|j7|d�\}}|�r�|�s�t8�9� �:|� t d��d}�z&|j�r| j�;|j| j� | j�r�|jj�r�| � � �r�tj�<|jj�}tj�=|tj�<| j�g��>|��s�| j?�@d||f � |j�r�| j�A|jddd� W dS tjB| j?|| || tC| d| jD| j| jE| jF|| jGd�\}}}}|j�r�|�r�| j�H|jd|� | j�A|j|d� |jI�r| j?�Jd|� W �n� tK�yX d| } | j?�J| � |j�rR| j�A|jdd| � Y �n� tL�y� } zFd|j| jMf } | j?�J| � |j�r�| j�A|jdd| � W Y d} ~ �nnd} ~ 0 tjN�y� } z"| j?�@d|| f � W Y d} ~ �n2d} ~ 0 tO�y� } z�|jP�rHt�Q| ��rHd| } | j?�J| � |j�r�| j�A|jdd| � nv| jR�r�d |tS| �f } | jTd 7 _T| j?�U| � |j�r�| j�A|jdd tV| �� n"|j�r�| j�A|jdd tS| �� � W Y d} ~ nRd} ~ 0 0 |j�r | j?�Jd |� |� � �r|jW|jX|j|jY|jZd � n t�[|j� | j\� | j]|7 _]W d � n1 �sH0 Y dS )!z@Worker function for performing the actual copy (and rm, for mv).)rQ �mvz<The %s command does not allow provider-only source URLs (%s)z/Cannot preserve POSIX attributes with a stream.z8Cannot upload from a stream when using gsutil -m option.NT)rR z,%s: "%s" and "%s" are the same file - abort.zP%s: a version-specific URL (%s) cannot be the destination for gsutil cp - abort.z<Cannot specify storage class for a non-cloud destination: %s)�atime�mtime�uid�gid�mode)�additionalProperties)r� r� r� zPThis sync will orphan file(s), please fix their permissions before trying again.r ziSkipping copy of source URL %s because it would be copied outside the expected destination directory: %s.�skipz4Would have copied outside the destination directory.)�src_obj_metadataZallow_splitting�headers�manifest� gzip_encoded� gzip_extsrR �use_stet�md5�OKz Created: %szSkipping existing item: %sz0Skipping item %s with unsupported object type %sz�Skipping copy of source URL %s because destination URL %s is already being copied by another gsutil process or thread (did you specify the same source URL twice?) zRejected (noclobber): %szError copying %s: %srI rZ zRemoving %s...)� generation�provider)^r r �GetCopyHelperOpts� perform_mv� command_name�source_storage_url�expanded_storage_url�names_container�is_multi_source_request� IsCloudUrl� IsProviderr � IsFileUrl�IsStream�parallel_operationsZInsistDstUrlNamesContainer� exp_dst_url�have_existing_dst_containerr � use_manifestr� Z WasSuccessful� url_string�recursion_requested�os�path�exists� object_name�makedirs�OSError�errno�EEXISTZConstructDstUrl�!is_multi_top_level_source_requestZFixWindowsNamingZCheckForDirFileConflictZ SrcDstSame� HasGeneration�dest_storage_class�expanded_resultr � JsonToMessage�apitools_messages�Object�statr" r% � MetadataValuer& �metadatar# r� � permissionsr' r� r� �logging� getLogger�criticalZ Initialize�abspath� commonpath� startswithrY r} Z SetResultZ PerformCopyrb r� r� r� r� �Set� print_ver�infor r! Zunsupported_typeZFileConcurrencySkipError� Exception� no_clobberZIsNoClobberServerException�continue_on_errorr[ r\ rZ r. � DeleteObject� bucket_namer� �scheme�unlink� stats_lock�total_bytes_transferred) r� Zcopy_object_inforQ rR � gsutil_api�copy_helper_optsZcmd_name�src_url� exp_src_urlZsrc_url_names_containerZhave_multiple_srcsra �dst_urlr� r� rO r� r� r� r� Z posix_attrs�custom_metadata�valid�err�bytes_transferred� containerZ result_urlr� �messagerA rA rE rS � s� ���� � ��� ���������� �� ������ � *��� � � zCpCommand.CopyFuncc c s� t �� }t||�D ]�\}}t j|| j| j| jd�\}}tt| j | j | j| j|| j pV|j | j| j | j| jpl| jt|�� |j|j| j|j | jd�d� t||��}| jp�|�� | _| jp�|�� | _| j� |j � t � | j |�| _ |V qd S )N)rY )rR Z delete_source�file_size_will_change)� project_id� all_versions�ignore_symlinksr� �bucket_listing_fields)!r r� �zipZExpandUrlToSingleBlrr� r� rY r r r� r] r� r� r� �exclude_symlinksr� r� r r� �skip_unsupported_objects� preserve_aclrT r� r � has_file_dstr� � has_cloud_dst�provider_types�addr� � itertools�chain�combined_src_urls) r� Zsrc_url_strs_iter� dst_url_strsr� � src_url_str� dst_url_strr� r� Z!name_expansion_iterator_dst_tuplerA rA rE �/_ConstructNameExpansionIteratorDstTupleIterator� sN �� ����z9CpCommand._ConstructNameExpansionIteratorDstTupleIteratorc C sh | � � }d| _t| jd �}|�� rz|jdks6|�� rz| jrDtd��|�� rXt |dd�nd }t � | �j | jd d� |d�S |j r�t| j�dkr�td ��t� g}n&t| j�d k r�td ��| jd d� g}| jd g}g | _d | _d | _t� | _t| �||�|j�}| jd d d d d �\}}t�|| j|| � d } |j �slt | j | j | � � | j| j �pV|j!| j"| j#| j$| j%d� } t&�'� | _(d| _)t*�*� } d} | j+t,|t-| | j. | d� | j/� d| j� t*�*� } | | | _0t1| j| j0�| _2t3| j| j| j2| j| j0| j4|jt5| j�d� | j t6k�r4| jdk�r4| j/�7d| j| j0t8| j2�� | j)�rd| j)dk�rLdnd} tdj9| j)| d���dS )Nr ����-z?Cannot preserve POSIX attributes with a stream or a named pipe.�wb)r� )� cat_out_fdrI z.Source URLs cannot be specified with -I option� z+Wrong number of arguments for "cp" command.F)� process_count� thread_count� parallel_operations_override�print_macos_warning)r� r� r� r� )r\ r� )� fail_on_error�seek_ahead_iteratorztotal_bytes_transferred: %d)r� r� �avg_throughputr� �total_elapsed_time�uses_fan�is_daisy_chainr� z;Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)�s� z5{count} file{pl}/object{pl} could not be transferred.)�count�pl):� _ParseOptsr� r rV r� r� �IsFiforT r r* r � CatHelper� CatUrlStrings�read_args_from_stdin�lenr, r� r� r� �setr� r r� � daisy_chain�_GetProcessAndThreadCountr Z.TriggerReauthForDestinationProviderIfNecessaryr� r r� r] �GetSeekAheadGsutilApir� r� r� r� r� r� r � CreateLockr� r\ �time�ApplyrW rb r� rY r� r/ Ztotal_bytes_per_secondr r� �listr r� r0 �format)r� r� r� r� Z src_url_strsr� �name_expansion_iteratorr� r� r� � start_time� shared_attrs�end_timeZ plural_strrA rA rE � RunCommand s� ��� ��� � � �� �� ��zCpCommand.RunCommandc C s� d}d| _ d}d| _d}d}d}d}d}d| _d }d | _d| _d| _d} d} d } d } d } d }d| _d | _| j�r | jD �]�\}}|dkr�|}d| _|dkr�d| _|dkr�d| _qv|dkr�d}qv|dkr�d| _ qv|dkr�|} qv|d kr�d}qv|d k�rd} d d � |� d �D �} qv|dk�r"d} t } qv|dk�r<d}t |�| _qv|dk�rLd}qv|dk�r\d}qv|dk�rld}qv|dk�r�d| _t � qv|dk�s�|dk�r�d| _ qv|dk�r�t|�}qv|dk�r�d| _qv|dk�r�d}qv|dk�r�d} dd � |� d �D �} qv|dk�rd} t } qv|d krvd| _qv|�r2|�r2td ��| j�rJ| j�rJtd ��| �r^| �r^td ��| �r�| �r�| �rztd ��ntd!��| �p�| | _| | _t||||||||| j| |d"� S )#NFr2 Tr1 r3 r4 r5 z--testcallbackfiler6 r8 c S s g | ] }|� � �qS rA ��strip�rB �xrA rA rE � <listcomp>� � z(CpCommand._ParseOpts.<locals>.<listcomp>�,r7 r9 z-Mr: r< r; rG rH r= z-Ur> r@ c S s g | ] }|� � �qS rA r r rA rA rE r � r r? z--stetz:Specifying both the -p and -a options together is invalid.z�The gsutil -m option is not supported with the cp -A flag, to ensure that object version ordering is preserved. Please re-run the command without the -m option.z@Specifying both the -j/-J and -z/-Z options together is invalid.z:Specifying both the -j and -J options together is invalid.z:Specifying both the -z and -Z options together is invalid.) r� r� r r r� r� r� � canned_aclr� �test_callback_filer� )r� r� rT �cannedr� r� r� r� rM �splitr r r$ r� r- r r� r� r� r )r� r� r� r r r� r� r� r r� Z gzip_localZ gzip_arg_extsZ gzip_arg_allr r� �o�arA rA rE r� � s� �� � �� �zCpCommand._ParseOpts)NF)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr � CP_SUB_ARGSr �XML�JSONr �%MakeZeroOrMoreCloudOrFileURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr� rS r� r r� � __classcell__rA rA r� rE rd � s> ��� i@xrd )N)fr# � __future__r r r r r� r� r� r� r r^ �apitools.base.pyr �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr Z gslib.metricsr �gslib.name_expansionr r r r r �gslib.storage_urlr r r �"gslib.third_party.storage_apitoolsr r� Z gslib.utilsr r r � gslib.utils.cloud_api_helperr �gslib.utils.constantsr r Zgslib.utils.copy_helperr r r r r r! Zgslib.utils.posix_utilr" r# r$ r% r& r' �gslib.utils.shim_utilr( r) Zgslib.utils.system_utilr* r+ r, Zgslib.utils.text_utilr- r. �gslib.utils.unit_utilr/ r0 r% Z_SYNOPSIS_TEXTZ_DESCRIPTION_TEXTZ_NAME_CONSTRUCTION_TEXTZ_SUBDIRECTORIES_TEXTZ_COPY_IN_CLOUD_TEXTZ_CHECKSUM_VALIDATION_TEXTZ_RETRY_HANDLING_TEXTZ_RESUMABLE_TRANSFERS_TEXTZ_STREAMING_TRANSFERS_TEXTZ _SLICED_OBJECT_DOWNLOADS_TEXTZ _PARALLEL_COMPOSITE_UPLOADS_TEXTZ _CHANGING_TEMP_DIRECTORIES_TEXTZ_COPYING_SPECIAL_FILES_TEXTZ _OPTIONS_TEXT�joinr, r& ZCP_AND_MV_SHIM_FLAG_MAPr �itemsr~ rP rW rb rc rd rA rA rA rE �<module> s� �32)% g�� �� commands/__pycache__/cat.cpython-39.pyc 0000644 00000010255 15025013616 0013757 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ejr�eZdZde d ZG dd� de �ZdS )zDImplementation of Unix-like cat command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�Command)�CommandArgument)� ApiSelector)�CommandException)� cat_helper)� constants)�GcloudStorageFlag)�GcloudStorageMapz gsutil cat [-h] url... z <B>SYNOPSIS</B> aJ <B>DESCRIPTION</B> The cat command outputs the contents of one or more URLs to stdout. While the cat command does not compute a checksum, it is otherwise equivalent to doing: gsutil cp url... - (The final '-' causes gsutil to stream the output to stdout.) WARNING: The gsutil cat command does not compute a checksum of the downloaded data. Therefore, we recommend that users either perform their own validation of the output of gsutil cat or use gsutil cp or rsync (both of which perform integrity checking automatically). <B>OPTIONS</B> -h Prints short header for each object. For example: gsutil cat -h gs://bucket/meeting_notes/2012_Feb/*.txt This would print a header with the object name before the contents of each text object that matched the wildcard. -r range Causes gsutil to output just the specified byte range of the object. Ranges can be of these forms: start-end (e.g., -r 256-5939) start- (e.g., -r 256-) -numbytes (e.g., -r -5) where offsets start at 0, start-end means to return bytes start through end (inclusive), start- means to return bytes start through the end of the object, and -numbytes means to return the last numbytes of the object. For example: gsutil cat -r 256-939 gs://bucket/object returns bytes 256 through 939, while: gsutil cat -r -5 gs://bucket/object returns the final 5 bytes of the object. c @ s� e Zd ZdZejdg edejdddde j e j ge j e � � gd� Zejdg dd ei d �Zed dged �ed �d�d�Zdd� ZdS )� CatCommandz%Implementation of gsutil cat command.�cat� zhr:Fr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz$Concatenate object content to stdout)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storagez-d�-r)�-hr$ )�gcloud_command�flag_mapc C s� d}d}d}d}| j r�| j D ]�\}}|dkr2d}q |dkr�|�� }|dkrLq t�d�}|�|�}|sptd | ��|�d �r�t|�d ��}|�d �r�t|�d ��}|�d �r�t|�d ��}q | �� q t � | �j | j |||d �S )z(Command entry point for the cat command.FNr r% Tr$ �-z;^(?P<start>[0-9]+)-(?P<end>[0-9]*)$|^(?P<endslice>-[0-9]+)$zInvalid range (%s)�start�endZendslice)� show_header� start_byte�end_byte) �sub_opts�strip�re�compile�matchr �group�long� RaiseInvalidArgumentExceptionr Z CatHelperZ CatUrlStrings�args) �selfr+ Z request_ranger, r- �o�aZ range_matcherZ range_match� r: �!/opt/gsutil/gslib/commands/cat.py� RunCommand� s< � �zCatCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �NO_MAXr �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r �gcloud_storage_mapr< r: r: r: r; r ] s< �� �� r ) r@ � __future__r r r r r0 �six� gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr Z gslib.utilsr r �gslib.utils.shim_utilr r �PY3�intr4 rB rI r r: r: r: r; �<module> s. ��2 commands/__pycache__/notification.cpython-39.pyc 0000644 00000062743 15025013616 0015707 0 ustar 00 a +(WgŇ � @ s d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl m Z ddl m Z dd l mZ dd l mZ dd l mZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl$m&Z& ddl'm(Z( ddl'm)Z) dZ*dZ+d Z,d Z-d Z.e,e+�/d � e*�/d � e- e.�/d � d Z0d Z1d!Z2d"Z3d#Z4d$Z5d%e3 e1 e2 d& e4 e5 d'j6e%j7d(� Z8d)Z9ee0e8�Z:ee,e3�Z;ee*e1�Z<ee+e2�Z=ee-e4�Z>ee.e5�Z?d*d+d,�Z@G d-d.� d.e�ZAdS )/z8This module provides the notification command to gsutil.� )�absolute_import)�print_function)�division)�unicode_literalsN)�datetime)�metrics)�AccessDeniedException)�NotFoundException)� PublishPermissionDeniedException)�Command)�NO_MAX)�CommandArgument)� ApiSelector)�CommandException)�CreateHelpText)�PopulateProjectId)� PubsubApi)�StorageUrlFromString)�Binding)� copy_helper)� shim_util)�GcloudStorageFlag)�GcloudStorageMapz2 gsutil notification list gs://<bucket_name>... zO gsutil notification delete (<notificationConfigName>|gs://<bucket_name>)... z� gsutil notification create -f (json|none) [-p <prefix>] [-t <topic>] \ [-m <key>:<value>]... [-e <eventType>]... gs://<bucket_name> zW gsutil notification watchbucket [-i <id>] [-t <token>] <app_url> gs://<bucket_name> z> gsutil notification stopchannel <channel_id> <resource_id> � a� <B>LIST</B> The list sub-command provides a list of notification configs belonging to a given bucket. The listed name of each notification config can be used with the delete sub-command to delete that specific notification config. For listing Object Change Notifications instead of Cloud Pub/Sub notification subscription configs, add a -o flag. <B>LIST EXAMPLES</B> Fetch the list of notification configs for the bucket example-bucket: gsutil notification list gs://example-bucket The same as above, but for Object Change Notifications instead of Cloud Pub/Sub notification subscription configs: gsutil notification list -o gs://example-bucket Fetch the notification configs in all buckets matching a wildcard: gsutil notification list gs://example-* Fetch all of the notification configs for buckets in the default project: gsutil notification list gs://* a� <B>DELETE</B> The delete sub-command deletes notification configs from a bucket. If a notification config name is passed as a parameter, that notification config alone is deleted. If a bucket name is passed, all notification configs associated with that bucket are deleted. Cloud Pub/Sub topics associated with this notification config are not deleted by this command. Those must be deleted separately, for example with the gcloud command `gcloud beta pubsub topics delete`. Object Change Notification subscriptions cannot be deleted with this command. For that, see the command `gsutil notification stopchannel`. <B>DELETE EXAMPLES</B> Delete a single notification config (with ID 3) in the bucket example-bucket: gsutil notification delete projects/_/buckets/example-bucket/notificationConfigs/3 Delete all notification configs in the bucket example-bucket: gsutil notification delete gs://example-bucket a� <B>CREATE</B> The create sub-command creates a notification config on a bucket, establishing a flow of event notifications from Cloud Storage to a Cloud Pub/Sub topic. As part of creating this flow, the create command also verifies that the destination Cloud Pub/Sub topic exists, creating it if necessary, and verifies that the Cloud Storage bucket has permission to publish events to that topic, granting the permission if necessary. If a destination Cloud Pub/Sub topic is not specified with the -t flag, Cloud Storage chooses a topic name in the default project whose ID is the same as the bucket name. For example, if the default project ID specified is 'default-project' and the bucket being configured is gs://example-bucket, the create command uses the Cloud Pub/Sub topic "projects/default-project/topics/example-bucket". In order to enable notifications, your project's `Cloud Storage service agent <https://cloud.google.com/storage/docs/projects#service-accounts>`_ must have the IAM permission "pubsub.topics.publish". This command checks to see if the destination Cloud Pub/Sub topic grants the service agent this permission. If not, the create command attempts to grant it. A bucket can have up to 100 total notification configurations and up to 10 notification configurations set to trigger for a specific event. <B>CREATE EXAMPLES</B> Begin sending notifications of all changes to the bucket example-bucket to the Cloud Pub/Sub topic projects/default-project/topics/example-bucket: gsutil notification create -f json gs://example-bucket The same as above, but specifies the destination topic ID 'files-to-process' in the default project: gsutil notification create -f json \ -t files-to-process gs://example-bucket The same as above, but specifies a Cloud Pub/Sub topic belonging to the specific cloud project 'example-project': gsutil notification create -f json \ -t projects/example-project/topics/files-to-process gs://example-bucket Create a notification config that only sends an event when a new object has been created: gsutil notification create -f json -e OBJECT_FINALIZE gs://example-bucket Create a topic and notification config that only sends an event when an object beginning with "photos/" is affected: gsutil notification create -p photos/ gs://example-bucket List all of the notificationConfigs in bucket example-bucket: gsutil notification list gs://example-bucket Delete all notitificationConfigs for bucket example-bucket: gsutil notification delete gs://example-bucket Delete one specific notificationConfig for bucket example-bucket: gsutil notification delete \ projects/_/buckets/example-bucket/notificationConfigs/1 <B>OPTIONS</B> The create sub-command has the following options -e Specify an event type filter for this notification config. Cloud Storage only sends notifications of this type. You may specify this parameter multiple times to allow multiple event types. If not specified, Cloud Storage sends notifications for all event types. The valid types are: OBJECT_FINALIZE - An object has been created. OBJECT_METADATA_UPDATE - The metadata of an object has changed. OBJECT_DELETE - An object has been permanently deleted. OBJECT_ARCHIVE - A live version of an object has become a noncurrent version. -f Specifies the payload format of notification messages. Must be either "json" for a payload matches the object metadata for the JSON API, or "none" to specify no payload at all. In either case, notification details are available in the message attributes. -m Specifies a key:value attribute that is appended to the set of attributes sent to Cloud Pub/Sub for all events associated with this notification config. You may specify this parameter multiple times to set multiple attributes. -p Specifies a prefix path filter for this notification config. Cloud Storage only sends notifications for objects in this bucket whose names begin with the specified prefix. -s Skips creation and permission assignment of the Cloud Pub/Sub topic. This is useful if the caller does not have permission to access the topic in question, or if the topic already exists and has the appropriate publish permission assigned. -t The Cloud Pub/Sub topic to which notifications should be sent. If not specified, this command chooses a topic whose project is your default project and whose ID is the same as the Cloud Storage bucket name. <B>NEXT STEPS</B> Once the create command has succeeded, Cloud Storage publishes a message to the specified Cloud Pub/Sub topic when eligible changes occur. In order to receive these messages, you must create a Pub/Sub subscription for your Pub/Sub topic. To learn more about creating Pub/Sub subscriptions, see `the Pub/Sub Subscriber Overview <https://cloud.google.com/pubsub/docs/subscriber>`_. You can create a simple Pub/Sub subscription using the ``gcloud`` command-line tool. For example, to create a new subscription on the topic "myNewTopic" and attempt to pull messages from it, you could run: gcloud beta pubsub subscriptions create --topic myNewTopic testSubscription gcloud beta pubsub subscriptions pull --auto-ack testSubscription a� <B>WATCHBUCKET</B> The watchbucket sub-command can be used to watch a bucket for object changes. A service account must be used when running this command. The app_url parameter must be an HTTPS URL to an application that will be notified of changes to any object in the bucket. The optional id parameter can be used to assign a unique identifier to the created notification channel. If not provided, a random UUID string is generated. The optional token parameter can be used to validate notifications events. To do this, set this custom token and store it to later verify that notification events contain the client token you expect. <B>WATCHBUCKET EXAMPLES</B> Watch the bucket example-bucket for changes and send notifications to an application server running at example.com: gsutil notification watchbucket https://example.com/notify \ gs://example-bucket Assign identifier my-channel-id to the created notification channel: gsutil notification watchbucket -i my-channel-id \ https://example.com/notify gs://example-bucket Set a custom client token that is included with each notification event: gsutil notification watchbucket -t my-client-token \ https://example.com/notify gs://example-bucket a� <B>STOPCHANNEL</B> The stopchannel sub-command can be used to stop sending change events to a notification channel. The channel_id and resource_id parameters should match the values from the response of a bucket watch request. <B>STOPCHANNEL EXAMPLES</B> Stop the notification event channel with channel identifier channel1 and resource identifier SoGqan08XDIFWr1Fv_nGpRJBHh8: gsutil notification stopchannel channel1 SoGqan08XDIFWr1Fv_nGpRJBHh8 a� You can use the ``notification`` command to configure `Pub/Sub notifications for Cloud Storage <https://cloud.google.com/storage/docs/pubsub-notifications>`_ and `Object change notification <https://cloud.google.com/storage/docs/object-change-notification>`_ channels. <B>CLOUD PUB/SUB</B> The "create", "list", and "delete" sub-commands deal with configuring Cloud Storage integration with Google Cloud Pub/Sub. aP <B>OBJECT CHANGE NOTIFICATIONS</B> Object change notification is a separate, older feature within Cloud Storage for generating notifications. This feature sends HTTPS messages to a client application that you've set up separately. This feature is generally not recommended, because Pub/Sub notifications are cheaper, easier to use, and more flexible. For more information, see `Object change notification <https://cloud.google.com/storage/docs/object-change-notification>`_. The "watchbucket" and "stopchannel" sub-commands enable and disable Object change notifications. aY <B>NOTIFICATIONS AND PARALLEL COMPOSITE UPLOADS</B> gsutil supports `parallel composite uploads <https://cloud.google.com/storage/docs/uploads-downloads#parallel-composite-uploads>`_. If enabled, an upload can result in multiple temporary component objects being uploaded before the actual intended object is created. Any subscriber to notifications for this bucket then sees a notification for each of these components being created and deleted. If this is a concern for you, note that parallel composite uploads can be disabled by setting "parallel_composite_upload_threshold = 0" in your .boto config file. Alternately, your subscriber code can filter out gsutil's parallel composite uploads by ignoring any notification about objects whose names contain (but do not start with) the following string: "{composite_namespace}". )Zcomposite_namespacea_ Watch bucket attempt failed: {watch_error} You attempted to watch a bucket with an application URL of: {watch_url} which is not authorized for your project. Please ensure that you are using Service Account authentication and that the Service Account's project is authorized for the application URL. Notification endpoint URLs must also be whitelisted in your Cloud Console project. To do that, the domain must also be verified using Google Webmaster Tools. For instructions, please see `Notification Authorization <https://cloud.google.com/storage/docs/object-change-notification#_Authorization>`_. �NONE� JSON_API_V1)�none�jsonc @ s� e Zd ZdZdd� ZdZejdg d�ede dd d d e j ge j e � � e �� gg e �� ge �� ge � � e �d �gd �d � Zejdg d �ddeeeeeed�d�Zeeg d�ede j j d�ede j j d�ed�ed�ed�ed�d�d�eg d �i d�eg d �i d d �d �i d�Z d!d"� Z!d#d$� Z"d%d&� Z#d'd(� Z$d)d*� Z%d9d+d,�Z&d-d.� Z'd/d0� Z(d1d2� Z)d3d4� Z*d5d6� Z+e$e'e)e!e"d�Z,d7d8� Z-dS ):�NotificationCommandz.Implementation of gsutil notification command.c C s t jst�d�t _t jS )Nz]/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/notificationConfigs/(?P<notification>[0-9]+))r �_notification_path_regex�re�compile��self� r$ �*/opt/gsutil/gslib/commands/notification.py�_GetNotificationPathRegex� s �z-NotificationCommand._GetNotificationPathRegexN� notification)�notify� notifyconfig� notificationsZnotif� zi:t:m:t:of:e:p:sF� )� watchbucket� stopchannel�list�delete�create) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)r- r. r) � command_helpz$Configure object change notification)r1 r/ r0 r- r. )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�bucketsr* r1 z--custom-attributes)� repeat_typez --event-typesz--object-prefixz--payload-formatz--skip-topic-setupz--topic)�-m�-e�-p�-f�-s�-t)�gcloud_command�flag_map)rD rE r* r0 )rD rE r* r/ z--human-readableT)rM rN �supports_output_translation)r1 r0 r/ c C st | � � d}d}| jr>| jD ] \}}|dkr0|}|dkr |}q |pLtt�� �}| jd }| jd }|�� �d�sxtd��t |�}|� � r�|j dks�td | j ��|� � s�td | j ��| j �d ||� z | jj|j||||j d �}W nB t�y& } z(| j �tjt| �|d �� � W Y d} ~ n d} ~ 0 0 |j} |j} |j}| j �d� | j �d| � | j �d| � | j �d|� dS )z/Creates a watch on a bucket given in self.args.Nz-irL r ���zhttps://z,The application URL must be an https:// URL.�gs�7The %s command can only be used with gs:// bucket URLs.�*URL must name a bucket for the %s command.z.Watching bucket %s with application URL %s ...)�token�provider)Z watch_error� watch_urlz0Successfully created watch notification channel.z Watch channel identifier: %sz%Canonicalized resource identifier: %szClient state token: %s)�CheckArguments�sub_opts�str�uuid�uuid4�args�lower� startswithr r �IsBucket�scheme� command_name�logger�info� gsutil_api� WatchBucket� bucket_namer �warn�)NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE�format�id� resourceIdrT ) r# � identifierZ client_token�o�arV � bucket_arg� bucket_url�channel�e� channel_id� resource_idr$ r$ r% � _WatchBucket� sd ���� � �� z NotificationCommand._WatchBucketc C sF | j d }| j d }| j�d||� | jj||dd� | j�d� dS )Nr r, z3Removing channel %s with resource identifier %s ...rQ �rU z Succesfully removed channel.)r\ rb rc rd � StopChannel)r# rs rt r$ r$ r% � _StopChannel( s � z NotificationCommand._StopChannelc C s� t |�}|�� r|jdks(td| j ��|�� s>td| j ��| jj|jdd�j}| j � d|j� t |�D ]x\}}| j � d|d � | j � d|j � | j � d |j � | j � d |j� | j � d |j� | j � d tt�|jd ��� qjdS )z<Lists active channel watches on a bucket given in self.args.rQ rR rS rv z?Bucket %s has the following active Object Change Notifications:z Notification channel %d:r, z Channel identifier: %sz Resource identifier: %sz Application URL: %sz Created by: %sz Creation time: %si� r )r r_ r` r ra rd � ListChannelsrf �itemsrb rc � enumeraters rt �push_url�subscriber_emailrY r � fromtimestamp�creation_time_ms)r# ro rp �channels�idxrq r$ r$ r% � _ListChannels3 s: ��� ���z!NotificationCommand._ListChannelsc C sl | � � d }d }i }g }d }d}| jr�| jD ]�\}}|dkrH|�|� q,|dkrV|}q,|dkr�d|vrntd��|�dd�\} } | || < q,|dkr�|}q,|d kr�d }q,|d kr,|}q,|tvr�td ��t| }| jd } t| �} | �� r�| � � �std| j | j | f ��| j dk�r td| j ��| j } | j�d| � | jj| dg| j d�}|j}|�sddtd �| f }|�d��s�dtd �|f }| j�d|� d }|�r�| jj|| j d�j}| j�d||� | �||�}tdd�D ]x}z2| jj| ||||�r�|nd || j d�}W �qLW n> t�yF |dk�r@|�r@| j�d � t � d � n� Y n0 �q�d | |j f }| j�d |� dS ) NTrH rJ rG �:zCCustom attributes specified with -m should be of the form key:valuer, rI rK FrL z@Must provide a payload format with -f of either 'json' or 'none'rP z.%s %s requires a GCS bucket name, but got '%s'rQ rR z#Creating notification for bucket %s� projectNumber)�fieldsrU zprojects/%s/topics/%sz projects/z Using Cloud Pub/Sub topic %srv z"Service account for project %d: %sr r+ )� pubsub_topic�payload_format�custom_attributes� event_types�object_name_prefixrU zfRetrying create notification in 10 seconds (new permissions may take up to 10 seconds to take effect.)� z,projects/_/buckets/%s/notificationConfigs/%sz Created notification config %s) rW rX �appendr �split�PAYLOAD_FORMAT_MAPr\ r � IsCloudUrlr_ ra �subcommand_namer` rf rb �debugrd � GetBucketr� r r^ �GetProjectServiceAccount� email_address� _CreateTopic�range�CreateNotificationConfigr rc �time�sleeprj )r# r� r� r� r� r� Zshould_setup_topicrm rn �key�valuero rp rf �bucket_metadata�bucket_project_numberZ just_modified_topic_permissions�service_accountZattempt_numberZcreate_responseZnotification_namer$ r$ r% �_CreateN s� � � �� ���� ���� � � �zNotificationCommand._Createc C s� t | jd�}z |j|d� | j�d|� W n: tyd | j�d|� |j|d� | j�d|� Y n0 |j|d�}tdd| gd�}||j vr�|j � |� |j ||d � d S | j�d |� d S d S )aa Assures that a topic exists, creating it if necessary. Also adds GCS as a publisher on that bucket, if necessary. Args: pubsub_topic: name of the Cloud Pub/Sub topic to use/create. service_account: the GCS service account that needs publish permission. Returns: true if we modified IAM permissions, otherwise false. )rb )� topic_namezTopic %s already existszCreating topic %sz Created Cloud Pub/Sub topic %szroles/pubsub.publisherzserviceAccount:%s)�role�members)r� �policyTz/GCS already has publish permission to topic %s.FN) r rb ZGetTopicr� r Z CreateTopicrc ZGetTopicIamPolicyr �bindingsr� ZSetTopicIamPolicy)r# r� r� Z pubsub_apir� �bindingr$ r$ r% r� � s( � �z NotificationCommand._CreateTopicc c sD | � � }| jD �].}|�|�}|r�|s<td| j| j|f ��|�d�}|�d�}d}| jj|dd�D ] }|j |krd||fV d} q�qd|s�t d| ��qt |�} | � � s�td | j ��| j dkr�td ��d } | �� r�d } n | �� r�|} | s�td | j ��| �| �jdgd�D ]0} | jj| jjdd�D ]}| jj|fV �q$�q qd S )ay Yields bucket/notification tuples from command-line args. Given a list of strings that are bucket names (gs://foo) or notification config IDs, yield tuples of bucket names and their associated notifications. Args: accept_notification_configs: whether notification configs are valid args. Yields: Tuples of the form (bucket_name, Notification) z4%s %s accepts only bucket names, but you provided %s�bucketr' FrQ rv Tz Could not find notification %szJThe %s command must be used on cloud buckets or notification config names.z+The %s command only works on gs:// buckets.Nzgs://*z<The %s command cannot be used on cloud objects, only bucketsrj )� bucket_fields)r& r\ �matchr ra r� �grouprd �ListNotificationConfigsrj r r r� r` � IsProviderr_ �WildcardIterator� IterBuckets� storage_urlrf ) r# �accept_notification_configsZ path_regexZ list_entryr� rf �notification_id�foundr' r� �path�blrr$ r$ r% � _EnumerateNotificationsFromArgs� sb �� � �� �� � � z3NotificationCommand._EnumerateNotificationsFromArgsc C sZ | � � | jr4dt| j�v rV| jD ]}| �|� q"n"| jdd�D ]\}}| �||� q@dS )Nz-oF)r� r )rW rX �dictr\ r� r� �_PrintNotificationDetails�r# rf r' r$ r$ r% �_List s �zNotificationCommand._Listc C s� t dj||j|jtd�d � d�� |jrTt d� |jjD ]}t d|j|jf � q:g }|j rt|� dd� |j � � |j r�|� d|j � |r�t d � |D ] }t |� q�| j �d � d S ) Nz\projects/_/buckets/{bucket}/notificationConfigs/{notification} Cloud Pub/Sub topic: {topic}z//pubsub.googleapis.com/)r� r' �topicz Custom attributes:z %s: %sz Event Types: %sz, z Object name prefix: '%s'z Filters:� )�printri rj r� �lenr� �additionalPropertiesr� r� r� r� �joinr� rb rc )r# r� r' �attr�filters�liner$ r$ r% r� ) s. � �� z-NotificationCommand._PrintNotificationDetailsc C s$ | � � D ]\}}| �||j� qdS )Nr )r� �_DeleteNotificationrj r� r$ r$ r% �_Delete@ s zNotificationCommand._Deletec C s | j j||dd� dS )NrQ )r' rU r )rd �DeleteNotificationConfig)r# rf r� r$ r$ r% r� E s �z'NotificationCommand._DeleteNotificationc C sT z2t � | j| jj�\| _| _tj| jd� || �W S t jyN | �� Y n0 d S )N)rX ) �getoptr\ � command_specr6 rX r �LogCommandParams� GetoptError� RaiseInvalidArgumentException)r# �funcr$ r$ r% �_RunSubCommandK s �� z"NotificationCommand._RunSubCommandc C sT | j �d�| _| jtjv r<tj| jgd� | �tj| j �S td| j| j f ��dS )z1Command entry point for the notification command.r )� subcommandsz+Invalid subcommand "%s" for the %s command.N) r\ �popr� r � SUBCOMMANDSr r� r� r ra r" r$ r$ r% � RunCommand_ s � �z NotificationCommand.RunCommand)T).�__name__� __module__� __qualname__�__doc__r& r r �CreateCommandSpec� _SYNOPSISr r �JSONr �MakeFreeTextArgument�%MakeZeroOrMoreCloudBucketURLsArgument� MakeZeroOrMoreCloudURLsArgument� MakeNCloudBucketURLsArgumentr� �HelpSpec�_DETAILED_HELP_TEXT�_create_help_text�_list_help_text�_delete_help_text�_watchbucket_help_text�_stopchannel_help_text� help_specr r r �RepeatFlagType�DICT�LIST�gcloud_storage_mapru rx r� r� r� r� r� r� r� r� r� r� r� r$ r$ r$ r% r y s� �����%���������-�15 k. 6 �r )Br� � __future__r r r r r� r r� rZ r �gslibr �gslib.cloud_apir r r � gslib.commandr r �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.help_providerr �gslib.project_idr Zgslib.pubsub_apir �gslib.storage_urlr Z4gslib.third_party.pubsub_apitools.pubsub_v1_messagesr Z gslib.utilsr r �gslib.utils.shim_utilr r �_LIST_SYNOPSIS�_DELETE_SYNOPSIS�_CREATE_SYNOPSISZ_WATCHBUCKET_SYNOPSISZ_STOPCHANNEL_SYNOPSIS�lstripr� �_LIST_DESCRIPTION�_DELETE_DESCRIPTION�_CREATE_DESCRIPTIONZ_WATCHBUCKET_DESCRIPTIONZ_STOPCHANNEL_DESCRIPTIONri � PARALLEL_UPLOAD_TEMP_NAMESPACE� _DESCRIPTIONrh r� r� r� r� r� r� r� r r$ r$ r$ r% �<module> s� ������ x" � � � �����' ������ commands/__pycache__/lifecycle.cpython-39.pyc 0000644 00000014576 15025013616 0015161 0 ustar 00 a +(Wg�" � @ s0 d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dZ dZ e e �!d� d Z"dZ#dZ$de# e$ d Z%ee"e%�Z&ee e#�Z'ee e$�Z(G dd� de �Z)dS ) zBImplementation of lifecycle configuration command for GCS buckets.� )�absolute_import)�print_function)�division)�unicode_literalsN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�UrlsAreForSingleProvider)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)�LifecycleTranslationz+ gsutil lifecycle get gs://<bucket_name> zA gsutil lifecycle set <config-json-file> gs://<bucket_name>... � aP <B>GET</B> Gets the lifecycle management configuration for a given bucket. You can get the lifecycle management configuration for only one bucket at a time. To update the configuration, you can redirect the output of the ``get`` command into a file, edit the file, and then set it on the bucket using the ``set`` sub-command. z� <B>SET</B> Sets the lifecycle management configuration on one or more buckets. The ``config-json-file`` specified on the command line should be a path to a local file containing the lifecycle configuration JSON document. aG You can use the ``lifecycle`` command to get or set lifecycle management policies for a given bucket. This command is supported for buckets only, not objects. For more information, see `Object Lifecycle Management <https://cloud.google.com/storage/docs/lifecycle>`_. The ``lifecycle`` command has two sub-commands: a� <B>EXAMPLES</B> The following lifecycle management configuration JSON document specifies that all objects in this bucket that are more than 365 days old are deleted automatically: { "rule": [ { "action": {"type": "Delete"}, "condition": {"age": 365} } ] } The following empty lifecycle management configuration JSON document removes all lifecycle configuration for a bucket: {} c s� e Zd ZdZejddgededdddej ej gej e � d�e � � ge �d�gd �d � Zejdd d gd deeed�d�Z� fdd�Zdd� Zdd� Zdd� Z� ZS )�LifecycleCommandz+Implementation of gsutil lifecycle command.� lifecycleZlifecycleconfig� � TF� )�set�get) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_argumentsZ getlifecycleZ setlifecycle� command_helpz/Get or set lifecycle configuration for a bucket)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s� | j d dkrZtdtdddd�| j d �g| j dd � i d �ii d �}| j d d� | _ n td tg d �i d �ii d �}t� �|�S ) Nr r �storage�buckets�updatez--lifecycle-file={}r r )�gcloud_command�flag_mapr )r, r- �describezo--format=gsutiljson[key=lifecycle_config,empty=' has no lifecycle configuration.',empty_prefix_key=storage_url])�argsr �format�super�get_gcloud_storage_args)�self�gcloud_storage_map�� __class__� �'/opt/gsutil/gslib/commands/lifecycle.pyr5 � s6 � ��� ��� �z(LifecycleCommand.get_gcloud_storage_argsc C s� | j d }| j dd� }t|�s.td| j ��t|d�}|�� }|�� d}|D ]�}| j|dgd�}|D ]l}|j} d }| j � d |� | j d kr�| j j || | j d � qht�|�} tj| d �} | j j| j| | j dgd� qhqP|s�ttt|� ��dS )z?Sets lifecycle configuration for a Google Cloud Storage bucket.r r Nz,"%s" command spanning providers not allowed.�rFr �� bucket_fieldsTz(Setting lifecycle configuration on %s...�s3��provider)r �id)rA �fields)r2 r r � command_name�open�read�close�GetBucketUrlIterFromArg� storage_url�logger�info�scheme� gsutil_api�XmlPassThroughSetLifecycler �JsonLifecycleToMessage�apitools_messages�Bucket� PatchBucket� bucket_namer �list) r6 Z lifecycle_arg�url_argsZlifecycle_fileZ lifecycle_txt� some_matched�url_str� bucket_iter�blr�urlr �bucket_metadatar: r: r; �_SetLifecycleConfig� sB � � � � z$LifecycleCommand._SetLifecycleConfigc C sx | j | jd dgd�\}}|jdkr@tj�| jj||jd�� n4|jrd|jj rdtj�t � |j�� ntj�d| � dS )z?Gets lifecycle configuration for a Google Cloud Storage bucket.r r r= r? r@ z#%s has no lifecycle configuration. ) �GetSingleBucketUrlFromArgr2 rL �sys�stdout�writerM �XmlPassThroughGetLifecycler �ruler �JsonLifecycleFromMessage)r6 � bucket_urlr[ r: r: r; �_GetLifecycleConfig� s" � ����z$LifecycleCommand._GetLifecycleConfigc C s^ | j �d�}|dkr*tj|gd� | �� S |dkrHtj|gd� | �� S td|| jf ��dS )z.Command entry point for the lifecycle command.r r )� subcommandsr z+Invalid subcommand "%s" for the %s command.N)r2 �popr �LogCommandParamsre r\ r rD )r6 � subcommandr: r: r; � RunCommand� s �zLifecycleCommand.RunCommand)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �JSON�XMLr �MakeNFileURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� MakeNCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr5 r\ re rj � __classcell__r: r: r8 r; r c sJ �� ����� &&r )*rn � __future__r r r r r^ �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr rP �gslib.utils.constantsr �gslib.utils.shim_utilr � gslib.utils.translation_helperr � _GET_SYNOPSIS� _SET_SYNOPSIS�lstriprp �_GET_DESCRIPTION�_SET_DESCRIPTION� _DESCRIPTIONrx ry rz r r: r: r: r; �<module> sB ��� commands/__pycache__/rsync.cpython-39.pyc 0000644 00000153561 15025013616 0014356 0 ustar 00 a +(Wg> � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl m Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlmZ dd lmZ ddlZdd lmZ dd l m Z dd l m Z dd l m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z1 ddl0m2Z2 ddl3m4Z4 ddl3m5Z5 ddl3m6Z6 ddl7m8Z9 dd l:m;Z; dd l:m<Z< dd l:m=Z= dd l>m?Z? dd l@mAZA dd!lBmCZC dd"lBmDZD dd#lBmEZE dd$lBmFZF dd%lGmHZH dd&lGmIZI dd'lGmJZJ dd(lGmKZK dd)lLmMZM dd*lLmNZN dd+lLmOZO dd,lPmQZQ dd-lPmRZR dd.lPmSZS dd/lPmTZT dd0lPmUZU dd1lPmVZV dd2lPmWZW dd3lPmXZX dd4lPmYZY dd5lPmZZZ dd6lPm[Z[ dd7lPm\Z\ dd8lPm]Z] dd9lPm^Z^ dd:lPm_Z_ dd;lPm`Z` dd<lPmaZa dd=lPmbZb dd>lPmcZc dd?lPmdZd dd@lemfZf ddAlemgZg ddBlhmiZi ddClhmjZj ddDlkmlZl ddElmmnZn ddFlompZp ddGlomqZq ddHlomrZr ddIlsmtZt eju�r�evZwdJZxdKex dL ZydMZzdNZ{dOZ|g Z}dPdQ� Z~dRdS� ZdTdU� Z�dVdW� Z�dtdXdY�Z�dZd[� Z�d\d]� Z�d^d_� Z�d`da� Z�dbdc� Z�ddde� Z�G dfdg� dge��Z�G dhdi� die��Z�G djdk� dke��Z�dudldm�Z�dndo� Z�dpdq� Z�G drds� dse �Z�dS )vz*Implementation of Unix-like rsync command.� )�absolute_import)�print_function)�division)�unicode_literalsN)�islice)�urllib)�config)�BucketListingObject)�NotFoundException)�ServiceException)�Command)�DummyArgChecker)�'ShimTranslatePredefinedAclSubOptForCopy)�CommandArgument)� ApiSelector)�CommandException)�LogPerformanceSummaryParams)�PluralityCheckableIterator)�SeekAheadResult)�GetCaughtSignals)�RegisterSignalHandler)�GenerationFromUrlAndString)�IsCloudSubdirPlaceholder)�StorageUrlFromString)�storage_v1_messages)� constants)� copy_helper)�parallelism_framework_util)�UsingCrcmodExtension)�GetCloudApiInstance)�CreateCopyHelperOpts)� GetSourceFieldsNeededForCopy)�GZIP_ALL_FILES)�SkipUnsupportedObjectError)�%CalculateB64EncodedCrc32cFromContents)�"CalculateB64EncodedMd5FromContents)�SLOW_CRCMOD_RSYNC_WARNING)�SLOW_CRCMOD_WARNING)�CreateCustomMetadata)� GetValueFromObjectCustomMetadata)�ObjectIsGzipEncoded)� ATIME_ATTR)�ConvertDatetimeToPOSIX)�ConvertModeToBase8)�+DeserializeFileAttributesFromObjectMetadata)�GID_ATTR)�InitializePreservePosixData)� MODE_ATTR)� MTIME_ATTR)�NA_ID)�NA_MODE)�NA_TIME)�NeedsPOSIXAttributeUpdate)�ParseAndSetPOSIXAttributes)�POSIXAttributes)�'SerializeFileAttributesToObjectMetadata)�UID_ATTR)� ValidateFilePermissionAccess)�WarnFutureTimestamp)�WarnInvalidValue)�WarnNegativeAttribute)� DiffAction)�RsyncDiffToApply)�GcloudStorageFlag)�GcloudStorageMap)� IS_WINDOWS)�CopyCustomMetadata)�CalculateThroughput)�SECONDS_PER_DAY)�TEN_MIB)�CreateWildcardIteratorz, gsutil rsync [OPTION]... src_url dst_url z <B>SYNOPSIS</B> a S <B>DESCRIPTION</B> The gsutil rsync command makes the contents under dst_url the same as the contents under src_url, by copying any missing files/objects (or those whose data has changed), and (if the -d option is specified) deleting any extra files/objects. src_url must specify a directory, bucket, or bucket subdirectory. For example, to sync the contents of the local directory "data" to the bucket gs://mybucket/data, you could do: gsutil rsync data gs://mybucket/data To recurse into directories use the -r option: gsutil rsync -r data gs://mybucket/data If you have a large number of objects to synchronize you might want to use the gsutil -m option (see "gsutil help options"), to perform parallel (multi-threaded/multi-processing) synchronization: gsutil -m rsync -r data gs://mybucket/data The -m option typically will provide a large performance boost if either the source or destination (or both) is a cloud URL. If both source and destination are file URLs the -m option will typically thrash the disk and slow synchronization down. Note 1: Shells (like bash, zsh) sometimes attempt to expand wildcards in ways that can be surprising. Also, attempting to copy files whose names contain wildcard characters can result in problems. For more details about these issues see `Wildcard behavior considerations <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. Note 2: If you are synchronizing a large amount of data between clouds you might consider setting up a `Google Compute Engine <https://cloud.google.com/products/compute-engine>`_ account and running gsutil there. Since cross-provider gsutil data transfers flow through the machine where gsutil is running, doing this can make your transfer run significantly faster than running gsutil on your local workstation. Note 3: rsync does not copy empty directory trees, since Cloud Storage uses a `flat namespace <https://cloud.google.com/storage/docs/folders>`_. <B>Using -d Option (with caution!) to mirror source and destination.</B> The rsync -d option is very useful and commonly used, because it provides a means of making the contents of a destination bucket or directory match those of a source bucket or directory. This is done by copying all data from the source to the destination and deleting all other data in the destination that is not in the source. Please exercise caution when you use this option: It's possible to delete large amounts of data accidentally if, for example, you erroneously reverse source and destination. To make the local directory my-data the same as the contents of gs://mybucket/data and delete objects in the local directory that are not in gs://mybucket/data: gsutil rsync -d -r gs://mybucket/data my-data To make the contents of gs://mybucket2 the same as gs://mybucket1 and delete objects in gs://mybucket2 that are not in gs://mybucket1: gsutil rsync -d -r gs://mybucket1 gs://mybucket2 You can also mirror data across local directories. This example will copy all objects from dir1 into dir2 and delete all objects in dir2 which are not in dir1: gsutil rsync -d -r dir1 dir2 To mirror your content across clouds: gsutil rsync -d -r gs://my-gs-bucket s3://my-s3-bucket Change detection works if the other Cloud provider is using md5 or CRC32. AWS multipart upload has an incompatible checksum. As mentioned above, using -d can be dangerous because of how quickly data can be deleted. For example, if you meant to synchronize a local directory from a bucket in the cloud but instead run the command: gsutil -m rsync -r -d ./your-dir gs://your-bucket and your-dir is currently empty, you will quickly delete all of the objects in gs://your-bucket. You can also cause large amounts of data to be lost quickly by specifying a subdirectory of the destination as the source of an rsync. For example, the command: gsutil -m rsync -r -d gs://your-bucket/data gs://your-bucket would cause most or all of the objects in gs://your-bucket to be deleted (some objects may survive if there are any with names that sort lower than "data" under gs://your-bucket/data). In addition to paying careful attention to the source and destination you specify with the rsync command, there are two more safety measures you can take when using gsutil rsync -d: 1. Try running the command with the rsync -n option first, to see what it would do without actually performing the operations. For example, if you run the command: gsutil -m rsync -r -d -n gs://your-bucket/data gs://your-bucket it will be immediately evident that running that command without the -n option would cause many objects to be deleted. 2. Enable object versioning in your bucket, which allows you to restore objects if you accidentally delete them. For more details see `Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_. <B>BE CAREFUL WHEN SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SYMLINKS, DEVICES, ETC.)</B> Running gsutil rsync over a directory containing operating system-specific file types (symbolic links, device files, sockets, named pipes, etc.) can cause various problems. For example, running a command like: gsutil rsync -r ./dir gs://my-bucket will cause gsutil to follow any symbolic links in ./dir, creating objects in my-bucket containing the data from the files to which the symlinks point. This can cause various problems: * If you use gsutil rsync as a simple way to backup a directory to a bucket, restoring from that bucket will result in files where the symlinks used to be. At best this is wasteful of space, and at worst it can result in outdated data or broken applications -- depending on what is consuming the symlinks. * If you use gsutil rsync over directories containing broken symlinks, gsutil rsync will abort (unless you pass the -e option). * gsutil rsync skips symlinks that point to directories. Since gsutil rsync is intended to support data operations (like moving a data set to the cloud for computational processing) and it needs to be compatible both in the cloud and across common operating systems, there are no plans for gsutil rsync to support operating system-specific file types like symlinks. We recommend that users do one of the following: * Don't use gsutil rsync over directories containing symlinks or other OS- specific file types. * Use the -e option to exclude symlinks or the -x option to exclude OS-specific file types by name. * Use a tool (such as tar) that preserves symlinks and other OS-specific file types, packaging up directories containing such files before uploading to the cloud. <B>EVENTUAL CONSISTENCY WITH NON-GOOGLE CLOUD PROVIDERS</B> While Google Cloud Storage is strongly consistent, some cloud providers only support eventual consistency. You may encounter scenarios where rsync synchronizes using stale listing data when working with these other cloud providers. For example, if you run rsync immediately after uploading an object to an eventually consistent cloud provider, the added object may not yet appear in the provider's listing. Consequently, rsync will miss adding the object to the destination. If this happens you can rerun the rsync operation again later (after the object listing has "caught up"). <B>FAILURE HANDLING</B> The rsync command retries failures when it is useful to do so, but if enough failures happen during a particular copy or delete operation, or if a failure isn't retryable, the overall command fails. If the -C option is provided, the command instead skips failing objects and moves on. At the end of the synchronization run, if any failures were not successfully retried, the rsync command reports the count of failures and exits with non-zero status. At this point you can run the rsync command again, and gsutil attempts any remaining needed copy and/or delete operations. For more details about gsutil's retry handling, see `Retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_. <B>CHANGE DETECTION ALGORITHM</B> To determine if a file or object has changed, gsutil rsync first checks whether the file modification time (mtime) of both the source and destination is available. If mtime is available at both source and destination, and the destination mtime is different than the source, or if the source and destination file size differ, gsutil rsync will update the destination. If the source is a cloud bucket and the destination is a local file system, and if mtime is not available for the source, gsutil rsync will use the time created for the cloud object as a substitute for mtime. Otherwise, if mtime is not available for either the source or the destination, gsutil rsync will fall back to using checksums. If the source and destination are both cloud buckets with checksums available, gsutil rsync will use these hashes instead of mtime. However, gsutil rsync will still update mtime at the destination if it is not present. If the source and destination have matching checksums and only the source has an mtime, gsutil rsync will copy the mtime to the destination. If neither mtime nor checksums are available, gsutil rsync will resort to comparing file sizes. Checksums will not be available when comparing composite Google Cloud Storage objects with objects at a cloud provider that does not support CRC32C (which is the only checksum available for composite objects). See 'gsutil help compose' for details about composite objects. <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> If both the source and destination URL are cloud URLs from the same provider, gsutil copies data "in the cloud" (i.e., without downloading to and uploading from the machine where you run gsutil). In addition to the performance and cost advantages of doing this, copying in the cloud preserves metadata (like Content-Type and Cache-Control). In contrast, when you download data from the cloud it ends up in a file, which has no associated metadata, other than file modification time (mtime). Thus, unless you have some way to hold on to or re-create that metadata, synchronizing a bucket to a directory in the local file system will not retain the metadata other than mtime. Note that by default, the gsutil rsync command does not copy the ACLs of objects being synchronized and instead will use the default bucket ACL (see "gsutil help defacl"). You can override this behavior with the -p option. See the `Options section <https://cloud.google.com/storage/docs/gsutil/commands/rsync#options>`_ to learn how. <B>LIMITATIONS</B> 1. The gsutil rsync command will only allow non-negative file modification times to be used in its comparisons. This means gsutil rsync will resort to using checksums for any file with a timestamp before 1970-01-01 UTC. 2. The gsutil rsync command considers only the live object version in the source and destination buckets when deciding what to copy / delete. If versioning is enabled in the destination bucket then gsutil rsync's replacing or deleting objects will end up creating versions, but the command doesn't try to make any noncurrent versions match in the source and destination buckets. 3. The gsutil rsync command does not support copying special file types such as sockets, device files, named pipes, or any other non-standard files intended to represent an operating system resource. If you run gsutil rsync on a source directory that includes such files (for example, copying the root directory on Linux that includes /dev ), you should use the -x flag to exclude these files. Otherwise, gsutil rsync may fail or hang. 4. The gsutil rsync command copies changed files in their entirety and does not employ the `rsync delta-transfer algorithm <https://rsync.samba.org/tech_report/>`_ to transfer portions of a changed file. This is because Cloud Storage objects are immutable and no facility exists to read partial object checksums or perform partial replacements. <B>OPTIONS</B> -a predef-acl Sets the specified predefined ACL on uploaded objects. See "gsutil help acls" for further details. Note that rsync will decide whether or not to perform a copy based only on object size and modification time, not current ACL state. Also see the -p option below. -c Causes the rsync command to compute and compare checksums (instead of comparing mtime) for files if the size of source and destination match. This option increases local disk I/O and run time if either src_url or dst_url are on the local file system. -C If an error occurs, continue to attempt to copy the remaining files. If errors occurred, gsutil's exit status will be non-zero even if this flag is set. This option is implicitly set when running "gsutil -m rsync...". NOTE: -C only applies to the actual copying operation. If an error occurs while iterating over the files in the local directory (e.g., invalid Unicode file name) gsutil will print an error message and abort. -d Delete extra files under dst_url not found under src_url. By default extra files are not deleted. NOTE: this option can delete data quickly if you specify the wrong source/destination combination. See the help section above, "BE CAREFUL WHEN USING -d OPTION!". -e Exclude symlinks. When specified, symbolic links will be ignored. Note that gsutil does not follow directory symlinks, regardless of whether -e is specified. -i Skip copying any files that already exist at the destination, regardless of their modification time. -j <ext,...> Applies gzip transport encoding to any file upload whose extension matches the -j extension list. This is useful when uploading files with compressible content (such as .js, .css, or .html files) because it saves network bandwidth while also leaving the data uncompressed in Google Cloud Storage. When you specify the -j option, files being uploaded are compressed in-memory and on-the-wire only. Both the local files and Cloud Storage objects remain uncompressed. The uploaded objects retain the Content-Type and name of the original files. Note that if you want to use the top-level -m option to parallelize copies along with the -j/-J options, your performance may be bottlenecked by the "max_upload_compression_buffer_size" boto config option, which is set to 2 GiB by default. This compression buffer size can be changed to a higher limit, e.g.: gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \ -m rsync -j html,txt /local/source/dir gs://bucket/path -J Applies gzip transport encoding to file uploads. This option works like the -j option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If you use this option and some of the source files don't compress well (e.g., that's often true of binary data), this option may result in longer uploads. -n Causes rsync to run in "dry run" mode, i.e., just outputting what would be copied or deleted without actually doing any copying/deleting. -p Causes ACLs to be preserved when objects are copied. Note that rsync will decide whether or not to perform a copy based only on object size and modification time, not current ACL state. Thus, if the source and destination differ in size or modification time and you run gsutil rsync -p, the file will be copied and ACL preserved. However, if the source and destination don't differ in size or checksum but have different ACLs, running gsutil rsync -p will have no effect. Note that this option has performance and cost implications when using the XML API, as it requires separate HTTP calls for interacting with ACLs. The performance issue can be mitigated to some degree by using gsutil -m rsync to cause parallel synchronization. Also, this option only works if you have OWNER access to all of the objects that are copied. You can avoid the additional performance and cost of using rsync -p if you want all objects in the destination bucket to end up with the same ACL by setting a default object ACL on that bucket instead of using rsync -p. See 'gsutil help defacl'. -P Causes POSIX attributes to be preserved when objects are copied. With this feature enabled, gsutil rsync will copy fields provided by stat. These are the user ID of the owner, the group ID of the owning group, the mode (permissions) of the file, and the access/modification timestamps of the file. For downloads, these attributes will only be set if the source objects were uploaded with this flag enabled. On Windows, this flag will only set and restore access time and modification time. This is because Windows doesn't have a notion of POSIX uid/gid/mode. -R, -r The -R and -r options are synonymous. Causes directories, buckets, and bucket subdirectories to be synchronized recursively. If you neglect to use this option gsutil will make only the top-level directory in the source and destination URLs match, skipping any sub-directories. -u When a file/object is present in both the source and destination, if mtime is available for both, do not perform the copy if the destination mtime is newer. -U Skip objects with unsupported object types instead of failing. Unsupported object types are Amazon S3 Objects in the GLACIER storage class. -x pattern Causes files/objects matching pattern to be excluded, i.e., any matching files/objects are not copied or deleted. Note that the pattern is a `Python regular expression <https://docs.python.org/3/howto/regex.html>`_, not a wildcard (so, matching any string ending in "abc" would be specified using ".*abc$" rather than "*abc"). Note also that the exclude path is always relative (similar to Unix rsync or tar exclude options). For example, if you run the command: gsutil rsync -x "data.[/\].*\.txt$" dir gs://my-bucket it skips the file dir/data1/a.txt. You can use regex alternation to specify multiple exclusions, for example: gsutil rsync -x ".*\.txt$|.*\.jpg$" dir gs://my-bucket skips all .txt and .jpg files in dir. NOTE: When using the Windows cmd.exe command line interpreter, use ``^`` as an escape character instead of ``\`` and escape the ``|`` character. When using Windows PowerShell, use ``'`` instead of ``"`` and surround the ``|`` character with ``"``. -y pattern Similar to the -x option, but the command will first skip directories/prefixes using the provided pattern and then exclude files/objects using the same pattern. This is usually much faster, but won't work as intended with negative lookahead patterns. For example, if you run the command: gsutil rsync -y "^(?!.*\.txt$).*" dir gs://my-bucket This would first exclude all subdirectories unless they end in .txt before excluding all files except those ending in .txt. Running the same command with the -x option would result in all .txt files being included, regardless of whether they appear in subdirectories that end in .txt. �-i i' c C s t � dS )zDCalled when rsync command is killed with SIGINT, SIGQUIT or SIGTERM.N)�CleanUpTempFiles)� signal_num�cur_stack_frame� rM �#/opt/gsutil/gslib/commands/rsync.py�_HandleSignals s rO c C sz t D ]} | js| �� qt D ]X} tj�| j�r zt�| j� W q tyr } z t � d| j|� W Y d}~q d}~0 0 q dS )a{ Cleans up temp files. This function allows the main (RunCommand) function to clean up at end of operation, or if gsutil rsync is interrupted (e.g., via ^C). This is necessary because tempfile.NamedTemporaryFile doesn't allow the created file to be re-opened in read mode on Windows, so we have to use tempfile.mkstemp, which doesn't automatically delete temp files. z;Failed to close and delete temp file "%s". Got an error: %sN) � _tmp_files�closed�close�os�path�isfile�name�unlink� Exception�logging�debug)�fileobj�erM rM rN rJ # s �rJ c C sL |j tjks| jsdS t|j�}|�� rHtj� |j �rH| j � d|� dS dS )z5Arg checker that skips symlinks if -e flag specified.Tz Skipping symbolic link %s...F) � diff_actionr? �REMOVE�exclude_symlinksr � src_url_str� IsFileUrlrS rT �islink� object_name�logger�info)�command_instance� diff_to_apply� exp_src_urlrM rM rN �_DiffToApplyArgChecker@ s � ri c C s� t |�} t |�} | �� r�|tks(| �� rr|tkr<| �d|� t| jd��} t| �}W d � q�1 sf0 Y nX|tks�| �� r�|tkr�| �d|� t| jd��} t| �}W d � n1 s�0 Y | �� �r�|tk�r*|tkr�| �d|� t| jd��} t| �}W d � n1 �s 0 Y nV|tk�r�|tk�rJ| �d|� t| jd��} t| �}W d � n1 �sv0 Y ||||fS )a� Computes any file checksums needed by _CompareObjects. Args: logger: logging.logger for outputting log messages. src_url_str: Source URL string. src_size: Source size src_crc32c: Source CRC32c. src_md5: Source MD5. dst_url_str: Destination URL string. dst_size: Destination size dst_crc32c: Destination CRC32c. dst_md5: Destination MD5. Returns: (src_crc32c, src_md5, dst_crc32c, dst_md5) zComputing CRC32C for %s...�rbNzComputing MD5 for %s...) r ra �_NArG re �openrc r$ r% ) rd r` �src_size� src_crc32c�src_md5� dst_url_str�dst_size� dst_crc32c�dst_md5�src_url�dst_url�fprM rM rN �_ComputeNeededFileChecksumsM s2 ( & * (rw c C s� t | |d�}|\}}}tj|dtjd�}ztt| |||�|� W nN ty� } z6| j� d||f � | j� t � � � d| _ W Y d}~n d}~0 0 |�� dS )ae Worker function for listing files/objects under to be sync'd. Outputs sorted list to out_file_name, formatted per _BuildTmpOutputLine. We sort the listed URLs because we don't want to depend on consistent sort order across file systems and cloud providers. Args: cls: Command instance. args_tuple: (base_url_str, out_file_name, desc), where base_url_str is top-level URL string to list; out_filename is name of file to which sorted output should be written; desc is 'source' or 'destination'. thread_state: gsutil Cloud API instance to use. �� thread_state�w��mode�encodingz3Caught non-retryable exception while listing %s: %s� N)r �iorl r �UTF8� _BatchSort�_FieldedListingIteratorrX rd �errorrZ � traceback� format_exc� non_retryable_listing_failuresrR ) �clsZ args_tuplery � gsutil_api� base_url_strZ out_filename�desc�out_filer\ rM rM rN �_ListUrlRootFunc{ s �� r� c c sB t �| j�D ]0}t j�| j|�}t j�|�r tt|�d�V q dS )a� A generator that yields a BLR for each file in a local directory. We use this function instead of WildcardIterator for listing a local directory without recursion, because the glob.globi implementation called by WildcardIterator skips "dot" files (which we don't want to do when synchronizing to or from a local directory). Args: base_url: URL for the directory over which to iterate. Yields: BucketListingObject for each file in the directory. N)rS �listdirrc rT �joinrU r r )�base_url�filenamerM rM rN �_LocalDirIterator� s r� c c sz t |�}|jdkr"| js"t|�}n�| jr8d|�d� }nd|�d� }ddddd d t g}| jr�|�d t d t d t d t g� | j d ur�|| j | j fnd }t||| j|| j| jd �j|d �}d} |D ]�} | j} t| | d�r�q�| jr�| �� r�tj�| j�r�q�| j �rD| jt|j�d � } | � | j ��r4| dd � } | j � | ��rDq�| d7 } | t dk�rj| j� d|| � t!| �V q�d S )aV Iterator over base_url_str formatting output per _BuildTmpOutputLine. Args: cls: Command instance. gsutil_api: gsutil Cloud API instance to use for bucket listing. base_url_str: The top-level URL string over which to iterate. desc: 'source' or 'destination'. Yields: Output line formatted per _BuildTmpOutputLine. �filez%s/**�/\z%s/*�crc32c�md5HashrV �size� timeCreatedz metadata/%sN)� project_id� exclude_tuple�ignore_symlinksrd )�bucket_listing_fieldsr )�blrr~ zAt %s listing %d...)"r �scheme�recursion_requestedr� �rstripr2 �preserve_posix_attrs�extendr+ r1 r/ r: �exclude_pattern� exclude_dirsrH r� r_ rd � IterObjects� storage_urlr ra rS rT rb rc � url_string�len� startswith�delim�match� _PROGRESS_REPORT_LISTING_COUNTre �_BuildTmpOutputLine) r� r� r� r� r� �iterator�wildcard�fieldsr� �ir� �url� str_to_checkrM rM rN r� � sp ������� � r� c C s� t }t}t}t}t}t }t }t}| j} | �� r�t�| j�\ }} } } }}} }}} t |�}t |�}t |�}|dk rrt }|dk r~t }n�| � � �rn| j j } | j jdu�rDt| j tt �\} } zDt | �}| r�|t kr�td| j� |t t�� �t kr�td| j� W n$ t�y td| j� t }Y n0 t| j | j�}|jj}|j}|j }|j }t | j j �}| j j �p\t}| j j!�pjt}nt"d| j# ��t$| j�| ||||||||g }dd� |D �}d�%|�d S ) a� Builds line to output to temp file for given BucketListingRef. Args: blr: The BucketListingRef. Returns: The output line, formatted as _EncodeUrl(URL)<sp>size<sp>time_created<sp>atime<sp>mtime<sp>mode<sp>uid<sp> gid<sp>crc32c<sp>md5 where md5 will only be present for cloud URLs that aren't composite objects. A missing field is populated with '-', or -1 in the case of atime/mtime/time_created. r N�mtimez Got unexpected URL type (%s)c S s g | ]}t �t|���qS rM )�six� ensure_text�str)�.0r� rM rM rN � <listcomp>R � z'_BuildTmpOutputLine.<locals>.<listcomp>� � )&r5 rk r3 r4 r� ra rS �statrc �longr- � IsCloudUrl� root_objectr� �metadatar) r2 r> r� �timerF r< � ValueErrorr= r. r| � permissions�atime�uid�gidr, r� r� r� r r� � _EncodeUrlr� )r� r� r� r� �md5r| r� � time_createdr� r� �_r� Zfound_mZ mtime_str� posix_attrs�attrsrM rM rN r� sr � � � r� c C s t �| �} tjj| dd�S )z�Encodes url_str with quote plus encoding and UTF8 character encoding. We use this for all URL encodings. Args: url_string (unicode): String URL to encode. Returns: (str) A string encoded using urllib's `quote_plus()` method. � ~)�safe)r� � ensure_strr �parse� quote_plus)r� rM rM rN r� V s r� c C s" t j�| �}tjr |�tj�}|S )z�Inverts encoding from `_EncodeUrl()`. Args: enc_url_string (str): String containing UTF-8-decodable characters that were encoded using urllib's `quote_plus()`. Returns: (unicode) A decoded URL. )r r� � unquote_plusr� �PY2�decoder r� )Zenc_url_stringr� rM rM rN � _DecodeUrli s r� c C s� t �ddd�}g }�z"z|tt| |��}|s,q�tjd|jt|�f dtj d�}|� |� |� t � d�|��� |�� |�d� q|�tj|� � W nF ty� } z.|jtjkr�td �t�d ����� W Y d }~n d }~0 0 W |D ]T}z|�� t� |j� W q� t �y2 } z t � d |j|� W Y d }~q�d }~0 0 q�n^|D ]V}z|�� t� |j� W n6 t �y� } z t � d |j|� W Y d }~n d }~0 0 �q<0 d S ) a� Sorts input lines from in_iter and outputs to out_file. Sorts in batches as input arrives, so input file does not need to be loaded into memory all at once. Derived from Python Recipe 466302: Sorting big files the Python 2.4 way by Nicolas Lehuen. Sorted format is per _BuildTmpOutputLine. We're sorting on the entire line when we could just sort on the first record (URL); but the sort order is identical either way. Args: in_iter: Input iterator. out_file: Output file. �GSUtil�rsync_buffer_linesi } z%s-%06izw+r{ � r r� z�Synchronization failed because too many open file handles were needed while building synchronization state. Please see the comments about rsync_buffer_lines in your .boto config file for a possible way to address this problem.Nz8Failed to remove rsync chunk file "%s". Got an error: %s) r �getint�sortedr r rl rV r� r r� �append�writer� � text_typer� �flush�seek� writelines�heapq�merge�IOError�errnoZEMFILEr �textwrap�wraprR rS �removerX rY rZ )Zin_iterr� � buffer_sizeZ chunk_filesZ current_chunkZ output_chunkr\ Z chunk_filerM rM rN r� z sR � ��� ��r� c @ s@ e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dS )� _DiffIteratorz7Iterator yielding sequence of RsyncDiffToApply objects.c C s� || _ |j| _|j| _|j| _| j j| _|| _|| _|j| _|j | _ |j | _ | j� d� t j ddd�}t j ddd�}|j| _|j| _t�|� t�|� |�� |�� t| jj| jdf| jj| jdfg�}d|_d g}|jt|t|t|jj d d � |j�rt d ��t | jd �| _ t | jd �| _ t�| j � t�| j � |�!� �rx|�"� �rx| j�rxt#t| j ��| _$| �%� | j �&d� t#t| j ��| _$t#t| j ��| _'d S )Nz!Building synchronization state...zgsutil-rsync-src-F)�prefix�deletezgsutil-rsync-dst-�source� destinationr r� T)� arg_checker� parallel_operations_override� fail_on_errorz/Caught non-retryable exception - aborting rsync�r)(� command_obj�compute_file_checksums� delete_extrasr� rd � base_src_url� base_dst_urlr� �preserve_posix�skip_old_files�ignore_existingre �tempfile�NamedTemporaryFilerV �sorted_list_src_file_name�sorted_list_dst_file_namerP r� rR �iterr� r� �Applyr� � _RootListingExceptionHandlerr �ParallelOverrideReason�SPEEDr rl �sorted_list_src_file�sorted_list_dst_filer� ra r �sorted_src_urls_it�_ValidateObjectAccessr� �sorted_dst_urls_it)�selfr� r� r� Z temp_src_fileZ temp_dst_fileZ args_iter� shared_attrsrM rM rN �__init__� s� �� ���� �� ��z_DiffIterator.__init__c C sz t �� }| jD ]D}| �|�\ }}}}}}}}}}t||||d�\}} |s|�| � q|rv|D ]} | j�| � q\td��dS )a Validates that the user won't lose access to the files if copied. Iterates over the src file list to check if access will be maintained. If at any point we would orphan a file, a list of errors is compiled and logged with an exception raised to the user. )r� r� r| zPThis sync will orphan file(s), please fix their permissions before trying again.N) � collections�dequer �_ParseTmpFileLiner; r� rd �criticalr ) r �errorsrt r` r� �src_mode�src_uid�src_gid�valid�errrM rM rN r s �� z#_DiffIterator._ValidateObjectAccessc C s\ |� dd�\ }}}}}}}} } } t|�t|�t|�t|�t|�t|�t|�t| �| | �� f S )a� Parses output from _BuildTmpOutputLine. Parses into tuple: (URL, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) where crc32c and/or md5 can be _NA and atime/mtime/time_created can be NA_TIME. Args: line: The line to parse. Returns: Parsed tuple: (url, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) N� )�rsplitr� �intr� �strip) r �lineZ encoded_urlr� r� r� r� r| r� r� r� r� rM rM rN r ! s ��z _DiffIterator._ParseTmpFileLinec C s2 t |��� r.|tkr.|tkr.| j�d|� dS dS )z�Warns if given url_str is a cloud URL and is missing both crc32c and md5. Args: url_str: Destination URL string. crc32c: Destination CRC32c. md5: Destination MD5. Returns: True if issued warning. zKFound no hashes to validate %s. Integrity cannot be assured without hashes.TF)r r� rk rd �warn)r �url_strr� r� rM rM rN �_WarnIfMissingCloudHash? s ��z%_DiffIterator._WarnIfMissingCloudHashc C s< |t k} |t k} | jp,t|��� o,t|��� } | jr>d| | fS | jr^| r^| r^||k r^d| | fS | s�| r�| r�||kpx||k| | fS ||kr�d| | fS t| j||||||| | � \}}} } |tkr�| tkr�| j� d||� || k| | fS |tk�r| tk�r| j� d||� || k| | fS | � |||��s2| � || | � d| | fS )ai Returns whether src should replace dst object, and if mtime is present. Uses mtime, size, or whatever checksums are available. Args: src_url_str: Source URL string. src_size: Source size. src_mtime: Source modification time. src_crc32c: Source CRC32c. src_md5: Source MD5. dst_url_str: Destination URL string. dst_size: Destination size. dst_mtime: Destination modification time. dst_crc32c: Destination CRC32c. dst_md5: Destination MD5. Returns: A 3-tuple indicating if src should replace dst, and if src and dst have mtime. FTzComparing md5 for %s and %sz Comparing crc32c for %s and %s) r5 r� r r� r� r� rw rd rk rZ r )r r` rm � src_mtimern ro rp rq � dst_mtimerr rs � has_src_mtime� has_dst_mtimeZ use_hashesrM rM rN �_CompareObjectsT sP * � � � � �z _DiffIterator._CompareObjectsc ! c s� t | jj�d��}t | jj�d��}d}d }}|du r�| j�� rHd}nt| �t| j��\ }}}}} } } } } }t || | | | d�}t ||d� � dd��}t j | jt|�ddd| jd| jd�j}|du �r| j�� �s| �t| j��\ }}}}}}}}}}t ||d� � dd��}|�r�qH|du �s,||k �rFt|||tj|�V d}q0||k�rt| j�rntd|t � tjd�V d}q0t|��� �r�t|��� �r�| tk�r�|} | �||| | ||||||� \} } } | �r�t|||tj|�V nb| j�r t||| || || || |� \}} | �r>t|||tj |�V n | �r>| �s>t|||tj |�V d}d}q0| j�sTdS |�rptd|t � tjd�V | jD ]:} | �| �\ }}}}}}}}}}td|t � tjd�V �qvdS ) zrIterates over src/dst URLs and produces a RsyncDiffToApply sequence. Yields: The RsyncDiffToApply. r� FNT)r� r� r� r� r| �\�/)rt rh �src_url_names_container�have_multiple_srcs�has_multiple_top_level_srcs� exp_dst_url�have_existing_dest_subdirr� ) r� r� r� r� r� r �IsEmptyr �nextr8 r� �replacer �ConstructDstUrlr r� r r@ r? �COPYr� r^ r� ra r5 r! r� r6 �POSIX_SRC_TO_DST�MTIME_SRC_TO_DST)!r Zbase_src_url_lenZbase_dst_url_lenZout_of_src_itemsr` rp rm Zsrc_time_created� src_atimer r r r rn ro r� Zsrc_url_str_to_checkZdst_url_str_would_copy_torq r� � dst_atimer �dst_mode�dst_uid�dst_gidrr rs Zdst_url_str_to_checkZshould_replacer r Z needs_updater rM rM rN �__iter__� s� � ��� ��� � �� � � �� � � �z_DiffIterator.__iter__N) �__name__� __module__� __qualname__�__doc__r r r r r! r5 rM rM rM rN r� � s V Qr� c @ s e Zd ZdZdd� Zdd� ZdS )�_SeekAheadDiffIteratorzGWraps _AvoidChecksumAndListingDiffIterator and yields SeekAheadResults.c C s || _ d S )N)�cloned_diff_iterator)r r; rM rM rN r s z _SeekAheadDiffIterator.__init__c c sB | j D ]6}|jpd}|jtjks,|jtjkr0d}t|d�V qd S )Nr )� data_bytes)r; � copy_sizer] r? r/ r. r )r rg Z bytes_to_copyrM rM rN r5 s �z _SeekAheadDiffIterator.__iter__N)r6 r7 r8 r9 r r5 rM rM rM rN r: s r: c @ s e Zd ZdZdd� ZdS )�$_AvoidChecksumAndListingDiffIteratorz�Iterator initialized from an existing _DiffIterator. This iterator yields RsyncDiffToApply objects used to estimate the total work that will be performed by the DiffIterator, while avoiding expensive computation. c C s� d| _ |j| _|j| _d| _t�d�| _|j| _|j| _|j | _ |j | _ t |j d�| _ t |jd�| _t�| j � t�| j� tt| j ��| _tt| j��| _d S )NF�dummyr� )r� r� r� r� rY � getLoggerrd r� r� r� r� rl r� r r� r rP r� r r� r r )r Zinitialized_diff_iteratorrM rM rN r 7 s. �� ��z-_AvoidChecksumAndListingDiffIterator.__init__N)r6 r7 r8 r9 r rM rM rM rN r> . s r> c C s� t | |d�}|j}t|�}|j}|jtjkr�| jrB| j� d|� n|| j� d|� |� � r�zt � |j � W q� ty� | j�d|� Y q�0 n2z |j|j|j |j|jd� W n ty� Y n0 �n�|jtjk�r.|j}t|�}| j�rr|� � �r^z2t|j d��} W d � n1 �s0 Y W n: t�y\ } z | j� d|j � � W Y d} ~ n d} ~ 0 0 | j� d ||� �q|�z|d} |�� �r�t||j�} |j|j|j | |j| j d �} t | ��r,| j� d | � ndt � � } |j t!t"�"� �t# k�r�t$d |j%� |�&� �s|�'� �r,|�'� �rd nd} | j�(d| |j%� | j)�r<| j)}nt j j*g d�}t+||| j,d� t � � }||_)t-|| dd� t.j/| j|||| t0| | j1d| j2| j3| j,d� }|du�r�|\} }} } | j4� | j5|7 _5W d � n1 �s�0 Y W n8 t6�y( } z | j� d|| j7� W Y d} ~ n d} ~ 0 0 �nN|jtj8k�rJt|j�}| j�r^| j� d|� n�| j� d|j%� |j }t � � }t9t:|i�|_)|�� �r4t|j�}t||j�}z |j;|j|j ||j|jd� W nf t<�y0 } zL| j�d|� | j� d|j%� t=| t>|j|j|tj|j?�|d� W Y d}~n d}~0 0 nt@|j || j,d� �n2|jtjAk�rnt|j�}| j�rz| j� d|� n�| j� d |j%� t � � }t j j*g d�|_)t+||j)dd� |�� �r|t||j�}|j|j|j ||jd gd �}z |j;|j|j ||j|jd� W nf t<�yj } zL| j�d|� | j� d|j%� t=| t>|j|j|tj|j?�|d� W Y d}~n d}~0 0 ntBd |j ��dS ) zEWorker function for performing the actual copy and remove operations.rx zWould remove %sz Removing %sz%s was already removed)� generation�providerrj NzCould not open %szWould copy %s to %s)rA rB r� a� %s has a compressed content-encoding, so it will be decompressed upon download; future executions of gsutil rsync with this source object will always download it. If you wish to synchronize such an object efficiently, compress the source objects in place before synchronizing them, rather than (for example) using gsutil cp -Z to compress them on-the-fly (which results in compressed content-encoding).r� ZStreamsz Named pipeszlWARNING: %s are not supported by gsutil rsync and will likely fail. Use the -x option to exclude %s by name.)�additionalProperties)r� T)�override)�src_obj_metadata�headers�is_rsync� gzip_encoded� gzip_extsr� z0Skipping item %s with unsupported object type %szWould set mtime for %sz$Copying mtime from src to dst for %s)rB rA z Error while trying to patch: %szkCopying whole file/object for %s instead of patching because you don't have patch permission on the object.z!Would set POSIX attributes for %sz/Copying POSIX attributes from src to dst for %s�aclz Got unexpected DiffAction (%d))Cr rp r �src_posix_attrsr] r? r^ �dryrunrd re ra rS rW rc �FileNotFoundErrorrZ � DeleteObject� bucket_namerA r� r r- r` rl rX r� r �GetObjectMetadata�source_metadata_fieldsr* �apitools_messages�Objectr� r� r� rF r< r� �IsFifo�IsStreamr r� � MetadataValuer9 r� rD r � PerformCopy�_RsyncExceptionHandlerrF rH rI � stats_lock�total_bytes_transferredr# �unsupported_typer/ r( r2 �PatchObjectMetadatar � _RsyncFuncr@ r= r7 r. r )r� rg ry r� rp ru r� r` rt r� r\ rE �src_generation� type_text�custom_metadataZtmp_obj_metadataZ copy_result�bytes_transferredr� � obj_metadataZdst_generationr �dst_obj_metadatarM rM rN r] ^ sj � $ �� �� ���� 2� � � ��� � ��� �� ��� �r] c C s | j �t|�� dS )zDSimple exception handler for exceptions during listing URLs to sync.N)rd r� r� �r� r\ rM rM rN r ! s r c C s4 | j �t|�� | jd7 _| j �dt�� � dS )z9Simple exception handler to allow post-completion status.r~ z* Encountered exception while syncing: %s N)rd r� r� �op_failure_countrZ r� r� rd rM rM rN rX & s �rX c s� e Zd ZdZejdg eddddddejej gej e � d�gd� Z ej dd d gd d ei d �Z� fdd�Zdd� Zdd� Zdd� Z� ZS )� RsyncCommandz'Implementation of gsutil rsync command.�rsync� za:cCdenpPriRuUx:y:j:JTFr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments�sync� synchronize� command_helpz.Synchronize content of two buckets/directories)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s� t | j� ddg}dd� | jD �}d|vr@|dg7 }| j�d� d|v r�| jd d �\}\}}t|��� r�t|��� r�| j�d � t|t d �t d �t d�t d�t d�t d�t d�t d�t d�t d�t d�t d�t d�t d�t d�t d�d�d �}t � � |�S ) N�storagerg c S s g | ] \}}|�qS rM rM )r� �flagr� rM rM rN r� M r� z8RsyncCommand.get_gcloud_storage_args.<locals>.<listcomp>�-ez--no-ignore-symlinksz�By default, gsutil copies file symlinks, but, by default, this command (run via the gcloud storage shim) does not copy any symlinks.�-PF)� should_update_sub_opts_and_argsz�For preserving POSIX with rsync downloads, gsutil aborts if a single download will result in invalid destination POSIX. However, this command (run via the gcloud storage shim) will skip invalid copies and still perform valid copies.z--predefined-aclz--checksums-onlyz--continue-on-errorz&--delete-unmatched-destination-objectsz--ignore-symlinksz --no-clobberz--gzip-in-flight-allz--gzip-in-flightz --dry-runz--preserve-posixz--preserve-aclz --recursivez--skip-unsupportedz --skip-if-dest-has-newer-mtimez --exclude)�-a�-c�-C�-dr �-i�-J�-j�-nr� �-p�-R�-r�-U�-u�-x)�gcloud_command�flag_map) r �sub_optsrd r � ParseSubOptsr r� ra rB rA �super�get_gcloud_storage_args)r r� Z flag_keysr� � source_pathZdestination_path�gcloud_storage_map�� __class__rM rN r� I sN �� ����z$RsyncCommand.get_gcloud_storage_argsc C s2 t j|| j| j|| jd�\}}|s.td| ��|S )aw Sanity checks that URL names an existing container. Args: url_str: URL string to check. treat_nonexistent_object_as_subdir: indicates if should treat a non-existent object as a subdir. Returns: URL for checked string. Raises: CommandException if url_str doesn't name an existing container. )rd z�arg (%s) does not name a directory, bucket, or bucket subdir. If there is an object with the same path, please add a trailing slash to specify the directory.)r �ExpandUrlToSingleBlrr� r� rd r )r r �"treat_nonexistent_object_as_subdirr� Zhave_existing_containerrM rM rN �_InsistContainerv s � ��z RsyncCommand._InsistContainerc C s� | � � d| _t�� | _t� s>| jr2| j�t � n | j�t � | � | j d d�}| � | j d d�}|� � o||� � o||j|jk}t|�� |� � |�� |� � || j|j|jgd� t|� � | j| jd| jd�| _d| _d}t� D ]}t|t� q�| jdd| j j dd �\}}t j || j || d � t!| ||�}t"t#|��} | j�$d � t%�%� } z"| j&t'|t(|t)d| d � W t*� nt*� 0 t%�%� } | | | _+t,| j| j+�| _-t| j-| j+| jd � | j�r�| j�r�dnd} t.d| j| | f ��dS )z*Command entry point for the rsync command.r Fr~ T)� has_file_src� has_cloud_src� has_file_dst� has_cloud_dst�is_daisy_chain�uses_fan�provider_types)rG r� )re rZ N)� process_count� thread_countr� �print_macos_warning)� worker_countzStarting synchronization...)r� r� �seek_ahead_iterator)�avg_throughput�total_elapsed_timerZ �sr� z/%d file%s/object%s could not be copied/removed.)/� _ParseOptsrZ r � CreateLockrY r r� rd r r' r& r� �argsr� r� r ra �parallel_operationsr! �skip_unsupported_objects� preserve_aclr� rQ re r r rO �_GetProcessAndThreadCountr r r �.TriggerReauthForDestinationProviderIfNecessaryr� r� r: r> re r� r� r] rX ri rJ r� rE �total_bytes_per_secondr ) r rt ru r� r rK r� r� Z diff_iteratorr� � start_time�end_time� plural_strrM rM rN � RunCommand� s� � �� � � � � �� �zRsyncCommand.RunCommandc C sF d| _ d| _d| _d| _d| _d| _d| _d| _d | _d| _ d| _ d| _ d }d | _ d}d }d }| j �r�| j D �]�\}}|dkr�|}d| _ |dkr�d| _ql|dkr�d| _ql|dkr�d| _ql|dkr�d| _ ql|dkr�d}d d � |�d �D �}ql|d k� r�d}t}ql|d k�rd| _ql|dk�r"d| _ql|dk�r@d| _t�s�t� ql|dk�sT|dk�r\d| _ql|dk�rnd| _ ql|dk�r�d| _ ql|dk�r�d| _ ql|dk�s�|dkrl|dk�r�d| _|�s�td��zt�|�| _W ql tj�y� td| ��Y ql0 ql| j�r|�rtd��|�r"|�r"td��|| _|�p0|| _t|| j| j d�S ) NFr� Tr� r� r� r r� c S s g | ] }|� � �qS rM )r )r� �xrM rM rN r� r� z+RsyncCommand._ParseOpts.<locals>.<listcomp>�,r� r� r� r� r� r� r� r� r� r� z-yz Invalid blank exclude filterzInvalid exclude filter (%s)z:Specifying both the -p and -a options together is invalid.z:Specifying both the -j and -J options together is invalid.)� canned_aclr� r� )r_ �continue_on_errorr� r� r� r� rL r� r� r� r� r� �cannedr� �splitr" rC r0 r� r �re�compiler� rH rI r )r r� rH � gzip_arg_exts� gzip_arg_all�o�arM rM rN r� � s� � � �zRsyncCommand._ParseOpts)r6 r7 r8 r9 r �CreateCommandSpec� _SYNOPSISr �XML�JSONr � MakeNCloudOrFileURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr� r� r� r� � __classcell__rM rM r� rN rf . s6 �� -\rf )N)N)�r9 � __future__r r r r r r� r� r � itertoolsr rY rS r� r� r� r� r� �sysr� � six.movesr �botor �crcmod�gslib.bucket_listing_refr �gslib.cloud_apir r � gslib.commandr r Zgslib.commands.cpr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr Z gslib.metricsr �"gslib.plurality_checkable_iteratorr �gslib.seek_ahead_threadr �gslib.sig_handlingr r �gslib.storage_urlr r r �"gslib.third_party.storage_apitoolsr rR Z gslib.utilsr r r Zgslib.utils.boto_utilr � gslib.utils.cloud_api_helperr Zgslib.utils.copy_helperr r! r" r# �gslib.utils.hashing_helperr$ r% r& r' �gslib.utils.metadata_utilr( r) r* �gslib.utils.posix_utilr+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> �gslib.utils.rsync_utilr? r@ �gslib.utils.shim_utilrA rB Zgslib.utils.system_utilrC � gslib.utils.translation_helperrD �gslib.utils.unit_utilrE rF rG Zgslib.wildcard_iteratorrH �PY3r r� r� r� rk Z_OUTPUT_BUFFER_SIZEr� rP rO rJ ri rw r� r� r� r� r� r� r� �objectr� r: r> r] r rX rf rM rM rM rN �<module> s� �� # . 'PP7 k0 D commands/__pycache__/update.cpython-39.pyc 0000644 00000030416 15025013616 0014473 0 ustar 00 a +(Wg�F � @ sN d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZ ddlZddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lm Z ddl m Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl$m&Z& ddl$m'Z' dZ(de( de'� Z)G dd� de�Z*dS )z5Implementation of update command for updating gsutil.� )�absolute_import)�print_function)�division)�unicode_literalsN)�input)�Command)� ApiSelector)�CommandException)�'CheckAndMaybePromptForAnalyticsEnabling)�RegisterSignalHandler)� system_util)�GetConfigFilePaths)� CERTIFICATE_VALIDATION_ENABLED)�RELEASE_NOTES_URL)�CompareVersions)� DisallowUpdateIfDataInGsutilDir)�LookUpGsutilVersion)�GsutilPubTarballz! gsutil update [-f] [-n] [url] z <B>SYNOPSIS</B> a <B>DESCRIPTION</B> NOTE: This command is not available if you're using a gsutil installation from a package manager or the Cloud SDK. When using the Cloud SDK, use ``gcloud components update``. The gsutil update command downloads the latest gsutil release, checks its version, and offers to let you update to it if it differs from the version you're currently running. Once you say "Y" to the prompt of whether to install the update, the gsutil update command locates where the running copy of gsutil is installed, unpacks the new version into an adjacent directory, moves the previous version aside, moves the new version to where the previous version was installed, and removes the moved-aside old version. Because of this, users are cautioned not to store data in the gsutil directory, since that data will be lost when you update gsutil. (Some users change directories into the gsutil directory to run the command. We advise against doing that, for this reason.) Note also that the gsutil update command will refuse to run if it finds user data in the gsutil directory. By default gsutil update will retrieve the new code from %s, but you can optionally specify a URL to use instead. This is primarily used for distributing pre-release versions of the code to a small group of early test users. NOTE: gsutil periodically checks whether a more recent software update is available. By default this check is performed every 30 days; you can change (or disable) this check by editing the software_update_check_period variable in the .boto config file. Note also that gsutil will only check for software updates if stdin, stdout, and stderr are all connected to a TTY, to avoid interfering with cron jobs, streaming transfers, and other cases where gsutil input or output are redirected from/to files or pipes. Software update periodic checks are also disabled by the gsutil -q option (see 'gsutil help options') <B>OPTIONS</B> -f Forces the update command to offer to let you update, even if you have the most current copy already. This can be useful if you have a corrupted local copy. -n Causes update command to run without prompting [Y/n] whether to continue if an update is available. c @ s� e Zd ZdZejddgeddddddejej gej d � Z ej ddgd d e i d �Z d d� Zg d�Zdd� Zdd� Zdd� Zdd� ZdS )� UpdateCommandz(Implementation of gsutil update command.�update�refreshr � �fnTF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api� command_helpz#Update to the latest gsutil release)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s� t js t jrdS t�� }t�tj�j|kr.dS t � }d� |�}| � |||� g }|D ]0}t t� t�|�tj ��} |�d| |f � qTtd� t�d��d||| jd� |�f dd��dS ) a� Explains what to do if sudo needed to update gsutil software. Happens if gsutil was previously installed by a different user (typically if someone originally installed in a shared file system location, using sudo). Args: tf: Opened TarFile. dirs_to_remove: List of directories to remove. old_cwd: Path to the working directory we should chdir back to if sudo is needed. It's possible that we've chdir'd to a temp directory that's been deleted, which can cause odd behavior (e.g. OSErrors when opening the metrics subprocess). If this is not truthy, we won't attempt to chdir back to this value. Raises: CommandException: if errors encountered. N� z sudo chmod %s %s� a Since it was installed by a different user previously, you will need to update using the following commands. You will be prompted for your password, and the install will run as "root". If you're unsure what this means please ask your system administrator for help:z; sudo chmod 0644 %s sudo env BOTO_CONFIG="%s" %s update%sT�� informational)r � IS_CYGWIN� IS_WINDOWS�os�getuid�stat�gslib� GSUTIL_DIR�st_uidr �join�_CleanUpUpdateCommand�oct�S_IMODE�ST_MODE�appendr �textwrap�wrap� gsutil_path) �self�tf�dirs_to_remove�old_cwd�user_idZconfig_file_listZ config_filesZ chmod_cmds� config_file�mode� rF �$/opt/gsutil/gslib/commands/update.py�_ExplainIfSudoNeeded� s2 ����� �z"UpdateCommand._ExplainIfSudoNeeded))Z applications�auto�binZbootZdesktop�devzdocuments and settings�etc�export�homeZkernel�libZlib32�libraryz lost+foundZ mach_kernel�mediaZmnt�net�null�network�opt�private�procz program filesZpython�rootZsbin�scriptsZsrv�sys�system�tmpZusersZusr�varZvolumes�win�win32�windowsZwinntc C s8 |D ].}|sd}|� tj��� | jv rtd| ��qdS )a� Raises Exception if any of dirs is known to be unsafe for gsutil update. This provides a fail-safe check to ensure we don't try to overwrite or delete any important directories. (That shouldn't happen given the way we construct tmp dirs, etc., but since the gsutil update cleanup uses shutil.rmtree() it's prudent to add extra checks.) Args: dirs: List of directories to check. Raises: CommandException: If unsafe directory encountered. rS zKEnsureDirsSafeForUpdate: encountered unsafe directory (%s); aborting updateN)�lstripr0 �sep�lower�unsafe_update_dirsr )r? �dirs�drF rF rG �_EnsureDirsSafeForUpdate� s �z&UpdateCommand._EnsureDirsSafeForUpdatec C st |r |� � | �|� |D ].}zt�|� W q tyF tjsB� Y q0 q|rpzt�|� W n tyn Y n0 dS )a Cleans up temp files etc. from running update command. Args: tf: Opened TarFile, or None if none currently open. dirs_to_remove: List of directories to remove. old_cwd: Path to the working directory we should chdir back to. It's possible that we've chdir'd to a temp directory that's been deleted, which can cause odd behavior (e.g. OSErrors when opening the metrics subprocess). If this is not truthy, we won't attempt to chdir back to this value. N) �closerg �shutil�rmtree�OSErrorr r/ r0 �chdir)r? r@ rA rB � directoryrF rF rG r7 � s z#UpdateCommand._CleanUpUpdateCommandc C s� t jrtd��t�� r td��t}|s.td��t� d}d}| jrj| jD ] \}}|dkr\d}|dkrHd}qHg }t� � }|� |� t � � }t � |� |s�| j�d� | j�r4| jd } | �d �s�td ��t| �| ��D ]\\} } | d kr�td ��| j} | �� �r| �� �s|�s0td ��q�| �� �r(| �� s�td��q�nt� } t| j| �} | �rRd}nT| � | �}|� � t t j � dd�d�� }|�!� �"� } W d � n1 �s�0 Y |�s�t j#| k�r�| �$|||� | j�r�td| dd��n tddd��|�sBt%� t&| t j#�\}}|�r0t'd� t(�)d| t j*t+f ��� nt'd| t j*f � | �,|||� |�r\d}nt-d�}|�r||�.� d dk�r�| �$|||� tddd��|�s�| � | �}t/t0j1t0j2� t j �3t j � t j*d ��}tj |d �}tj |d �}|� |� |� |� | �4|� z|j |d � W n@ t5�yR } z&| �$|||� td | ��W Y d}~n d}~0 0 tj6�s`t �7|�D ]�\}}}|D ]H}t � t j � ||�t j8�}t �9|t:j;t:j<B t:j=B t:j>B � t �?|� �qt|D ]N} t � t j � || �t j8�}t �9|t:j@t:jAB t:jBB t:j=B t:j>B � t �?|� �qqft � t j � |dd�t j8�}t �9|t:j@t:j=B t:jAB t:j>B t:jBB � t �?|� t �Ct j*t j � |d �� t �Ct j � |d�t j*� | �$|||� t/t0j1t0jD� | j�d!� d S )"z+Command entry point for the update command.z�The update command is only available for gsutil installed from a tarball. If you installed gsutil via another method, use the same method to update it.aP The update command is disabled for Cloud SDK installs. Please run "gcloud components update" to update it. Note: the Cloud SDK incorporates updates to the underlying tools approximately every 2 weeks, so if you are attempting to update to a recently created release / pre-release of gsutil it may not yet be available via the Cloud SDK.z�Your boto configuration has https_validate_certificates = False. The update command cannot be run this way, for security reasons.Fz-fTz-nz Checking for software update...r z.tar.gzz0The update command only works with tar.gz files.z4Invalid update URL. Must name a single .tar.gz file.zG"update" command does not support "file://" URLs without the -f option.z;Invalid update object URL. Must name a single .tar.gz file.N�gsutil�VERSION�rz You already have %s installed.r, z5You already have the latest gsutil release installed.r+ z�This command will update to the "%s" version of gsutil at %s. NOTE: This a major new version, so it is strongly recommended that you review the release note details at %s before updating to this version, especially if you use gsutil in scripts.z<This command will update to the "%s" version of gsutil at %s�yzProceed? [y/N] zNot running update.z..)�dir)�pathzUpdate failed: %s.�oldzUpdate complete.)Er3 �IS_PACKAGE_INSTALLr r �InvokedViaCloudSdkr r �sub_opts�tempfile�mkdtempr; r0 �getcwdrl �logger�info�args�endswith� enumerate�WildcardIterator� storage_url� IsFileUrl� IsDirectory� IsCloudUrl�IsObjectr r � gsutil_api�_FetchAndOpenGsutilTarball� extractall�openrs r6 �read�stripro r7 r r �printr< r= r4 r rH r rc r �signal�SIGINT�SIG_IGN�normpathrg � Exceptionr/ �walk�O_RDONLY�fchmodr2 �S_IWRITE�S_IRUSR�S_IRGRP�S_IROTHrh �S_IRWXU�S_IXGRP�S_IXOTH�rename�SIG_DFL) r? �https_validate_certificatesZ force_updateZ no_prompt�o�unused_arA Ztmp_dirrB �update_from_url_str�i�resultr� Ztarball_versionr@ Zver_file�_�major�answerZgsutil_bin_parent_dirZold_dirZnew_dir�e�dirnameZsubdirs� filenames�filename�fd�subdirrF rF rG � RunCommand s ��� ���� ,��� ���� � "����� zUpdateCommand.RunCommandc C s2 | j jd|dg| j| jdd� t�d�}d|_|S )N�cpzfile://gsutil.tar.gzT)�skip_update_checkz gsutil.tar.gzr )�command_runner�RunNamedCommand�headers�debug�tarfiler� Z errorlevel)r? r� r@ rF rF rG r� � s � z(UpdateCommand._FetchAndOpenGsutilTarballN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �XML�JSON� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specrH rd rg r7 r� r� rF rF rF rG r f s: �� 5, 1r )+r� � __future__r r r r r0 ri r� r2 rZ r� rx r< � six.movesr r3 � gslib.commandr �gslib.cs_api_mapr �gslib.exceptionr Z gslib.metricsr �gslib.sig_handlingr Z gslib.utilsr Zgslib.utils.boto_utilr r �gslib.utils.constantsr Zgslib.utils.text_utilr �gslib.utils.update_utilr r r r� r� r rF rF rF rG �<module> sH �.��3 commands/__pycache__/pap.cpython-39.pyc 0000644 00000015034 15025013616 0013770 0 ustar 00 a +(Wg% � @ s$ d Z ddlmZ ddlmZ ddlZddlmZ ddlmZ ddl m Z ddl m Z dd l mZ dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ dZdZ ee � d� Z dZ dZ de e Z!ee e!�Z"eee �Z#ee e �Z$ej%j&Z&de�'� d d d Z(G dd� de�Z)dS ) z/This module provides the pap command to gsutil.� )�absolute_import)�print_functionN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)� shim_utilz= gsutil pap set (enforced|inherited) gs://<bucket_name>... z( gsutil pap get gs://<bucket_name>... � a <B>SET</B> The ``pap set`` command configures public access prevention for Cloud Storage buckets. If you set a bucket to be ``inherited``, it uses public access prevention only if the bucket is subject to the `public access prevention <https://cloud.google.com/storage/docs/org-policy-constraints#public-access-prevention>`_ organization policy constraint. <B>SET EXAMPLES</B> Configure ``redbucket`` and ``bluebucket`` to use public access prevention: gsutil pap set enforced gs://redbucket gs://bluebucket a <B>GET</B> The ``pap get`` command returns public access prevention values for the specified Cloud Storage buckets. <B>GET EXAMPLES</B> Check if ``redbucket`` and ``bluebucket`` are using public access prevention: gsutil pap get gs://redbucket gs://bluebucket z� The ``pap`` command is used to retrieve or configure the `public access prevention <https://cloud.google.com/storage/docs/public-access-prevention>`_ setting of Cloud Storage buckets. This command has two sub-commands: ``get`` and ``set``. z)--format=value[separator=": "](name.sub("z ","gs://"),z.iamConfiguration.publicAccessPrevention.yesno(zno="inherited"))c @ s� e Zd ZdZejddgededdddej gej e � d�ge dd d gd �e � � gd �d � Z ejddgddeeed �d�Zeedddedgi dd�eeg d�i d�eg d�i d�d�i d�d �i d�Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%S )&� PapCommandz"Implements the gsutil pap command.ZpapZpublicaccessprevention� � F� �mode�enforced� inherited)�choices��get�set) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz"Configure public access prevention)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�buckets�listz--rawT)�gcloud_command�flag_map�supports_output_translation)r- r. �updatez--public-access-prevention)r0 r1 )r- r. r3 z --no-public-access-prevention)r r c C s |j jdkrtd| j ��d S )N�gsz7The %s command can only be used with gs:// bucket URLs.)� storage_url�schemer � command_name)�self�blr� r: �!/opt/gsutil/gslib/commands/pap.py�,_ValidateBucketListingRefAndReturnBucketName� s ��z7PapCommand._ValidateBucketListingRefAndReturnBucketNamec C sP |j }| jj|jdg|jd�}|j}|jp,d}t|��d�}t d||f � dS )z7Gets the public access prevention setting for a bucket.�iamConfiguration��fields�providerr �/z%s: %sN) r5 � gsutil_api� GetBucket� bucket_namer6 r= �publicAccessPrevention�str�rstrip�print)r8 r9 � bucket_url�bucket_metadata� iam_config�public_access_prevention�bucketr: r: r; �_GetPublicAccessPrevention� s � z%PapCommand._GetPublicAccessPreventionc C sV |j }t� }||_tj|d�}td|t|��d�f � | jj |j |dg|j d� dS )zMSets the Public Access Prevention setting for a bucket enforced or inherited.)r= z*Setting Public Access Prevention %s for %srA r= r> r ) r5 �IamConfigurationValuerE �apitools_messages�BucketrH rF rG rB � PatchBucketrD r6 )r8 r9 � setting_argrI rK rJ r: r: r; �_SetPublicAccessPrevention� s � �z%PapCommand._SetPublicAccessPreventionc C s | j �d�}|dvr td��d}g }d}|dkr8| j}n$|dkr\| j}| j �d�}|�|� | j�d�tj kr�td� t � d | j ����d }| j }|s�| �� |D ]\}| �|�}|D ]H} | j�| jj�tj kr�td� t � d | j ����d }|| g|�R � q�q�|�sttt|� ��dS ) z-Handles pap command on Cloud Storage buckets.r r zpap only supports get|setNr r r4 r z=The "%s" command can only be with the Cloud Storage JSON API.Fz2The "%s" command can only be used for GCS Buckets.T)�args�popr rN rT �appendrB �GetApiSelectorr �JSON�join�textwrap�wrapr7 �$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr5 r6 r r/ ) r8 � subcommand�subcommand_func�subcommand_argsrS � some_matched�url_args�url_str� bucket_iter�bucket_listing_refr: r: r; �_Pap� sP �� ����zPapCommand._Papc C s\ | j d }| jdd� |dks&|dkrLtj| jd� tj|gd� | �� n td| ��d S ) z(Command entry point for the pap command.r T)� check_argsr r )�sub_opts)� subcommandsz-Invalid subcommand "%s", use get|set instead.N)rU � ParseSubOptsr �LogCommandParamsri rg r )r8 �action_subcommandr: r: r; � RunCommand� s �zPapCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rY r �MakeNCloudURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �_GCLOUD_LIST_FORMAT�gcloud_storage_mapr< rN rT rg rn r: r: r: r; r \ sp ����� ��� ����$�( -r )*rr � __future__r r r[ �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rP �gslib.utils.constantsr �gslib.utils.shim_utilr Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprt �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONry r{ rz rQ rO �get_format_flag_caretr} r r: r: r: r; �<module> sL �� ���� commands/__pycache__/rb.cpython-39.pyc 0000644 00000007441 15025013616 0013616 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ dZde d ZG dd� de �ZdS )z@Implementation of rb command for deleting cloud storage buckets.� )�absolute_import)�print_function)�division)�unicode_literals)�NotEmptyException)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�StorageUrlFromString)�NO_MAX)�GcloudStorageFlag)�GcloudStorageMapz( gsutil rb [-f] gs://<bucket_name>... z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> Delete one or more buckets. Buckets must be empty before you can delete them. Be certain you want to delete a bucket before you do so, as once it is deleted the name becomes available and another user may create a bucket with that name. (But see also "DOMAIN NAMED BUCKETS" under "gsutil help naming" for help carving out parts of the bucket name space.) <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when removing buckets. If some buckets couldn't be removed, gsutil's exit status will be non-zero even if this flag is set. If no buckets could be removed, the command raises a "no matches" error. c @ s� e Zd ZdZejdg d�ededdddej ej gej e � � gd� Z ejdg d�d d ei d �Zeg d �d ed�id�Zdd� ZdS )� RbCommandz$Implementation of gsutil rb command.�rb)Z deletebucketZ removebucketZ removebuckets�rmdir� �fFr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzRemove buckets)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�buckets�delete�-fz--continue-on-error)�gcloud_command�flag_mapc C sh d| _ | jr*| jD ]\}}|dkrd| _ qd}d}| jD �]}t|�}|�� rVtd��z t| �|�jdgd��}W n d}| j r�Y q8n� Y n0 |D ]�}|j } | j � d| � z| j j | j| jd� W nr t�y } z@d}| j r�W Y d } ~ q�n d | jv �r td | ��n� W Y d } ~ n"d } ~ 0 d}| j �s4� Y n0 d}q�q8|�sZttt| j� ��|�rdd S d S )z'Command entry point for the rb command.Fr* Tz."rb" command requires a provider or bucket URL�id)� bucket_fieldszRemoving %s...)�providerN�VersionedBucketNotEmptyzqBucket is not empty. Note: this is a versioned bucket, so to delete all objects you need to use: gsutil rm -r %sr r )�continue_on_error�sub_opts�argsr �IsObjectr �list�WildcardIterator� IterBuckets� storage_url�logger�info� gsutil_api� DeleteBucket� bucket_name�schemer �reasonr ) �self�o�unused_a� did_some_workZ some_failed�url_str� wildcard_urlZblrs�blr�url�e� rI � /opt/gsutil/gslib/commands/rb.py� RunCommandn sT � �zRbCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r �gcloud_storage_maprK rI rI rI rJ r < s@ ������r N)rO � __future__r r r r �gslib.cloud_apir � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.storage_urlr �gslib.utils.constantsr �gslib.utils.shim_utilr r rQ rW r rI rI rI rJ �<module> s* �� commands/__pycache__/cors.cpython-39.pyc 0000644 00000015075 15025013616 0014163 0 ustar 00 a +(Wg," � @ sN d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m Z dZ!dZ"dZ#dZ$e"e!�%d� d Z&dd�'e#e$g� d Z(ee&e(�Z)ee!e#�Z*ee"e$�Z+G d d � d e �Z,dS ) z=Implementation of cors configuration command for GCS buckets.� )�absolute_import)�print_function)�division)�unicode_literalsN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�StorageUrlFromString)�UrlsAreForSingleProvider)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)�CorsTranslation)�REMOVE_CORS_CONFIGz& gsutil cors get gs://<bucket_name> z: gsutil cors set <cors-json-file> gs://<bucket_name>... z� <B>GET</B> Gets the CORS configuration for a single bucket. The output from ``cors get`` can be redirected into a file, edited and then updated using ``cors set``. z� <B>SET</B> Sets the CORS configuration for one or more buckets. The ``cors-json-file`` specified on the command line should be a path to a local file containing a JSON-formatted CORS configuration, such as the example described above. � z a Gets or sets the Cross-Origin Resource Sharing (CORS) configuration on one or more buckets. This command is supported for buckets only, not objects. An example CORS JSON file looks like the following: [ { "origin": ["http://origin1.example.com"], "responseHeader": ["Content-Type"], "method": ["GET"], "maxAgeSeconds": 3600 } ] The above CORS configuration explicitly allows cross-origin GET requests from http://origin1.example.com and may include the Content-Type response header. The preflight request may be cached for 1 hour. Note that requests to the authenticated browser download endpoint ``storage.cloud.google.com`` do not allow CORS requests. For more information about supported endpoints for CORS, see `Cloud Storage CORS support <https://cloud.google.com/storage/docs/cross-origin#server-side-support>`_. The following (empty) CORS JSON file removes any CORS configuration for a bucket: [] The cors command has two sub-commands: z� For more info about CORS generally, see https://www.w3.org/TR/cors/. For more info about CORS in Cloud Storage, see the `CORS concept page <https://cloud.google.com/storage/docs/cross-origin>`_. c @ s� e Zd ZdZejdddgededdddej ej gej e � d�e � � ge �d�gd �d � Zejdg d �d d eeed�d�Zeeg d�i d�eg d�i d�d�i d�Zdd� Zdd� Zdd� Zdd� ZdS ) � CorsCommandz&Implementation of gsutil cors command.�cors�getcors�setcors� � F� )�set�get) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)r r z cross-origin� command_helpz7Get or set a CORS configuration for one or more buckets)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�buckets�describeze--format=gsutiljson[key=cors_config,empty=' has no CORS configuration.',empty_prefix_key=storage_url])�gcloud_command�flag_map)r0 r1 �updatez --cors-filec C s, | j s| �� | j d �� dkr$dS dS d S )Nr r r r )�args�$RaiseWrongNumberOfArgumentsException�lower)�self� r: �"/opt/gsutil/gslib/commands/cors.py�_CalculateUrlsStartArg� s z"CorsCommand._CalculateUrlsStartArgc C s | j d }| j dd� }t|�s.td| j ��t|d�}|�� }|�� | j�t |d �j �| _ d}|D ]�}| j |dgd�}|D ]t}|j } d }| j�d |� | j d kr�| jj|| | j d � q�t�|�} | s�t} tj| d �} | jj| j| | j dgd� q�qh|�sttt|� ��dS )z9Sets CORS configuration on a Google Cloud Storage bucket.r r Nz,"%s" command spanning providers not allowed.�rF�id�� bucket_fieldsTzSetting CORS on %s...�s3��provider)r )rC �fields)r6 r r � command_name�open�read�close� gsutil_api�GetApiSelectorr �scheme�api�GetBucketUrlIterFromArg� storage_url�logger�info�XmlPassThroughSetCorsr �JsonCorsToMessageEntriesr �apitools_messages�Bucket� PatchBucket� bucket_namer �list) r9 Zcors_arg�url_argsZ cors_fileZcors_txt� some_matched�url_str� bucket_iter�blr�urlr �bucket_metadatar: r: r; �_SetCors� sH � � � � zCorsCommand._SetCorsc C sp | j | jd dgd�\}}|jdkr@tj�| jj||jd�� n,|jr\tj�t � |j�� ntj�d| � dS )z:Gets CORS configuration for a Google Cloud Storage bucket.r r r? rA rB z %s has no CORS configuration. ) �GetSingleBucketUrlFromArgr6 rK �sys�stdout�writerI �XmlPassThroughGetCorsr r �MessageEntriesToJson)r9 � bucket_urlr^ r: r: r; �_GetCors� s � �� �zCorsCommand._GetCorsc C sR | j �d�}|dkr | j}n"|dkr,| j}ntd|| jf ��tj|gd� |� S )z)Command entry point for the cors command.r r r zCInvalid subcommand "%s" for the %s command. See "gsutil help cors".)� subcommands)r6 �poprg r_ r rE r �LogCommandParams)r9 �action_subcommand�funcr: r: r; � RunCommand� s ��zCorsCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �MakeNFileURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� MakeNCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �gcloud_storage_mapr< r_ rg rm r: r: r: r; r g sZ � ����� ���*r )-rq � __future__r r r r ra �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �gslib.storage_urlr r �"gslib.third_party.storage_apitoolsr rS �gslib.utils.constantsr �gslib.utils.shim_utilr � gslib.utils.translation_helperr r � _GET_SYNOPSIS� _SET_SYNOPSIS�_GET_DESCRIPTION�_SET_DESCRIPTION�lstriprs �join� _DESCRIPTIONr{ r| r} r r: r: r: r; �<module> sB � �" commands/__pycache__/ubla.cpython-39.pyc 0000644 00000015725 15025013616 0014142 0 ustar 00 a +(Wg4( � @ sV d Z ddlmZ ddlmZ ddlZddlZddlmZ ddlm Z ddl m Z ddl m Z dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddl m Z dZ dZ e e � d� Z!dZ"dZ#de# e" Z$ee!e$�Z%ee e"�Z&ee e#�Z'ej(j)Z)e)j*Z+de �,� d d d d e �,� d d d Z-G d d � d e �Z.dS )!z+This module provides the command to gsutil.� )�absolute_import)�print_functionN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)� InsistOnOrOff)� shim_utilz2 gsutil ubla set (on|off) gs://<bucket_name>... z! gsutil ubla get bucket_url... � a~ <B>SET</B> The ``ubla set`` command enables or disables uniform bucket-level access for Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use uniform bucket-level access: gsutil ubla set on gs://redbucket gs://bluebucket Configure your buckets to NOT use uniform bucket-level access: gsutil ubla set off gs://redbucket gs://bluebucket a <B>GET</B> The ``ubla get`` command shows whether uniform bucket-level access is enabled for the specified Cloud Storage bucket(s). <B>GET EXAMPLES</B> Check if your buckets are using uniform bucket-level access: gsutil ubla get gs://redbucket gs://bluebucket z� The ``ubla`` command is used to retrieve or configure the `uniform bucket-level access <https://cloud.google.com/storage/docs/bucket-policy-only>`_ setting of Cloud Storage bucket(s). This command has two sub-commands, ``get`` and ``set``. z--format=multi[terminator="z2"](name:format="value(format('Uniform bucket-levelz! access setting for gs://{}:'))",zD iamConfiguration.uniformBucketLevelAccess.enabled.yesno(no="False")z:format="value[terminator='z '](format(' Enabled: {}'))",zC iamConfiguration.uniformBucketLevelAccess.lockedTime.sub("T", " ")z,:format="value(format(' LockedTime: {}'))")c @ s� e Zd ZdZejddgededdddej gej e � d�ge dd d gd �e � � gd �d � Z ejddgddeeed �d�Zeedddedgi d�eeg d�i d�eg d�i d�d�i d�d �i d�Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#S )$� UblaCommandz#Implements the gsutil ubla command.�ublaZuniformbucketlevelaccess� � F� �mode�on�off)�choices��get�set) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz%Configure Uniform bucket-level access)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�buckets�listz--raw)�gcloud_command�flag_map)r/ r0 �updatez --uniform-bucket-level-access)r/ r0 r4 z --no-uniform-bucket-level-access)r r c C s |j jdkrtd| j ��d S )N�gsz7The %s command can only be used with gs:// bucket URLs.)� storage_url�schemer � command_name)�self�blr� r; �"/opt/gsutil/gslib/commands/ubla.py�,_ValidateBucketListingRefAndReturnBucketName� s ��z8UblaCommand._ValidateBucketListingRefAndReturnBucketNamec C s� | � |� |j}| jj|jdg|jd�}|j}|j}t|�� d�|j d�}d}|j rb|j |d< d}|r~t d| j f i |��� d S ) z:Gets the Uniform bucket-level access setting for a bucket.�iamConfiguration��fields�provider�/)�bucket�enabledr � locked_timez LockedTime: {locked_time} zGUniform bucket-level access setting for {bucket}: Enabled: {enabled} N)r= r6 � gsutil_api� GetBucket� bucket_namer7 r> �bucketPolicyOnly�str�rstriprD � lockedTime�print�format)r9 r: � bucket_url�bucket_metadata� iam_configZuniform_bucket_level_accessr@ �locked_time_liner; r; r< �_GetUbla� s* � � ��zUblaCommand._GetUblac C s~ | � |� |j}t� }t� |_|dk|j_tj|d�}|dkrBdnd}td|t |�� d�f � | j j |j |dg|jd� d S ) zDSets the Uniform bucket-level access setting for a bucket on or off.r )r> �Enabling� Disablingz(%s Uniform bucket-level access for %s...rB r> r? r )r= r6 �IamConfigurationValue� uniformBucketLevelAccessValuerI rD �apitools_messages�BucketrM rJ rK rF � PatchBucketrH r7 )r9 r: � setting_argrO rQ rP � setting_verbr; r; r< �_SetUbla� s � �zUblaCommand._SetUblac C s� | j �d�}|dvr td��d}g }d}|dkr8| j}n.|dkrf| j}| j �d�}t|d� |�|� d}| j }|s|| �� |D ],}| �|�}|D ]} d }|| g|�R � q�q�|s�tt t |� ��dS ) z/Handles ubla command on a Cloud Storage bucket.r r zubla only supports get|setNr r z-Only on and off values allowed for set optionFT) �args�popr rS r] r �append�$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr r1 ) r9 � subcommand�subcommand_func�subcommand_argsr[ � some_matched�url_args�url_str� bucket_iter�bucket_listing_refr; r; r< �_Ubla� s6 � zUblaCommand._Ublac C s� | j jdd�tjkr.td�t�d| j ����| j d }| j dd� |dksT|d krzt j | j d � t j |gd � | �� n td | ��d S )z)Command entry point for the ubla command.r5 )rA r zBThe "%s" command can only be used with the Cloud Storage JSON API.r T)� check_argsr r )�sub_opts)� subcommandsz-Invalid subcommand "%s", use get|set instead.N)rF �GetApiSelectorr �JSONr �join�textwrap�wrapr8 r^ � ParseSubOptsr �LogCommandParamsrm rk )r9 �action_subcommandr; r; r< � RunCommand s" ��� �zUblaCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rp r �MakeNCloudURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �_GCLOUD_FORMAT_STRING�gcloud_storage_mapr= rS r] rk rw r; r; r; r< r c sp ����� ��� ����$�($r )/r{ � __future__r r �getoptrr �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rX �gslib.utils.constantsr �gslib.utils.shim_utilr �gslib.utils.text_utilr Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstripr} �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONr� r� r� rY rV �BucketPolicyOnlyValuerW �get_format_flag_newliner� r r; r; r; r< �<module> sd �� ��������� commands/__pycache__/setmeta.cpython-39.pyc 0000644 00000026375 15025013616 0014664 0 ustar 00 a +(Wgs? � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l m Z dd l mZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl(m*Z* ddl+m,Z, dd l+m-Z- dd l.m/Z/ dd l.m0Z0 dd l.m1Z1 e!j2Z3d Z4d!e4 d" Z5g d#�Z6g d$�Z7d%d&� Z8d+d'd(�Z9G d)d*� d*e�Z:dS ),zDImplementation of setmeta command for setting cloud object metadata.� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�AccessDeniedException)�PreconditionException)� Preconditions)�Command)�CommandArgument)� ApiSelector)�CommandException)�NameExpansionIterator)� SeekAheadNameExpansionIterator)�StorageUrlFromString)�storage_v1_messages)�MetadataMessage)� constants)�parallelism_framework_util)�GetCloudApiInstance)�IsCustomMetadataHeader)�Retry)�GcloudStorageFlag)�GcloudStorageMap)�InsistAsciiHeader)�InsistAsciiHeaderValue)�CopyObjectMetadata)�ObjectMetadataFromHeaders)�PreconditionsFromHeadersz6 gsutil setmeta -h [header:value|header] ... url... z <B>SYNOPSIS</B> u� <B>DESCRIPTION</B> The gsutil setmeta command allows you to set or remove the metadata on one or more objects. It takes one or more header arguments followed by one or more URLs, where each header argument is in one of two forms: - If you specify ``header:value``, it sets the provided value for the given header on all applicable objects. - If you specify ``header`` (with no value), it removes the given header from all applicable objects. For example, the following command sets the ``Content-Type`` and ``Cache-Control`` headers while also removing the ``Content-Disposition`` header on the specified objects: gsutil setmeta -h "Content-Type:text/html" \ -h "Cache-Control:public, max-age=3600" \ -h "Content-Disposition" gs://bucket/*.html If you have a large number of objects to update you might want to use the gsutil -m option, to perform a parallel (multi-threaded/multi-processing) update: gsutil -m setmeta -h "Content-Type:text/html" \ -h "Cache-Control:public, max-age=3600" \ -h "Content-Disposition" gs://bucket/*.html You can also use the setmeta command to set custom metadata on an object: gsutil setmeta -h "x-goog-meta-icecreamflavor:vanilla" gs://bucket/object Custom metadata is always prefixed in gsutil with ``x-goog-meta-``. This distinguishes it from standard request headers. Other tools that send and receive object metadata by using the request body do not use this prefix. While gsutil supports custom metadata with arbitrary Unicode values, note that when setting metadata using the XML API, which sends metadata as HTTP headers, Unicode characters are encoded using UTF-8, then url-encoded to ASCII. For example: gsutil setmeta -h "x-goog-meta-foo: ã" gs://bucket/object stores the custom metadata key-value pair of ``foo`` and ``%C3%A3``. Subsequently, running ``ls -L`` using the JSON API to list the object's metadata prints ``%C3%A3``, while ``ls -L`` using the XML API url-decodes this value automatically, printing the character ``ã``. The setmeta command reads each object's current generation and metageneration and uses those as preconditions unless they are otherwise specified by top-level arguments. For example, the following command sets the custom metadata ``icecreamflavor:vanilla`` if the current live object has a metageneration of 2: gsutil -h "x-goog-if-metageneration-match:2" setmeta -h "x-goog-meta-icecreamflavor:vanilla" See `Object metadata <https://cloud.google.com/storage/docs/metadata>`_ for more information about object metadata. <B>OPTIONS</B> -h Specifies a header:value to be added, or header to be removed, from each named object. )z cache-controlzcontent-dispositionzcontent-encodingzcontent-languagez content-typez custom-time)�storage�objects�updatec C s | j �|� d| _dS )zDException handler that maintains state about post-completion status.FN)�logger�error�everything_set_okay)�cls�e� r' �%/opt/gsutil/gslib/commands/setmeta.py� _SetMetadataExceptionHandler� s r) c C s | j ||d� d S )N�� thread_state)�SetMetadataFunc)r% �name_expansion_resultr+ r' r' r( �_SetMetadataFuncWrapper� s r. c s� e Zd ZdZejddgedejdddde j e j ge j e � � gd� Zejddgdd ei d �Zeei d �Z� fd d �Zdd� Zeeddd�ddd��Zdd� Zdd� Z� Z S )�SetMetaCommandz)Implementation of gsutil setmeta command.�setmetaZ setheader� zh:rRF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz(Set metadata on already uploaded objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text��gcloud_command�flag_mapc s� g }| j D ]\}}|dv r dg} q&q | �| �� �\}}g | _ | jdd� |D �dd�}| j|dd�}t| | | }t|i d�} t� �| �S ) N)z-rz-Rz --recursivec S s i | ] }|d �qS )Nr' )�.0Z clear_keyr' r' r( � <dictcomp>� � z:SetMetaCommand.get_gcloud_storage_args.<locals>.<dictcomp>T)�unsetFrD )�sub_opts�_ParseMetadataHeaders� _GetHeaderStringsFromSubOpts�_translate_headers� _GCLOUD_OBJECTS_UPDATE_COMMANDr �super�get_gcloud_storage_args) �selfZrecursive_flag�o�_Z clear_setZset_dict� clear_flags� set_flags�command�gcloud_storage_map�� __class__r' r( rQ � s* �� ��z&SetMetaCommand.get_gcloud_storage_argsc C sN | � | �� �\}}|| _|D ]}d| j|< q | js:td��t| j�dkr~| js~t| jd �}|�� rl|� � s~td| jd ��d| _ t | j �| _ t| j| j| j| j| j| j| j| jg d�d� }t| j| j| �� | j| j| j| jd �}z| jt|td|d � W n< t�y8 } z"|j d k�r"| � � � W Y d }~n d }~0 0 | j �sJtd ��dS )z,Command entry point for the setmeta command.� z|gsutil setmeta requires one or more headers to be provided with the -h flag. See "gsutil help setmeta" for more information.r1 r z URL (%s) must name an objectT)� generation�metadata�metageneration)� all_versions�continue_on_error�bucket_listing_fields)r_ � project_id)� fail_on_error�seek_ahead_iteratori� Nz+Metadata for some objects could not be set.) rL rM �metadata_changer �len�args�recursion_requestedr � IsCloudUrl�IsObjectr$ r �headers� preconditionsr � command_name�debugr" � gsutil_apir_ �parallel_operationsr �GetSeekAheadGsutilApirb �Applyr. r) r �status�_WarnServiceAccounts)rR �metadata_minus� metadata_plus�header�url�name_expansion_iteratorrd r& r' r' r( � RunCommand� sd � � � � � zSetMetaCommand.RunCommand� )�tries� timeout_secsNc C s� t | |d�}|j}| j�d|� t�tj|j�}t | j j | j j d�}|j du rV|j |_ |j du rh|j|_ t| j�}|j|jd�}|tjkr�n(|tjkr�t||dd� |}d|_ d|_|j|j|j||j ||jdgd � t|jt t � � d �� dS ) z�Sets metadata on an object. Args: name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. r* zSetting metadata on %s...)� gen_match�meta_gen_matchN)�providerT)�override�id)r\ rl r� �fields)� message_time) r �expanded_storage_urlr" �infor � JsonToMessage�apitools_messages�Object�expanded_resultr rl r~ r r\ r^ r re �GetApiSelector�schemer �XML�JSONr �PatchObjectMetadata� bucket_name� object_name�_PutToQueueWithTimeout� status_queuer �time) rR r- r+ ro � exp_src_url�cloud_obj_metadatarl �patch_obj_metadata�apir' r' r( r, sD �� � �z SetMetaCommand.SetMetadataFuncc C sL | j s g S g }| j D ]2\}}|dkrd|v s4d|v r<td��|�|� q|S )a: Gets header values from after the "setmeta" part of the command. Example: $ gsutil -h not:parsed setmeta is:parsed gs://bucket/object -> ["is:parsed"] Returns: List[str]: Headers without the "-h" but not yet split on colons. Raises: CommandException Found canned ACL. z-hz x-goog-aclz x-amz-aclzWgsutil setmeta no longer allows canned ACLs. Use gsutil acl set ... to set canned ACLs.)rK r �append)rR rk rS �ar' r' r( rM @ s � z+SetMetaCommand._GetHeaderStringsFromSubOptsc C sV t � }t � }i }i }d}d}d}d} |D ]�} | �d�} | \} } }t| � | �� }t|�}|s||tvr|td| ttt��f ��|r�|r�||| < |d7 }q�t | |� t |�}|||< |d7 }q(|r�|� | � | d7 } q(|� |� |d7 }q(|t |�k�s2|t |�k�s2|t |�k�s2| t |�k�s2|� t |�� ���r:td��|�|� |�|� ||fS )z�Validates and parses metadata changes from the headers argument. Args: headers: Header dict to validate and parse. Returns: (metadata_plus, metadata_minus): Tuple of header sets to add and remove. r �:zhInvalid or disallowed header (%s). Only these fields (plus x-goog-meta-* fields) can be set or unset: %sr1 z%Each header must appear at most once.)�set� partitionr �lowerr �SETTABLE_FIELDSr �sorted�listr �str�addrf � intersection�keysr! )rR rk ru Zcust_metadata_minusrv Zcust_metadata_plusZnum_metadata_plus_elemsZ num_cust_metadata_plus_elemsZnum_metadata_minus_elemsZ num_cust_metadata_minus_elemsZmd_arg�partsrw rT �valueZlowercase_headerZis_custom_metar' r' r( rL X s\ �� � � �� z$SetMetaCommand._ParseMetadataHeaders)N) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �NO_MAXr r� r� r � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr rO rX rQ rz r r r, rM rL � __classcell__r' r' rY r( r/ � sH ���� � ? 1r/ )N);r� � __future__r r r r r� �apitools.base.pyr �gslib.cloud_apir r r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.name_expansionr r �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr r� �gslib.thread_messager Z gslib.utilsr r � gslib.utils.cloud_api_helperr �gslib.utils.metadata_utilr �gslib.utils.retry_utilr �gslib.utils.shim_utilr r Zgslib.utils.text_utilr r � gslib.utils.translation_helperr r r �PutToQueueWithTimeoutr� r� r� r� rO r) r. r/ r' r' r' r( �<module> sT ��H commands/__pycache__/__init__.cpython-39.pyc 0000644 00000000503 15025013616 0014742 0 ustar 00 a +(Wg � @ s8 d Z ddlmZ ddlmZ ddlmZ ddlmZ dS )zPackage marker file.� )�absolute_import)�print_function)�division)�unicode_literalsN)�__doc__� __future__r r r r � r r �&/opt/gsutil/gslib/commands/__init__.py�<module> s commands/__pycache__/stat.cpython-39.pyc 0000644 00000012055 15025013616 0014163 0 ustar 00 a +(Wg� � @ s4 d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd l m Z dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z ddl m Z ddl m Z ddl!m"Z" dZ#de# d Z$G dd � d e�Z%dS ) zEImplementation of Unix-like stat command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�BucketListingObject)�AccessDeniedException)�EncryptionException)�NotFoundException)�Command)�CommandArgument)� ApiSelector)�CommandException)�InvalidUrlError)�NO_URLS_MATCHED_TARGET)�ContainsWildcard)�StorageUrlFromString)�NO_MAX)�ENCRYPTED_FIELDS)�PrintFullInfoAboutObject)� UNENCRYPTED_FULL_LISTING_FIELDS)�GcloudStorageMapz gsutil stat url... z <B>SYNOPSIS</B> a <B>DESCRIPTION</B> The stat command outputs `object metadata <https://cloud.google.com/storage/docs/metadata>`_ for the specified object URLs. It is similar to running: gsutil ls -L gs://some-bucket/some-object but is more efficient because it avoids listing objects and gets the minimum necessary amount of object metadata. Moreover, because it avoids listing objects (which for some storage providers are eventually consistent) the ``gsutil stat`` command provides a strongly consistent way to check for the existence (and read the metadata) of an object. The ``gsutil stat`` command does, however, make listing object requests if you specify URLs using wildcards. If run with the ``gsutil -q`` option nothing is printed, e.g.: gsutil -q stat gs://some-bucket/some-object This behavior can be useful when writing scripts: even though nothing is printed from the command, it still has an exit status of 0 for an existing object and 1 for a non-existent object. NOTE: Unlike the gsutil ls command, the stat command does not support operations on sub-directories. For example, if you run the command: gsutil -q stat gs://some-bucket/some-subdir/ gsutil looks for information about an object called "some-subdir/" (with a trailing slash) inside the bucket "some-bucket", as opposed to operating on objects nested under gs://some-bucket/some-subdir/. Unless you actually have an object with that name, the operation fails. However, you can use the stat command on objects within subdirectories. For example, this command works as expected: gsutil -q stat gs://some-bucket/some-subdir/file.txt c @ sp e Zd ZdZejdg ededdddej ej gej e � � gd� Z ejdg dd ei d �Zeg d �i d �Zd d� ZdS )� StatCommandz&Implementation of gsutil stat command.�stat� � Fr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzDisplay object status)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�objects�listz --fetch-encrypted-object-hashesz--stat)�gcloud_command�flag_mapc C s� t t }d}| jD �]h}d}t|�}|�� s4td��z�t|�rR| �|�j|d�}nbz"| j j |j |j |j |j|d�}W n0 ty� | j j |j |j |j |jtd�}Y n0 t||d�g}|D ]0}|�� r�|d7 }t�� �tj�r�t|dd� q�W n` t�y& t�� �tj��r"tj�d | d � Y n( t�y: � Y n t�yL Y n0 |st�� �tj��rxtj�t | d � d }q|�r�dS dS ) z%Command entry point for stat command.Fr z,The stat command only works with object URLs)�bucket_listing_fields)� generation�provider�fields)� root_objectr )�incl_aclz+You aren't authorized to read %s - skipping� T) r r �argsr �IsObjectr r �WildcardIterator� IterObjects� gsutil_api�GetObjectMetadata� bucket_name� object_namer3 �schemer r �logging� getLogger� isEnabledFor�INFOr r �sys�stderr�writer r r ) �selfZ stat_fieldsZfound_nonmatching_arg�url_strZ arg_matches�url�blr_iterZ single_obj�blr� rN �"/opt/gsutil/gslib/commands/stat.py� RunCommand� sh �� � �� zStatCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr �gcloud_storage_maprP rN rN rN rO r ] s> ���� � r )&rT � __future__r r r r rB rF �gslib.bucket_listing_refr �gslib.cloud_apir r r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r r �gslib.storage_urlr r �gslib.utils.constantsr Zgslib.utils.ls_helperr r r �gslib.utils.shim_utilr rV r\ r rN rN rN rO �<module> s< ��. commands/__pycache__/test.cpython-39.pyc 0000644 00000043351 15025013616 0014172 0 ustar 00 a +(Wg q � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlmZ ddlZdd lmZ dd lmZ dd lmZ dd lmZ dd l m Z ddl m Z ddl m!Z! ddl m"Z" ddl m#Z# ddl$m%Z% ddl$m&Z& ddl'm(Z( z ddl)Z)W n e*�yX dZ)Y n0 ej+�rfe,Z-dZ.dZ/dZ0dZ1de1 de. Z2edd�Z3d d � Z4d d � Z5d d!� Z6d"d#� Z7d$d%� Z8d*d&d'�Z9G d(d)� d)e�Z:dS )+�&Implementation of gsutil test command.� )�absolute_import)�print_function)�division)�unicode_literals)� namedtupleN)�range)�ProjectIdException)�Command)�ResetFailureCount)�CommandException)�PopulateProjectId)� GetTestNames)�InvokedFromParFile)�unittest)�NO_MAX)�UTF8)� IS_WINDOWS� �2 �sequential_onlyz3 gsutil test [-l] [-u] [-f] [command command...] z <B>SYNOPSIS</B> a <B>DESCRIPTION</B> The gsutil test command runs the gsutil unit tests and integration tests. The unit tests use an in-memory mock storage service implementation, while the integration tests send requests to the production service using the `preferred API <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ set in the boto configuration file. CAUTION: The ``test`` command creates test buckets and objects in your project. Force quitting the ``test`` command can leave behind stale buckets, objects, and HMAC keys in your project. To run both the unit tests and integration tests, run the command with no arguments: gsutil test To run the unit tests only (which run quickly): gsutil test -u Tests run in parallel regardless of whether the top-level -m flag is present. To limit the number of tests run in parallel to 10 at a time: gsutil test -p 10 To force tests to run sequentially: gsutil test -p 1 To have sequentially-run tests stop running immediately when an error occurs: gsutil test -f To run tests for one or more individual commands add those commands as arguments. For example, the following command will run the cp and mv command tests: gsutil test cp mv To list available tests, run the test command with the -l argument: gsutil test -l The tests are defined in the code under the gslib/tests module. Each test file is of the format test_[name].py where [name] is the test name you can pass to this command. For example, running "gsutil test ls" would run the tests in "gslib/tests/test_ls.py". You can also run an individual test class or function name by passing the test module followed by the class name and optionally a test name. For example, to run the an entire test class by name: gsutil test naming.GsutilNamingTests or an individual test function: gsutil test cp.TestCp.test_streaming You can list the available tests under a module or class by passing arguments with the -l option. For example, to list all available test functions in the cp module: gsutil test -l cp To output test coverage: gsutil test -c -p 500 coverage html This will output an HTML report to a directory named 'htmlcov'. Test coverage is compatible with v4.1 of the coverage module (https://pypi.python.org/pypi/coverage). <B>OPTIONS</B> -b Run tests against multi-regional US buckets. By default, tests run against regional buckets in us-central1. -c Output coverage information. -f Exit on first sequential test failure. -l List available tests. -p N Run at most N tests in parallel. The default value is %d. -s Run tests against S3 instead of GS. -u Only run unit tests. �TestProcessDataz name return_code stdout stderrc s G � �fdd�dt j�� � S )z�Creates a closure of CustomTestResult. Args: total_tests: The total number of tests being run. Returns: An instance of CustomTestResult. c s$ e Zd ZdZ�� �fdd�Z� ZS )z3MakeCustomTestResultClass.<locals>.CustomTestResultzDA subclass of unittest.TextTestResult that prints a progress report.c s� t � | ��|� | jr�d�|�� �d�dd � �}d| j�t| j�t| j �t| j �|f }|d d� }|� d�}| j � d| � d S )N�.���z( %d/%d finished - E[%d] F[%d] s[%d] - %s�I z%s - )�super� startTest�dots�join�id�splitZtestsRun�len�errorsZfailures�skipped�ljust�stream�write)�self�testZtest_id�message)�CustomTestResult� __class__� total_tests� �"/opt/gsutil/gslib/commands/test.pyr � s �� z=MakeCustomTestResultClass.<locals>.CustomTestResult.startTest)�__name__� __module__� __qualname__�__doc__r � __classcell__r- �r* r, )r+ r. r* � s r* )r ZTextTestResult)r, r- r4 r. �MakeCustomTestResultClass� s r5 c C sX | g}g }|rT|� � }|D ]6}t|tj�r6|�|� q|�|�� td�d� � qq |S )zGTakes a list of test suites and returns a list of contained test names.�gslib.tests.test_N)�pop� isinstancer � TestSuite�appendr r! )� test_suiteZsuites� test_names�suiter( r- r- r. �GetTestNamesFromSuites� s r> c C s t | j��d�d d | j S )z<Converts a python.unittest to its gsutil test-callable name.�'� r )�strr+ r �_testMethodName)� test_caser- r- r. �TestCaseToName� s �rD c C s ddl m} g }g }g }g }| g}g }|rp|�� }t|tjj�rX|jD ]} |�| � qFq&t|tj �r&|�|� q&|D ]�} t | | j d�} t | dd�r�|�t | �� qtt | dd�s�|�t | �� qtt | dd�s�|�t | �� qtt| |�r�|�t | �� qt|�t | �� qtt |�t |�t |�t |�fS )a� Splits a test suite into groups with different running properties. Args: test_suite: A python unittest test suite. Returns: 4-part tuple of lists of test names: (tests that must be run sequentially, tests that must be isolated in a separate process but can be run either sequentially or in parallel, unit tests that can be run in parallel, integration tests that can run in parallel) r )�GsUtilUnitTestCaseNZrequires_isolationFZis_parallelizableT)Z"gslib.tests.testcase.unit_testcaserE r7 r8 r r= r9 Z_testsr: ZTestCase�getattrrB rD �sorted) r; rE �isolated_tests�sequential_testsZ parallelizable_integration_testsZparallelizable_unit_testsZitems_to_evaluateZcases_to_evaluateZ suite_or_case�itemrC Z test_methodr- r- r. � SplitParallelizableTestSuite� s: �rK c C s d}| D ]}|s|d7 }q|S )z*Counts number of falses in the input list.r r@ r- )Z input_listZ num_falserJ r- r- r. �CountFalseInList s rL c C s� |}t � stjgng }tjjr$dgng }tjjr6dgng } g } zddt� g} W n tyb Y n0 t � � } | } t |�|k �r�|t | �k �r�t j �� } |r�|| d< t� }dd� t|tjg | dg | | d t g | | t d �d � g �D �}t�| �D ] \}}t�|�|t�|�< q�|�tj|tjtj|d �� |d 7 }|�d� t � � | dkrptd|| t |�t | �f � t � � } qp|t | �k�r�tdt |�t | �f � |S )ag Creates test processes to run tests in parallel. Args: parallel_tests: List of all parallel tests. test_index: List index of last created test before this function call. process_list: List of running subprocesses. Created processes are appended to this list. process_done: List of booleans indicating process completion. One 'False' will be added per process created. max_parallel_tests: Maximum number of tests to run in parallel. root_coverage_file: The root .coverage filename if coverage is requested. Returns: Index of last created test. �-s�-bz-oz GSUtil:default_project_id=%sZGSUTIL_COVERAGE_OUTPUT_FILEc S s g | ]}t �|��qS r- )�six� ensure_str)�.0�partr- r- r. � <listcomp>H s z'CreateTestProcesses.<locals>.<listcomp>r( �--r6 N)�stdout�stderr�envr@ Fr z.Created %d new processes (total %d/%d created)z.Test process creation finished (%d/%d created)) r �sys� executable�tests�util� RUN_S3_TESTS�USE_MULTIREGIONAL_BUCKETSr r �timerL r! �os�environ�copy�dict�list�gslib� GSUTIL_PATH�_SEQUENTIAL_ISOLATION_FLAGrO � iteritemsrP r: � subprocess�Popen�PIPE�print)Zparallel_tests� test_index� process_list� process_done�max_parallel_tests�root_coverage_fileZorig_test_indexZexecutable_prefixZ s3_argumentZmultiregional_bucketsZproject_id_argZprocess_create_start_time� last_log_timerW Zenvstr�cmd�k�vr- r- r. �CreateTestProcesses sz � � ��������� �� �� �ru c @ sp e Zd ZdZejdg ededdddegd� Z ej dg dd e i d �Z d d � Z d d� Ze� e� fdd�Zdd� ZdS )� TestCommandr r( r zbuflp:scTF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�supported_private_args� command_helpz2Run gsutil unit/integration tests (for developers))� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s< g }g }g }d}d}d} t |�} t�� } } t|d||||d�} t |�| k �r�tt |��D ]�}|| sZ|| �� du rxqZd||< || �� \}}|| j�� || j�� || j dkr�|d7 }|� t || || j ||d�� qZt |�| k �rt|| ||||d�} t |�| k r@t�� | dk�r�t dt |�| |f � t |�| k�rR|d7 }n t |�} d}|d k�r�g }tt |��D ] }|| �sx|� || � �qxt�� | }t d j tt|��|d �� t�� } t�d� q@t�� }|�r0|D ]J}|jdk�r�|j�d �}t d |j � |D ]}t |�t��� � �q�q�||| fS )aZ Executes the parallel/isolated portion of the test suite. Args: parallel_integration_tests: List of tests to execute. max_parallel_tests: Maximum number of parallel tests to run at once. coverage_filename: If not None, filename for coverage output. Returns: (int number of test failures, float elapsed time) r )rp NTr@ )�name� return_coderU rV r z %d/%d finished - %d failures� zL{sec} seconds elapsed since beginning parallel tests. Still running: {procs})�secZprocs� zResults for failed test %s:)r! r^ ru r �poll� communicaterU �closerV � returncoder: r rk �formatrA �int�sleepr� r r� �decoder �strip)r' �parallel_integration_testsro Zcoverage_filenamerm rn Zprocess_results�num_parallel_failuresZ progress_less_logging_cyclesZcompleted_as_of_last_log�num_parallel_testsZparallel_start_timerq rl Zproc_numrU rV Z still_running�elapsedZprocess_run_finish_time�resultZ new_stderr�liner- r- r. �RunParallelTests� s� ���� �� � �z TestCommand.RunParallelTestsc C s~ t d|| |f � t d|| t|| �|t|�|t|�f � | �|� t � |sb|rbt d� n|rnt d� |szt d� dS )z6Prints test results for parallel and sequential tests.z-Parallel tests complete. Success: %s Fail: %szDRan %d tests in %.3fs (%d sequential in %.3fs, %d parallel in %.3fs)�OKzFAILED (parallel tests)zFAILED (sequential tests)N)rk �float�PrintSkippedTests)r' �num_sequential_tests�sequential_success�sequential_skipped�sequential_time_elapsedr� r� �parallel_time_elapsedr- r- r. �PrintTestResults� s( � ��� z TestCommand.PrintTestResultsc C sh t |�dkst |�dkrdt|�}t|�}|�|�}td� |D ]$\}}td|�� � td| � q>dS )a� Prints all skipped tests, and the reasons they were skipped. Takes the union of sequentual_skipped and parallel_skipped, and pretty-prints the resulting methods and reasons. Note that these two arguments are lists of tuples from TestResult.skipped as described here: https://docs.python.org/2/library/unittest.html#unittest.TestResult.skipped Args: sequentual_skipped: An instance of TestResult.skipped. parallel_skipped: An instance of TestResult.skipped. r zTests skipped:z z Reason: N)r! �set�unionrk r )r' r� Zparallel_skippedZ all_skipped�method�reasonr- r- r. r� � s z TestCommand.PrintSkippedTestsc ) C sn d}d}t }d}d}| jr�| jD ]�\}}|dkr:dtj_q |dkrHd}q |dkrVd}q |dkrdd}q |dt krvd}q |dkr�t|�}q |d kr�tjjs�td ��dtj_ q |d kr dtj_ q |r�t s�td ��tjj r�|t kr�| j �d t � t }tt� �}|�r.| j�s.tdt|� � tdd�t|��� dS | j�r�g } | jD ]>} | |v �sb| �d�d |v �rr| �d| � n | �| � �q@ndd� |D �} t�� t�� } | �rlt�� } | D ]�} z| �| �}| � |� W n� t t f�yf } zzd| t |�f }z t | � W nH t!�yH } z.t"�#� }t$�%dd|�}|d| 7 }W Y d}~n d}~0 0 t|��W Y d}~n d}~0 0 �q�|�r�t&| �}tdt|� � tdd�t|��� dS t'�(� �)� t'j*k�r�d}nd }t'�+t'j,� |�r t j d gd d t-�.� d gd!�}|�/� |�0� d}d}t1| �\}}}}tjj �s&g }t'�2d"|� t'�2d#|� t'�2d$|� t'�2d%|� |�pnt|�dk�on| }d&t3j4d'< |�r�| �5� } t6| �} tj7|| |d(�} | �8| �} | �9� }�nf|dk�r�||7 }g }t:�:� } t|�t|� �rXtd)t|�t|� � || }!| �;td*d� |!D ���} | �5� }"t6|"�} tj7|| |d(�} | �8| �} | �9� }| j<}#nd}"d}t:�:� | }$||7 }t|�}%|%�s�n�g }#t=||%�}&|%dk�r�|dk�r�d+}'|&t k�r�|'d,7 }'td-�t>�?|'|%|&f ��� n td.|% � | �@|||�r�|jAjBnd�\}}(| �C|"||#|$|%||(� |�rJ|�D� |�E� |�F� td/|jAjB � d0t3j4d'< |�rj|�sjtG� dS dS )1z)Command entry point for the test command.FrN Tz-cz-fz-lrT z-prM zbS3 tests require S3 credentials. Please add appropriate credentials to your .boto file and re-run.z-uzrCoverage has been requested but the coverage module was not found. You can install it with "pip install coverage".zCReducing parallel tests to %d due to S3 maximum bucket limitations.zFound %d test names:� z r r �gslib.tests.test_%sc S s g | ] }d | �qS )r� r- )rQ r� r- r- r. rS H � z*TestCommand.RunCommand.<locals>.<listcomp>zLFailed to import test code from file %s. TestLoader provided this error: %sz\nz z Additional traceback: %sNr@ � rd zgslib/third_party/*z gslib/tests/*�*)�sourceZomitzSequential tests to run: %szIsolated tests to run: %sz Parallel unit tests to run: %sz%Parallel integration tests to run: %s�1�GSUTIL_TEST_ANALYTICS)� verbosity� resultclass�failfastz Running %d tests sequentially.c S s g | ]}|�qS r- r- )rQ � test_namer- r- r. rS � r� z1Running %d tests in parallel mode (%d processes).z� Please be patient while your CPU is incinerated. If your machine becomes unresponsive, consider reducing the amount of parallel test processes by running 'gsutil test -p <num_processes>'.� z4Running %d tests sequentially in isolated processes.z%Coverage information was saved to: %s�0)H� _DEFAULT_TEST_PARALLEL_PROCESSES�sub_optsrZ r[ r] rf �longZ HAS_S3_CREDSr r\ ZRUN_INTEGRATION_TESTS�coverage�#_DEFAULT_S3_TEST_PARALLEL_PROCESSES�logger�warnrG r �argsrk r! r r r: r ZinstallHandlerZ TestLoaderr9 ZloadTestsFromNameZaddTests� ImportError�AttributeErrorrA � __import__� Exception� traceback� format_exc�re�subr> �logging� getLogger�getEffectiveLevel�INFO�disable�ERROR�tempfile� gettempdirZerase�startrK �debugr_ r` ZcountTestCasesr5 ZTextTestRunner�runZ wasSuccessfulr^ ZloadTestsFromNamesr# �min�textwrap�wrapr� Z data_files�filenamer� �stop�combine�saver ))r' r� Z list_testsro Zperform_coverager �o�ar< Zcommands_to_testr� �loaderr= � command_nameZsuite_for_current_command�e�msg� stack_trace�errr� Zcoverage_controllerr� r� rI rH Zparallel_unit_testsr� Zrun_tests_sequentiallyr, r� Zrunner�retZsequential_start_timeZsequential_tests_to_runr� r� r� r� � num_processesr) r� r- r- r. � RunCommand sd ��� � "" ��� �� � ��� ���� ��� zTestCommand.RunCommandN)r/ r0 r1 r2 r �CreateCommandSpec� _SYNOPSISr rf � command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr� r� r� r� r� r- r- r- r. rv h s2 � � Zrv )N);r2 � __future__r r r r � collectionsr r� r_ rh r� rX r� r� r^ r� rO � six.movesr rd �gslib.cloud_apir � gslib.commandr r �gslib.exceptionr �gslib.project_idr Z gslib.testsrZ Zgslib.tests.utilr r r �gslib.utils.constantsr r Zgslib.utils.system_utilr r� r� �PY3r� r� r� r� rf r� r� r r5 r> rD rK rL ru rv r- r- r- r. �<module> sr �^��b�5 � N commands/__pycache__/web.cpython-39.pyc 0000644 00000017203 15025013616 0013765 0 ustar 00 a +(Wgx' � @ s( d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddlm Z dZ dZ e e � d� Z!dZ"dZ#de" e# Z$ee!e$�Z%ee e#�Z&ee e"�Z'G dd� de �Z(dS )z<Implementation of website configuration command for buckets.� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageFlag)�GcloudStorageMapzR gsutil web set [-m <main_page_suffix>] [-e <error_page>] gs://<bucket_name>... z% gsutil web get gs://<bucket_name> � a <B>SET</B> The ``gsutil web set`` command allows you to configure or disable the website configuration on your buckets. The ``set`` sub-command has the following options (omit both options to unset the configuration): <B>SET OPTIONS</B> -m <index.html> Specifies the object name to serve when a bucket listing is requested via a custom domain. -e <404.html> Specifies the error page to serve when a request is made for a non-existent object via a custom domain a4 <B>GET</B> The ``gsutil web get`` command retrieves the web semantics configuration for a bucket and displays a JSON representation of the configuration. In Google Cloud Storage, this would look like the following: { "notFoundPage": "404.html", "mainPageSuffix": "index.html" } ay Cloud Storage allows you to configure a bucket to behave like a static website when the bucket is accessed through a `custom domain <https://cloud.google.com/storage/docs/request-endpoints#custom-domains>`_. For example, if you set a ``main_page_suffix``, a subsequent GET bucket request through a custom domain serves the specified "main" page instead of performing the usual bucket listing. Similarly, if you set an ``error_page``, a subsequent GET object request through a custom domain for a non-existent object serves the specified error page instead of the standard Cloud Storage error. Requests to a bucket through other `endpoints <https://cloud.google.com/storage/docs/request-endpoints>`_ are unaffected by the bucket's website configuration. See `Static website examples and tips <https://cloud.google.com/storage/docs/static-website>`_ for additional examples and information. Notes: 1. Because the main page is only served when a bucket listing request is made via a custom domain endpoint, you can continue to use ``gsutil ls`` as you normally would for bucket listing. 2. The main_page_suffix applies to each subdirectory of the bucket. For example, with the main_page_suffix configured to be index.html, a GET request for ``http://www.example.com`` retrieves ``http://www.example.com/index.html`` but shows ``http://www.example.com`` as the URL to the requester, and a GET request for ``http://www.example.com/photos`` retrieves ``http://www.example.com/photos/index.html`` but shows ``http://www.example.com/photos`` as the URL to the requester. 3. There is just one 404.html page. For example, a GET request for ``http://www.example.com/photos/missing`` retrieves ``http://www.example.com/404.html``, not ``http://www.example.com/photos/404.html``. The web command has two sub-commands: c s� e Zd ZdZejdddgededdddej ej gej e � � ge � d�gd �d � Zejdddgd d eeed �d�Zedeg d�i dd�ii d�Z� fdd�Zdd� Zdd� Zdd� Z� ZS ) � WebCommandz%Implementation of gsutil web command.�web� setwebcfg� getwebcfg� zm:e:F� )�set�get) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz(Set a website configuration for a bucket)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textr )�storage�buckets�describezk--format=gsutiljson[key=website_config,empty=' has no website configuration.',empty_prefix_key=storage_url]T)�gcloud_command�flag_map�supports_output_translation�r0 r1 c s| | j d dkrjtdtg d�td�td�d�d�ii d�}d| j v sdd | j v sd|jd jd d g7 _|}ntj}t� �|�S ) Nr r )r- r. �updatez--web-error-pagez--web-main-page-suffix)�-e�-mr3 r5 r6 z--clear-web-error-pagez --clear-web-main-page-suffix)�argsr r r0 r �gcloud_storage_map�super�get_gcloud_storage_args)�selfZset_command_mapr8 �� __class__� �!/opt/gsutil/gslib/commands/web.pyr: � s& ��� � �z"WebCommand.get_gcloud_storage_argsc C s� | j | jd dgd�\}}|jdkr@tj�| jj||jd�� nD|jrt|jj sV|jj rttj�t t � |j��d � ntj�d| � dS )z(Gets website configuration for a bucket.r �website�� bucket_fields�s3)�providerr z!%s has no website configuration. )�GetSingleBucketUrlFromArgr7 �scheme�sys�stdout�write� gsutil_api�XmlPassThroughGetWebsiter@ �mainPageSuffix� notFoundPage�strr � MessageToJson)r; � bucket_url�bucket_metadatar> r> r? �_GetWeb� s" � ����zWebCommand._GetWebc C s� d}d}| j r8| j D ]"\}}|dkr*|}q|dkr|}q| j}tjj||d�}d}|D ]\}| j|dgd�} | D ]B} | j} d}| j�d | � tj|d �} | j j | j | | j dgd � qnqV|s�t tt|� ��d S ) z(Sets website configuration for a bucket.Nr6 r5 )rL rM F�idrA Tz&Setting website configuration on %s...)r@ )rD �fieldsr )�sub_optsr7 �apitools_messages�Bucket� WebsiteValue�GetBucketUrlIterFromArg� storage_url�logger�inforJ � PatchBucket� bucket_namerF r r �list) r; �main_page_suffix� error_page�o�a�url_argsr@ � some_matched�url_str� bucket_iter�blr�urlrQ r> r> r? �_SetWeb� s8 � � zWebCommand._SetWebc C sb | j �d�}| jdd� |dkr(| j}n"|dkr8| j}ntd|| jf ��tj|g| j d� |� S )z(Command entry point for the web command.r T)� check_argsr r zBInvalid subcommand "%s" for the %s command. See "gsutil help web".)� subcommandsrU ) r7 �pop� ParseSubOptsrR rj r � command_namer �LogCommandParamsrU )r; �action_subcommand�funcr> r> r? � RunCommand s ���zWebCommand.RunCommand)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �%MakeZeroOrMoreCloudBucketURLsArgument� MakeNCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr r8 r: rR rj rs � __classcell__r> r> r< r? r } sR ���� ��� "r ))rw � __future__r r r r rG �apitools.base.pyr �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rV �gslib.utils.constantsr �gslib.utils.shim_utilr r � _SET_SYNOPSIS� _GET_SYNOPSIS�lstripry �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONr� r� r� r r> r> r> r? �<module> s> (�(�* commands/__pycache__/label.cpython-39.pyc 0000644 00000024726 15025013616 0014277 0 ustar 00 a +(Wg\? � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl m Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl$m&Z& ddl'm(Z( dZ)dZ*dZ+dZ,d Z-d Z.e)e*�/d � e+�/d � d Z0d e, e- e. Z1ee0e1�Z2ee*e,�Z3ee)e-�Z4ee+e.�Z5G d!d"� d"e�Z6dS )#z<Implementation of label command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�metrics)�PreconditionException)� Preconditions)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)� shim_util)�NO_MAX)�UTF8)�Retry)�GcloudStorageFlag)�GcloudStorageMap)�LabelTranslationz< gsutil label set <label-json-file> gs://<bucket_name>... z' gsutil label get gs://<bucket_name> z� gsutil label ch <label_modifier>... gs://<bucket_name>... where each <label_modifier> is one of the following forms: -l <key>:<value> -d <key> z� <B>GET</B> The "label get" command gets the `labels <https://cloud.google.com/storage/docs/tags-and-labels#bucket-labels>`_ applied to a bucket, which you can save and edit for use with the "label set" command. ah <B>SET</B> The "label set" command allows you to set the labels on one or more buckets. You can retrieve a bucket's labels using the "label get" command, save the output to a file, edit the file, and then use the "label set" command to apply those labels to the specified bucket(s). For example: gsutil label get gs://bucket > labels.json Make changes to labels.json, such as adding an additional label, then: gsutil label set labels.json gs://example-bucket Note that you can set these labels on multiple buckets at once: gsutil label set labels.json gs://bucket-foo gs://bucket-bar al <B>CH</B> The "label ch" command updates a bucket's label configuration, applying the label changes specified by the -l and -d flags. You can specify multiple label changes in a single command run; all changes will be made atomically to each bucket. <B>CH EXAMPLES</B> Examples for "ch" sub-command: Add the label "key-foo:value-bar" to the bucket "example-bucket": gsutil label ch -l key-foo:value-bar gs://example-bucket Change the above label to have a new value: gsutil label ch -l key-foo:other-value gs://example-bucket Add a new label and delete the old one from above: gsutil label ch -l new-key:new-value -d key-foo gs://example-bucket <B>CH OPTIONS</B> The "ch" sub-command has the following options -l Add or update a label with the specified key and value. -d Remove the label with the specified key. � z ag Gets, sets, or changes the label configuration (also called the tagging configuration by other storage providers) of one or more buckets. An example label JSON document looks like the following: { "your_label_key": "your_label_value", "your_other_label_key": "your_other_label_value" } The label command has three sub-commands: c @ s� e Zd ZdZejdededdddej ej gej e � d�e � � ge �d�ge � � gd�d� Zejdg d d eeeed �d �Zeeg d �i d�eg d�i d�eg d�edejjd�edejjd�d�d�d �i d�Z dd� Z dd� Z dd� Z dd � Z d d � Z!d S ) � LabelCommandz'Implementation of gsutil label command.�label� zl:d:F� )�set�get�ch) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz8Get, set, or change the label configuration of a bucket.)r r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�buckets�describezj--format=gsutiljson[key=labels,empty=' has no label configuration.',empty_prefix_key=storage_url,indent=2])�gcloud_command�flag_map)r0 r1 �updatez --labels-file)r0 r1 r5 z--remove-labels)� repeat_typez--update-labels)�-d�-lc C s( | j s| �� | j d �� dkr$dS dS )Nr r r r )�args�$RaiseWrongNumberOfArgumentsException�lower��self� r> �#/opt/gsutil/gslib/commands/label.py�_CalculateUrlsStartArg� s z#LabelCommand._CalculateUrlsStartArgc s� t �j�dk r��� �jd }tj�|�s8td| ��t�|dt ��}|� � � W d � n1 sd0 Y t t ddd�� �fd d ��}d }�jdd� }|D ]*}�j |d gd �}|D ]}d}||� q�q�|s�ttt|� ��dS )z8Parses options and sets labels on the specified buckets.r r z Could not find the file "%s".�rN� r ��tries� timeout_secsc s | j }�j�d| � |jdkr4�jj� ||jd� n�d }d }t�� �}�j�|j�t j krj |j ddgd�\}}|j }i }|jr�t�t�|j��}tdd� t�|�D ��}|�|� t�|�}n t�|�}t|d �} tj|d �}�jj|j|| |jd gd � d S ) N�$Setting label configuration on %s...�s3��provider�labels�metageneration�� bucket_fieldsc s s | ]\}}|d fV qd S �Nr> )�.0�key�_r> r> r? � <genexpr> s zELabelCommand._SetLabel.<locals>._SetLabelForBucket.<locals>.<genexpr>��meta_gen_match�rJ �id�� preconditionsrI �fields)� storage_url�logger�info�scheme� gsutil_api�XmlPassThroughSetTagging�json�loads�GetApiSelectorr �JSON�GetSingleBucketUrlFromArg� url_stringrK rJ r �JsonFromMessage�dict�six� iteritemsr5 � DictToMessager �apitools_messages�Bucket� PatchBucket� bucket_name) �blr�url�labels_messagerK Znew_label_jsonrQ �bucket_metadata� label_jsonZ merged_labelsrX �Z label_textr= r> r? �_SetLabelForBucket� sD � � �� �z2LabelCommand._SetLabel.<locals>._SetLabelForBucketFrV rL T)�lenr9 r: �os�path�isfiler �codecs�openr �readr r �GetBucketUrlIterFromArgr �list) r= Zlabel_filenameZ label_fileru � some_matched�url_args�url_str� bucket_iter�bucket_listing_refr> rt r? � _SetLabel� s$ & * zLabelCommand._SetLabelc s0 i � _ d� _� jr�� jD ]�\}}|dkr^|�d�}t|�dkrJtd| ��|d � j |d < q|dkr�|�d�}t|�dkr�td| ��d � j |< � jd7 _q� �� q� j s�td ��ttd dd �� fd d��}d}� j }|s� � � |D ]&}� � |�} | D ]} d}|| � �q q�|�s,tt t |� ��d S )z;Parses options and changes labels on the specified buckets.r r8 �:r zrFound incorrectly formatted option for "gsutil label ch": "%s". To add a label, please use the form <key>:<value>.r r7 zhFound incorrectly formatted option for "gsutil label ch": "%s". To delete a label, provide only its key.NzAPlease specify at least one label change with the -l or -d flags.rB rC c s2 | j }� j�d| � d }d }� j�|j�tjkr�� j}� j rz� j |j ddgd�\}}|j sz|j }tdd� t�� j�D ��}t�|�}nv� j |j ddgd�\}}|j }i }|j r�t�t�|j ��}t�� j�D ]$\}} | s�||v r�||= q�| ||< q�t�|�}t|d�} tj|d�}� jj|j|| |jd gd � d S ) NrF rJ rK rL c s s | ]\}}|r||fV qd S rN r> )rO �k�vr> r> r? rR Q s zCLabelCommand._ChLabel.<locals>._ChLabelForBucket.<locals>.<genexpr>rS rU rV rW )rZ r[ r\ r^ rb r] r rc � label_changes� num_deletionsrd re rJ rK rg rh ri r rj r` ra rf r rk rl rm rn ) ro rp rq rK Zcorrected_changesrQ rr rs rP �valuerX r< r> r? �_ChLabelForBucket= sL � � � � �z0LabelCommand._ChLabel.<locals>._ChLabelForBucketFT)r� r� �sub_opts�splitrv r � RaiseInvalidArgumentExceptionr r r9 r: r} r r~ ) r= �o�aZ label_splitZ val_splitr� r r� r� r� r� r> r< r? �_ChLabel sR �� �� � 3 zLabelCommand._ChLabelc C sb | j |dgd�\}}|jdkr6t| jj||jd�� n(|jrRttj|jdd�� n td| � dS ) z.Gets and prints the labels for a cloud bucket.rJ rL rG rH T)� pretty_printz %s has no label configuration.N)rd r] �printr^ �XmlPassThroughGetTaggingrJ r rf )r= Z bucket_arg� bucket_urlrr r> r> r? �_GetAndPrintLabel s � � � z LabelCommand._GetAndPrintLabelc C s� | j �d�}| jdd� tj| jd� |dkrNtj|gd� | �| j d � nV|dkrntj|gd� | �� n6|dkr�tj|gd� | �� nt d || j | j f ��dS ) z*Command entry point for the label command.r T)� check_args)r� r )� subcommandsr r zAInvalid subcommand "%s" for the %s command. See "gsutil help %s".) r9 �pop� ParseSubOptsr �LogCommandParamsr� r� r� r� r � command_name)r= �action_subcommandr> r> r? � RunCommand� s$ ��zLabelCommand.RunCommandN)"�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XMLrc r �MakeNFileURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument�MakeNCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� _ch_help_text� help_specr r r �RepeatFlagType�LIST�DICT�gcloud_storage_mapr@ r� r� r� r� r> r> r> r? r � sv � ����� ������ � C_r )7r� � __future__r r r r rz r` rw rh �gslibr �gslib.cloud_apir r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rk Z gslib.utilsr �gslib.utils.constantsr r �gslib.utils.retry_utilr �gslib.utils.shim_utilr r � gslib.utils.translation_helperr � _SET_SYNOPSIS� _GET_SYNOPSIS� _CH_SYNOPSIS�_GET_DESCRIPTION�_SET_DESCRIPTION�_CH_DESCRIPTION�lstripr� � _DESCRIPTIONr� r� r� r� r r> r> r> r? �<module> s` �� � � � commands/__pycache__/help.cpython-39.pyc 0000644 00000014645 15025013616 0014147 0 ustar 00 a +(Wg�$ � @ s$ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z ddl m Z ddlZdd lmZ dd lmZ ddlZdd lmZ dd lmZ dd lmZ ddlmZ ddlm Z ddlm Z ddlm Z ddlm Z dZ de d Z!dZ"G dd� de�Z#dS )�&Implementation of gsutil help command.� )�absolute_import)�print_function)�division)�unicode_literalsN)�PIPE)�Popen)�Command)� OLD_ALIAS_MAP)�CommandException)� HelpProvider)�MAX_HELP_NAME_LEN)� constants)� IS_WINDOWS)�IsRunningInteractively)� GetTermLines)� text_utilz" gsutil help [command or topic] z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> The following command provides a summary of gsutil commands and additional topics on which help is available: gsutil help The following command provides help about the specified command or topic: gsutil help [command or topic] The following command provides help about the specified sub-command: gsutil help [command] [sub-command] For example, running the following provides help about the "set" sub-command of the "acl" command: gsutil help acl set If you set the PAGER environment variable to the path to a pager program (such as /bin/less on Linux), long help sections are piped through the specified pager. z�Usage: gsutil [-D] [-DD] [-h header]... [-i service_account] [-m] [-o section:flag=value]... [-q] [-u user_project] [command [opts...] args...]c @ s` e Zd ZdZejdddgeddddd dd � Zejddgd d e i d �Z dd� Z dd� Z dd� Z dS )� HelpCommandr �help�?Zmanr � � TF)�command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg� command_helpz"Get help about commands and topics)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s< | � � \}}g }| js�|�dt � dtt� d }t|d dd� d�D ] }|�||jj|jj f � qH|�d� t|d d d� d�D ] }|�||jj|jj f � q�|�d � �ntd }| jd }||vr�|�d| � �nL|| }d}t | j�dk�r�|jj } | �r8| jd | v �r8|d | jd }| | jd } nbd}| �sV|�d||f � nDg } | D ]} | �d|| f � �q^|�d| jd ||d� | �f � |�s(|�s�|jj}|jj } |�d� |�d||jj f � |�d� |�| �d�� t�|dg�d } | �r(d|| f }|�d� |�|� | �d � |�� d S ) z)Command entry point for the help command.z%s Available commands: z %-zs%s r c S s | j jS �N�� help_specr! ��hp� r, �"/opt/gsutil/gslib/commands/help.py�<lambda>r � z(HelpCommand.RunCommand.<locals>.<lambda>)�keyz Additional help topics: �additional_helpc S s | j jS r' r( r* r, r, r- r. w r/ z6 Use gsutil help <command or topic> for detailed help.Fr zNo help available for "%s"N� � Tz`The "%s" command has no subcommands. You can ask for the full help by running: gsutil help %s z gsutil help %s %sz�Subcommand "%s" does not exist for command "%s". You can either ask for the full help about the command by running: gsutil help %s Or you can ask for help about one of the subcommands: %s� z <B>NAME</B> z %s - %s z zr The "%s" alias is deprecated, and will eventually be removed completely. Please use the "%s" command instead.z <B>DEPRECATION WARNING</B> r )� _LoadHelpMaps�args�append�top_level_usage_string�strr �sortedr) r! r$ �lenr& �joinr% �stripr �get� _OutputHelp)�self� help_type_map� help_name_map�output� format_str� help_provZinvalid_subcommand�argr! Zsubcommand_mapr% Zsubcommand_examples� subcommandZ new_aliasZdeprecation_warningr, r, r- � RunCommandj s� � � � � ��� �� � � zHelpCommand.RunCommandc C s t s t� s4t�dd|�}t�dd|�}t�|� dS t�dd|�}t�dd|�}t|�d��}dtj v �r|t � k�rtj d �d �}|d � d �r�|� d � z2t jr�|�tj�}n|}t|td d�j|d� W n< t�y } z"tdd �|�|f ��W Y d}~n d}~0 0 n t�|� dS )z�Outputs simply formatted string. This function paginates if the string is too long, PAGER is defined, and the output is a tty. Args: help_str: String to format. z<B>r z</B>Nz[1mz[0;0mr4 ZPAGERr3 r Zlessz-rT)�stdin�universal_newlines)�inputz Unable to open pager (%s): %s)r r �re�subr � print_to_fdr; �split�os�environr �endswithr7 �six�PY2�encoder �UTF8r r � communicate�OSErrorr r< )r@ Zhelp_str� num_linesZpagerZinput_for_pager�er, r, r- r? � s4 �� � zHelpCommand._OutputHelpc C s� t �tjj�D ]\}}}td| � qt �tjj�D ]\}}}td| � q4i }i }tjjD ] }g ||< q\t � t � � t � � �D ]P}|t u r�q~tj�||� |||jj< |jjD ] }|||< q�||jj �|� q~||fS )a. Returns tuple of help type and help name. help type is a dict with key: help type value: list of HelpProviders help name is a dict with key: help command name or alias value: HelpProvider Returns: (help type, help name) zgslib.commands.%szgslib.addlhelp.%s)�pkgutil� iter_modules�gslib�commands�__path__� __import__Zaddlhelp� help_provider�ALL_HELP_TYPES� itertools�chainr �__subclasses__r � SanityCheckr) r! r" r# r7 )r@ �_� module_namerA rB �srE r" r, r, r- r5 � s& � zHelpCommand._LoadHelpMapsN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSIS� command_spec�HelpSpec�_DETAILED_HELP_TEXTr) rH r? r5 r, r, r, r- r Q s. � � D%r )$rm � __future__r r r r rc rP r[ rL rS � subprocessr r Zgslib.addlhelpr] � gslib.commandr r �gslib.commands�gslib.exceptionr �gslib.help_providerr r Z gslib.utilsr Zgslib.utils.system_utilr r r r ro rr r8 r r, r, r, r- �<module> s@ �� � commands/__pycache__/versioning.cpython-39.pyc 0000644 00000013013 15025013616 0015366 0 ustar 00 a +(Wg]! � @ s, d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ dZ dZ e e � d� Z dZ dZ!de e! Z"ee e"�Z#de�$� d d d Z%ee e!�Z&ee e �Z'G dd � d e �Z(d S ) z?Implementation of versioning configuration command for buckets.� )�absolute_import)�print_function)�division)�unicode_literals)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)� shim_utilz8 gsutil versioning set (on|off) gs://<bucket_name>... z/ gsutil versioning get gs://<bucket_name>... � z� <B>SET</B> The "set" sub-command requires an additional sub-command, either "on" or "off", which, respectively, will enable or disable versioning for the specified bucket(s). z� <B>GET</B> The "get" sub-command gets the versioning configuration for a bucket and displays whether or not it is enabled. z� The `Versioning Configuration <https://cloud.google.com/storage/docs/object-versioning>`_ feature enables you to configure a Google Cloud Storage bucket to keep old versions of objects. The gsutil versioning command has two sub-commands: z'--format=value[separator=""](name.sub("z", "gs://").sub("$", ": "),zversioning.enabledz .yesno("Enabled", "Suspended"))c @ s� e Zd ZdZejdddgededdddej ej gej e dd d gd �e � � ge � � gd �d � Z ejdddgddeeed�d�Zeeddddegi dd�eeg d�i d�eg d�i d�d�i d�d�i d�Zd d � Zd d � Zd d!� Zd"d#� Zd$S )%�VersioningCommandz,Implementation of gsutil versioning command.� versioning� setversioning� getversioning� � F�mode�on�off)�choices)�set�get) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz4Enable or suspend versioning for one or more buckets)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�buckets�listz--rawT)�gcloud_command�flag_map�supports_output_translation)r0 r1 �updatez --versioning)r3 r4 )r0 r1 r6 z--no-versioning�r r c C s, | j s| �� | j d �� dkr$dS dS d S )Nr r r � )�args�$RaiseWrongNumberOfArgumentsException�lower)�self� r= �(/opt/gsutil/gslib/commands/versioning.py�_CalculateUrlsStartArg� s z(VersioningCommand._CalculateUrlsStartArgc C s� | j d �� }|dvr$td| j ��| j dd� }|s>| �� d}|D ]�}| j|dgd�}|D ]p}|j}d }tjtj� � d �}|d kr�| j � d |� d |j _ n| j � d |� d|j _ | jj|j||jdgd� q^qF|s�ttt|� ��dS )z+Gets versioning configuration for a bucket.r r7 z,Argument to "%s set" must be either <on|off>r8 NF�id�� bucket_fieldsT)r r z Enabling versioning for %s...z Suspending versioning for %s...)�provider�fields)r9 r; r � command_namer: �GetBucketUrlIterFromArg� storage_url�apitools_messages�Bucket�VersioningValue�logger�infor �enabled� gsutil_api� PatchBucket� bucket_name�schemer r2 ) r< �versioning_arg�url_args� some_matched�url_str� bucket_iter�blr�url�bucket_metadatar= r= r> �_SetVersioning� s: �� � z VersioningCommand._SetVersioningc C s� | j }d}|D ]^}| j|dgd�}|D ]D}d}|jjrV|jjjrVtd|j�d� � q&td|j�d� � q&q|s�tt t |� ��dS ) z6Gets versioning configuration for one or more buckets.Fr rA Tz %s: Enabled�/z %s: SuspendedN) r9 rF � root_objectr rM �print� url_string�rstripr r r2 )r< rS rT rU rV rW r= r= r> �_GetVersioning� s �z VersioningCommand._GetVersioningc C s� | j �d�}|dkr*| j}tj|gd� nL|dkr`| j}| j d �� }|dv rvtj||gd� ntd|| j| jf ��|� dS )z/Command entry point for the versioning command.r r )� subcommandsr r7 zAInvalid subcommand "%s" for the %s command. See "gsutil help %s".) r9 �popr` r �LogCommandParamsrZ r; r rE )r< �action_subcommand�funcrR r= r= r> � RunCommand� s$ � ��z VersioningCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �_GCLOUD_FORMAT_STRING�gcloud_storage_mapr? rZ r` rf r= r= r= r> r N st �������� � ����%�)!r N))rj � __future__r r r r �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rH �gslib.utils.constantsr �gslib.utils.shim_utilr Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprl �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONrr �get_format_flag_caretrv rs rt r r= r= r= r> �<module> sL �� ���� commands/__pycache__/autoclass.cpython-39.pyc 0000644 00000014015 15025013616 0015204 0 ustar 00 a +(Wg�# � @ s4 d Z ddlmZ ddlmZ ddlZddlmZ ddlmZ ddl m Z ddl m Z dd l mZ dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ dZ dZ e e � d� Z dZ dZ!de! e Z"ee e"�Z#ee e �Z$ee e!�Z%de�&� d d d d d d Z'G d d � d e�Z(dS ) z5This module provides the autoclass command to gsutil.� )�absolute_import)�print_functionN)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)� text_util)�NO_MAX)�GcloudStorageMap)� shim_utilz7 gsutil autoclass set (on|off) gs://<bucket_name>... z. gsutil autoclass get gs://<bucket_name>... � z� <B>SET</B> The ``set`` sub-command requires an additional sub-command, either ``on`` or ``off``, which enables or disables Autoclass for the specified bucket(s). a+ <B>GET</B> The ``get`` sub-command gets the current Autoclass configuration for a bucket. The returned configuration has the following fields: ``enabled``: a boolean field indicating whether the feature is on or off. ``toggleTime``: a timestamp indicating when the enabled field was set. z� The `Autoclass <https://cloud.google.com/storage/docs/autoclass>`_ feature automatically selects the best storage class for objects based on access patterns. This command has two sub-commands, ``get`` and ``set``. z--format=value[separator="z"](zformat("gs://{}:", name),z format(" Enabled: {}",z&autoclass.enabled.yesno(True, False)),z format(" Toggle Time: {}",zautoclass.toggleTime))c @ s� e Zd ZdZejdg ededdddej gej e � d�ge ddd gd �e � � gd �d � Z ejdg d deeed �d�Zeedddedgi dd�eeg d�i d�eg d�i d�d�i d�d �i d�Zdd� Zd d � Zd d � Zd d!� Zd"S )#�AutoclassCommandz(Implements the gsutil autoclass command.� autoclass� � F� �mode�on�off)�choices��get�set) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzConfigure Autoclass feature)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�buckets�listz--rawT)�gcloud_command�flag_map�supports_output_translation)r/ r0 �updatez--enable-autoclass)r2 r3 )r/ r0 r5 z--no-enable-autoclass)r r c C sn |j }| jj|jdg|jd�}t|��d�}|jrPt|jdd�}t|jdd�}nd}d}t d� |||�� dS ) z(Gets the autoclass setting for a bucket.r ��fields�provider�/�enabledF� toggleTimeNz#{}: Enabled: {} Toggle Time: {}) � storage_url� gsutil_api� GetBucket� bucket_name�scheme�str�rstripr �getattr�print�format)�self�blr� bucket_url�bucket_metadata�bucketr: Z toggle_time� rK �'/opt/gsutil/gslib/commands/autoclass.py�_get_autoclass� s ��z AutoclassCommand._get_autoclassc C s^ |j }tj�� }|dk|_tj|d�}td|t|��d�f � | jj |j |dg|j d� dS )z'Turns autoclass on or off for a bucket.r )r zSetting Autoclass %s for %sr9 r r6 r ) r< �apitools_messages�Bucket�AutoclassValuer: rD rA rB r= � PatchBucketr? r@ )rF rG � setting_argrH Zautoclass_configrI rK rK rL �_set_autoclass� s � �z AutoclassCommand._set_autoclassc C s& | j �d�}|dvr td��d}g }d}|dkr8| j}n0|dkrh| j}| j �d�}t�|d� |�|� | j� d�t j kr�td � t �d | j ����d }| j }|s�| �� |D ]\}| �|�}|D ]H} | j� | jj�t j kr�td � t �d | j ����d }|| g|�R � q�q�|�s"ttt|� ��dS )z3Handles autoclass command on Cloud Storage buckets.r r z autoclass only supports get|setNr r z-Only on and off values allowed for set option�gsr z=The "%s" command can only be with the Cloud Storage JSON API.Fz2The "%s" command can only be used for GCS Buckets.T)�args�popr rM rS r � InsistOnOrOff�appendr= �GetApiSelectorr �JSON�join�textwrap�wrap� command_name�$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr< r@ r r1 ) rF Z subcommandZsubcommand_funcZsubcommand_argsrR Z some_matched�url_args�url_strZ bucket_iter�bucket_listing_refrK rK rL � _autoclass� sV � �� ����zAutoclassCommand._autoclassc C sZ | j d }| jdd� |dks&|dkrJtj| jd� tj|gd� | �� S td| ��d S ) z.Command entry point for the autoclass command.r T)� check_argsr r )�sub_opts)� subcommandsz-Invalid subcommand "%s", use get|set instead.N)rU � ParseSubOptsr �LogCommandParamsrf rd r )rF �action_subcommandrK rK rL � RunCommand� s �zAutoclassCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rZ r �MakeNCloudURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �_GCLOUD_FORMAT_STRING�gcloud_storage_maprM rS rd rk rK rK rK rL r R sp ����� �� � ����%�)/r ))ro � __future__r r r\ �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rN Z gslib.utilsr �gslib.utils.constantsr �gslib.utils.shim_utilr r � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprq �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONrv rx rw �get_format_flag_newlinerz r rK rK rK rL �<module> sX �� ������� commands/__pycache__/ls.cpython-39.pyc 0000644 00000045471 15025013616 0013636 0 ustar 00 a +(Wg�j � @ s~ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddlm Z ddl m Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl%m'Z' ddl(m)Z) e�*d �Z+d Z,d e, d Z-G d d!� d!e.�Z/G d"d#� d#e �Z0dS )$zCImplementation of Unix-like ls command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�NotFoundException)�Command)�CommandArgument)� ApiSelector)�CommandException)�ContainsWildcard)�StorageUrlFromString)�NO_MAX)�S3_DELETE_MARKER_GUID)�UTF8)�ENCRYPTED_FIELDS)�LsHelper)�PrintFullInfoAboutObject)� UNENCRYPTED_FULL_LISTING_FIELDS)�GcloudStorageFlag)�GcloudStorageMap)� InsistAscii)� text_util)�AclTranslation)�LabelTranslation)�MakeHumanReadablez([^\s]*)\s([^\.\+]*).*z? gsutil ls [-a] [-b] [-d] [-l] [-L] [-r] [-p proj_id] url... z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> Retrieves a list of providers, buckets, or objects matching the criteria, ordered in the list lexicographically by name. <B>LISTING PROVIDERS, BUCKETS, SUBDIRECTORIES, AND OBJECTS</B> If you run ``gsutil ls`` without URLs, it lists all of the Google Cloud Storage buckets under your default project ID (or all of the Cloud Storage buckets under the project you specify with the ``-p`` flag): gsutil ls If you specify one or more provider URLs, ``gsutil ls`` lists buckets at each listed provider: gsutil ls gs:// gsutil currently supports ``gs://`` and ``s3://`` as valid providers If you specify bucket URLs, or use `URI wildcards <https://cloud.google.com/storage/docs/wildcards>`_ to capture a set of buckets, ``gsutil ls`` lists objects at the top level of each bucket, along with the names of each subdirectory. For example: gsutil ls gs://bucket might produce output like: gs://bucket/obj1.htm gs://bucket/obj2.htm gs://bucket/images1/ gs://bucket/images2/ The "/" at the end of the last 2 URLs tells you they are subdirectories, which you can list using: gsutil ls gs://bucket/images* If you specify object URLs, ``gsutil ls`` lists the specified objects. For example: gsutil ls gs://bucket/*.txt lists all files whose name matches the above wildcard at the top level of the bucket. For more details, see `URI wildcards <https://cloud.google.com/storage/docs/wildcards>`_. <B>DIRECTORY BY DIRECTORY, FLAT, and RECURSIVE LISTINGS</B> Listing a bucket or subdirectory (as illustrated near the end of the previous section) only shows the objects and names of subdirectories it contains. You can list all objects in a bucket by using the -r option. For example: gsutil ls -r gs://bucket lists the top-level objects and buckets, then the objects and buckets under gs://bucket/images1, then those under gs://bucket/images2, etc. If you want to see all objects in the bucket in one "flat" listing use the recursive ("**") wildcard, like: gsutil ls -r gs://bucket/** or, for a flat listing of a subdirectory: gsutil ls -r gs://bucket/dir/** If you want to see only the subdirectory itself, use the -d option: gsutil ls -d gs://bucket/dir <B>LISTING OBJECT DETAILS</B> If you specify the -l option, gsutil outputs additional information about each matching provider, bucket, subdirectory, or object. For example: gsutil ls -l gs://bucket/*.html gs://bucket/*.txt prints the object size, creation time stamp, and name of each matching object, along with the total count and sum of sizes of all matching objects: 2276224 2020-03-02T19:25:17Z gs://bucket/obj1.html 3914624 2020-03-02T19:30:27Z gs://bucket/obj2.html 131 2020-03-02T19:37:45Z gs://bucket/obj3.txt TOTAL: 3 objects, 6190979 bytes (5.9 MiB) Note that the total listed in parentheses above is in mebibytes (or gibibytes, tebibytes, etc.), which corresponds to the unit of billing measurement for Google Cloud Storage. You can get a listing of all the objects in the top-level bucket directory (along with the total count and sum of sizes) using a command like: gsutil ls -l gs://bucket To print additional detail about objects and buckets use the gsutil ls -L option. For example: gsutil ls -L gs://bucket/obj1 prints something like: gs://bucket/obj1: Creation time: Fri, 26 May 2017 22:55:44 GMT Update time: Tue, 18 Jul 2017 12:31:18 GMT Storage class: STANDARD Content-Length: 60183 Content-Type: image/jpeg Hash (crc32c): zlUhtg== Hash (md5): Bv86IAzFzrD1Z2io/c7yqA== ETag: 5ca67960a586723b7344afffc81 Generation: 1378862725952000 Metageneration: 1 ACL: [ { "entity": "project-owners-867484910061", "projectTeam": { "projectNumber": "867484910061", "team": "owners" }, "role": "OWNER" }, { "email": "jane@gmail.com", "entity": "user-jane@gmail.com", "role": "OWNER" } ] TOTAL: 1 objects, 60183 bytes (58.77 KiB) Note that results may contain additional fields, such as custom metadata or a storage class update time, if they are applicable to the object. Also note that some fields, such as update time, are not available with the (non-default) XML API. See also "gsutil help acl" for getting a more readable version of the ACL. <B>LISTING BUCKET DETAILS</B> If you want to see information about the bucket itself, use the -b option. For example: gsutil ls -L -b gs://bucket prints something like: gs://bucket/ : Storage class: STANDARD Location constraint: US Versioning enabled: False Logging configuration: None Website configuration: None CORS configuration: None Lifecycle configuration: None Requester Pays enabled: True Labels: None Default KMS key: None Time created: Thu, 14 Jan 2016 19:25:17 GMT Time updated: Thu, 08 Jun 2017 21:17:59 GMT Metageneration: 1 Bucket Policy Only enabled: False ACL: [ { "entity": "project-owners-867489160491", "projectTeam": { "projectNumber": "867489160491", "team": "owners" }, "role": "OWNER" } ] Default ACL: [ { "entity": "project-owners-867489160491", "projectTeam": { "projectNumber": "867489160491", "team": "owners" }, "role": "OWNER" } ] Note that some fields above (time created, time updated, metageneration) are not available with the (non-default) XML API. <B>OPTIONS</B> -l Prints long listing (owner, length). -L Prints even more detail than -l. Note: If you use this option with the (non-default) XML API it generates an additional request per object being listed, which makes the -L option run much more slowly and cost more than the default JSON API. -d List matching subdirectory names instead of contents, and do not recurse into matching subdirectories even if the -R option is specified. -b Prints info about the bucket when used with a bucket URL. -h When used with -l, prints object sizes in human readable format (e.g., 1 KiB, 234 MiB, 2 GiB, etc.) -p proj_id Specifies the project ID or project number to use for listing buckets. -R, -r Requests a recursive listing, performing at least one listing operation per subdirectory. If you have a large number of subdirectories and do not require recursive-style output ordering, you may be able to instead use wildcards to perform a flat listing, e.g. ``gsutil ls gs://mybucket/**``, which generally performs fewer listing operations. -a Includes non-current object versions / generations in the listing (only useful with a versioning-enabled bucket). If combined with -l option also prints metageneration for each listed object. -e Include ETag in long listing (-l) output. c @ s e Zd ZdZdZdZdZdS )� ListingStylez(Enum class for specifying listing style.�SHORT�LONG� LONG_LONGN)�__name__� __module__� __qualname__�__doc__r r r � r# r# � /opt/gsutil/gslib/commands/ls.pyr s r c @ s� e Zd ZdZejdddgededdddej ej gej e � � gd � Z ejdddgd d ei d �Zeg d �ed�ed�ed�ed�ed�ed�ed�ed�ed�d� d�Zdd� Zdd� Zd d � Zd S ) � LsCommandz$Implementation of gsutil ls command.�ls�dir�listr z aebdlLhp:rRFT) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz#List providers, buckets, or objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storager& z --fetch-encrypted-object-hashes�-r�-l�-L�-b�-e�-az--readable-sizesz --project) r< �-Rr= r> r? r@ rA �-h�-p)�gcloud_command�flag_mapc C s� |t jks|t jkr"t�|� dS |j}|j}|j}|j||t � |j �t � |j �|j o\|j j|jrfdnd|jrrdnd|jr~dnd|jr�dnd|jo�|jjd� }|jr�d|d< |jr�tj |jdd�|d< nd|d< |jr�|jjr�|jj|d < nd|d < |jr�dnd|d < |j�r*|jj�r*|jj� d �|d < |j �r<|j |d < |j �rP|j j |d< |j �rb|j |d< |j!�rz|j!� d�|d< |j"�r�|j"� d�|d< |j#�r�|j#|d< |j$�r�|j$j%�r�|j$j%j}||d< |j$j&�r�|j$j&|d< |j'�r�|j'|d< |j(�r|j(|d< |D ]R}|| } t)| t*j+��rd| v�r.�q| �,dd�} | �-d��sNd| } | ||< �qd} d} d} d}d}d}d}d}d}d}d}d}d |v �r�d} d |v �r�d } d|v �r�d } d|v �r�d }d|v �r�d }d|v �r�d }d|v �r�d!}d|v �r�d"}d|v �rd#}d|v �rd$}d|v �r$d%}d|v �r2d&}t�d'| d( | d) | | d* d+ | | | | | | | | d, j.f i |��� |j/j0d-k�r�t�d.� dS )/a Print listing info for given bucket. Args: bucket_blr: BucketListingReference for the bucket being listed listing_style: ListingStyle enum describing type of output desired. Returns: Tuple (total objects, total bytes) in the bucket. NZPresent�None) �bucket� storage_class�location_constraint�aclZ default_acl� versioning�website_config�logging_config� cors_config�lifecycle_config�requester_paysZretention_policyT)� pretty_print�labelsZdefault_kms_key�encryption_configz %a, %d %b %YZautoclass_enabled_dateZ location_typeZcustom_placement_locations�metagenerationz%a, %d %b %Y %H:%M:%S GMT� time_created�updatedZdefault_eventbased_holdZbucket_policy_only_enabledZpublic_access_prevention�rpoZ satisfies_pzs� z � z2 Autoclass: Enabled on {autoclass_enabled_date} z" Location type: {location_type} z4 Placement locations: {custom_placement_locations} z$ Metageneration: {metageneration} z Time created: {time_created} z Time updated: {updated} z5 Default Event-Based Hold: {default_eventbased_hold} z' Retention Policy: {retention_policy} z: Bucket Policy Only enabled: {bucket_policy_only_enabled} z6 Public access prevention: {public_access_prevention} z RPO: {rpo} z" Satisfies PZS: {satisfies_pzs} z-{bucket} : Storage class: {storage_class} z- Location constraint: {location_constraint} z� Versioning enabled: {versioning} Logging configuration: {logging_config} Website configuration: {website_config} CORS configuration: {cors_config} Lifecycle configuration: {lifecycle_config} Requester Pays enabled: {requester_pays} z Labels: {labels} z% Default KMS key: {default_kms_key} z, ACL: {acl} Default ACL: {default_acl}�s3z�Note: this is an S3 bucket so configuration values may be blank. To retrieve bucket configuration values, use individual configuration commands such as gsutil acl get <bucket>.)1r r r r � print_to_fd� root_object�location� storageClass� url_stringr �JsonFromMessagerK �defaultObjectAclrL �enabled�website�logging�cors� lifecycle�billing� requesterPays�retentionPolicyrS r � encryption�defaultKmsKeyName� autoclass� toggleTime�strftime� locationType�customPlacementConfig� dataLocationsrU � timeCreatedrW �defaultEventBasedHold�iamConfiguration�bucketPolicyOnly�publicAccessPreventionrX � satisfiesPZS� isinstance�six� string_types�replace� startswith�format� storage_url�scheme)�selfZ bucket_blr� listing_stylerH rJ rI �fieldsrc �keyZprevious_value� new_valueZautoclass_lineZlocation_type_lineZ custom_placement_locations_lineZmetageneration_lineZtime_created_lineZtime_updated_lineZ default_eventbased_hold_lineZretention_policy_lineZ bucket_policy_only_enabled_lineZ public_access_prevention_lineZrpo_lineZsatisifies_pzs_liner# r# r$ �_PrintBucketInfo^ s2 � � � � � � �� � � � ���� � � � � � � ����������zLsCommand._PrintBucketInfoc C s� |j }|j}|jr4t|jjv r4d}d}d}|d7 }n$| jrDt|j�nt|j�}|j}d}t � dt|j ��}d}d} d} | j r�|d7 }t|j �} | jr�|d 7 }|j} |||| | d �} t�|| � ||fS ) z(Prints an object with ListingStyle.LONG.�0r z<DeleteMarker>� z\1T\2Zz"%(size)10s %(timestamp)s %(url)sNz# metageneration=%(metageneration)sz etag=%(etag)s)�size� timestamp�urlrU �etag)r] r` �metadatar �additionalProperties�human_readabler r� �str�JSON_TIMESTAMP_RE�subrs � all_versionsrU � include_etagr� r r\ ) r� �bucket_listing_ref�obj�url_str� size_string� num_bytes�num_objsr� ZprintstrZ encoded_etagZencoded_metagenZ format_argsr# r# r$ �_PrintLongListing� sB � � � �zLsCommand._PrintLongListingc s� d}d}t j}d}d� _d� _d� _d� _d� _� jr� jD ]�\}}|dkrTd� _q<|dkrdd� _q<|dkrrd}q<|dkr�d� _q<|dkr�t j}q<|dkr�t j }q<|d kr�t |d � |� _ q<|d ks�|d kr�d� _q<|d kr<d� _q<� j s�dg� _ d}d}� fdd�} | } � j D �]>} t | �} | �� �r6td� j ��d} |t jk�sR|t jk�rZdg} n|t j k�rng d�} | �� �r�� �d| j �j| d�D ]}� �||� �q��q| �� �r.|�r.d}� �| �j| d�D ]F}t| ��s�|j�s�� jj|jj dg| jd� � �||� |d7 }�q�t| ��sN|�sNd}�qdd� }|t jk�rbt � j� j � j| � j� jd �}n�|t jk�r�g d �}� j�r�|� d d g� � j�r�|� d � t � j� j � j!|| � j� j|� jd!� }nF|t j k�r t"t# }t � j� j t$|| � j� j|� jd!� }n td"| ��|�%| �\}}}| �&� �r>|dk�r>|dk�r>d}||7 }||7 }�q|�r�|t jk�r�t'�(d#||t)t*|��f � |�r�td$��|�r�t+d%��dS )&z'Command entry point for the ls command.FTrA r@ r? rC r= r> rD z/Invalid non-ASCII character found in project IDr< rB z-dzgs://r c s( t � j�dkr$t�dt�| j� � d S )Nr� z%s:)�len�argsr r\ rz � ensure_textr` ��blr�r� r# r$ �MaybePrintBucketHeaderG s z4LsCommand.RunCommand.<locals>.MaybePrintBucketHeaderz$Only cloud URLs are supported for %sN�id)rK rm rh rf rq rb rk ru rS r^ rp re rg rU rj rt rX rx r_ rs rW rL rd z%s://*)� bucket_fields)r� �providerr� c S s t �ddt�| j�f � d S )Nz%-33s%srZ )r r\ rz r� r` r� r# r# r$ �_PrintPrefixLong� s �z.LsCommand.RunCommand.<locals>._PrintPrefixLong)r� �print_bucket_header_func�should_recurse�list_subdir_contents)�namer� rs rW � generationrU r� )�print_object_func�print_dir_funcr� r� r� r� r� zUnknown listing style: %sz TOTAL: %d objects, %d bytes (%s)z$One or more URLs matched no objects.z+One or more bucket URLs matched no buckets.),r r �recursion_requestedr� r� r� r� �sub_optsr r r � project_idr� r � IsFileUrlr � command_name� IsProvider�WildcardIteratorr� � IterBucketsr� �IsBucketr r] � gsutil_api� GetBucketr � bucket_namer �logger�extend�appendr� r r r �ExpandUrlAndPrint�IsObjectr r\ r �floatr )r� �got_nomatch_errorsZgot_bucket_nomatch_errorsr� Zget_bucket_info�o�aZ total_objs� total_bytesr� Zprint_bucket_headerr� r r� r� Z total_bucketsr� �listing_helper�bucket_listing_fieldsZexp_dirs�exp_objs� exp_bytesr# r� r$ � RunCommand s � � ��� � � � � � �� � ��zLsCommand.RunCommandN)r r r! r" r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r �gcloud_storage_mapr� r� r� r# r# r# r$ r% ' s^ ������ �� $r% )1r" � __future__r r r r �rerz �gslib.cloud_apir � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.storage_urlr r �gslib.utils.constantsr r r Zgslib.utils.ls_helperr r r r �gslib.utils.shim_utilr r Zgslib.utils.text_utilr Z gslib.utilsr � gslib.utils.translation_helperr r �gslib.utils.unit_utilr �compiler� r� r� �objectr r% r# r# r# r$ �<module> sJ �� i commands/__pycache__/rewrite.cpython-39.pyc 0000644 00000040174 15025013616 0014674 0 ustar 00 a +(Wg�f � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl%m'Z' ddl(m)Z) ddl(m*Z* ddl(m+Z+ ddl(m,Z, ddl-m.Z. dd l-m/Z/ dd l0m1Z1 dd l2m3Z3 dd l2m4Z4 dd l5m6Z6 dd!l7m8Z8 d"Z9d#Z:d$e: d% Z;d&d'� Z<d2d(d)�Z=d*d+� Z>G d,d-� d-e?�Z@G d.d/� d/e�ZAd0d1� ZBdS )3zIImplementation of rewrite command (in-place cloud object transformation).� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�config)�EncryptionException)�Command)�CommandArgument)� ApiSelector)�CommandException)�NameExpansionIterator)� SeekAheadNameExpansionIterator)�FileProgressCallbackHandler)�StorageUrlFromString)�storage_v1_messages)� FileMessage)�GetCloudApiInstance)�NO_MAX)�UTF8)� CryptoKeyType)�CryptoKeyWrapperFromKey)�GetEncryptionKeyWrapper)�MAX_DECRYPTION_KEYS)�GcloudStorageFlag)�GcloudStorageMap)� StdinIterator)� ConvertRecursiveToFlatWildcard)�NormalizeStorageClass)� text_util)�PreconditionsFromHeaders�A z� gsutil rewrite -k [-O] [-f] [-r] [-s] url... gsutil rewrite -k [-O] [-f] [-r] [-s] -I gsutil rewrite -s [-k] [-O] [-f] [-r] url... gsutil rewrite -s [-k] [-O] [-f] [-r] -I z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> The gsutil rewrite command rewrites cloud objects, applying the specified transformations to them. The transformation(s) are atomic for each affected object and applied based on the input transformation flags. Object metadata values are preserved unless altered by a transformation. At least one transformation flag, -k or -s, must be included in the command. The -k flag is supported to add, rotate, or remove encryption keys on objects. For example, the command: gsutil rewrite -k -r gs://bucket updates all objects in gs://bucket with the current encryption key from your boto config file, which may either be a base64-encoded CSEK or the fully-qualified name of a Cloud KMS key. The rewrite command acts only on live object versions, so specifying a URL with a generation number fails. If you want to rewrite a noncurrent version, first copy it to the live version, then rewrite it, for example: gsutil cp gs://bucket/object#123 gs://bucket/object gsutil rewrite -k gs://bucket/object You can use the -s option to specify a new storage class for objects. For example, the command: gsutil rewrite -s nearline gs://bucket/foo rewrites the object, changing its storage class to nearline. If you specify the -k option and you have an encryption key set in your boto configuration file, the rewrite command skips objects that are already encrypted with the specified key. For example, if you run: gsutil rewrite -k -r gs://bucket and gs://bucket contains objects encrypted with the key specified in your boto configuration file, gsutil skips rewriting those objects and only rewrites objects that are not encrypted with the specified key. This avoids the cost of performing redundant rewrite operations. If you specify the -k option and you do not have an encryption key set in your boto configuration file, gsutil always rewrites each object, without explicitly specifying an encryption key. This results in rewritten objects being encrypted with either the bucket's default KMS key (if one is set) or Google-managed encryption (no CSEK or CMEK). Gsutil does not attempt to determine whether the operation is redundant (and thus skippable) because gsutil cannot be sure how the object is encrypted after the rewrite. Note that if your goal is to encrypt objects with a bucket's default KMS key, you can avoid redundant rewrite costs by specifying the bucket's default KMS key in your boto configuration file; this allows gsutil to perform an accurate comparison of the objects' current and desired encryption configurations and skip rewrites for objects already encrypted with that key. If have an encryption key set in your boto configuration file and specify multiple transformations, gsutil only skips those that would not change the object's state. For example, if you run: gsutil rewrite -s nearline -k -r gs://bucket and gs://bucket contains objects that already match the encryption configuration but have a storage class of standard, the only transformation applied to those objects would be the change in storage class. You can pass a list of URLs (one per line) to rewrite on stdin instead of as command line arguments by using the -I option. This allows you to use gsutil in a pipeline to rewrite objects identified by a program, such as: some_program | gsutil -m rewrite -k -I The contents of stdin can name cloud URLs and wildcards of cloud URLs. The rewrite command requires OWNER permissions on each object to preserve object ACLs. You can bypass this by using the -O flag, which causes gsutil not to read the object's ACL and instead apply the default object ACL to the rewritten object: gsutil rewrite -k -O -r gs://bucket <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when rewriting multiple objects. If some of the objects could not be rewritten, gsutil's exit status is non-zero even if this flag is set. This option is implicitly set when running "gsutil -m rewrite ...". -I Causes gsutil to read the list of objects to rewrite from stdin. This allows you to run a program that generates the list of objects to rewrite. -k Rewrite objects with the current encryption key specified in your boto configuration file. The value for encryption_key may be either a base64-encoded CSEK or a fully-qualified KMS key name. If no value is specified for encryption_key, gsutil ignores this flag. Instead, rewritten objects are encrypted with the bucket's default KMS key, if one is set, or Google-managed encryption, if no default KMS key is set. -O When a bucket has uniform bucket-level access (UBLA) enabled, the -O flag is required and skips all ACL checks. When a bucket has UBLA disabled, the -O flag rewrites objects with the bucket's default object ACL instead of the existing object ACL. This is needed if you do not have OWNER permission on the object. -R, -r The -R and -r options are synonymous. Causes bucket or bucket subdirectory contents to be rewritten recursively. -s <class> Rewrite objects using the specified storage class. c C s( | j s| j�t|�� | jd7 _dS )z9Simple exception handler to allow post-completion status.� N)�continue_on_error�logger�error�str�op_failure_count)�cls�e� r* �%/opt/gsutil/gslib/commands/rewrite.py�_RewriteExceptionHandler� s r, c C s | j ||d� d S )N�� thread_state)� RewriteFunc)r( �name_expansion_resultr. r* r* r+ �_RewriteFuncWrapper� s r1 c c s. | D ]$}t |�jdur"td| ��|V qdS )zAGenerator function that ensures generation-less (live) arguments.Nz-"rewrite" called on URL with generation (%s).)r � generationr )�url_strs�url_strr* r* r+ �GenerationCheckGenerator� s �r5 c @ s e Zd ZdZdZdZdS )�_TransformTypesz Enum class for valid transforms.� crypto_key� storage_classN)�__name__� __module__� __qualname__�__doc__� CRYPTO_KEY� STORAGE_CLASSr* r* r* r+ r6 � s r6 c @ s� e Zd ZdZejdg ededdddej gej e � � gd� Z ej dddgd d ei d �Zeg d �ed �ed�e�ddd�rzdned�ed�ed�ed�ed�d�d�Zdd� Zdd� Zd d d �ZdS ) �RewriteCommandz)Implementation of gsutil rewrite command.�rewriter zfkIrROs:F) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_argumentsZrekey�rotate� command_helpzRewrite objects)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�objects�update�-Iz--continue-on-error�GSUtil�encryption_keyNz--clear-encryption-keyz--no-preserve-acl�-r�-s)rW �-f�-k�-OrZ �-Rr[ )�gcloud_command�flag_mapc C s |j dkrtdt|� ��d S )N�gsz5"rewrite" called on URL with unsupported provider: %s)�schemer r& )�self�urlr* r* r+ � CheckProvider s �z RewriteCommand.CheckProviderc C s� | j | _i | _d| _d| _d| _ddg| _t� | _d| _ t t �| _ | j rP| j j nd| _| jr�| jD ]�\}}|dkrzd| _qb|dkr�| j�tj� qb|dkr�d| _qb|d kr�d| _qb|d ks�|d kr�d| _d| _qb|dkrb| j�tj� t|�| _qb| j�r| j�r td ��t� }n| j�s"td ��| j}| j�s>td| j ��t| j�pJi �| _ t |�}| j�rht |�}t | j | j!| j"| j#|| j| j$| j�p�| j ddgd� }d}| j�s�t |�}t%| j | j!| �&� || j| j| j$d�}t'dt(�D ]R}|d } t)t �*ddt+| � d��} | du �r �q2| j,t-j.k�r�| | j| j < �q�| jdu�rL| j | j| j< | j du �rzd�/t0�1d��} t2d| t3j4d� | j5t6|t7| j dg|d� | j �r�| j �r�d nd } td | j | | f ��dS ) z,Command entry point for the rewrite command.NFr] r[ r r\ TrW r^ rZ r_ z&No arguments allowed with the -I flag.z:The rewrite command (without -I) expects at least one URL.zgrewrite command requires at least one transformation flag. Currently supported transformation flags: %s�name�size)� project_idr# �bucket_listing_fields)� all_versionsri r" rX zdecryption_key%s� a> NOTE: No encryption_key was specified in the boto configuration file, so gsutil will not provide an encryption key in its rewrite API requests. This will decrypt the objects unless they are in buckets with a default KMS key set, in which case the service will automatically encrypt the rewritten objects with that key.z%s )�filer' )� fail_on_error� shared_attrs�seek_ahead_iterator�s� z*%d file%s/object%s could not be rewritten.)8�parallel_operationsr# �csek_hash_to_keywrapper�dest_storage_class�no_preserve_acl�read_args_from_stdinZ supported_transformation_flags�set�transform_typesr' r r � boto_file_encryption_keywrapper�crypto_key_sha256Zboto_file_encryption_sha256�sub_opts�addr6 r= �recursion_requestedrk r> r �argsr r r �headers� preconditionsr5 r r � command_name�debugr$ � gsutil_apiri r �GetSeekAheadGsutilApi�ranger r �getr& � crypto_typer �CSEK�join�textwrap�wrap�print�sys�stderr�Applyr1 r, ) rd �o�ar3 Zurl_strs_generator�name_expansion_iteratorrp Zseek_ahead_url_strs�i� key_number� keywrapper�msg� plural_strr* r* r+ � RunCommand s� � � �� � � � � ��� �zRewriteCommand.RunCommandc C s t | |d�}|j}| �|� |j|j|j|j|jd�}| jrDg |_ n|j sVt d| ��|j rb|j nd }d }|j r�|j j r�|j j }|�d�}|d up�|d u}d } | jd ur�| jjtjkr�| jj} d } | jd ur�| jjtjkr�| jj} | jd u} || ko�|| k} tj| jv�r | �s td||f ��g } tj| jv �rN| jt |j �k�rN| � d� tj| jv �rr| �rr| �rr| � d� t | �t | j�k�r�| j �!d|| f � d S t"�#t$j%t"�&|��}d |_d |_'d |_ d |_ tj| jv �r�| j|_ | d u�r�| |_ d }|d u�r&|| j(v �r| j(| }ntd ||f ��d }tj| jv �rr|�rP| �rP| �srd }n"|�rb| �sbd }n|�sr| �rrd }t)j*�+t,||j-�� t)j*�.� |j/�0t1|d t2�2� d|j3t1j4d�� t5|j/||d�j6}|j7|||j| j8||| j|jg d� |j/�0t1|d t2�2� d|j3t1j4d�� d S )Nr- )r2 �providera No OWNER permission found for object %s. If your bucket has uniform bucket-level access (UBLA) enabled, include the -O option in your command to avoid this error. If your bucket does not use UBLA, you can use the -O option to apply the bucket's default object ACL when rewriting.�asciiz�The "-k" flag was not passed to the rewrite command, but the encryption_key value in your boto config file did not match the key used to encrypt the object "%s" (hash: %s). To encrypt the object using a different key, you must specify the "-k" flag.z storage classzencryption keyz3Skipping %s, all transformations were redundant: %szOMissing decryption key with SHA256 hash %s. No decryption key matches object %sZ RewritingZRotatingZ DecryptingZ EncryptingF)�finishedrh � message_type)�src_url�operation_name)�src_generationr� �progress_callback�decryption_tuple�encryption_tupler� �fieldsT)9r �expanded_storage_urlrf �GetObjectMetadata� bucket_name� object_namer2 rc rv �aclr � kmsKeyName�customerEncryption� keySha256�encoderz r� r �CMEKr7 r� r{ r6 r= ry r r> ru r � storageClass�append�lenr$ �infor �PyValueToMessage�apitools_messages�Object�MessageToPyValue�idrt r� r� �write�_ConstructAnnounceText� url_string�flush� status_queue�putr �timerh � FILE_REWRITEr �call� CopyObjectr� )rd r0 r. r� Z transform_urlZ src_metadataZsrc_encryption_kms_keyZsrc_encryption_sha256Zsrc_was_encryptedZdest_encryption_kms_keyZdest_encryption_sha256Zshould_encrypt_destZencryption_unchangedZredundant_transformsZ dest_metadata�decryption_keywrapperr� r� r* r* r+ r/ � s ������ � � �� ���� �� �� � � ��� � ���� ��zRewriteCommand.RewriteFunc)N)r9 r: r; r< r �CreateCommandSpec� _SYNOPSISr r �JSONr � MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r r r� �gcloud_storage_maprf r� r/ r* r* r* r+ r? � sN �� �� r? c C sz | dd� � d�}t|�}td�}|t|� | tkr\td�}d|t| | | d� }d||f }dt }|�|�S ) a� Constructs announce text for ongoing operations on url_string. This truncates the text to a maximum of MAX_PROGRESS_INDICATOR_COLUMNS, and informs the rewrite-related operation ('Encrypting', 'Rotating', or 'Decrypting'). Args: operation_name: String describing the operation, i.e. 'Rotating' or 'Encrypting'. url_string: String describing the file/object being processed. Returns: Formatted announce text for outputting operation progress. N� � z: z...z...%sz%s%s:z{0:%ds})�ljustr� � MAX_PROGRESS_INDICATOR_COLUMNS�format)r� r� Zjustified_op_stringZ start_lenZend_lenZ ellipsis_lenZbase_announce_text� format_strr* r* r+ r� E s ��� r� )N)Cr< � __future__r r r r r� r� r� �apitools.base.pyr �botor �gslib.cloud_apir � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.name_expansionr r �gslib.progress_callbackr �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr r� �gslib.thread_messager � gslib.utils.cloud_api_helperr �gslib.utils.constantsr r � gslib.utils.encryption_helperr r r r �gslib.utils.shim_utilr r Zgslib.utils.system_utilr Zgslib.utils.text_utilr r Z gslib.utilsr � gslib.utils.translation_helperr r� r� r� r, r1 r5 �objectr6 r? r� r* r* r* r+ �<module> sb ��v w commands/__pycache__/hmac.cpython-39.pyc 0000644 00000030621 15025013616 0014117 0 ustar 00 a +(Wg�B � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z dZ dZ dZ dZ dZ!dZ"dZ#dZ$dZ%dZ&e e �'d � e �'d � e �'d � e!�'d � d Z(d d �)e"e#e$e%e&g� Z*ee(e*�Z+d d gZ,d!Z-ee e"�Z.ee e#�Z/ee e$�Z0ee e%�Z1ee!e&�Z2d"d#� Z3d$d%� Z4d&e �5� d' d( d) Z6d&e �5� d* d+ d, d- d. d/ d0 d1 d2 Z7d3Z8ed4�Z9ed5d6d7e6gd8e9id9�Z:eg d:�d8e9id9�Z;ed5d6d;e7gd8e9id9�Z<ed5d6d<e8ged=�ed>�e9d?�d9�Z=ed5d6d<e7ged=�ed@�ed>�e9dA�d9�Z>ed5d6dBe7gedCdDdE��edF�e9dG�d9�Z?G dHdI� dIe�Z@dJS )Kz� Implementation of HMAC key management command for GCS. NOTE: Any modification to this file or corresponding HMAC logic should be submitted in its own PR and release to avoid concurrency issues in testing. � )�absolute_import)�division)�print_function)�unicode_literals)�Command)�CommandArgument)� ApiSelector��CommandException)�CreateHelpText)�LogCommandParams)�PopulateProjectId)�GetCloudApiInstance)�GcloudStorageFlag)�GcloudStorageMap)� InsistAscii)� shim_utilz= gsutil hmac create [-p <project>] <service_account_email> z1 gsutil hmac delete [-p <project>] <access_id> z. gsutil hmac get [-p <project>] <access_id> zJ gsutil hmac list [-a] [-l] [-p <project>] [-u <service_account_email>] zR gsutil hmac update -s (ACTIVE|INACTIVE) [-e <etag>] [-p <project>] <access_id> a <B>CREATE</B> The ``hmac create`` command creates an HMAC key for the specified service account: gsutil hmac create test.service.account@test_project.iam.gserviceaccount.com The secret key material is only available upon creation, so be sure to store the returned secret along with the access_id. <B>CREATE OPTIONS</B> The ``create`` sub-command has the following option -p <project> Specify the ID or number of the project in which to create a key. a� <B>DELETE</B> The ``hmac delete`` command permanently deletes the specified HMAC key: gsutil hmac delete GOOG56JBMFZX6PMPTQ62VD2 Note that keys must be updated to be in the ``INACTIVE`` state before they can be deleted. <B>DELETE OPTIONS</B> The ``delete`` sub-command has the following option -p <project> Specify the ID or number of the project from which to delete a key. a� <B>GET</B> The ``hmac get`` command retrieves the specified HMAC key's metadata: gsutil hmac get GOOG56JBMFZX6PMPTQ62VD2 Note that there is no option to retrieve a key's secret material after it has been created. <B>GET OPTIONS</B> The ``get`` sub-command has the following option -p <project> Specify the ID or number of the project from which to get a key. a� <B>LIST</B> The ``hmac list`` command lists the HMAC key metadata for keys in the specified project. If no project is specified in the command, the default project is used. <B>LIST OPTIONS</B> The ``list`` sub-command has the following options -a Show all keys, including recently deleted keys. -l Use long listing format. Shows each key's full metadata excluding the secret. -p <project> Specify the ID or number of the project from which to list keys. -u <service_account_email> Filter keys for a single service account. a� <B>UPDATE</B> The ``hmac update`` command sets the state of the specified key: gsutil hmac update -s INACTIVE -e M42da= GOOG56JBMFZX6PMPTQ62VD2 Valid state arguments are ``ACTIVE`` and ``INACTIVE``. To set a key to state ``DELETED``, use the ``hmac delete`` command on an ``INACTIVE`` key. If an etag is set in the command, it will only succeed if the provided etag matches the etag of the stored key. <B>UPDATE OPTIONS</B> The ``update`` sub-command has the following options -s <ACTIVE|INACTIVE> Sets the state of the specified key to either ``ACTIVE`` or ``INACTIVE``. -e <etag> If provided, the update will only be performed if the specified etag matches the etag of the stored key. -p <project> Specify the ID or number of the project in which to update a key. � z z� You can use the ``hmac`` command to interact with service account `HMAC keys <https://cloud.google.com/storage/docs/authentication/hmackeys>`_. The ``hmac`` command has five sub-commands: �INACTIVE�ACTIVEz%a, %d %b %Y %H:%M:%S GMTc C s t d| ||f �S )NzD%s %s requires an Access ID to be specified as the last argument. %sr )� command_name� subcommand�synopsis� r �"/opt/gsutil/gslib/commands/hmac.py�_AccessIdException� s ��r c C s� d dd�}d| j }||d| j�7 }||d| j�7 }||d| j�7 }||d| j�t��7 }||d | j�t��7 }||d | jd d �7 }|S )z4Format the key metadata for printing to the console.Tc S s& d}d|| d |f }|r"|d7 }|S )z=Format the metadata name-value pair into two aligned columns.� z %-*s %s�:r r )�name�value�new_line�widthZinfo_strr r r � FormatInfo� s z&_KeyMetadataOutput.<locals>.FormatInfozAccess ID %s: �StatezService Account�Projectz Time CreatedzTime Last UpdatedZEtagF)r )T) �accessId�state�serviceAccountEmail� projectId� timeCreated�strftime� _TIME_FORMAT�updated�etag)�metadatar" �messager r r �_KeyMetadataOutput� s � �r0 z--format=value[separator="z"](z-format("Access ID: {}", metadata.accessId),z"format("Secret: {}", secret))z%"](format("Access ID {}:", accessId),z,format(" State: {}", state),z:format(" Service Account: {}", serviceAccountEmail),z0format(" Project: {}", projectId),z$format(" Time Created: {}",z6 timeCreated.date(format="%a %d %b %Y %H:%M:%S GMT")),z$format(" Time Last Updated: {}",z2 updated.date(format="%a %d %b %Y %H:%M:%S GMT")),z+format(" Etag: {}", etag))zW--format=table[no-heading](format("{} ", accessId),state:width=11, serviceAccountEmail)z --project�storage�hmac�create�-p��gcloud_command�flag_map)r1 r2 �delete�describe�listz--allz--service-account)�-a�-ur4 z--long)r; �-lr< r4 �updatez --activatez --deactivate)r r z--etag)�-s�-er4 c s� e Zd ZdZejddddddejgejee � � ge � � ge � � ge � � ge � � gd�d� Z ej dg d d e eeeeed�d �Z� fd d �Zddd�Zd dd�Zd dd�Zd dd�Zd dd�Zdd� Z� ZS ) � HmacCommandz&Implementation of gsutil hmac command.r2 � � z ae:lp:s:u:T�r3 r8 �getr: r> ) �min_args�max_args�supported_sub_args� file_url_ok�urls_start_arg�gs_api_support�gs_default_api�usage_synopsis�argparse_arguments� command_helpz-CRUD operations on service account HMAC keys.)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc sN | j d dkr*d| j v r*tdtii d�}nttttttd�i d�}t� � |�S )Nr r: r= r5 )r3 r8 r> rE r: ) �argsr �LIST_COMMAND_LONG_FORMAT�CREATE_COMMAND�DELETE_COMMAND�UPDATE_COMMAND� GET_COMMAND� LIST_COMMAND�super�get_gcloud_storage_args)�self�gcloud_storage_map�� __class__r r r^ < s ��� z#HmacCommand.get_gcloud_storage_argsNc C sx | j r| j d | _nd}t|| j| jtf ��t| |d�}|j| j| jdd�}t dd|j j f � t dd|j f � d S ) z'Creates HMAC key for a service account.r zI%s %s requires a service account to be specified as the last argument. %s�� thread_state�gs��providerz%-12s %sz Access ID:zSecret:N) rV �service_account_emailr r �action_subcommand�_CREATE_SYNOPSISr � CreateHmacKey� project_id�printr. r% �secret)r_ rd �err_msg� gsutil_api�responser r r �_CreateHmacKeyP s �� �zHmacCommand._CreateHmacKeyc C sD | j r| j d }nt| j| jt��t| |d�}|j| j|dd� dS )zDeletes an HMAC key.r rc re rf N)rV r r ri �_DELETE_SYNOPSISr � DeleteHmacKeyrl )r_ rd � access_idrp r r r �_DeleteHmacKeyd s � zHmacCommand._DeleteHmacKeyc C sP | j r| j d }nt| j| jt��t| |d�}|j| j|dd�}tt |�� dS )z!Gets HMAC key from its Access Id.r rc re rf N) rV r r ri � _GET_SYNOPSISr � GetHmacKeyrl rm r0 �r_ rd ru rp rq r r r � _GetHmacKeyp s � zHmacCommand._GetHmacKeyc C s� | j r td| j| jtf ��t| |d�}|j| j| j| j dd�}d}| j rh|D ]}t t |�� t � qNn"|D ] }t ||j |j|jf � qldS )z1Lists HMAC keys for a project or service account.z'%s %s received unexpected arguments. %src re rf z %s %-12s %sN)rV r r ri �_LIST_SYNOPSISr � ListHmacKeysrl rh �show_all� long_listrm r0 r% r& r' )r_ rd rp rq Zshort_list_format�itemr r r � _ListHmacKeys~ s* �� � �zHmacCommand._ListHmacKeysc C s� | j s td| j| jtf ��n | j tvr:tdd�t� ��| jrL| jd }nt| j| jt��t | |d�}|j | j || j | j dd�}t t|�� dS ) zUpdate an HMAC key's state.z*A state flag must be supplied for %s %s %sz&The state flag value must be one of %sz, r rc re rf N)r& r r ri �_UPDATE_SYNOPSIS�_VALID_UPDATE_STATES�joinrV r r � UpdateHmacKeyrl r- rm r0 ry r r r �_UpdateHmacKey� s. �� � � �zHmacCommand._UpdateHmacKeyc C sF | j jdd�tjkr td��| j�d�| _| jdd� t | j d� d| _ d| _ d | _ d | _d| _| j r�| j D ]p\}}|d kr�|| _ ql|d kr�t|d � || _ql|d kr�|| _ ql|dkr�d| _ ql|dkr�d| _ql|dkrl|| _ql| js�td�| _| j| j| j| j| jd�}| j|v�r(td| j| jf ��t | jgd� || j � dS )z)Command entry point for the hmac command.re rf z9The "hmac" command can only be used with the GCS JSON APIr T)� check_args)�sub_optsNFr< r4 z/Invalid non-ASCII character found in project IDr? r; r= r@ rD zCInvalid subcommand "%s" for the %s command. See "gsutil help hmac".)� subcommands)rp �GetApiSelectorr �JSONr rV �popri � ParseSubOptsr r� rh r& r} r~ r- r rl r rr rv rz r� r� r )r_ �o�aZmethod_for_argr r r � RunCommand� sT � � � zHmacCommand.RunCommand)N)N)N)N)N) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpecr r� � _SYNOPSISr �%MakeZeroOrMoreCloudOrFileURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_create_help_text�_delete_help_text�_get_help_text�_list_help_text�_update_help_text� help_specr^ rr rv rz r� r� r� � __classcell__r r ra r rA sL ���� rA N)Ar� � __future__r r r r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.help_providerr Z gslib.metricsr �gslib.project_idr � gslib.utils.cloud_api_helperr �gslib.utils.shim_utilr r Zgslib.utils.text_utilr Z gslib.utilsr rj rs rw r{ r� Z_CREATE_DESCRIPTIONZ_DELETE_DESCRIPTION�_GET_DESCRIPTIONZ_LIST_DESCRIPTIONZ_UPDATE_DESCRIPTION�lstripr� r� � _DESCRIPTIONr� r� r+ r� r� r� r� r� r r0 �get_format_flag_newlineZ_CREATE_COMMAND_FORMATZ_DESCRIBE_COMMAND_FORMATZ_LIST_COMMAND_SHORT_FORMATZ _PROJECT_FLAGrX rY r[ r\ rW rZ rA r r r r �<module> s� ������ ���� �������� �� � ���� � �� �� ��� commands/__pycache__/config.cpython-39.pyc 0000644 00000121737 15025013616 0014465 0 ustar 00 a +(Wg�� � @ s@ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlmZ ddlmZ ddlZdd lmZ ddlZdd lmZ dd lmZ dd l m Z dd l m Z ddl m!Z! ddl m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl'm)Z) ddl*m+Z+ ddl*m,Z, ddl*m-Z- ddl*m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 dZ5d e5 d Z6d Z7d Z8e0� \Z9Z:e9�r�d Z;d!Z<ne=e�>� d"�Z;d#Z<d$Z?d%Z@d&ZAd'ZBd(ZCd)ZDd*ZEd+ZFd,ZGd-e,e-e+e.e(jHe;e<e?e@e?e@eCe eeDeEd.� ZId/ZJG d0d1� d1e�ZKd2d3� ZLdS )4zJImplementation of config command for creating a gsutil configuration file.� )�absolute_import)�print_function)�division)�unicode_literalsN)�input)�ResponseNotReady)�Provider)�Command)�!DEFAULT_TASK_ESTIMATION_THRESHOLD)�MAX_COMPOSE_ARITY)� CredTypes��AbortException)�CommandException)�'CheckAndMaybePromptForAnalyticsEnabling)�RegisterSignalHandler)� constants)� system_util)�CHECK_HASH_ALWAYS)� CHECK_HASH_IF_FAST_ELSE_FAIL)� CHECK_HASH_IF_FAST_ELSE_SKIP)�CHECK_HASH_NEVER)� ShouldProhibitMultiprocessing)�ServerNotFoundError)� HAS_CRYPTOz/ gsutil [-D] config (-a|-e) [-n] [-o <file>] z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> The ``gsutil config`` command generally applies to users who have legacy standalone installations of gsutil. If you installed gsutil via the Cloud SDK, ``gsutil config`` fails unless you are specifically using the ``-a`` flag or have configured gcloud to not pass its managed credentials to gsutil (via the command ``gcloud config set pass_credentials_to_gsutil false``). For all other use cases, Cloud SDK users should use the ``gcloud auth`` group of commands instead, which configures OAuth2 credentials that gcloud implicitly passes to gsutil at runtime. To check if you are using gsutil from the Cloud SDK or as a legacy standalone, use ``gsutil version -l`` and in the output look for "using cloud sdk". Important: The default behavior for the ``gsutil config`` command is to obtain user account credentials for authentication. However, user account credentials are no longer supported for standalone gsutil. For this reason, running the default ``gsutil config`` command fails, and using any of the following flags causes the command to fail: ``-b``, ``-f``, ``-r``, ``--reauth``, ``-s``, ``-w``. When using standalone gsutil, it's recommended that you use service account credentials via the ``-e`` flag. The ``gsutil config`` command obtains access credentials for Cloud Storage and writes a `boto/gsutil configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_ containing the obtained credentials along with a number of other configuration- controllable values. Unless specified otherwise (see OPTIONS), the configuration file is written to ~/.boto (i.e., the file .boto under the user's home directory). If the default file already exists, an attempt is made to rename the existing file to ~/.boto.bak; if that attempt fails the command exits. A different destination file can be specified with the ``-o`` option (see OPTIONS). Because the boto configuration file contains your credentials you should keep its file permissions set so no one but you has read access. (The file is created read-only when you run ``gsutil config``.) <B>CONFIGURING SERVICE ACCOUNT CREDENTIALS</B> Service accounts are useful for authenticating on behalf of a service or application (as opposed to a user). If you use gsutil as a legacy stand-alone tool, you configure credentials for service accounts using the ``-e`` option: gsutil config -e Note that if you use gsutil through the Cloud SDK, you instead activate your service account via the `gcloud auth activate-service-account <https://cloud.google.com/sdk/gcloud/reference/auth/activate-service-account>`_ command. When you run ``gsutil config -e``, you are prompted for the path to your private key file and, if not using a JSON key file, your service account email address and key file password. To get this data, follow the instructions on `Service Accounts <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`_. Using this information, gsutil populates the "gs_service_key_file" attribute in the boto configuration file. If not using a JSON key file, gsutil also populates the "gs_service_client_id" and "gs_service_key_file_password" attributes. Note that your service account is NOT considered an Owner for the purposes of API access (see "gsutil help creds" for more information about this). See https://developers.google.com/identity/protocols/OAuth2ServiceAccount for further information on service account authentication. If you want to use credentials based on access key and secret (the older authentication method before OAuth2 was supported), see the ``-a`` option in the OPTIONS section. If you wish to use gsutil with other providers (or to copy data back and forth between multiple providers) you can edit their credentials into the [Credentials] section after creating the initial boto configuration file. <B>OPTIONS</B> -a Prompt for Google Cloud Storage access key and secret (the older authentication method before OAuth2 was supported) instead of obtaining an OAuth2 token. -e Prompt for service account credentials. This option requires that ``-a`` is not set. -n Write the configuration file without authentication configured. This flag is mutually exclusive with all flags other than ``-o``. -o <file> Write the configuration to <file> instead of ~/.boto. Use ``-`` for stdout. z)https://cloud.google.com/console#/projectaY # This file contains credentials and other configuration information needed # by the boto library, used by gsutil. You can edit this file (e.g., to add # credentials) but be careful not to mis-edit any of the variable names (like # "gs_access_key_id") or remove important markers (like the "[Credentials]" and # "[Boto]" section delimiters). # � � � � �0Z50MZ150MZ200M� Z2G� aK [Boto] # http_socket_timeout specifies the timeout (in seconds) used to tell httplib # how long to wait for socket timeouts. The default is 70 seconds. Note that # this timeout only applies to httplib, not to httplib2 (which is used for # OAuth2 refresh/access token exchanges). #http_socket_timeout = 70 # The following two options control the use of a secure transport for requests # to S3 and Google Cloud Storage. It is highly recommended to set both options # to True in production environments, especially when using OAuth2 bearer token # authentication with Google Cloud Storage. # Set 'https_validate_certificates' to False to disable server certificate # checking. The default for this option in the boto library is currently # 'False' (to avoid breaking apps that depend on invalid certificates); it is # therefore strongly recommended to always set this option explicitly to True # in configuration files, to protect against "man-in-the-middle" attacks. https_validate_certificates = True # 'debug' controls the level of debug messages printed for the XML API only: # 0 for none, 1 for basic boto debug, 2 for all boto debug plus HTTP # requests/responses. #debug = <0, 1, or 2> # 'num_retries' controls the number of retry attempts made when errors occur # during data transfers. The default is 6. # Note 1: You can cause gsutil to retry failures effectively infinitely by # setting this value to a large number (like 10000). Doing that could be useful # in cases where your network connection occasionally fails and is down for an # extended period of time, because when it comes back up gsutil will continue # retrying. However, in general we recommend not setting the value above 10, # because otherwise gsutil could appear to "hang" due to excessive retries # (since unless you run gsutil -D you won't see any logged evidence that gsutil # is retrying). # Note 2: Don't set this value to 0, as it will cause boto to fail when reusing # HTTP connections. #num_retries = <integer value> # 'max_retry_delay' controls the max delay (in seconds) between retries. The # default value is 60, so the backoff sequence will be 1 seconds, 2 seconds, 4, # 8, 16, 32, and then 60 for all subsequent retries for a given HTTP request. # Note: At present this value only impacts the XML API and the JSON API uses a # fixed value of 60. #max_retry_delay = <integer value> aB [GoogleCompute] # 'service_account' specifies the a Google Compute Engine service account to # use for credentials. This value is intended for use only on Google Compute # Engine virtual machines and usually lives in /etc/boto.cfg. Most users # shouldn't need to edit this part of the config. #service_account = default az0 [GSUtil] # 'resumable_threshold' specifies the smallest file size [bytes] for which # resumable Google Cloud Storage uploads are attempted. The default is 8388608 # (8 MiB). #resumable_threshold = %(resumable_threshold)d # 'rsync_buffer_lines' specifies the number of lines of bucket or directory # listings saved in each temp file during sorting. (The complete set is # split across temp files and separately sorted/merged, to avoid needing to # fit everything in memory at once.) If you are trying to synchronize very # large directories/buckets (e.g., containing millions or more objects), # having too small a value here can cause gsutil to run out of open file # handles. If that happens, you can try to increase the number of open file # handles your system allows (e.g., see 'man ulimit' on Linux; see also # http://docs.python.org/2/library/resource.html). If you can't do that (or # if you're already at the upper limit), increasing rsync_buffer_lines will # cause gsutil to use fewer file handles, but at the cost of more memory. With # rsync_buffer_lines set to 32000 and assuming a typical URL is 100 bytes # long, gsutil will require approximately 10 MiB of memory while building # the synchronization state, and will require approximately 60 open file # descriptors to build the synchronization state over all 1M source and 1M # destination URLs. Memory and file descriptors are only consumed while # building the state; once the state is built, it resides in two temp files that # are read and processed incrementally during the actual copy/delete # operations. #rsync_buffer_lines = 32000 # 'state_dir' specifies the base location where files that # need a static location are stored, such as pointers to credentials, # resumable transfer tracker files, and the last software update check. # By default these files are stored in ~/.gsutil #state_dir = <file_path> # gsutil periodically checks whether a new version of the gsutil software is # available. 'software_update_check_period' specifies the number of days # between such checks. The default is 30. Setting the value to 0 disables # periodic software update checks. #software_update_check_period = 30 # 'tab_completion_timeout' controls the timeout (in seconds) for tab # completions that involve remote requests (such as bucket or object names). # If tab completion does not succeed within this timeout, no tab completion # suggestions will be returned. # A value of 0 will disable completions that involve remote requests. #tab_completion_timeout = 5 # 'parallel_process_count' and 'parallel_thread_count' specify the number # of OS processes and Python threads, respectively, to use when executing # operations in parallel. The default settings should work well as configured, # however, to enhance performance for transfers involving large numbers of # files, you may experiment with hand tuning these values to optimize # performance for your particular system configuration. #parallel_process_count = %(parallel_process_count)d #parallel_thread_count = %(parallel_thread_count)d # 'parallel_composite_upload_threshold' specifies the maximum size of a file to # upload in a single stream. Files larger than this threshold will be # partitioned into component parts and uploaded in parallel and then composed # into a single object. # The number of components will be the smaller of # ceil(file_size / parallel_composite_upload_component_size) and # MAX_COMPOSE_ARITY. The current value of MAX_COMPOSE_ARITY is # %(max_compose_arity)d. # If 'parallel_composite_upload_threshold' is set to 0, then automatic parallel # uploads will never occur. # Setting an extremely low threshold is unadvisable. The vast majority of # environments will see degraded performance for thresholds below 80M, and it # is almost never advantageous to have a threshold below 20M. # 'parallel_composite_upload_component_size' specifies the ideal size of a # component in bytes, which will act as an upper bound to the size of the # components if ceil(file_size / parallel_composite_upload_component_size) is # less than MAX_COMPOSE_ARITY. # Values can be provided either in bytes or as human-readable values # (e.g., "150M" to represent 150 mebibytes) # # Note: At present parallel composite uploads are disabled by default, because # using composite objects requires a compiled crcmod (see "gsutil help crcmod"), # and for operating systems that don't already have this package installed this # makes gsutil harder to use. Google is actively working with a number of the # Linux distributions to get crcmod included with the stock distribution. Once # that is done we will re-enable parallel composite uploads by default in # gsutil. # # Note: Parallel composite uploads should not be used with NEARLINE, COLDLINE, # or ARCHIVE storage class buckets, as doing this incurs an early deletion # charge for each component object. # # Note: Parallel composite uploads are not enabled with Cloud KMS encrypted # objects as a source or destination, as composition with KMS objects is not yet # supported. #parallel_composite_upload_threshold = %(parallel_composite_upload_threshold)s #parallel_composite_upload_component_size = %(parallel_composite_upload_component_size)s # # 'parallel_composite_upload_bypass_kms_check' removes the object/bucket KMS checks # used to guard composition of KMS objects. #disable_parallel_composite_upload_kms_check = False # 'sliced_object_download_threshold' and # 'sliced_object_download_component_size' have analogous functionality to # their respective parallel_composite_upload config values. # 'sliced_object_download_max_components' specifies the maximum number of # slices to be used when performing a sliced object download. #sliced_object_download_threshold = %(sliced_object_download_threshold)s #sliced_object_download_component_size = %(sliced_object_download_component_size)s #sliced_object_download_max_components = %(sliced_object_download_max_components)s # Compressed transport encoded uploads buffer chunks of compressed data. When # running a composite upload and/or many uploads in parallel, compression may # consume more memory than available. This setting restricts the number of # compressed transport encoded uploads running in parallel such that they # don't consume more memory than set here. This is 2GiB by default. # Values can be provided either in bytes or as human-readable values # (e.g., "2G" to represent 2 gibibytes) #max_upload_compression_buffer_size = %(max_upload_compression_buffer_size)s # GZIP compression level, if using compression. Reducing this can have # a dramatic impact on compression speed with minor size increases. # This is a value from 0-9, with 9 being max compression. # A good level to try is 6, which is the default used by the gzip tool. #gzip_compression_level = %(gzip_compression_level)s # 'task_estimation_threshold' controls how many files or objects gsutil # processes before it attempts to estimate the total work that will be # performed by the command. Estimation makes extra directory listing or API # list calls and is performed only if multiple processes and/or threads are # used. Estimation can slightly increase cost due to extra # listing calls; to disable it entirely, set this value to 0. #task_estimation_threshold=%(task_estimation_threshold)s # 'use_magicfile' specifies if the 'file --mime <filename>' command should be # used to guess content types instead of the default filename extension-based # mechanism. Available on UNIX and macOS (and possibly on Windows, if you're # running Cygwin or some other package that provides implementations of # UNIX-like commands). When available and enabled use_magicfile should be more # robust because it analyzes file contents in addition to extensions. #use_magicfile = False # Service account emails for testing the hmac command. If these fields are not # populated with distinct service accounts the tests for the hmac command will # not be run. Primarily useful for tool developers. #test_hmac_service_account = #test_hmac_alt_service_account = #test_hmac_list_service_account = # Service account emails for testing impersonation credentials. If this field is # not populated with a service account the tests for service account # impersonation will not run. Primarily useful for tool developers. #test_impersonate_service_account = # 'content_language' specifies the ISO 639-1 language code of the content, to be # passed in the Content-Language header. By default no Content-Language is sent. # See the ISO 639-1 column of # http://www.loc.gov/standards/iso639-2/php/code_list.php for a list of # language codes. content_language = en # 'check_hashes' specifies how strictly to require integrity checking for # downloaded data. Legal values are: # '%(hash_fast_else_fail)s' - (default) Only integrity check if the digest # will run efficiently (using compiled code), else fail the download. # '%(hash_fast_else_skip)s' - Only integrity check if the server supplies a # hash and the local digest computation will run quickly, else skip the # check. # '%(hash_always)s' - Always check download integrity regardless of possible # performance costs. # '%(hash_never)s' - Don't perform download integrity checks. This setting is # not recommended except for special cases such as measuring download # performance excluding time for integrity checking. # This option exists to assist users who wish to download a GCS composite object # and are unable to install crcmod with the C-extension. CRC32c is the only # available integrity check for composite objects, and without the C-extension, # download performance can be significantly degraded by the digest computation. # This option is ignored for daisy-chain copies, which don't compute hashes but # instead (inexpensively) compare the cloud source and destination hashes. #check_hashes = if_fast_else_fail # 'encryption_key' specifies a single customer-supplied encryption key that # will be used for all data written to Google Cloud Storage. See # "gsutil help encryption" for more information # Encryption key: RFC 4648 section 4 base64-encoded AES256 string # Warning: If decrypt_key is specified without an encrypt_key, objects will be # decrypted when copied in the cloud. #encryption_key= # Each 'decryption_key' entry specifies a customer-supplied decryption key that # will be used to access and Google Cloud Storage objects encrypted with # the corresponding key. # Decryption keys: Up to 100 RFC 4648 section 4 base64-encoded AES256 strings # in ascending numerical order, starting with 1. #decryption_key1= #decryption_key2= #decryption_key3= # The ability to specify an alternative JSON API version is primarily for cloud # storage service developers. #json_api_version = v1 # Specifies the API to use when interacting with cloud storage providers. If the # gsutil command supports this API for the provider, it will be used instead of # the default API. Commands typically default to XML for S3 and JSON for GCS. # Note that if any encryption configuration options are set (see above), the # JSON API will be used for interacting with Google Cloud Storage buckets even # if XML is preferred, as gsutil does not currently support this functionality # when using the XML API. #prefer_api = json #prefer_api = xml # Disables the prompt asking for opt-in to data collection for analytics. #disable_analytics_prompt = True # The "test" command runs tests against regional buckets (unless you supply the # `-b` option). By default, the region used is us-central1, but you can change # the default region using this option. #test_cmd_regional_bucket_location = us-central1 # Tests for the "notification watchbucket" command require a notification URL. # If this option is not supplied, those tests will be skipped. #test_notification_url = https://yourdomain.url/notification-endpoint # Used in conjunction with --stet flag on cp command for end-to-end encryption. # STET binary path. If not specified, gsutil checks PATH for "stet". #stet_binary_path = <Path to binary "/usr/local/bin/stet"> # STET config path. If not specified, the STET binary will run with its default # settings. #stet_config_path = ~/.config/my_config.yaml # Adds an API call before parallel operations that triggers a reauth challenge. #trigger_reauth_challenge_for_parallel_operations = False )Zhash_fast_else_failZhash_fast_else_skipZ hash_alwaysZ hash_never�resumable_threshold�parallel_process_count�parallel_thread_count�#parallel_composite_upload_threshold�(parallel_composite_upload_component_size� sliced_object_download_threshold�%sliced_object_download_component_size�%sliced_object_download_max_componentsZmax_compose_arity�task_estimation_threshold�"max_upload_compression_buffer_sizeZgzip_compression_levela0 [OAuth2] # This section specifies options used with OAuth2 authentication. # 'token_cache' specifies how the OAuth2 client should cache access tokens. # Valid values are: # 'in_memory': an in-memory cache is used. This is only useful if the boto # client instance (and with it the OAuth2 plugin instance) persists # across multiple requests. # 'file_system' : access tokens will be cached in the file system, in files # whose names include a key derived from the refresh token the access token # based on. # The default is 'file_system'. #token_cache = file_system #token_cache = in_memory # 'token_cache_path_pattern' specifies a path pattern for token cache files. # This option is only relevant if token_cache = file_system. # The value of this option should be a path, with place-holders '%(key)s' (which # will be replaced with a key derived from the refresh token the cached access # token was based on), and (optionally), %(uid)s (which will be replaced with # the UID of the current user, if available via os.getuid()). # Note that the config parser itself interpolates '%' placeholders, and hence # the above placeholders need to be escaped as '%%(key)s'. # The default value of this option is # token_cache_path_pattern = <tmpdir>/oauth2client-tokencache.%%(uid)s.%%(key)s # where <tmpdir> is the system-dependent default temp directory. # The following options specify the label and endpoint URIs for the OAUth2 # authorization provider being used. Primarily useful for tool developers. #provider_label = Google #provider_authorization_uri = https://accounts.google.com/o/oauth2/auth #provider_token_uri = https://oauth2.googleapis.com/token # 'oauth2_refresh_retries' controls the number of retry attempts made when # rate limiting errors occur for OAuth2 requests to retrieve an access token. # The default value is 6. #oauth2_refresh_retries = <integer value> # The following options specify the OAuth2 client identity and secret that is # used when requesting and using OAuth2 tokens. If not specified, a default # OAuth2 client for the gsutil tool is used; for uses of the boto library (with # OAuth2 authentication plugin) in other client software, it is recommended to # use a tool/client-specific OAuth2 client. For more information on OAuth2, see # http://code.google.com/apis/accounts/docs/OAuth2.html c @ s� e Zd ZdZejdg d�eddddgdddd� Zejdg d �d d e i d �Z d d� Z dd� Z dd� Z dd� Zejdfdd�Zdd� ZdS )� ConfigCommandz(Implementation of gsutil config command.�config)�cfg�conf� configurer z abefhno:rs:w�reauthF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args�supported_private_args� file_url_ok�provider_url_ok�urls_start_arg)r. r/ r0 �aws�s3� command_helpz0Obtain credentials and create configuration file)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s~ t jt jB t jB }tt d�r&|t jO }zt �||d�}W n8 ttfyp } z t d||f ��W Y d}~n d}~0 0 t � |d�S )a Creates and opens a configuration file for writing. The file is created with mode 0600, and attempts to open existing files will fail (the latter is important to prevent symlink attacks). It is the caller's responsibility to close the file. Args: file_path: Path of the file to be created. Returns: A writable file object for the opened file. Raises: CommandException: if an error occurred when opening the file (including when the file already exists). � O_NOINHERITi� z!Failed to open %s for writing: %sN�w) �os�O_RDWR�O_CREAT�O_EXCL�hasattrrD �open�OSError�IOErrorr �fdopen)�self� file_path�flags�fd�e� rT �$/opt/gsutil/gslib/commands/config.py�_OpenConfigFileV s � z ConfigCommand._OpenConfigFilec C s� t jr dS t�|�}ttjtjB |j@ �r�| j� d� t d�}|dv r�z t� |d� | j� d� W q� t y� } z| j� d� W Y d}~q�d}~0 0 n | j� d� dS ) aA Checks that the file has reasonable permissions for a private key. In particular, check that the filename provided by the user is not world- or group-readable. If either of these are true, we issue a warning and offer to fix the permissions. Args: file_path: The name of the private key file. Nz� Your private key file is readable by people other than yourself. This is a security risk, since anyone with this information can use your service account. zDWould you like gsutil to change the file permissions for you? (y/N) )�y�Y� z� The permissions on your file have been successfully modified. The only access allowed is readability by the user (permissions 0400 in chmod).z� We were unable to modify the permissions on your file. If you would like to fix this yourself, consider running: "sudo chmod 400 </path/to/key>" for improved security.z� You have chosen to allow this file to be readable by others. If you would like to fix this yourself, consider running: "sudo chmod 400 </path/to/key>" for improved security.)r � IS_WINDOWSrF �stat�bool�S_IRGRP�S_IROTH�st_mode�logger�warnr �chmod�info� Exception)rO rP �stZfix_it�_rT rT rU � _CheckPrivateKeyFilePermissionss s* � �� �z-ConfigCommand._CheckPrivateKeyFilePermissionsc C s* |sd| }d| }|� d||f � dS )a� Writes proxy name/value pair or comment line to config file. Writes proxy name/value pair if value is not None. Otherwise writes comment line. Args: config_file: File object to which the resulting config file will be written. name: The config variable name. value: The value, or None. desc: Human readable description (for comment). z#%sz<%s>z%s = %s N)�write)rO � config_file�name�value�descrT rT rU � _WriteConfigLineMaybeCommented� s z,ConfigCommand._WriteConfigLineMaybeCommentedc C s� t j}|�d� | �|d|�ddd�d� | �|d|�ddd�d� | �|d|�ddd�d � | �|d |�dd d�d � | �|d |�dd d�d � | �|d|�ddd�d� dS )z�Writes proxy section of configuration file. Args: config_file: File object to which the resulting config file will be written. a? # To use a proxy, edit and uncomment the proxy and proxy_port lines. # If you need a user/password with this proxy, edit and uncomment # those lines as well. If your organization also disallows DNS # lookups by client machines, set proxy_rdns to True (the default). # If you have installed gsutil through the Cloud SDK and have # configured proxy settings in gcloud, those proxy settings will # override any other options (including those set here, along with # any settings in proxy-related environment variables). Otherwise, # if proxy_host and proxy_port are not specified in this file and # one of the OS environment variables http_proxy, https_proxy, or # HTTPS_PROXY is defined, gsutil will use the proxy server specified # in these environment variables, in order of precedence according # to how they are listed above. �proxy�BotoNz proxy host� proxy_typez4proxy type (socks4, socks5, http) | Defaults to http� proxy_portz proxy port� proxy_userz proxy user� proxy_passzproxy password� proxy_rdnsFzLlet proxy server perform DNS lookups (True,False); socks proxy not supported)�botor- rh rm � get_value)rO ri r- rT rT rU � _WriteProxyConfigFileSection� s< � ����� ��z*ConfigCommand._WriteProxyConfigFileSectionTc C s� ddd�}ddd�}i }i }d}|�rJ|t jkr�td�} z@t| d�� } t�| �� � W d � n1 sh0 Y d }W n ty� ts�t d ��Y n0 |s�td �} td � t � d ��d �} | � | � n�|t jkr�t d��nn|t jk�rJd} |D ]J}|dkr�td| �||< td| �||< d } || �r2|| s�t d��q�| �sJt d��|�t�� � |�dtjtj�� �d�f � |�d� |�d� |�r|t jk�r�|�d� |�d| � |�s |�d| � | �s�|�d � |�d � n|�d | � n |�d � nt�� �r|�d � |D ]�}|| }|| }||v �r�||v �r�|�d!||f � |�d"||| f � |�d#||| f � n|�d$||||||f � tj| }|�d%|||f � |dk�r|�d&|||f � |�d'� �q|�t � d(�� |�d)t � | � |� |�t � |�t � |�d*� d+}|t jk�r@d,}|�d-| � t�� �s�td.��!� }d/}|�svt d0��|�d1||f � t"� |�t#� |�d2� dS )3a Creates a boto config file interactively. Needed credentials are obtained interactively, either by asking the user for access key and secret, or by walking the user through the OAuth2 approval flow. Args: config_file: File object to which the resulting config file will be written. cred_type: There are three options: - for HMAC, ask the user for access key and secret - for OAUTH2_USER_ACCOUNT, raise an error - for OAUTH2_SERVICE_ACCOUNT, prompt the user for OAuth2 for service account email address and private key file (and if the file is a .p12 file, the password for that file). configure_auth: Boolean, whether or not to configure authentication in the generated file. r; �gs)r; �googler< Fz0What is the full path to your private key file? �rbNTz�Service account authentication via a .p12 file requires either PyOpenSSL or PyCrypto 2.6 or later. Please install either of these to proceed, use a JSON-format key file, or configure a different type of credentials.z,What is your service account email address? � zjWhat is the password for your service key file [if you haven't set one explicitly, leave this line blank]?� z�The user account authentication flow no longer works as of February 1, 2023. Tokens generated before this date will continue to work. To authenticate with your user account, install gsutil via Cloud SDK and run "gcloud auth login"ry z What is your %s access key ID? z#What is your %s secret access key? z2Incomplete credentials provided. Please try again.z*No credentials provided. Please try again.z4# This file was created by gsutil version %s at %s. z%Y-%m-%d %H:%M:%Szl# # You can create additional configuration files by running # gsutil config [options] [-o <config-file>] z[Credentials] z@# Google OAuth2 service account credentials (for "gs://" URIs): zgs_service_key_file = %s zgs_service_client_id = %s a� # If you would like to set your password, you can do so # using the following commands (replaced with your # information): # "openssl pkcs12 -in cert1.p12 -out temp_cert.pem" # "openssl pkcs12 -export -in temp_cert.pem -out cert2.p12" # "rm -f temp_cert.pem" # Your initial password is "notasecret" - for more # information, please see # http://www.openssl.org/docs/apps/pkcs12.html. z!#gs_service_key_file_password = z#gs_service_key_file_password = %s z�# To add Google OAuth2 credentials ("gs://" URIs), edit and uncomment the # following line: #gs_oauth2_refresh_token = <your OAuth2 refresh token> zg# Google OAuth2 credentials are managed by the Cloud SDK and # do not need to be present in this file. z!# %s credentials ("%s://" URIs): z%s_access_key_id = %s z%s_secret_access_key = %s z�# To add HMAC %s credentials for "%s://" URIs, edit and uncomment the # following two lines: #%s_access_key_id = <your %s access key ID> #%s_secret_access_key = <your %s secret access key> a� # The ability to specify an alternate storage host and port # is primarily for cloud storage service developers. # Setting a non-default gs_host only works if prefer_api=xml. #%s_host = <alternate storage host address> #%s_port = <alternate storage host port> # In some cases, (e.g. VPC requests) the "host" HTTP header should # be different than the host used in the request URL. #%s_host_header = <alternate storage host header> z�#%s_json_host = <alternate JSON API storage host address> #%s_json_port = <alternate JSON API storage host port> #%s_json_host_header = <alternate JSON API storage host header> z�# To impersonate a service account for "%s://" URIs over # JSON API, edit and uncomment the following line: #%s_impersonate_service_account = <service account email> a� # This configuration setting enables or disables mutual TLS # authentication. The default value for this setting is "false". When # set to "true", gsutil uses the configured client certificate as # transport credential to access the APIs. The use of mTLS ensures that # the access originates from a trusted enterprise device. When enabled, # the client certificate is auto discovered using the endpoint # verification agent. When set to "true" but no client certificate or # key is found, users receive an error. #use_client_certificate = False # The command line to execute, which prints the # certificate, private key, or password to use in # conjunction with "use_client_certificate = True". #cert_provider_command = <Absolute path to command to run for # certification. Ex: "/scripts/gen_cert.sh"> z%s z� # 'default_api_version' specifies the default Google Cloud Storage XML API # version to use. If not set below gsutil defaults to API version 1. � r zdefault_api_version = %d zWhat is your project-id? z� # 'default_project_id' specifies the default Google Cloud Storage project ID to # use with the 'mb' and 'ls' commands. This default can be overridden by # specifying the -p option to the 'mb' and 'ls' commands. zlNo default project ID entered. The default project ID is needed by the ls and mb commands; please try again.z %sdefault_project_id = %s zH#client_id = <OAuth2 client id> #client_secret = <OAuth2 client secret> )$r �OAUTH2_SERVICE_ACCOUNTr rK �json�loads�read� ValueErrorr r �join�textwrap�wraprg �OAUTH2_USER_ACCOUNT�HMACrh �CONFIG_PRELUDE_CONTENT�lstrip�gslib�VERSION�datetime�now�strftimer �InvokedViaCloudSdkr � HostKeyMap�dedent�CONFIG_BOTO_SECTION_CONTENTrw �$CONFIG_GOOGLECOMPUTE_SECTION_CONTENT�'CONFIG_INPUTLESS_GSUTIL_SECTION_CONTENT�stripr � CONFIG_OAUTH2_CONFIG_CONTENT)rO ri � cred_type�configure_authZ provider_mapZuri_mapZkey_idsZsec_keysZservice_account_key_is_json�gs_service_key_fileZ key_file_fp�gs_service_client_id� gs_service_key_file_passwordZ got_creds�provider� key_prefixZ uri_schemeZhost_key� api_version�default_project_idZproject_id_section_preluderT rT rU �_WriteBotoConfigFile� s , � ��� � � � ���� �� �� �� � ��� �� ���� �� z"ConfigCommand._WriteBotoConfigFilec C sF t j}d}d}d}d}| jD ]T\}}|dkr8t j}d}q |dkrLt j}d}q |dkrZd}q |dkrh|}q | �� q |r�|r�td��|s�|s�|r�td ��t�� r�t� � r�|s�|r�td � d d d � t j d ddd��d � t j dddd��g���t�� �r|�rt j�d � t � d��d � d}|�s�tj�dd�} | �r8| } ntj�tj� dd��} tj�| ��sb| }n�| d }tj�|��r�td| |f ��nbz$t j�d| |f � t�| |� W n8 t�y� } z td| || f ��W Y d} ~ n d} ~ 0 0 | }|dk�r�t j} n*| �|�} t j�d � t � d | ��d � ttj t � z| j | ||d � W n� t�y� } z�t | t �} | �rvt j�d � |dk�r�| �!� t�"|� z&|�r�t j�d | � t�||� W n* t�y� } z| �W Y d} ~ n d} ~ 0 0 � W Y d} ~ n d} ~ 0 0 |dk�rB| �!� t#j$�%d d!��sBt j�d d � t � d"| �� d � d#S )$z+Command entry point for the config command.NFTz-az-ez-nz-ozYBoth -a and -e cannot be specified. Please see "gsutil help config" for more information.zoThe -a and -e flags cannot be specified with the -n flag. Please see "gsutil help config" for more information.r{ zDOAuth2 is the preferred authentication mechanism with the Cloud SDK.z<Run "gcloud auth login" to configure authentication, unless:z�You don't want gsutil to use OAuth2 credentials from the Cloud SDK, but instead want to manage credentials with .boto files generated by running "gsutil config"; in which case run "gcloud config set pass_credentials_to_gsutil false".z- z )�initial_indent�subsequent_indentzbYou want to authenticate with an HMAC access key and secret, in which case run "gsutil config -a".z�This command will configure HMAC credentials, but gsutil will use OAuth2 credentials from the Cloud SDK by default. To make sure the HMAC credentials are used, run: "gcloud config set pass_credentials_to_gsutil false".z Z BOTO_CONFIG�~z.botoz.bakzDCannot back up existing config file "%s": backup file exists ("%s").z0Backing up existing config file "%s" to "%s"... z:Failed to back up existing config file ("%s" -> "%s"): %s.�-z�This command will create a boto config file at %s containing your credentials, based on your responses to the following questions.)r� r� z Caught ^C; cleaning up z'Restoring previous backed up file (%s) ro rn zzBoto config file "%s" created. If you need to use a proxy to access the Internet please see the instructions in that file.r )&r r� �sub_optsr� r~ � RaiseInvalidArgumentExceptionr r r� �CloudSdkCredPassingEnabledr� r� r� �sys�stderrrh rF �environ�get�path� expanduser�exists�renamerd �stdoutrV r �signal�SIGINT�_CleanupHandlerr� � isinstancer �close�unlinkru r- � has_option)rO r� Zoutput_file_nameZhas_aZhas_er� �optZopt_argZdefault_config_path_bakZboto_config_from_envZdefault_config_pathrS � output_fileZ user_abortedrT rT rU � RunCommand� s ��������� �������� ���� � � ����zConfigCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSIS� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specrV rg rm rw r r� r� r� rT rT rT rU r, ; s< � � ,.� dr, c C s t d��d S )Nz User interrupted config commandr )Zunused_signalnumZunused_handlerrT rT rU r� R s r� )Mr� � __future__r r r r r� r �multiprocessingrF r� �socketr[ r� r� �time� webbrowser� six.movesr �six.moves.http_clientr ru � boto.providerr r� � gslib.commandr r Zgslib.commands.composer �gslib.cred_typesr �gslib.exceptionr r Z gslib.metricsr �gslib.sig_handlingr Z gslib.utilsr r �gslib.utils.hashing_helperr r r r �&gslib.utils.parallelism_framework_utilr �httplib2r �oauth2client.clientr r� r� ZGOOG_CLOUD_CONSOLE_URIr� � should_prohibit_multiprocessingZ unused_os� DEFAULT_PARALLEL_PROCESS_COUNT� DEFAULT_PARALLEL_THREAD_COUNT�min� cpu_countZ+DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLDZ0DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZEZ(DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLDZ-DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZEZ-DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTSZ*DEFAULT_MAX_UPLOAD_COMPRESSION_BUFFER_SIZEZ DEFAULT_GZIP_COMPRESSION_LEVELr� r� �RESUMABLE_THRESHOLD_Br� r� r, r� rT rT rT rU �<module> s� ��] 0 k�� � 0 commands/__pycache__/signurl.cpython-39.pyc 0000644 00000051764 15025013616 0014705 0 ustar 00 a +(Wg�s � @ sx d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlmZ ddlm Z dd lm Z ddl Z ddl Z ddl Z ddlZddlZdd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlm Z ddl m Z ddl m Z ddl!m"Z" ddl!m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z-m.Z. z8ddl/m0Z0 ddl/m1Z1 ddl/m2Z2 dd l/m3Z3 d Z4W n$ e5�y� dZ1dZ2d Z4dZ0Y n0 zDdd l6m7Z7 dd l8m9Z9 dd!l:m;Z; dd"l<m=Z= dd#l>m?Z? d Z@W n e5�y d Z@d$ZAY n0 d%ZBe d&d'�ZCe d(d)�ZDd*ZEd+eE d, ZFd-d.� ZGd/d0� ZHd9d1d2�ZId3d4� ZJd:d5d6�ZKG d7d8� d8e�ZLdS );zpImplementation of URL Signing workflow. see: https://cloud.google.com/storage/docs/access-control#Signed-URLs) � )�absolute_import)�print_function)�division)�unicode_literalsN)�datetime)� timedelta)�timezone)�urllib)� HttpError)� MakeRequest)�Request)�config)�Command)�CommandArgument)� ApiSelector)�CommandException)�ContainsWildcard)�StorageUrlFromString)� constants)� GetNewHttp)�GcloudStorageMap�GcloudStorageFlag)� CreatePayload� GetFinalUrl�to_bytes)� FILETYPE_PEM)�load_privatekey)�sign)�PKeyTF)�hashes)�padding)� RSAPrivateKey)�pkcs12)�NameOIDzapyca/cryptography is not available. Either install it, or please consider using the .json keyfile�auto� ��days� ��hoursz� gsutil signurl [-c <content_type>] [-d <duration>] [-m <http_method>] \ [-p <password>] [-r <region>] [-b <project>] (-u | <private-key-file>) \ (gs://<bucket_name> | gs://<bucket_name>/<object_name>)... z <B>SYNOPSIS</B> aG <B>DESCRIPTION</B> The signurl command will generate a signed URL that embeds authentication data so the URL can be used by someone who does not have a Google account. Please see the `Signed URLs documentation <https://cloud.google.com/storage/docs/access-control/signed-urls>`_ for background about signed URLs. Multiple gs:// URLs may be provided and may contain wildcards. A signed URL will be produced for each provided URL, authorized for the specified HTTP method and valid for the given duration. NOTE: Unlike the gsutil ls command, the signurl command does not support operations on sub-directories. For example, unless you have an object named ``some-directory/`` stored inside the bucket ``some-bucket``, the following command returns an error: ``gsutil signurl <private-key-file> gs://some-bucket/some-directory/`` The signurl command uses the private key for a service account (the '<private-key-file>' argument) to generate the cryptographic signature for the generated URL. The private key file must be in PKCS12 or JSON format. If the private key is encrypted the signed URL command will prompt for the passphrase used to protect the private key file (default 'notasecret'). For more information regarding generating a private key for use with the signurl command please see the `Authentication documentation. <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`_ If you used `service account credentials <https://cloud.google.com/storage/docs/gsutil/addlhelp/CredentialTypesSupportingVariousUseCases#supported-credential-types_1>`_ for authentication, you can replace the <private-key-file> argument with the -u or --use-service-account option to use the system-managed private key directly. This avoids the need to store a private key file locally, but prior to using this flag you must `configure <https://cloud.google.com/sdk/gcloud/reference/auth/activate-service-account>`_ ``gcloud`` to use your service account credentials. <B>OPTIONS</B> -b <project> Allows you to specify a user project that will be billed for requests that use the signed URL. This is useful for generating presigned links for buckets that use requester pays. Note that it's not valid to specify both the ``-b`` and ``--use-service-account`` options together. -c Specifies the content type for which the signed URL is valid for. -d Specifies the duration that the signed URL should be valid for, default duration is 1 hour. Times may be specified with no suffix (default hours), or with s = seconds, m = minutes, h = hours, d = days. This option may be specified multiple times, in which case the duration the link remains valid is the sum of all the duration options. The max duration allowed is 7 days when ``private-key-file`` is used. The max duration allowed is 12 hours when -u option is used. This limitation exists because the system-managed key used to sign the URL may not remain valid after 12 hours. -m Specifies the HTTP method to be authorized for use with the signed URL, default is GET. You may also specify RESUMABLE to create a signed resumable upload start URL. When using a signed URL to start a resumable upload session, you will need to specify the 'x-goog-resumable:start' header in the request or else signature validation will fail. -p Specify the private key password instead of prompting. -r <region> Specifies the `region <https://cloud.google.com/storage/docs/locations>`_ in which the resources for which you are creating signed URLs are stored. Default value is 'auto' which will cause gsutil to fetch the region for the resource. When auto-detecting the region, the current gsutil user's credentials, not the credentials from the private-key-file, are used to fetch the bucket's metadata. This option must be specified and not 'auto' when generating a signed URL to create a bucket. -u Use service account credentials instead of a private key file to sign the URL. You can also use the ``--use-service-account`` option, which is equivalent to ``-u``. Note that both options have a maximum allowed duration of 12 hours for a valid link. <B>USAGE</B> Create a signed URL for downloading an object valid for 10 minutes: gsutil signurl -d 10m <private-key-file> gs://<bucket>/<object> Create a signed URL, valid for one hour, for uploading a plain text file via HTTP PUT: gsutil signurl -m PUT -d 1h -c text/plain <private-key-file> \ gs://<bucket>/<obj> To construct a signed URL that allows anyone in possession of the URL to PUT to the specified bucket for one day, creating an object of Content-Type image/jpg, run: gsutil signurl -m PUT -d 1d -c image/jpg <private-key-file> \ gs://<bucket>/<obj> To construct a signed URL that allows anyone in possession of the URL to POST a resumable upload to the specified bucket for one day, creating an object of Content-Type image/jpg, run: gsutil signurl -m RESUMABLE -d 1d -c image/jpg <private-key-file> \ gs://bucket/<obj> c C s t jtjd�jdd�S )z2Returns the current utc time as a datetime object.��tzN��tzinfo)r �nowr �utc�replace� r2 r2 �%/opt/gsutil/gslib/commands/signurl.py�_NowUTC� s r4 c C s� t �d| �}|std��|�d�\} }t| �} |�� }|dkrJt| d�}n:|dkr^t| d�}n&|dkrrt| d�}n|d kr�t| d �}|S ) z>Parses the given duration and returns an equivalent timedelta.z^(\d+)([dDhHmMsS])?$z Unable to parse duration string�h�dr& r) �m)�minutes�s��seconds)�re�matchr �groups�int�lowerr )�durationr= �modifier�retr2 r2 r3 �_DurationToTimeDelta� s rD c C s� t �ddd�}d|i}|dkr8d}d|d< | s8|�d � | rD| |d < |rf|j||||| || || d � }n|tjrrd }nd }t||||| || || | d� \}}t| t�r�t | ||�}n&t s�t t ��| � t |�t�� t�� �}t||||�}|S )a1 Construct a string to sign with the provided key. Args: key: The private key to use for signing the URL. api: The CloudApiDelegator instance use_service_account: If True, use the service account credentials instead of using the key file to sign the URL provider: Cloud storage provider to connect to. If not present, class-wide default is used. client_id: Client ID signing this URL. method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. gcs_path: String path to the bucket of object for signing, in the form 'bucket' or 'bucket/object'. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. content_type: Optional Content-Type for the signed URL. HTTP requests using the URL must match this Content-Type. billing_project: Specify a user project to be billed for the request. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. generation: If not None, specifies a version of an object for signing. Returns: The complete URL (string). � Credentials�gs_hostzstorage.googleapis.com�host� RESUMABLE�POST�startzx-goog-resumablez�Warning: no Content-Type header was specified with the -c flag, so uploads to the resulting Signed URL must not specify a Content-Type.z content-type) �provider�methodrA �path� generation�logger�region�signed_headers�string_to_sign_debugs RSA-SHA256z RSA-SHA256) � client_idrL rA rM rN rO rP rQ �billing_projectrR )r �get�warn�SignUrl�six�PY2r � isinstancer r � HAVE_CRYPTOr �_CRYPTO_IMPORT_ERRORr r �PKCS1v15r �SHA256r )�key�api�use_service_accountrK rS rL rA �gcs_pathrO rP � content_typerT rR rN rF rQ � final_url�digest�string_to_sign�canonical_query_string� raw_signaturer2 r2 r3 � _GenSignedUrl� sX ) � � �ri c C sf t | �} t |�}ts tt��z0t�| |�\}}}|j�tj �d j }||fW S td��Y n0 d S )Nr z<Unable to load the keyfile, Invalid password or PKCS12 data.) r r[ r r\ r" �load_key_and_certificates�subject�get_attributes_for_oidr# � COMMON_NAME�value)� key_string�passwdr_ �cert�add� client_emailr2 r2 r3 � _ReadKeystoreR s rt c C s\ t �t�| ��}d|vs d|vr(td��|d }|rFtt|d |�}ntt|d �}||fS )a; Read the client email and private key from a JSON keystore. Assumes this keystore was downloaded from the Cloud Platform Console. By default, JSON keystore private keys from the Cloud Platform Console aren't encrypted so the passwd is optional as load_privatekey will prompt for the PEM passphrase if the key is encrypted. Arguments: ks_contents: JSON formatted string representing the keystore contents. Must be a valid JSON string and contain the fields 'private_key' and 'client_email'. passwd: Passphrase for encrypted private keys. Returns: key: Parsed private key from the keystore. client_email: The email address for the service account. Raises: ValueError: If unable to parse ks_contents or keystore is missing required fields. rs � private_keyz-JSON keystore doesn't contain required fields)�json�loadsrX � ensure_str� ValueErrorr r )Z ks_contentsrp �ksrs r_ r2 r2 r3 �_ReadJSONKeystore_ s r{ c s� e Zd ZdZejdddgedejddgddde j e j ge j e � � e �� gd � Zejdddgd d ei d �Z� fd d�Zdd� Zdd� Zdd� Zdd� Z� ZS )�UrlSignCommandz*Implementation of gsutil url_sign command.ZsignurlZ signedurlZ queryauth� z m:d:b:c:p:r:uzuse-service-accountF) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args�supported_private_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpzCreate a signed URL)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s� t �| j�}t �| j�}dddd| jd g}| jdd � | _d }d }d }d }t| j�D ]B\}\} } | dkrr|}qX| dkr�|}qX| d kr�|}qX| d krX|}qX|d ur�ttt| j| d ��� ��d } d| f| j|< |d u�r| j| d d k�rd | j|< |dg7 }|d u�r0| j| d } d d| f| j|< |d u�rZ| j| d } d d| f| j|< t � � t |t d�t d�t d�t d�t d�t d�d�d��}|| _|| _|S )N�storagezsign-urlz�--format=csv[separator="\t"](resource:label="URL", http_verb:label="HTTP Method", expiration:label="Expiration", signed_url:label="Signed URL")z--private-key-file=r r} �-d�-m�-c�-br9 rH )r� rI z --headers=x-goog-resumable=startz content-type=z userProject=z --http-verbz --durationz--query-paramsz --headersz--regionz--private-key-password)r� r� r� r� �-r�-p)�gcloud_command�flag_map) �copy�deepcopy�args�sub_opts� enumerate�strr? rD � total_seconds�super�get_gcloud_storage_argsr r )�selfZ original_argsZoriginal_sub_optsr� Zduration_arg_idxZhttp_verb_arg_idxZcontent_type_arg_idxZbilling_project_arg_idx�i�flag�_r; Zcontent_type_valueZ project_valueZfully_translated_command�� __class__r2 r3 r� � sv � ���� � � ���z&UrlSignCommand.get_gcloud_storage_argsc C sp d }d}d}d }t }d}d }| jD ]�\}} tjrD| �tjjp@tj �} |dkrl|d urb|t | �7 }q�t | �}q"|dkrz| }q"|dkr�| }q"|dkr�| }q"|dkr�| }q"|d ks�|d kr�d }q"|d kr�| }q"| � � q"|d u r�t d d�}n4|�r|t k�rtdt ��n|tk�rtdt ��|dv�r,td��|�sJt| j�dk �rJtd��|�r^|�r^td��|||||||fS )N�GET� Fr� r� r� r� r� z-uz--use-service-accountTr� r} r) z�Max valid duration allowed is %s when -u flag is used. For longer duration, consider using the private-key-file instead of the -u option.z Max valid duration allowed is %s)r� �PUT�DELETE�HEADrH z9HTTP method must be one of[GET|HEAD|PUT|DELETE|RESUMABLE]� z�The command requires a key file argument and one or more URL arguments if the --use-service-account flag is missing. Run `gsutil help signurl` for more infozLSpecifying both the -b and --use-service-account options together isinvalid.)�_AUTO_DETECT_REGIONr� rX rY �decode�sys�stdin�encodingr �UTF8rD � RaiseInvalidArgumentExceptionr �!_MAX_EXPIRATION_TIME_WITH_MINUS_Ur �_MAX_EXPIRATION_TIME�lenr� ) r� �deltarL rc rp rP ra rT �o�vr2 r2 r3 �_ParseAndCheckSubOpts� sd �� � � �z$UrlSignCommand._ParseAndCheckSubOptsc C s� t || j|||dtdd�||||| dd� } z6t� } t| d�} t| | �} | jdvrZt�| ��| jW S ty� } zL|� d�r�|j }d|j }|j r�|d |j 7 }nd | }t |��W Y d }~n d }~0 0 d S ) zFPerforms a head request against a signed URL to check for read access.r� �< r: T) r_ r` ra rK rS rL rA rb rN rO rP rT rR )�� � � �responsezbUnexpected HTTP response code %s while querying object readability. Is your system clock accurate?z Content: %szbExpected an HTTP response code of 200 while querying object readability, but received an error: %sN) ri � gsutil_apir r r r � status_coder � FromResponseZhas_attrr� �contentr )r� r_ ra rK rs rb rN rO rP rT Z signed_urlr5 �reqr� � http_errorZerror_responseZ error_stringr2 r2 r3 � _ProbeObjectAccessWithClient1 sB � ��z+UrlSignCommand._ProbeObjectAccessWithClientc C sD g }|D ]6}t |�r0|�dd� | �|�D �� q|�t|�� q|S )Nc S s g | ] }|j �qS r2 )� storage_url)�.0�blrr2 r2 r3 � <listcomp>b � z8UrlSignCommand._EnumerateStorageUrls.<locals>.<listcomp>)r �extend�WildcardIterator�appendr )r� Zin_urlsrC �url_strr2 r2 r3 �_EnumerateStorageUrls] s z$UrlSignCommand._EnumerateStorageUrlsc C sB t s td��| �� \}}}}}}}|r*dnd}| �| j|d� �} i } d} |s�z"tt| jd d��� |�\} } W q� ty� |s�t � d�}z"t t| jd d��� |�\} } W n& ty� td� | jd ���Y n0 Y q�0 n| j j dd �} td � | D �]B} | jdk�rtd ��| �� �rH|tk�r.td ��| j}|d k�rltd��n$d� | jtjj| j�tj�dd��}|tk�r| j| v �r�| | j }nrz | jd� | j�dgd�\}}W n< t�y� } z"td� |j j | j���W Y d}~n d}~0 0 |j � � }|| | j< n|}t | | j || j| |||| j!| j"|||dd�}t#�$t%j&t'j(d�j)dd�| �*� �}t%�+|�}|�,d�}t-j.�r~|�/tj�}d� | j0|||�}t-j.�r�|�tj�}t|� | �1| || j| || j!| j"||� }|d k�r | �� �r�|d k�r�td � | ���n"|d k�r<|d k�r<td � | ���q�|d kr�| j"�2d!| �p6d"| � q�dS )#z(Command entry point for signurl command.zhThe signurl command requires the pyopenssl library (try pip install pyopenssl or easy_install pyopenssl)r r} N�rbzKeystore password:z$Unable to parse private key from {0}�gs)rK z%URL HTTP Method Expiration Signed URLz+Can only create signed urls from gs:// urlsz�Generating signed URLs for creating buckets requires a region be specified via the -r option. Run `gsutil help signurl` for more information about the '-r' option.rH z-Resumable signed URLs require an object name.z{0}/{1}s /~)�safezgs://{}�location)� bucket_fieldsz�{}: Failed to auto-detect location for bucket '{}'. Please ensure you have storage.buckets.get permission on the bucket or specify the bucket's location using the '-r' option.T)r_ r` ra rK rS rL rA rb rN rO rP rc rT rR r+ r- z%Y-%m-%d %H:%M:%Sz{0} {1} {2} {3}r� r� zkBucket {0} does not exist. Please create a bucket with that name before a creating signed URL to access it.zsObject {0} does not exist. Please create/upload an object with that name before a creating signed URL to access it.r� z�%s does not have permissions on %s, using this link will likely result in a 403 error until at least READ permissions are grantedz The account)3� HAVE_OPENSSLr r� r� r� r{ �open�readry �getpassrt �formatr� �GetServiceAccountId�print�scheme�IsBucketr� � bucket_namer �parse�quote� object_name�encoder r� �GetSingleBucketUrlFromArg� Exceptionr� �__name__r� r@ ri rN rO �calendar�timegmr r/ r r0 r1 � utctimetuple� fromtimestamp�strftimerX rY r� � url_stringr� rV )r� rL r� rc rp rP ra rT Zarg_start_indexZ storage_urlsZ region_cacher_ rs �urlrb Z bucket_regionr� �bucket�erd � expirationZ expiration_dtZtime_strZ url_info_str� response_coder2 r2 r3 � RunCommandh s� �� � � � �� � �� �$ � � ���� �zUrlSignCommand.RunCommand)r� � __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �NO_MAXr �XML�JSONr � MakeZeroOrMoreFileURLsArgument� MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr� r� r� r� r� � __classcell__r2 r2 r� r3 r| � sB ���� GB, r| )NNFN)N)Mr� � __future__r r r r r� r� r r r r� rv r< r� rX � six.movesr Zapitools.base.py.exceptionsr Z apitools.base.py.http_wrapperr r �botor � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.storage_urlr r Z gslib.utilsr Zgslib.utils.boto_utilr �gslib.utils.shim_utilr r �gslib.utils.signurl_helperr r r ZOpenSSL.cryptor r r r r� � ImportError� cryptography.hazmat.primitivesr �)cryptography.hazmat.primitives.asymmetricr Z-cryptography.hazmat.primitives.asymmetric.rsar! �,cryptography.hazmat.primitives.serializationr" �cryptography.hazmat._oidr# r[ r\ r� r� r� r� r r4 rD ri rt r{ r| r2 r2 r2 r3 �<module> s� ��! � ^ & commands/__pycache__/hash.cpython-39.pyc 0000644 00000017577 15025013616 0014151 0 ustar 00 a +(WgD' � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z ddl mZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlm Z ddlm Z ddlm Z ddlm Z ddl m!Z! ddl m"Z" ddlm#Z# e j$Z%dZ&de& d Z'de#�(� d d e#�(� d d d d d! e#�)� d" d# d$ d% d! e#�)� d& d' Z*G d(d)� d)e �Z+dS )*zEImplementation of hash command for calculating hashes of local files.� )�absolute_import)�print_function)�division)�unicode_literalsN)�Command)�CommandArgument)� ApiSelector)�CommandException)�FileProgressCallbackHandler)�ProgressCallbackWithTimeout)�StorageUrlFromString)� FileMessage)� FinalMessage)� boto_util)� constants)�hashing_helper)�parallelism_framework_util)�GcloudStorageFlag)�GcloudStorageMap)� shim_utilz* gsutil hash [-c] [-h] [-m] filename... z <B>SYNOPSIS</B> a <B>DESCRIPTION</B> Calculate hashes on local files, which can be used to compare with ``gsutil ls -L`` output. If a specific hash option is not provided, this command calculates all gsutil-supported hashes for the files. Note that gsutil automatically performs hash validation when uploading or downloading files, so this command is only needed if you want to write a script that separately checks the hash. If you calculate a CRC32c hash for files without a precompiled crcmod installation, hashing will be very slow. See "gsutil help crcmod" for details. <B>OPTIONS</B> -c Calculate a CRC32c hash for the specified files. -h Output hashes in hex format. By default, gsutil uses base64. -m Calculate a MD5 hash for the specified files. z>--format=value[separator="",terminator=""](digest_format.sub("z", "Hashes ["),z url.sub("z ", "] for ").sub("$", ":^\^n"),z,md5_hash.yesno(yes=" Hash (md5): ", no=""),zmd5_hash.yesno(no=""),z$md5_hash.yesno(yes="NEWLINE", no="")z.sub("NEWLINE", "z"),z2crc32c_hash.yesno(yes=" Hash (crc32c): ", no=""),zcrc32c_hash.yesno(no=""),z'crc32c_hash.yesno(yes="NEWLINE", no="")z")�)c s� e Zd ZdZejdg edejdddde j ge j e � � gd� Z ejdd gd d ei d �Z� fd d�Zedd� �Zdd� Zdd� Z� ZS )� HashCommandz&Implementation of gsutil hash command.�hash� ZchmTFr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_argumentsZchecksum� command_helpzCalculate file hashes)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s� t ddtgtd�d d d�d�}t| jdd� | jD � �}d|v rZd |vrZ| jd g7 _n d |v rzd|vrz| jd g7 _t� �|�S ) N�storager z--hex)�-h�-c�-m)�gcloud_command�flag_mapc S s g | ] \}}|�qS � r2 )�.0�flag�_r2 r2 �"/opt/gsutil/gslib/commands/hash.py� <listcomp>� � z7HashCommand.get_gcloud_storage_args.<locals>.<listcomp>r. r/ z --skip-md5z --skip-crc32c) r �_GCLOUD_FORMAT_STRINGr �set�args�sub_optsr0 �super�get_gcloud_storage_args)�self�gcloud_storage_mapZargs_set�� __class__r2 r6 r> t s ��� z#HashCommand.get_gcloud_storage_argsc C s� d}d}dd� }dd� }d}d}|rr|D ]H\} } | dkrBd}d}q(| dkr`d }d d� }d d� }q(| d kr(d}d}q(|s~d}d}|r�t �� s�|�tj� |||||fS ) aZ Returns behavior variables based on input options. Args: sub_opts: getopt sub-arguments for the command. logger: logging.Logger for the command. Returns: Tuple of calc_crc32c: Boolean, if True, command should calculate a CRC32c checksum. calc_md5: Boolean, if True, command should calculate an MD5 hash. format_func: Function used for formatting the hash in the desired format. cloud_format_func: Function used for formatting the hash in the desired format. output_format: String describing the hash output format. Fc S s t �| �� �S �N)r �Base64EncodeHash� hexdigest��digestr2 r2 r6 �<lambda>� r8 z(HashCommand._ParseOpts.<locals>.<lambda>c S s | S rC r2 rF r2 r2 r6 rH � r8 �base64r. Tr- �hexc S s | � � S rC )rE rF r2 r2 r6 rH � r8 c S s t �| ��d�S )N�ascii)r �Base64ToHexHash�decoderF r2 r2 r6 rH � s r/ )r �UsingCrcmodExtension�warnr �SLOW_CRCMOD_WARNING) �clsr< �logger� calc_crc32c�calc_md5� format_func�cloud_format_funcZfound_hash_option� output_format�o�unused_ar2 r2 r6 � _ParseOpts� s2 � zHashCommand._ParseOptsc C s, i }|rt j�d�|d< |r(t�� |d< |S )a4 Constructs the dictionary of hashes to compute based on the arguments. Args: calc_crc32c: If True, CRC32c should be included. calc_md5: If True, MD5 should be included. Returns: Dictionary of {string: hash digester}, where string the name of the digester algorithm. zcrc-32c�crc32c�md5)�crcmod� predefined�Crcr �GetMd5)r? rS rT � hash_dictr2 r2 r6 �_GetHashClassesFromArgs� s z#HashCommand._GetHashClassesFromArgsc C s� | � | j| j�\}}}}}d}| jD �]�}| �|�jg d�d�D �]�}d}t|�} |jj} t|�� � �r"t j � | �} | j j�t| dt�� | dtjd�� t| t| j jt|�dd�j�} | �||�} t| d �� }tj|| | d � W d � n1 s�0 Y | j j�t| dt�� | dtjd�� nbi } |j}|j} |j du}|j du}|�sd|�sdt � � � d |� q@|�rt|j | d < |�r�|j | d < t!d|| f � t"�#| �D ]0\}}t!d|| � � �r�||�n||�f � �q�q@q$|�s�t$d��t%| j jt&t�� �� dS )z)Command entry point for the hash command.F)r[ �customerEncryption�md5Hash�size)�bucket_listing_fieldsTN)re �finished� message_type�Hashing)�src_url�operation_name�rb)�callback_processorzNo hashes present for %sr\ r[ zHashes [%s] for %s:z Hash (%s): %szNo files matchedr )'rZ r< rR r; �WildcardIterator� IterObjectsr � storage_url� object_name� IsFileUrl�os�path�getsize� gsutil_api� status_queue�putr �time� FILE_HASHr r �callrb �openr �CalculateHashesFromContents� root_objectre rd r[ �logging� getLoggerrO �print�six� iteritemsr �_PutToQueueWithTimeoutr )r? rS rT rU rV rW Z matched_one�url_strZfile_ref�url� file_name� file_sizerm ra �fp� obj_metadataZ md5_presentZcrc32c_present�namerG r2 r2 r6 � RunCommand� s� � � ���� �$�� � �zHashCommand.RunCommand)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr �NO_MAXr �JSONr � MakeZeroOrMoreFileURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr> � classmethodrZ rb r� � __classcell__r2 r2 rA r6 r Y s8 �� 0r ),r� � __future__r r r r r rs ry r] r� � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.progress_callbackr r �gslib.storage_urlr �gslib.thread_messager r Z gslib.utilsr r r r �gslib.utils.shim_utilr r r �PutToQueueWithTimeoutr� r� r� �get_format_flag_caret�get_format_flag_newliner9 r r2 r2 r2 r6 �<module> s� ��������������������� commands/__pycache__/defstorageclass.cpython-39.pyc 0000644 00000013437 15025013616 0016366 0 ustar 00 a +(Wg� � @ s^ d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddl m Z dZ dZ e e � d� Z!dZ"dZ#dd�$e"e# g� Z%ee!e%�Z&ee e#�Z'ee e"�Z(de �)� d Z*eddd e*d gi d �Z+eg d �i d �Z,G d d!� d!e �Z-d"S )#zGImpl. of default bucket storage class command for Google Cloud Storage.� )�absolute_import)�print_function)�division)�unicode_literals)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)�NormalizeStorageClass)� shim_utilzD gsutil defstorageclass set <storage-class> gs://<bucket_name>... z4 gsutil defstorageclass get gs://<bucket_name>... � a{ <B>SET</B> The "defstorageclass set" command sets the default `storage class <https://cloud.google.com/storage/docs/storage-classes>`_ for the specified bucket(s). If you specify a default storage class for a certain bucket, Google Cloud Storage applies the default storage class to all new objects uploaded to that bucket, except when the storage class is overridden by individual upload requests. Setting a default storage class on a bucket provides a convenient way to ensure newly uploaded objects have a specific storage class. If you don't set the bucket's default storage class, it will default to Standard. z; <B>GET</B> Gets the default storage class for a bucket. z5 The defstorageclass command has two sub-commands: z)--format=value[separator=": "](name.sub("z", "gs://"),storageClass)�storage�buckets�listz--raw��gcloud_command�flag_map)r r �updatez--default-storage-classc @ s� e Zd ZdZejdededdddej ej gej e � � e � � ge � � gd�d� Zejddgd d eeed �d �Zeeed �i d �Zdd� Zdd� Zdd� Zdd� Zdd� Z dS )�DefStorageClassCommandz1Implementation of gsutil defstorageclass command.Zdefstorageclass� � F)�set�get) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_argumentsZdefaultstorageclass� command_helpz/Get or set the default storage class on buckets)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textr c C s |� d�s td| j|f ��d S )Nzgs://zD"%s" does not support the URL "%s". Did you mean to use a gs:// URL?)� startswithr � command_name)�self�url_str� r4 �-/opt/gsutil/gslib/commands/defstorageclass.py� _CheckIsGsUrl� s ��z$DefStorageClassCommand._CheckIsGsUrlc C s, | j s| �� | j d �� dkr$dS dS d S )Nr r r � )�args�$RaiseWrongNumberOfArgumentsException�lower)r2 r4 r4 r5 �_CalculateUrlsStartArg� s z-DefStorageClassCommand._CalculateUrlsStartArgc C s� t | jd �}| jdd� }|s(| �� d}|D ]t}| �|� | j|dgd�}|D ]P}d}t�� }| j�d||j � d �f � ||_ | j j |jj||jjdgd � qRq0|s�ttt|� ��dS ) z,Sets the default storage class for a bucket.r r7 NF�id�� bucket_fieldsTz3Setting default storage class to "%s" for bucket %s�/)�provider�fields)r r8 r9 r6 �GetBucketUrlIterFromArg�apitools_messages�Bucket�logger�info� url_string�rstrip� storageClass� gsutil_api� PatchBucket� storage_url� bucket_name�schemer r r )r2 �normalized_storage_class�url_args� some_matchedr3 � bucket_iter�blr�bucket_metadatar4 r4 r5 �_SetDefStorageClass� s, � � z*DefStorageClassCommand._SetDefStorageClassc C sp | j }d}|D ]H}| �|� | j|dgd�}|D ]$}d}td|j�d�|jjf � q0q|sltt t |� ��dS )z,Gets the default storage class for a bucket.FrI r= Tz%s: %sr? N) r8 r6 rB �printrG rH � root_objectrI r r r )r2 rP rQ r3 rR rS r4 r4 r5 �_GetDefStorageClass� s �� z*DefStorageClassCommand._GetDefStorageClassc C sv | j �d�}|g}|dkr"| j}n>|dkrJ| j}t| j d �}|�|� ntd|| j| jf ��tj |d� |� dS )z4Command entry point for the defstorageclass command.r r r zAInvalid subcommand "%s" for the %s command. See "gsutil help %s".)� subcommands) r8 �poprX rU r �appendr r1 r �LogCommandParams)r2 �action_subcommand�subcommand_args�funcrO r4 r4 r5 � RunCommand� s �� z!DefStorageClassCommand.RunCommandN) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �MakeFreeTextArgument�%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �SHIM_GET_COMMAND_MAP�SHIM_SET_COMMAND_MAP�gcloud_storage_mapr6 r; rU rX r` r4 r4 r4 r5 r \ sN ����� ��r N).rd � __future__r r r r �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rC �gslib.utils.constantsr �gslib.utils.shim_utilr Zgslib.utils.text_utilr Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprf �_SET_DESCRIPTION�_GET_DESCRIPTION�join� _DESCRIPTIONrm rn ro �get_format_flag_caret�_GCLOUD_FORMAT_STRINGrq rr r r4 r4 r4 r5 �<module> sT � �� ��� commands/__pycache__/bucketpolicyonly.cpython-39.pyc 0000644 00000015375 15025013616 0016617 0 ustar 00 a +(Wg�" � @ s d Z ddlmZ ddlmZ ddlZddlZddlmZ ddlm Z ddl m Z ddl m Z dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ dZ dZ e e � d� Z dZ dZ!de! e Z"ee e"�Z#ee e �Z$ee e!�Z%ej&j'Z'e'j(Z(G dd� de �Z)dS )z+This module provides the command to gsutil.� )�absolute_import)�print_functionN)�metrics)�Command)�CommandArgument)�ubla)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)� InsistOnOrOffz> gsutil bucketpolicyonly set (on|off) gs://<bucket_name>... z5 gsutil bucketpolicyonly get gs://<bucket_name>... � a� <B>SET</B> The ``bucketpolicyonly set`` command enables or disables the uniform bucket-level access feature on Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use uniform bucket-level access: gsutil bucketpolicyonly set on gs://redbucket gs://bluebucket Configure your buckets to NOT use uniform bucket-level access: gsutil bucketpolicyonly set off gs://redbucket gs://bluebucket a& <B>GET</B> The ``bucketpolicyonly get`` command shows whether uniform bucket-level access is enabled for the specified Cloud Storage bucket. <B>GET EXAMPLES</B> Check if your buckets are using uniform bucket-level access: gsutil bucketpolicyonly get gs://redbucket gs://bluebucket a The Bucket Policy Only feature is now known as `uniform bucket-level access <https://cloud.google.com/storage/docs/uniform-bucket-level-access>`_. The ``bucketpolicyonly`` command is still supported, but we recommend using the equivalent ``ubla`` `command <https://cloud.google.com/storage/docs/gsutil/commands/ubla>`_. The ``bucketpolicyonly`` command is used to retrieve or configure the uniform bucket-level access setting of Cloud Storage buckets. This command has two sub-commands, ``get`` and ``set``. c @ s� e Zd ZdZejdededdddej gej e � d�ge ddd gd �e � � gd �d � Z ejdg d deeed �d�ZejjZejd jd Ze�dd�ejd jd< dd� Zdd� Zdd� Zdd� Z d d � Z d S ) �BucketPolicyOnlyCommandz/Implements the gsutil bucketpolicyonly command.Zbucketpolicyonly� � F� �mode�on�off)�choices��get�set) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz%Configure uniform bucket-level access)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textr � zUniform bucket-level accesszBucket Policy Onlyc C s |j jdkrtd| j ��d S )N�gsz7The %s command can only be used with gs:// bucket URLs.)� storage_url�schemer � command_name)�self�blr� r3 �./opt/gsutil/gslib/commands/bucketpolicyonly.py�,_ValidateBucketListingRefAndReturnBucketName� s ��zDBucketPolicyOnlyCommand._ValidateBucketListingRefAndReturnBucketNamec C s� | � |� |j}| jj|jdg|jd�}|j}|j}t|�� d�|j d�}d}|j rb|j |d< d}|r~t d| j f i |��� d S ) z1Gets the Bucket Policy Only setting for a bucket.�iamConfiguration��fields�provider�/)�bucket�enabledr Z locked_timez LockedTime: {locked_time} z>Bucket Policy Only setting for {bucket}: Enabled: {enabled} N)r5 r. � gsutil_api� GetBucket� bucket_namer/ r6 �bucketPolicyOnly�str�rstripr< � lockedTime�print�format)r1 r2 � bucket_url�bucket_metadata� iam_configZbucket_policy_onlyr8 Zlocked_time_liner3 r3 r4 �_GetBucketPolicyOnly� s* � � ��z,BucketPolicyOnlyCommand._GetBucketPolicyOnlyc C s~ | � |� |j}t� }t� |_|dk|j_tj|d�}|dkrBdnd}td|t |�� d�f � | j j |j |dg|jd� d S ) z;Sets the Bucket Policy Only setting for a bucket on or off.r )r6 ZEnablingZ Disablingz %s Bucket Policy Only for %s...r: r6 r7 r )r5 r. �IamConfigurationValue�BucketPolicyOnlyValuer@ r< �apitools_messages�BucketrD rA rB r= � PatchBucketr? r/ )r1 r2 � setting_argrF rH rG Z setting_verbr3 r3 r4 �_SetBucketPolicyOnly� s � �z,BucketPolicyOnlyCommand._SetBucketPolicyOnlyc C s� | j �d�}|dvr td��d}g }d}|dkr8| j}n.|dkrf| j}| j �d�}t|d� |�|� d}| j }|s|| �� |D ],}| �|�}|D ]} d }|| g|�R � q�q�|s�tt t |� ��dS ) z;Handles bucketpolicyonly command on a Cloud Storage bucket.r r z&bucketpolicyonly only supports get|setNr r z-Only on and off values allowed for set optionFT) �args�popr rI rP r �append�$RaiseWrongNumberOfArgumentsException�GetBucketUrlIterFromArgr �list) r1 � subcommand�subcommand_func�subcommand_argsrO � some_matched�url_args�url_str� bucket_iter�bucket_listing_refr3 r3 r4 �_BucketPolicyOnly� s6 � z)BucketPolicyOnlyCommand._BucketPolicyOnlyc C s� | j jdd�tjkr.td�t�d| j ����| j d }| j dd� |dksT|d krzt j | j d � t j |gd � | �� n td | ��d S )z5Command entry point for the bucketpolicyonly command.r- )r9 r zBThe "%s" command can only be used with the Cloud Storage JSON API.r T)� check_argsr r )�sub_opts)� subcommandsz-Invalid subcommand "%s", use get|set instead.N)r= �GetApiSelectorr �JSONr �join�textwrap�wrapr0 rQ � ParseSubOptsr �LogCommandParamsra r_ )r1 �action_subcommandr3 r3 r4 � RunCommand� s" ��� �z"BucketPolicyOnlyCommand.RunCommandN) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rd r �MakeNCloudURLsArgument�%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr Z UblaCommand�gcloud_storage_map�gcloud_commandZ format_flag�replacer5 rI rP r_ rk r3 r3 r3 r4 r \ sR ������� �$r )*ro � __future__r r �getoptrf �gslibr � gslib.commandr �gslib.command_argumentr �gslib.commandsr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rL �gslib.utils.constantsr Zgslib.utils.text_utilr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstriprq �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONrv rx rw rM rJ rK r r3 r3 r3 r4 �<module> s> � � commands/__pycache__/retention.cpython-39.pyc 0000644 00000050031 15025013616 0015213 0 ustar 00 a +(Wg w � @ s< d Z ddlmZ ddlZddlmZ ddlmZ ddlm Z ddlm Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl&m(Z( ddl&m)Z) ddl&m*Z* ddl&m+Z+ ddl&m,Z, dd l&m-Z- dd l&m.Z. dd l/m0Z0 dd l1m2Z2 d Z3d!Z4d"Z5d#Z6d$Z7d%Z8d&Z9d'Z:d(Z;d)Z<d*Z=d+Z>d,Z?d-Z@e3e4 e5 e6 e7 e8 e9 ZAe:e; e< e= e> e? e@ ZBeeAeB�ZCee3e:�ZDee4e;�ZEee5e<�ZFee6e=�ZGee7e>�ZHee8e?�ZIee9e@�ZJG d.d/� d/e �ZKdS )0zEImplementation of Retention Policy configuration command for buckets.� )�absolute_importN)�encoding)�metrics)�AccessDeniedException)� Preconditions)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�NameExpansionIterator)� SeekAheadNameExpansionIterator)�StorageUrlFromString)�storage_v1_messages)�MetadataMessage)�GetCloudApiInstance)�NO_MAX)�PutToQueueWithTimeout)�ConfirmLockRequest)�ReleaseEventHoldFuncWrapper)�ReleaseTempHoldFuncWrapper)�RetentionInSeconds)�RetentionPolicyToString)�SetEventHoldFuncWrapper)�SetTempHoldFuncWrapper)�$UpdateObjectMetadataExceptionHandler)�GcloudStorageMap)�PreconditionsFromHeaderszA gsutil retention set <retention_period> gs://<bucket_name>... z0 gsutil retention clear gs://<bucket_name>... z. gsutil retention get gs://<bucket_name>... z/ gsutil retention lock gs://<bucket_name>... zF gsutil retention event-default (set|release) gs://<bucket_name>... zL gsutil retention event (set|release) gs://<bucket_name>/<object_name>... zK gsutil retention temp (set|release) gs://<bucket_name>/<object_name>... a� <B>SET</B> You can configure a data retention policy for a Cloud Storage bucket that governs how long objects in the bucket must be retained. You can also lock the data retention policy, permanently preventing the policy from being reduced or removed. For more information, see `Retention policies and Bucket Lock <https://cloud.google.com/storage/docs/bucket-lock>`_. The ``gsutil retention set`` command allows you to set or update the retention policy on one or more buckets. To remove an unlocked retention policy from one or more buckets, use the ``gsutil retention clear`` command. The ``set`` sub-command can set a retention policy with the following formats: <B>SET FORMATS</B> Formats for the ``set`` subcommand include: <number>s Specifies retention period of <number> seconds for objects in this bucket. <number>d Specifies retention period of <number> days for objects in this bucket. <number>m Specifies retention period of <number> months for objects in this bucket. <number>y Specifies retention period of <number> years for objects in this bucket. GCS JSON API accepts retention periods as number of seconds. Durations provided in terms of days, months or years are converted to their rough equivalent values in seconds, using the following conversions: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Retention periods must be greater than 0 and less than 100 years. Retention durations must be in only one form (seconds, days, months, or years), and not a combination of them. Note that while it is possible to specify retention durations shorter than a day (using seconds), enforcement of such retention periods is not guaranteed. Such durations may only be used for testing purposes. <B>EXAMPLES</B> Setting a retention policy with a duration of 1 year on a bucket: gsutil retention set 1y gs://my-bucket Setting a retention policy with a duration of 36 months on a bucket: gsutil retention set 36m gs://some-bucket You can also provide a precondition on a bucket's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks a bucket's metageneration before setting the retention policy on the bucket: gsutil -h "x-goog-if-metageneration-match: 1" \ retention set 1y gs://my-bucket a4 <B>CLEAR</B> The ``gsutil retention clear`` command removes an unlocked retention policy from one or more buckets. You cannot remove or reduce the duration of a locked retention policy. <B>EXAMPLES</B> Clearing an unlocked retention policy from a bucket: gsutil retention clear gs://my-bucket z� <B>GET</B> The ``gsutil retention get`` command retrieves the retention policy for a given bucket and displays a human-readable representation of the configuration. a1 <B>LOCK</B> The ``gsutil retention lock`` command PERMANENTLY locks an unlocked retention policy on one or more buckets. CAUTION: A locked retention policy cannot be removed from a bucket or reduced in duration. Once locked, deleting the bucket is the only way to "remove" a retention policy. a� <B>EVENT-DEFAULT</B> The ``gsutil retention event-default`` command sets the default value for an event-based hold on one or more buckets. By setting the default event-based hold on a bucket, newly-created objects inherit that value as their event-based hold (it is not applied retroactively). <B>EXAMPLES</B> Setting the default event-based hold on a bucket: gsutil retention event-default set gs://my-bucket Releasing the default event-based hold on a bucket: gsutil retention event-default release gs://my-bucket You can also provide a precondition on a bucket's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks a bucket's metageneration before setting the default event-based hold on the bucket: gsutil -h "x-goog-if-metageneration-match: 1" \ retention event-default set gs://my-bucket a[ <B>EVENT</B> The ``gsutil retention event`` command enables or disables an event-based hold on an object. <B>EXAMPLES</B> Setting the event-based hold on an object: gsutil retention event set gs://my-bucket/my-object Releasing the event-based hold on an object: gsutil retention event release gs://my-bucket/my-object You can also provide a precondition on an object's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks an object's metageneration before setting the event-based hold on the object: gsutil -h "x-goog-if-metageneration-match: 1" \ retention event set gs://my-bucket/my-object If you want to set or release an event-based hold on a large number of objects, then you might want to use the top-level '-m' option to perform a parallel update. For example, the following command sets an event-based hold on objects ending with .jpg in parallel, in the root folder: gsutil -m retention event set gs://my-bucket/*.jpg aA <B>TEMP</B> The ``gsutil retention temp`` command enables or disables a temporary hold on an object. <B>EXAMPLES</B> Setting the temporary hold on an object: gsutil retention temp set gs://my-bucket/my-object Releasing the temporary hold on an object: gsutil retention temp release gs://my-bucket/my-object You can also provide a precondition on an object's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks an object's metageneration before setting the temporary hold on the object: gsutil -h "x-goog-if-metageneration-match: 1" \ retention temp set gs://my-bucket/my-object If you want to set or release a temporary hold on a large number of objects, then you might want to use the top-level '-m' option to perform a parallel update. For example, the following command sets a temporary hold on objects ending with .jpg in parallel, in the root folder: gsutil -m retention temp set gs://bucket/*.jpg c s> e Zd ZdZejdg ededddej gej e � � ge � � ge � d�ge � � ge � � ge � � gd�e � � ge � � gd�e � � ge � � gd�d�d� Zejdg d d eeeeeeeed �d �Z� fd d�Zdd� Zdd� Zd,dd�Z dd� Z dd� Z dd� Z d d � Z d d � Z!d d!� Z"d"d#� Z#d$d%� Z$d&d'� Z%d(d)� Z&d*d+� Z'� Z(S )-�RetentionCommandz+Implementation of gsutil retention command.� retention� F� ��set�release)r$ �clear�get�lock� event-default�event�temp) �command_name_aliases�usage_synopsis�min_args�max_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz=Provides utilities to interact with Retention Policy feature.)r' r$ r& r( r) r* r+ )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc s | j d dkr^tdtdddd�t| j d ��g| j dd � i d �ii d �}| j d d� | _ n�ttg d �i d �ttg d �i d �tg d �i d �d �i d �ttg d�i d �tg d�i d �d �i d �tg d�i d �tg d�i d �ttg d�i d �tg d�i d �d �i d �d�i d �}t� �|�S )Nr r$ �storage�buckets�updatez--retention-period={}sr" r! )�gcloud_command�flag_map)r= r> r? z--clear-retention-period)r= �objectsr? z--event-based-hold)r= rB r? z--no-event-based-holdr# )r= r> r? z--default-event-based-hold)r= r> r? z --no-default-event-based-hold)r= r> �describez --format=yaml(retentionPolicy)z--raw)r= r> r? z--lock-retention-period)r= rB r? z--temporary-hold)r= rB r? z--no-temporary-hold)r& r* r) r' r( r+ )�argsr �formatr �super�get_gcloud_storage_args)�self�gcloud_storage_map�� __class__� �'/opt/gsutil/gslib/commands/retention.pyrG I s� �� ��� � ����������������Z�]z(RetentionCommand.get_gcloud_storage_argsc C s� | j �d�tjkr"td�| j���t| j�| _ | j � d�}| j dd� |dkrV| j }nr|dkrf| j}nb|dkrv| j}nR|d kr�| j}nB|d kr�| j}n2|d kr�| j}n"|d kr�| j}ntd �|| j���tj|g| jd� |� S )z.Command entry point for the retention command.�gsz�The {} command can only be used with the GCS JSON API. If you have only supplied hmac credentials in your boto file, please instead supply a credential type that can be used with the JSON API.r T)� check_argsr$ r& r' r( r) r* r+ zHInvalid subcommand "{}" for the {} command. See "gsutil help retention".)� subcommands�sub_opts)� gsutil_api�GetApiSelectorr �JSONr rE � command_namer �headers� preconditionsrD �pop� ParseSubOpts� _SetRetention�_ClearRetention� _GetRetention�_LockRetention�_DefaultEventHold� _EventHold� _TempHoldr �LogCommandParamsrQ )rH �action_subcommand�funcrL rL rM � RunCommand� s> � ���zRetentionCommand.RunCommandc C s� t | jjd�}d}|D ]P}| j|dgd�}|D ]6} | j} d}| j�|| � | jj| j ||| j |d� q.q|s|t t t |� ��d S )N)�meta_gen_matchF�id�� bucket_fieldsT)rW �provider�fields)r rW re �GetBucketUrlIterFromArg� storage_url�logger�inforR � PatchBucket� bucket_name�schemer r �list) rH �url_args�bucket_metadata_updaterj �log_msg_templaterW � some_matched�url_str� bucket_iter�blr�urlrL rL rM �BucketUpdateFunc� s$ � � z!RetentionCommand.BucketUpdateFuncNc C s� t | |d�}|j}| j�||� t�tj|j�}t | j j | j j d�}|j du rV|j |_ |j du rh|j|_ |j|j|j||j ||jdgd� t|jtt�� d�� dS )a� Updates metadata on an object using PatchObjectMetadata. Args: patch_obj_metadata: Metadata changes that should be applied to the existing object. log_template: The log template that should be printed for each object. name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. )� thread_state)� gen_matchre Nrf )� generationrW ri rj )� message_time)r �expanded_storage_urlrm rn r � JsonToMessage�apitools_messages�Object�expanded_resultr rW r} re r~ �metageneration�PatchObjectMetadatarp � object_namerq r � status_queuer �time) rH Zpatch_obj_metadata� log_template�name_expansion_resultr| rR � exp_src_urlZcloud_obj_metadatarW rL rL rM �ObjectUpdateMetadataFunc� s2 �� � �z)RetentionCommand.ObjectUpdateMetadataFuncc C s, t | j| j| j| j|| j| j| jddgd� S )Nr~ r� )� all_versions�continue_on_error�bucket_listing_fields)r rU �debugrm rR �recursion_requestedr� �parallel_operations�rH rs rL rL rM � _GetObjectNameExpansionIterator! s �z0RetentionCommand._GetObjectNameExpansionIteratorc C s$ t | j| j| �� || j| j| jd�S )N)r� � project_id)r rU r� �GetSeekAheadGsutilApir� r� r� r� rL rL rM �"_GetSeekAheadNameExpansionIterator- s �z3RetentionCommand._GetSeekAheadNameExpansionIteratorc C sT t | jd �}tjj|d�}d}tj|d�}| jdd� }| j||ddg|d � dS ) z6Set retention retention_period on one or more buckets.r ��retentionPeriodz!Setting Retention Policy on %s...��retentionPolicyr" Nrf r� �rj ru )r rD r� �Bucket�RetentionPolicyValuer{ )rH �seconds�retention_policyru rt rs rL rL rM rZ 6 s ���z RetentionCommand._SetRetentionc C s> t jjdd�}d}t j|d�}| j}| j||ddg|d� dS ) z8Clear retention retention_period on one or more buckets.Nr� z"Clearing Retention Policy on %s...r� rf r� r� r )r� r� r� rD r{ )rH r� ru rt rs rL rL rM r[ G s ���z RetentionCommand._ClearRetentionc C s. | j | jd dgd�\}}tt|j|�� dS )z)Get Retention Policy for a single bucket.r r� rg )�GetSingleBucketUrlFromArgrD �printr r� )rH � bucket_url�bucket_metadatarL rL rM r\ U s � z RetentionCommand._GetRetentionc C s� | j }d}|D ]�}| j|dgd�}|D ]�}|j}d}| jj|j|jg d�d�}|jr\|jjsnt d� |j���q&|jj du r�| j � d|� q&t|j|j�r�| j �d |� | jj|j|j|jd � q&| j � d � |�� q&q|s�t tt|� ��d S ) z-Lock Retention Policy on one or more buckets.Frf rg T)rf r� r� )ri rj z7Bucket "{}" does not have an Unlocked Retention Policy.z+Retention Policy on "%s" is already locked.z!Locking Retention Policy on %s...)ri z& Abort Locking Retention Policy on {}r )rD rk rl rR � GetBucketrp rq r� r� r rE �isLockedrm �errorr rn �LockRetentionPolicyr� r rr )rH rs rv rw rx ry rz r� rL rL rM r] \ sH ���� � ��z RetentionCommand._LockRetentionc C s� d}| j rJ| j d �� dkr"d}n(| j d �� dkr:d}ntd�| j���|rRdnd }d �|�}tj|d �}| j d d� }| j||d dg|d� dS )z?Sets default value for Event-Based Hold on one or more buckets.Nr r$ Tr% FzeInvalid subcommand "{}" for the "retention event-default" command. See "gsutil help retention event".ZSettingZ Releasingz${} default Event-Based Hold on %s...)�defaultEventBasedHoldr" rf r� r� )rD �lowerr rE rQ r� r� r{ )rH �hold�verbru rt rs rL rL rM r^ � s. �� ��z"RetentionCommand._DefaultEventHoldc C s> d}d}| � |�}| jdd� }|r(tnt}| �|||� dS )z7Sets or unsets Event-Based Hold on one or more objects.r* z Event-Basedr" Nr )�_ProcessHoldArgsrD r r �_SetHold�rH �sub_command_name�sub_command_full_namer� rs �obj_metadata_update_wrapperrL rL rM r_ � s ��zRetentionCommand._EventHoldc C s> d}d}| � |�}| jdd� }|r(tnt}| �|||� dS )z5Sets or unsets Temporary Hold on one or more objects.r+ Z Temporaryr" Nr )r� rD r r r� r� rL rL rM r` � s ��zRetentionCommand._TempHoldc C sP d}| j d �� dkr d}n0| j d �� dkr4d}ntd�| j d ||���|S )a Processes command args for Temporary and Event-Based Hold sub-command. Args: sub_command_name: The name of the subcommand: "temp" / "event" Returns: Returns a boolean value indicating whether to set (True) or release (False)the Hold. Nr r$ Tr% FzWInvalid subcommand "{}" for the "retention {}" command. See "gsutil help retention {}".)rD r� r rE )rH r� r� rL rL rM r� � s ��z!RetentionCommand._ProcessHoldArgsc C s� t |�dkr@| js@t|d �}|�� r.|�� s@td�|d ���| �|�}| �|�}d| _ z| j ||t d|d� W n8 t y� } z |j dkr�| �� � W Y d}~n d}~0 0 | j s�td�|���dS ) ap Common logic to set or unset Event-Based/Temporary Hold on objects. Args: obj_metadata_update_wrapper: The function for updating related fields in Object metadata. url_args: List of object URIs. sub_command_full_name: The full name for sub-command: "Temporary" / "Event-Based" r" r z URL ({}) must name an objectT)� fail_on_error�seek_ahead_iteratori� Nz*{} Hold for some objects could not be set.)�lenr� r � IsCloudUrl�IsObjectr rE r� r� �everything_set_okay�Applyr r �status�_WarnServiceAccounts)rH r� rs r� rz �name_expansion_iteratorr� �erL rL rM r� � s4 � � ��zRetentionCommand._SetHold)N))�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r rT r �%MakeZeroOrMoreCloudBucketURLsArgument� MakeNCloudBucketURLsArgument� MakeZeroOrMoreCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text�_clear_help_text�_lock_help_text�_event_default_help_text�_event_help_text�_temp_help_text� help_specrG rd r{ r� r� r� rZ r[ r\ r] r^ r_ r` r� r� � __classcell__rL rL rJ rM r st ����� �� p* � ( % r )Lr� � __future__r r� �apitools.base.pyr �gslibr �gslib.cloud_apir r � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �gslib.name_expansionr r �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr r� �gslib.thread_messager � gslib.utils.cloud_api_helperr �gslib.utils.constantsr �&gslib.utils.parallelism_framework_utilr �gslib.utils.retention_utilr r r r r r r r �gslib.utils.shim_utilr � gslib.utils.translation_helperr � _SET_SYNOPSISZ_CLEAR_SYNOPSIS� _GET_SYNOPSISZ_LOCK_SYNOPSISZ_EVENT_DEFAULT_SYNOPSISZ_EVENT_SYNOPSISZ_TEMP_SYNOPSIS�_SET_DESCRIPTIONZ_CLEAR_DESCRIPTION�_GET_DESCRIPTIONZ_LOCK_DESCRIPTIONZ_EVENT_DEFAULT_DESCRIPTIONZ_EVENT_DESCRIPTIONZ_TEMP_DESCRIPTIONr� � _DESCRIPTIONr� r� r� r� r� r� r� r� r rL rL rL rM �<module> s� @ ��� ���� � commands/__pycache__/mb.cpython-39.pyc 0000644 00000030360 15025013616 0013605 0 ustar 00 a +(WgkB � @ sx d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z m Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl m"Z" ddl#m$Z$ ddl#m%Z% ddl#m&Z& ddl'm(Z( d�)e�Z*de* dj)ed � Z+e�,d �Z-e�,d �Z.d Z/ej0j1Z1e1j2Z2G d d!� d!e �Z3dS )"z@Implementation of mb command for creating cloud storage buckets.� )�absolute_import)�print_function)�division)�unicode_literalsN)�AccessDeniedException�BadRequestException)�Command)�CommandArgument)�VALID_RPO_VALUES)�VALID_RPO_VALUES_STRING)� ApiSelector)�CommandException)�InvalidUrlError)�StorageUrlFromString)�storage_v1_messages)�NO_MAX)�RetentionInSeconds)�GcloudStorageFlag)�GcloudStorageMap)� InsistAscii)� InsistOnOrOff)�NormalizeStorageClass)� ValidateCMEKz� gsutil mb [-b (on|off)] [-c <class>] [-k <key>] [-l <location>] [-p <project>] [--autoclass] [--retention <time>] [--pap <setting>] [--placement <region1>,<region2>] [--rpo {}] gs://<bucket_name>... z <B>SYNOPSIS</B> a� <B>DESCRIPTION</B> Create one or more new buckets. Google Cloud Storage has a single namespace, so you are not allowed to create a bucket with a name already in use by another user. You can, however, carve out parts of the bucket name space corresponding to your company's domain name (see "gsutil help naming"). If you don't specify a project ID or project number using the -p option, the buckets are created using the default project ID specified in your `gsutil configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_. The -l option specifies the location for the buckets. Once a bucket is created in a given location, it cannot be moved to a different location. Instead, you need to create a new bucket, move the data over, and then delete the original bucket. <B>BUCKET STORAGE CLASSES</B> You can specify one of the `storage classes <https://cloud.google.com/storage/docs/storage-classes>`_ for a bucket with the -c option. Example: gsutil mb -c nearline gs://some-bucket See online documentation for `pricing <https://cloud.google.com/storage/pricing>`_ and `SLA <https://cloud.google.com/storage/sla>`_ details. If you don't specify a -c option, the bucket is created with the default storage class Standard Storage. <B>BUCKET LOCATIONS</B> You can specify one of the `available locations <https://cloud.google.com/storage/docs/locations>`_ for a bucket with the -l option. Examples: gsutil mb -l asia gs://some-bucket gsutil mb -c standard -l us-east1 gs://some-bucket If you don't specify a -l option, the bucket is created in the default location (US). <B>Retention Policy</B> You can specify retention period in one of the following formats: --retention <number>s Specifies retention period of <number> seconds for objects in this bucket. --retention <number>d Specifies retention period of <number> days for objects in this bucket. --retention <number>m Specifies retention period of <number> months for objects in this bucket. --retention <number>y Specifies retention period of <number> years for objects in this bucket. Examples: gsutil mb --retention 1y gs://some-bucket gsutil mb --retention 36m gs://some-bucket If you don't specify a --retention option, the bucket is created with no retention policy. <B>OPTIONS</B> --autoclass Enables the Autoclass feature that automatically sets object storage classes. -b <on|off> Specifies the uniform bucket-level access setting. When "on", ACLs assigned to objects in the bucket are not evaluated. Consequently, only IAM policies grant access to objects in these buckets. Default is "off". -c class Specifies the default storage class. Default is ``Standard``. See `Available storage classes <https://cloud.google.com/storage/docs/storage-classes#classes>`_ for a list of possible values. -k <key> Set the default KMS key using the full path to the key, which has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -l location Can be any supported location. See https://cloud.google.com/storage/docs/locations for a discussion of this distinction. Default is US. Locations are case insensitive. -p project Specifies the project ID or project number to create the bucket under. -s class Same as -c. --retention time Specifies the retention policy. Default is no retention policy. This can only be set on gs:// buckets and requires using the JSON API. For more details about retention policy see "gsutil help retention" --pap setting Specifies the public access prevention setting. Valid values are "enforced" or "inherited". When "enforced", objects in this bucket cannot be made publicly accessible. Default is "inherited". --placement reg1,reg2 Two regions that form the custom dual-region. Only regions within the same continent are or will ever be valid. Invalid location pairs (such as mixed-continent, or with unsupported regions) will return an error. --rpo setting Specifies the `replication setting <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_. This flag is not valid for single-region buckets, and multi-region buckets only accept a value of DEFAULT. Valid values for dual region buckets are {rpo_values}. If unspecified, DEFAULT is applied for dual-region and multi-region buckets. )Z rpo_valuesz.^[a-zA-Z0-9][a-zA-Z0-9\._-]{1,253}[a-zA-Z0-9]$z[-_a-z0-9]{64}z --retentionc s� e Zd ZdZejdg d�ededg d�dddej ej gej e � � gd � Z ejdg d �d d ei d �Zeg d�deddd��ded�ded�ded�ded�ded dd ��d ed �eed �d ed!�i d"�Z� fd#d$�Zd%d&� Z� ZS )'� MbCommandz$Implementation of gsutil mb command.�mb)� makebucket� createbucket�md�mkdir� z b:c:l:p:s:k:)� autoclassz retention=zpap=z placement=zrpo=Fr ) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args�supported_private_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)r r r r �locationZdraZdrasZreduced_availabilityZ durable_reduced_availability�rr�reduced_redundancy�standardz storage class�nearline�nl� command_helpz Make buckets)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�buckets�create�-bz --uniform-bucket-level-accessN)�on�off�-cz--default-storage-class�-kz--default-encryption-key�-lz --location�-pz --project�--papz--public-access-prevention)ZenforcedZ inherited� --placementz--retention-period�--rpoz--recovery-point-objective)�gcloud_command�flag_mapc sp d}|t | j�k r0| j| d tkr&q0|d7 }q|t | j�k rbttt| j| d ��d f| j|< t� �tj�S )Nr r �s) �len�sub_opts�_RETENTION_FLAG�strr �super�get_gcloud_storage_argsr �gcloud_storage_map)�selfZretention_arg_idx�� __class__� � /opt/gsutil/gslib/commands/mb.pyrO s � z!MbCommand.get_gcloud_storage_argsc C s� d}d}d}d}d}d}d}d}d} g } | j �r~| j D �]D\} } | dkrXd}| �| � q6| dkrx| }t|� | �| � q6| dkr�| }q6| dkr�t| d� | | _q6| d ks�| d kr�t| �}q6| tkr�t| �}q6| d k�r| �� } | t vr�t d � t | ���| �| � q6| d k�r*t | d� | dk}| �| � q6| dk�rD| }| �| � q6| dkr6| �d�}t|�dk�rrt d� | ���| �| � q6tj|| |d�} |�r�tjj|d�| _|�s�|�r�t� | _| j}|�r�t� |_||j_|�r�||_|�r�tj�� }||_|| _ |�rtj� � }||_ || _ | j D �]�}t!|�}|du�r^|j"dk�rJt d��tjj#|d�}|| _$| �r�| j%�&|j"�t'j(k�r�t d� d�)| ����|�*� �s�t d | ��t+�,|j-��r�t.�/|j-��r�t0d |j- ��| j1�2d |� z | j%j3|j-| j| |j"d � W n� t4�yf } zR|j5}d |v �rP|d!7 }|d"| 7 }| j�rF|d#| j 7 }t |��n� W Y d}~n�d}~0 t6�y� } zl|j7d$k�r�|j5d%k�r�|j"dk�r�|j-}||�8d&�d' d� }t d(�)t9�:d)||f ����n� W Y d}~n d}~0 0 �q d*S )+z'Command entry point for the mb command.FNz --autoclassTrA rB rC z/Invalid non-ASCII character found in project IDr@ z-srF z9Invalid value for --rpo. Must be one of: {}, provided: {}r= z,Only on and off values allowed for -b optionr> rD rE �,� zJPlease specify two regions separated by comma without space. Specified: {})r- �rpo� storageClass)�enabled�gsz7Retention policy can only be specified for GCS buckets.)�retentionPeriodzCThe {} option(s) can only be used for GCS Buckets with the JSON APIz, zIThe mb command requires a URL that specifies a bucket. "%s" is not valid.z Invalid bucket name in URL "%s"zCreating %s...)� project_id�metadata�provider�keyz* To authorize, run: gsutil kms authorizez \ -k %sz \ -p %si� ZDotfulBucketNameNotUnderTld�.r � z�Buckets with "." in the name must be valid DNS names. The bucket you are attempting to create (%s) is not a valid DNS name, because the final component (%s) is not currently a valid part of the top-level DNS tree.r );rK �appendr r r] r rL r �stripr r �formatr r �splitrJ �apitools_messages�Bucket�AutoclassValuer �IamConfigurationValue�iamConfiguration�BucketPolicyOnlyValue�bucketPolicyOnlyrZ �publicAccessPrevention�EncryptionValue�defaultKmsKeyName� encryption�CustomPlacementConfigValue� dataLocations�customPlacementConfig�argsr �scheme�RetentionPolicyValue�retentionPolicy� gsutil_api�GetApiSelectorr �JSON�join�IsBucket�BUCKET_NAME_RE�match� bucket_name�TOO_LONG_DNS_NAME_COMP�searchr �logger�info� CreateBucketr �reasonr �status�rfind�textwrap�wrap)rQ r �bucket_policy_only�kms_keyr- � storage_class�secondsZ placements�public_access_preventionrX Zjson_only_flags_in_command�o�a�bucket_metadata� iam_configrq Zplacement_configZbucket_url_str� bucket_url�retention_policy�e�messager� Z final_comprT rT rU � RunCommand s �� �� �� � ��� � �� � ���� zMbCommand.RunCommand)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XMLr{ r �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specr r rL rP rO r� � __classcell__rT rT rR rU r � sn ������� r )4r� � __future__r r r r �rer� �gslib.cloud_apir r � gslib.commandr �gslib.command_argumentr Zgslib.commands.rpor r �gslib.cs_api_mapr �gslib.exceptionr r �gslib.storage_urlr �"gslib.third_party.storage_apitoolsr rg �gslib.utils.constantsr Zgslib.utils.retention_utilr �gslib.utils.shim_utilr r Zgslib.utils.text_utilr r r � gslib.utils.encryption_helperr re r� r� �compiler~ r� rL rh rj rl r rT rT rT rU �<module> sR ��|�� commands/__pycache__/requesterpays.cpython-39.pyc 0000644 00000013036 15025013616 0016124 0 ustar 00 a +(Wg ! � @ s( d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z ddl m Z dd l m Z dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ dZ dZ e e � d� Z dZ dZ!de e! Z"ee e"�Z#ee e!�Z$ee e �Z%de�&� d d Z'G dd� de �Z(d S ) zBImplementation of requesterpays configuration command for buckets.� )�absolute_import)�print_function)�division)�unicode_literals)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�storage_v1_messages)�NO_MAX)�GcloudStorageMap)� shim_utilz; gsutil requesterpays set (on|off) gs://<bucket_name>... z2 gsutil requesterpays get gs://<bucket_name>... � z� <B>SET</B> The "set" sub-command requires an additional sub-command, either "on" or "off", which, respectively, will enable or disable Requester Pays for the specified bucket. z� <B>GET</B> The "get" sub-command gets the Requester Pays configuration for a bucket and displays whether or not it is enabled. a% The `Requester Pays <https://cloud.google.com/storage/docs/requester-pays>`_ feature enables you to configure a Google Cloud Storage bucket so that the requester pays all costs related to accessing the bucket and its objects. The gsutil requesterpays command has two sub-commands: z)--format=value[separator=": "](name.sub("z ", "gs://"),z,requester_pays.yesno("Enabled", "Disabled"))c @ s� e Zd ZdZejdededdddej gej e dddgd �e � � ge � � gd �d � Z ej dg d d eeed�d�Zeedddegi dd�eeg d�i d�eg d�i d�d�i d�d�i d�Zdd� Zdd � Zd d � Zd d � Zd!S )"�RequesterPaysCommandz/Implementation of gsutil requesterpays command.Z requesterpays� � F�mode�on�off)�choices)�set�get) �usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments� command_helpz8Enable or disable requester pays for one or more buckets)r r )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text�storage�buckets�listT)�gcloud_command�flag_map�supports_output_translation)r, r- �updatez--requester-pays)r/ r0 )r, r- r2 z--no-requester-pays�r r c C s, | j s| �� | j d �� dkr$dS dS d S )Nr r r � )�args�$RaiseWrongNumberOfArgumentsException�lower)�self� r9 �+/opt/gsutil/gslib/commands/requesterpays.py�_CalculateUrlsStartArg� s z+RequesterPaysCommand._CalculateUrlsStartArgc C s� | j d �� }|dvr$td| j ��| j dd� }|s>| �� d}|D ]�}| j|dgd�}|D ]p}|j}d }tjtj� � d �}|d kr�| j � d |� d |j _ n| j � d |� d|j _ | jj|j||jdgd� q^qF|s�ttt|� ��dS )z.Gets requesterpays configuration for a bucket.r r3 z,Argument to "%s set" must be either <on|off>r4 NF�id�� bucket_fieldsT)�billingr z!Enabling requester pays for %s...z"Disabling requester pays for %s...)�provider�fields)r5 r7 r � command_namer6 �GetBucketUrlIterFromArg� storage_url�apitools_messages�Bucket� BillingValue�logger�infor? � requesterPays� gsutil_api� PatchBucket� bucket_name�schemer r. ) r8 �requesterpays_arg�url_args� some_matched�url_str� bucket_iter�blr�url�bucket_metadatar9 r9 r: �_SetRequesterPays� s: �� � z&RequesterPaysCommand._SetRequesterPaysc C s� | j }d}|D ]^}| j|dgd�}|D ]D}d}|jjrV|jjjrVtd|j�d� � q&td|j�d� � q&q|s�tt t |� ��dS ) z9Gets requesterpays configuration for one or more buckets.Fr? r= Tz %s: Enabled�/z %s: DisabledN) r5 rC � root_objectr? rJ �print� url_string�rstripr r r. )r8 rP rQ rR rS rT r9 r9 r: �_GetRequesterPays� s �z&RequesterPaysCommand._GetRequesterPaysc C s� | j �d�}|dkr*| j}tj|gd� nL|dkr`| j}| j d �� }|dv rvtj||gd� ntd|| j| jf ��|� dS )z2Command entry point for the requesterpays command.r r )� subcommandsr r3 zAInvalid subcommand "%s" for the %s command. See "gsutil help %s".) r5 �popr] r �LogCommandParamsrW r7 r rB )r8 �action_subcommand�funcrO r9 r9 r: � RunCommand� s$ � ��z RequesterPaysCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �JSONr �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr �_GCLOUD_FORMAT_STRING�gcloud_storage_mapr; rW r] rc r9 r9 r9 r: r M sn ������ �������"�&!r N))rg � __future__r r r r �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �"gslib.third_party.storage_apitoolsr rE �gslib.utils.constantsr �gslib.utils.shim_utilr Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstripri �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONrn ro rp �get_format_flag_caretrr r r9 r9 r9 r: �<module> sH �� ��� commands/__pycache__/logging.cpython-39.pyc 0000644 00000022054 15025013616 0014636 0 ustar 00 a +(Wg�3 � @ sT d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m Z ddl!m"Z" dZ#dZ$e#e$�%d� d Z&dZ'dZ(de' e( d Z)ee&e)�Z*ee$e(�Z+ee#e'�Z,G d d � d e �Z-dS ) z<Implementation of logging configuration command for buckets.� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�metrics)�Command)�CommandArgument)� ApiSelector)�CommandException)�NO_URLS_MATCHED_TARGET)�CreateHelpText)�StorageUrlFromString)�UrlsAreForSingleProvider)�storage_v1_messages)�NO_MAX)�GcloudStorageFlag)�GcloudStorageMap)� text_utilz� gsutil logging set on -b <logging_bucket_name> [-o <log_object_prefix>] gs://<bucket_name>... gsutil logging set off gs://<bucket_name>... z) gsutil logging get gs://<bucket_name> � a� <B>SET</B> The ``set`` sub-command has two sub-commands: <B>ON</B> The ``gsutil logging set on`` command enables usage and storage logging for the buckets named by the specified URLs, outputting log files to the bucket specified with the ``-b`` flag. Cloud Storage doesn't validate the existence of the output bucket, so users should ensure it already exists, and all URLs must name Cloud Storage buckets (e.g., ``gs://bucket``). The optional ``-o`` flag specifies the prefix for log object names. The default prefix is the bucket name. For example, the command: gsutil logging set on -b gs://my_logging_bucket -o UsageLog \ gs://my_bucket1 gs://my_bucket2 causes all read and write activity to objects in ``gs://mybucket1`` and ``gs://mybucket2`` to be logged to objects prefixed with the name ``UsageLog``, with those log objects written to the bucket ``gs://my_logging_bucket``. In addition to enabling logging on your bucket(s), you also need to grant cloud-storage-analytics@google.com write access to the log bucket, using this command: gsutil acl ch -g cloud-storage-analytics@google.com:W gs://my_logging_bucket Note that log data may contain sensitive information, so you should make sure to set an appropriate default bucket ACL to protect that data. (See "gsutil help defacl".) <B>OFF</B> This command disables usage and storage logging for the buckets named by the specified URLs. All URLs must name Cloud Storage buckets (e.g., ``gs://bucket``). No logging data is removed from the log buckets when you disable logging, but Google Cloud Storage stops delivering new logs once you have run this command. a6 <B>GET</B> If logging is enabled for the specified bucket url, the server responds with a JSON document that looks something like this: { "logBucket": "my_logging_bucket", "logObjectPrefix": "UsageLog" } You can download log data from your log bucket using the gsutil cp command. aM Google Cloud Storage offers `usage logs and storage logs <https://cloud.google.com/storage/docs/access-logs>`_ in the form of CSV files that you can download and view. Usage logs provide information for all of the requests made on a specified bucket and are created hourly. Storage logs provide information about the storage consumption of that bucket for the last day and are created daily. Once set up, usage logs and storage logs are automatically created as new objects in a bucket that you specify. Usage logs and storage logs are subject to the same pricing as other objects stored in Cloud Storage. For a complete list of usage log fields and storage data fields, see `Usage and storage log format <https://cloud.google.com/storage/docs/access-logs#format>`_. The ``logging`` command has two sub-commands: a� <B>OPTIONS</B> -b bucket_name Specifies the bucket that stores the generated logs. This flag is only available for the ``set on`` command and is required for that command. -o log_prefix Specifies a common prefix for the names of generated logs. This flag is only available for the ``set on`` command and is optional for that command. c @ s� e Zd ZdZejdg d�ededdddej ej gej e dd d gd �e � � gd � Z ejdg d �ddeeed�d�Zeeg d�i d�eeg d�ed�ed�d�d�eg d�i d�d�i d�d�i d�Zdd� Zd d � Zd d � Zd d!� Zd"S )#�LoggingCommandz)Implementation of gsutil logging command.�logging)�disablelogging� enablelogging� getlogging� zb:o:Fr �mode�on�off)�choices) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)Z loggingconfigZlogs�logr r r � command_helpz(Configure or retrieve logging on buckets)�get�set)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_text)�storage�buckets�describezk--format=gsutiljson[key=logging_config,empty=' has no logging configuration.',empty_prefix_key=storage_url])�gcloud_command�flag_map)r5 r6 �updatez --log-bucketz--log-object-prefix)�-b�-o)r5 r6 r: z--clear-log-bucketz--clear-log-object-prefix)r r c C s� | j | jd dgd�\}}|jdkrBtj| jj||jd�dd� n<|jrp|jjrp|jj rpt�t t � |j��� nt�d| � dS ) z(Gets logging configuration for a bucket.r r �� bucket_fields�s3)�provider� )�endz %s has no logging configuration.) �GetSingleBucketUrlFromArg�args�schemer � print_to_fd� gsutil_api�XmlPassThroughGetLoggingr � logBucket�logObjectPrefix�strr � MessageToJson)�self� bucket_url�bucket_metadata� rP �%/opt/gsutil/gslib/commands/logging.py�_Get� s" � ����zLoggingCommand._Getc C s� t | j�std��d}d}| jD ]$\}}|dkr8t|�}|dkr |}q |sRtd��|�� sbtd��d}| jD ]t}| j|dgd �}|D ]Z}|j} d }| j� d |� t j j |j |p�| j d �} t j | d �} | jj| j | | jdgd� q�ql|s�ttt| j� ��dS )z+Enables logging configuration for a bucket.z8"logging set on" command spanning providers not allowed.Nr; r< z2"logging set on" requires '-b <log_bucket>' optionz$-b option must specify a bucket URL.F�idr= TzEnabling logging on %s...)rI rJ �r �r@ �fieldsr )r rD r �sub_optsr �IsBucket�GetBucketUrlIterFromArg� storage_url�logger�info�apitools_messages�Bucket� LoggingValue� bucket_namerG � PatchBucketrE r �list) rM Ztarget_bucket_url� target_prefix�opt�opt_arg� some_matched�url_str� bucket_iter�blr�urlr rO rP rP rQ �_Enable� sB � � zLoggingCommand._Enablec C s� d}| j D ]f}| j|dgd�}|D ]L}|j}d}| j�d|� tj�� }tj|d�}| jj |j ||j dgd� q"q |s�t t t| j � ��dS ) z,Disables logging configuration for a bucket.FrS r= TzDisabling logging on %s...rT rU r )rD rY rZ r[ r\ r] r^ r_ rG ra r` rE r r rb )rM rf rg rh ri rj r rO rP rP rQ �_Disable# s" � zLoggingCommand._Disablec C s� | j �d�}|dkr*| j}tj|gd� n�|dkr�| j �d�}| j sL| �� |dkrl| j}tj||gd� q�|dkr�| j}tj||gd� q�td|| j |f ��ntd|| j f ��| j d d � tj| j d � |� dS ) z,Command entry point for the logging command.r r- )� subcommandsr. r r zKInvalid subcommand "%s" for the "%s %s" command. See "gsutil help logging".zFInvalid subcommand "%s" for the %s command. See "gsutil help logging".T)� check_args)rW ) rD �poprR r �LogCommandParams�$RaiseWrongNumberOfArgumentsExceptionrk rl r � command_name� ParseSubOptsrW )rM �action_subcommand�funcZstate_subcommandrP rP rQ � RunCommand8 s< �� ��� zLoggingCommand.RunCommandN)�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr �%MakeZeroOrMoreCloudBucketURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� help_specr r �gcloud_storage_maprR rk rl rv rP rP rP rQ r � sj ����� ������+�/*r ).rz � __future__r r r r �sys�apitools.base.pyr �gslibr � gslib.commandr �gslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr r �gslib.help_providerr �gslib.storage_urlr r �"gslib.third_party.storage_apitoolsr r] �gslib.utils.constantsr �gslib.utils.shim_utilr r Z gslib.utilsr � _SET_SYNOPSIS� _GET_SYNOPSIS�lstripr| �_SET_DESCRIPTION�_GET_DESCRIPTION� _DESCRIPTIONr� r� r� r rP rP rP rQ �<module> sH *��� commands/__pycache__/version.cpython-39.pyc 0000644 00000012404 15025013616 0014673 0 ustar 00 a +(Wgy � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZ ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ dZde d Z G dd� de�Z dd� Z dS )�)Implementation of gsutil version command.� )�absolute_import)�print_function)�division)�unicode_literalsN)�Command)� system_util)�GetFriendlyConfigFilePaths)�UsingCrcmodExtension)�UTF8)�GetMd5)�$CheckMultiprocessingAvailableAndInitz gsutil version z <B>SYNOPSIS</B> aS <B>DESCRIPTION</B> Prints information about the version of gsutil. <B>OPTIONS</B> -l Prints additional information, such as the version of Python being used, the version of the Boto library, a checksum of the code, the path to gsutil, and the path to gsutil's configuration file. c @ sV e Zd ZdZejddgeddddddd� Zejddgdd e i d �Z d d � Z d d� Z dS )�VersionCommandr �version�verr �lF)�command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg� command_helpz Print version info about gsutil)� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C s d}| j r&| j D ]\}}|dkrd}qd�t� �}tj}z | �� }W n tyZ d}Y n0 ||krjd}nd| }tj� dtj � |r�d }tj� |j ||t j tj�d d �d t�� t�� f t� jt�� t�� |t� p�tjt� tjtj t j � d dd�d�� dS )z,Command entry point for the version command.Fz-lTz, z MISSING FILES�OKz!= %szgsutil version: %s a� checksum: {checksum} ({checksum_ok}) boto version: {boto_version} python version: {python_version} OS: {os_version} multiprocessing available: {multiprocessing_available} using cloud sdk: {cloud_sdk} pass cloud sdk credentials to gsutil: {cloud_sdk_credentials} config path(s): {config_paths} gsutil path: {gsutil_path} compiled crcmod: {compiled_crcmod} installed via package manager: {is_package_install} editable install: {is_editable_install} shim enabled: {is_shim_enabled} � � z%s %s�GSUtil�use_gcloud_storage)�checksumZ checksum_ok� boto_version�python_version� os_versionZmultiprocessing_availableZ cloud_sdkZcloud_sdk_credentials� config_paths� gsutil_pathZcompiled_crcmodZis_package_installZis_editable_installZis_shim_enabledr ) �sub_opts�joinr �gslib�CHECKSUM�_ComputeCodeChecksum�IOError�sys�stdout�write�VERSION�format�boto� __version__r �replace�platform�system�releaser � is_availabler �InvokedViaCloudSdk�CloudSdkCredPassingEnabled�"GetCloudSdkGsutilWrapperScriptPath� GSUTIL_PATHr �IS_PACKAGE_INSTALL�IS_EDITABLE_INSTALL�config�getbool) �selfZ long_form�o�_r* Zshipped_checksumZ cur_checksumZchecksum_ok_strZlong_form_output� rI �%/opt/gsutil/gslib/commands/version.py� RunCommandT sP � ���zVersionCommand.RunCommandc C s� t jr dS t� }t jg}t�t j�D ]2\}}}|D ]"}|�d�r2|�tj � ||�� q2q$t |�D ]z}t j r�t|d�}|�� }t�dd|�}|�|� |�� q`t|dtd�}|�� }t�dd|�}|�|�t�� |�� q`|�� S ) af Computes a checksum of gsutil code. This checksum can be used to determine if users locally modified gsutil when requesting support. (It's fine for users to make local mods, but when users ask for support we ask them to run a stock version of gsutil so we can reduce possible variables.) Returns: MD5 checksum of gsutil code. Z.PACKAGED_GSUTIL_INSTALLS_DO_NOT_HAVE_CHECKSUMSz.py�rbz (\r\n|\r|\n)� �r)�encodingr" )r. rB r rA �os�walk� GSLIB_DIR�endswith�append�pathr- �sorted�six�PY2�open�read�re�sub�update�closer �encode� hexdigest) rF �mZfiles_to_checksum�rootrH �files�filepath�f�contentrI rI rJ r0 � s* z#VersionCommand._ComputeCodeChecksumN) �__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSIS� command_spec�HelpSpec�_DETAILED_HELP_TEXT� help_specrK r0 rI rI rI rJ r ; s, � � =r c C sZ t j} t�� rVtj�ddd�}| �|�rVtj�| d| �|�� dd�}tj� |�rV|S dS )a If gsutil was invoked via the Cloud SDK, find its gsutil wrapper script. Returns: (str) The path to the Cloud SDK's gsutil wrapper script, or an empty string if gsutil was not invoked via the Cloud SDK or the wrapper script could not be found at its expected path. r: �gsutilr �binr# ) r. rA r r> rP rU r- rS �rfind�exists)r+ Zplatform_path_suffixZbin_pathrI rI rJ r@ � s � r@ ) rj � __future__r r r r rP r: r[ r2 rW r7 �crcmodr. � gslib.commandr Z gslib.utilsr Zgslib.utils.boto_utilr r �gslib.utils.constantsr �gslib.utils.hashing_helperr �&gslib.utils.parallelism_framework_utilr rl ro r r@ rI rI rI rJ �<module> s6 ��| commands/__pycache__/acl.cpython-39.pyc 0000644 00000045414 15025013616 0013754 0 ustar 00 a +(Wg#_ � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z dd l m Z dd l m Z dd l mZ dd l mZ dd l mZ ddl mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m Z ddl m!Z! ddl"m#Z$ ddl%m&Z& ddl'm(Z( dd l)m*Z* dd l+m,Z, dd l+m-Z- d Z.d Z/d!Z0d"Z1d#Z2d$Z3e.e/�4d%� e0�4d%� d& Z5d'd%�6e1e2e3g� Z7e e5e7�Z8e e/e1�Z9e e.e2�Z:e e0e3�Z;d(d)� Z<d.d*d+�Z=G d,d-� d-e�Z>dS )/z:Implementation of acl command for cloud storage providers.� )�absolute_import)�print_function)�division)�unicode_literalsN)�encoding)�metrics)� gcs_json_api)�AccessDeniedException)�BadRequestException)�PreconditionException)� Preconditions)�ServiceException)�Command)�SetAclExceptionHandler)�SetAclFuncWrapper)�CommandArgument)� ApiSelector)�CommandException)�CreateHelpText��StorageUrlFromString)�UrlsAreForSingleProvider)�)RaiseErrorIfUrlsAreMixOfBucketsAndObjects)�storage_v1_messages)� acl_helper)�NO_MAX)�Retry)�GcloudStorageFlag)�GcloudStorageMapzG gsutil acl set [-f] [-r] [-a] (<file-path>|<predefined-acl>) url... z gsutil acl get url aK gsutil acl ch [-f] [-r] <grant>... url... where each <grant> is one of the following forms: -u <id>|<email>:<permission> -g <id>|<email>|<domain>|All|AllAuth:<permission> -p (viewers|editors|owners)-<project number>:<permission> -d <id>|<email>|<domain>|All|AllAuth|(viewers|editors|owners)-<project number> z� <B>GET</B> The ``acl get`` command gets the ACL text for a bucket or object, which you can save and edit for the acl set command. aU <B>SET</B> The ``acl set`` command allows you to set an Access Control List on one or more buckets and objects. As part of the command, you must specify either a predefined ACL or the path to a file that contains ACL text. The simplest way to use the ``acl set`` command is to specify one of the predefined ACLs, e.g.,: gsutil acl set private gs://example-bucket/example-object If you want to make an object or bucket publicly readable or writable, it is recommended to use ``acl ch``, to avoid accidentally removing OWNER permissions. See the ``acl ch`` section for details. See `Predefined ACLs <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ for a list of predefined ACLs. If you want to define more fine-grained control over your data, you can retrieve an ACL using the ``acl get`` command, save the output to a file, edit the file, and then use the ``acl set`` command to set that ACL on the buckets and/or objects. For example: gsutil acl get gs://bucket/file.txt > acl.txt Make changes to acl.txt such as adding an additional grant, then: gsutil acl set acl.txt gs://cats/file.txt Note that you can set an ACL on multiple buckets or objects at once. For example, to set ACLs on all .jpg files found in a bucket: gsutil acl set acl.txt gs://bucket/**.jpg If you have a large number of ACLs to update you might want to use the gsutil -m option, to perform a parallel (multi-threaded/multi-processing) update: gsutil -m acl set acl.txt gs://bucket/**.jpg Note that multi-threading/multi-processing is only done when the named URLs refer to objects, which happens either if you name specific objects or if you enumerate objects by using an object wildcard or specifying the acl -r flag. <B>SET OPTIONS</B> The "set" sub-command has the following options -R, -r Performs "acl set" request recursively, to all objects under the specified URL. -a Performs "acl set" request on all object versions. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. If some of the ACLs couldn't be set, gsutil's exit status will be non-zero even if this flag is set. This option is implicitly set when running "gsutil -m acl...". a <B>CH</B> The "acl ch" (or "acl change") command updates access control lists, similar in spirit to the Linux chmod command. You can specify multiple access grant additions and deletions in a single command run; all changes will be made atomically to each object in turn. For example, if the command requests deleting one grant and adding a different grant, the ACLs being updated will never be left in an intermediate state where one grant has been deleted but the second grant not yet added. Each change specifies a user or group grant to add or delete, and for grant additions, one of R, W, O (for the permission to be granted). A more formal description is provided in a later section; below we provide examples. <B>CH EXAMPLES</B> Examples for "ch" sub-command: Grant anyone on the internet READ access to the object example-object: gsutil acl ch -u allUsers:R gs://example-bucket/example-object NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see "gsutil help setmeta". Grant the user john.doe@example.com READ access to all objects in example-bucket that begin with folder/: gsutil acl ch -r -u john.doe@example.com:R gs://example-bucket/folder/ Grant the group admins@example.com OWNER access to all jpg files in example-bucket: gsutil acl ch -g admins@example.com:O gs://example-bucket/**.jpg Grant the owners of project example-project WRITE access to the bucket example-bucket: gsutil acl ch -p owners-example-project:W gs://example-bucket NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access to a project's viewers/editors respectively. Remove access to the bucket example-bucket for the viewers of project number 12345: gsutil acl ch -d viewers-12345 gs://example-bucket NOTE: You cannot remove the project owners group from ACLs of gs:// buckets in the given project. Attempts to do so will appear to succeed, but the service will add the project owners group into the new set of ACLs before applying it. Note that removing a project requires you to reference the project by its number (which you can see with the acl get command) as opposed to its project ID string. Grant the service account foo@developer.gserviceaccount.com WRITE access to the bucket example-bucket: gsutil acl ch -u foo@developer.gserviceaccount.com:W gs://example-bucket Grant all users from the `G Suite <https://www.google.com/work/apps/business/>`_ domain my-domain.org READ access to the bucket gcs.my-domain.org: gsutil acl ch -g my-domain.org:R gs://gcs.my-domain.org Remove any current access by john.doe@example.com from the bucket example-bucket: gsutil acl ch -d john.doe@example.com gs://example-bucket If you have a large number of objects to update, enabling multi-threading with the gsutil -m flag can significantly improve performance. The following command adds OWNER for admin@example.org using multi-threading: gsutil -m acl ch -r -u admin@example.org:O gs://example-bucket Grant READ access to everyone from my-domain.org and to all authenticated users, and grant OWNER to admin@mydomain.org, for the buckets my-bucket and my-other-bucket, with multi-threading enabled: gsutil -m acl ch -r -g my-domain.org:R -g AllAuth:R \ -u admin@mydomain.org:O gs://my-bucket/ gs://my-other-bucket <B>CH ROLES</B> You may specify the following roles with either their shorthand or their full name: R: READ W: WRITE O: OWNER For more information on these roles and the access they grant, see the permissions section of the `Access Control Lists page <https://cloud.google.com/storage/docs/access-control/lists#permissions>`_. <B>CH ENTITIES</B> There are four different entity types: Users, Groups, All Authenticated Users, and All Users. Users are added with -u and a plain ID or email address, as in "-u john-doe@gmail.com:r". Note: Service Accounts are considered to be users. Groups are like users, but specified with the -g flag, as in "-g power-users@example.com:O". Groups may also be specified as a full domain, as in "-g my-company.com:r". allAuthenticatedUsers and allUsers are specified directly, as in "-g allUsers:R" or "-g allAuthenticatedUsers:O". These are case insensitive, and may be shortened to "all" and "allauth", respectively. Removing roles is specified with the -d flag and an ID, email address, domain, or one of allUsers or allAuthenticatedUsers. Many entities' roles can be specified on the same command line, allowing bundled changes to be executed in a single run. This will reduce the number of requests made to the server. <B>CH OPTIONS</B> The "ch" sub-command has the following options -d Remove all roles associated with the matching entity. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. With this option the gsutil exit status will be 0 even if some ACLs couldn't be changed. -g Add or modify a group entity's role. -p Add or modify a project viewers/editors/owners role. -R, -r Performs acl ch request recursively, to all objects under the specified URL. -u Add or modify a user entity's role. � z z+ The acl command has three sub-commands: c C s | j �d|� d| _d S )NzEncountered a problem: %sF)�logger�error�everything_set_okay)�cls� exception� r% �!/opt/gsutil/gslib/commands/acl.py�_ApplyExceptionHandler" s r' c C s | j ||d� d S )N)� thread_state)�ApplyAclChanges)r# Zurl_or_expansion_resultr( r% r% r& �_ApplyAclChangesWrapper' s r* c s� e Zd ZdZejdg d�ededdddej ej gej e � � e � � ge �d�ge � � gd�d � Zejdg d �d d eeeed �d�Zdd� Z� fdd�Zdd� Zdd� Zdd� Zdd� Ze e ddd �d&d d ��Z e e ddd �d d!� �Z d"d#� Z!d$d%� Z"� Z#S )'� AclCommandz%Implementation of gsutil acl command.�acl)�getacl�setacl�chacl� z afRrg:u:d:p:F� )�set�get�ch) �command_name_aliases�usage_synopsis�min_args�max_args�supported_sub_args� file_url_ok�provider_url_ok�urls_start_arg�gs_api_support�gs_default_api�argparse_arguments)r- r. �chmodr/ � command_helpz-Get, set, or change bucket and/or object ACLs)r3 r2 r4 )� help_name�help_name_aliases� help_type�help_one_line_summary� help_text�subcommand_help_textc C sZ dd� | j D �}d}| jD ]\}}|dv rd} q4qt||� |d �� rR|sRdS dS d S ) Nc S s g | ] }t |��qS r% r )�.0�urlr% r% r& � <listcomp>R � z6AclCommand._get_shim_command_group.<locals>.<listcomp>F)�-r�-RTr �buckets�objects)�args�sub_optsr �IsBucket)�selfZobject_or_bucket_urls�recurseZflag_key�_r% r% r& �_get_shim_command_groupQ s z"AclCommand._get_shim_command_groupc sN | j �d�}|dkrFt| j d ��� r,d}nd}td|ddgi d�}n�|d kr�| �� | j �d�}tj�|�rxd | }n"|t j v r�t j | }n|}d | }| � � }td|d g|g t d �t d�t d�t d�d�d�}nl|dk�rB| �� t �| j�| _| � � }td|d gt d�t d�t d�t d�t d �t d�t d�t d�d�d�}t� �|�S )Nr r3 rO rN �storageZdescribez --format=multi(acl:format=json))�gcloud_command�flag_mapr2 z --acl-file=z--predefined-acl=�updatez--all-versionsz--continue-on-errorz --recursive)�-a�-frM rL r4 z--add-acl-grantz--remove-acl-grant)�-g�-p�-u�-dr[ r\ rM rL )rP �popr �IsObjectr � ParseSubOpts�os�path�isfiler �+FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATIONrV r r Ztranslate_sub_opts_for_shimrQ �super�get_gcloud_storage_args)rS � sub_commandZ command_group�gcloud_storage_mapZacl_file_or_predefined_aclZacl_flag�predefined_acl�� __class__r% r& ri _ sf �� ����� �� z"AclCommand.get_gcloud_storage_argsc C s6 | j s| �� | j d �� dks*| jdkr.dS dS d S )Nr r2 r. r1 )rP �$RaiseWrongNumberOfArgumentsException�lower�command_alias_used)rS r% r% r& �_CalculateUrlsStartArg� s z!AclCommand._CalculateUrlsStartArgc C s� d| _ | jr\| jD ]H\}}|dkr*d| _q|dkr:d| _ q|dksJ|dkrRd| _q| �� qz| �tt� W n. ty� } z| � � � W Y d}~n d}~0 0 | j s�t d��dS ) z>Parses options and sets ACLs on the specified buckets/objects.Fr[ Tr\ rL rM N�'ACLs for some objects could not be set.) �continue_on_errorrQ � all_versions�recursion_requested� RaiseInvalidArgumentException�SetAclCommandHelperr r r �_WarnServiceAccountsr" r )rS �o�unused_aZunused_er% r% r& �_SetAcl� s" zAclCommand._SetAclc C sb d| _ g | _d| _| jr�| jD ]�\}}|dkr6d| _q |dkrjd|v rNtd��| j�tj|tjj d�� q |dkr�| j�tj|tjj d�� q |d kr�| j�tj|tjj d�� q |d kr�| j�t� |�� q |d ks�|d kr�d| _ q | �� q | js�td ��t| j��r t| jd �jdk�r0td�| j���d| _| jtt| jg d�d� | j�s^td��dS )zAParses options and changes ACLs on the specified buckets/objects.TFr\ r] zgserviceaccount.comznService accounts are considered users, not groups; please use "gsutil acl ch -u" instead of "gsutil acl ch -g")� scope_typer^ r_ r` rL rM zFPlease specify at least one access change with the -g, -u, or -d flagsr �gsz2The "{0}" command can only be used with gs:// URLs�r, � generation�metageneration)� object_fieldsrs N)Zparse_versions�changesrt rQ r �appendr Z AclChangeZ ChangeTypeZGROUP�PROJECT�USERZAclDelrv rw r rP r �scheme�format� command_namer" � ApplyAclFuncr* r' )rS rz �ar% r% r& �_ChAcl� s\ ���� ����zAclCommand._ChAclc C s | � � td| ��d S )NzTFailed to set acl for %s. Please ensure you have OWNER-role access to this resource.)ry r )rS rI r% r% r& �_RaiseForAccessDenied� s �z AclCommand._RaiseForAccessDenied� )�tries� timeout_secsNc C s� |r |}n| j }|j}|�� r>|j|j|jddgd�}|j}n |�� r\t� t j |j �}|j}|sj| � |� | �||�dkr�| j�d|� dS z�|�� r�t|jd�}t j|d�} |j|j| ||jd gd � nvt|j|jd �}t j |d�} z&|j|j|j| ||j|jd gd � W n2 t�y> } z| �||� W Y d} ~ n d} ~ 0 0 | j�d |� W n� t�y� } z tdt | � ��W Y d} ~ ndd} ~ 0 t �y� | � |� Y n@ t�y� } z&|�� �r�tt | ���| �W Y d} ~ n d} ~ 0 0 dS )z�Applies the changes in self.changes to the provided URL. Args: name_expansion_result: NameExpansionResult describing the target object. thread_state: If present, gsutil Cloud API instance to apply the changes. r, r� ��provider�fieldsr �No changes to %sN)�meta_gen_match�r, �id)� preconditionsr� r� �� gen_matchr� �r� r� r� r� zUpdated ACL on %sz$Received bad request from server: %s) � gsutil_api�expanded_storage_urlrR � GetBucket� bucket_namer� r, rb r � JsonToMessage�apitools_messages�Object�expanded_resultr� �$_ApplyAclChangesAndReturnChangeCountr �infor r� �Bucket� PatchBucketr� �PatchObjectMetadata� object_namer �(_RefetchObjectMetadataAndApplyAclChangesr r �strr ) rS �name_expansion_resultr( r� rI �bucket� current_acl� gcs_objectr� �bucket_metadata�object_metadata�er% r% r& r) � sl �� �� � "$ zAclCommand.ApplyAclChangesc C s� |j |j|j|jg d�d�}|j}| �||�dkrD| j�d|� dS tj |d�}t |j |j d�}|j |j|j|||j|j dgd � dS ) z<Reattempts object ACL changes after a PreconditionException.r r� r r� Nr� r� r� r� )�GetObjectMetadatar� r� r� r, r� r r� r� r� r r� r� r� )rS rI r� r� r� r� r� r% r% r& r� 3 s, � ��z3AclCommand._RefetchObjectMetadataAndApplyAclChangesc C s* d}| j D ]}||�||d| j�7 }q |S )Nr r, )r� �Executer )rS � storage_url� acl_messageZmodification_count�changer% r% r& r� L s �z/AclCommand._ApplyAclChangesAndReturnChangeCountc C s� | j �d�}| jdd� tj| jd� d| _|dkrTtj|gd� | �| j d � nR|dkrttj|gd� | �� n2|d v r�tj|gd� | � � nt d || j f ��dS ) z(Command entry point for the acl command.r T)� check_args)rQ Fr3 )� subcommandsr2 )r4 r� zBInvalid subcommand "%s" for the %s command. See "gsutil help acl".) rP ra rc r �LogCommandParamsrQ �def_acl�GetAndPrintAclr| r� r r� )rS Zaction_subcommandr% r% r& � RunCommandS s& ��zAclCommand.RunCommand)N)$�__name__� __module__� __qualname__�__doc__r �CreateCommandSpec� _SYNOPSISr r �XML�JSONr Z MakeFileURLOrCannedACLArgumentZ MakeZeroOrMoreCloudURLsArgumentZMakeNCloudURLsArgument� command_spec�HelpSpec�_DETAILED_HELP_TEXT�_get_help_text�_set_help_text� _ch_help_text� help_specrV ri rr r| r� r� r r r) r r� r� r� � __classcell__r% r% rm r& r+ + sV � ���� <0 E r+ )N)?r� � __future__r r r r rd �apitools.base.pyr �gslibr r �gslib.cloud_apir r r r r � gslib.commandr r r Zgslib.command_argumentr �gslib.cs_api_mapr �gslib.exceptionr �gslib.help_providerr �gslib.storage_urlr r r �"gslib.third_party.storage_apitoolsr r� Z gslib.utilsr �gslib.utils.constantsr �gslib.utils.retry_utilr �gslib.utils.shim_utilr r Z _SET_SYNOPSISZ _GET_SYNOPSISZ _CH_SYNOPSISZ_GET_DESCRIPTIONZ_SET_DESCRIPTIONZ_CH_DESCRIPTION�lstripr� �joinZ _DESCRIPTIONr� r� r� r� r' r* r+ r% r% r% r& �<module> sf = ��� commands/ubla.py 0000644 00000024064 15025013616 0007647 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import getopt import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistOnOrOff from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil ubla set (on|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil ubla get bucket_url... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``ubla set`` command enables or disables uniform bucket-level access for Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use uniform bucket-level access: gsutil ubla set on gs://redbucket gs://bluebucket Configure your buckets to NOT use uniform bucket-level access: gsutil ubla set off gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``ubla get`` command shows whether uniform bucket-level access is enabled for the specified Cloud Storage bucket(s). <B>GET EXAMPLES</B> Check if your buckets are using uniform bucket-level access: gsutil ubla get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The ``ubla`` command is used to retrieve or configure the `uniform bucket-level access <https://cloud.google.com/storage/docs/bucket-policy-only>`_ setting of Cloud Storage bucket(s). This command has two sub-commands, ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) # Aliases to make these more likely to fit on one line. IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue uniformBucketLevelAccessValue = IamConfigurationValue.BucketPolicyOnlyValue _GCLOUD_FORMAT_STRING = ( '--format=' + 'multi[terminator="' + shim_util.get_format_flag_newline() + '"](name:format="value(format(\'Uniform bucket-level' + ' access setting for gs://{}:\'))",' + ' iamConfiguration.uniformBucketLevelAccess.enabled.yesno(no="False")' + ':format="value[terminator=\'' + shim_util.get_format_flag_newline() + '\'](format(\' Enabled: {}\'))",' + ' iamConfiguration.uniformBucketLevelAccess.lockedTime.sub("T", " ")' + ':format="value(format(\' LockedTime: {}\'))")') class UblaCommand(Command): """Implements the gsutil ubla command.""" command_spec = Command.CreateCommandSpec( 'ubla', command_name_aliases=['uniformbucketlevelaccess'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='ubla', help_name_aliases=['uniformbucketlevelaccess'], help_type='command_help', help_one_line_summary='Configure Uniform bucket-level access', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING, '--raw' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--uniform-bucket-level-access', ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-uniform-bucket-level-access', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetUbla(self, blr): """Gets the Uniform bucket-level access setting for a bucket.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['iamConfiguration'], provider=bucket_url.scheme) iam_config = bucket_metadata.iamConfiguration # TODO(mynameisrafe): Replace bucketPolicyOnly with uniformBucketLevelAccess # when the property is live. uniform_bucket_level_access = iam_config.bucketPolicyOnly fields = { 'bucket': str(bucket_url).rstrip('/'), 'enabled': uniform_bucket_level_access.enabled } locked_time_line = '' if uniform_bucket_level_access.lockedTime: fields['locked_time'] = uniform_bucket_level_access.lockedTime locked_time_line = ' LockedTime: {locked_time}\n' if uniform_bucket_level_access: print(('Uniform bucket-level access setting for {bucket}:\n' ' Enabled: {enabled}\n' + locked_time_line).format(**fields)) def _SetUbla(self, blr, setting_arg): """Sets the Uniform bucket-level access setting for a bucket on or off.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url iam_config = IamConfigurationValue() # TODO(mynameisrafe): Replace bucketPolicyOnly with uniformBucketLevelAccess # when the property is live. iam_config.bucketPolicyOnly = uniformBucketLevelAccessValue() iam_config.bucketPolicyOnly.enabled = (setting_arg == 'on') bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config) setting_verb = 'Enabling' if setting_arg == 'on' else 'Disabling' print('%s Uniform bucket-level access for %s...' % (setting_verb, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['iamConfiguration'], provider=bucket_url.scheme) return 0 def _Ubla(self): """Handles ubla command on a Cloud Storage bucket.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('ubla only supports get|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._GetUbla elif subcommand == 'set': subcommand_func = self._SetUbla setting_arg = self.args.pop(0) InsistOnOrOff(setting_arg, 'Only on and off values allowed for set option') subcommand_args.append(setting_arg) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the ubla command.""" if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the Cloud Storage JSON API.' % self.command_name))) action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) self._Ubla() else: raise CommandException('Invalid subcommand "%s", use get|set instead.' % action_subcommand) commands/autoclass.py 0000644 00000021613 15025013616 0010717 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the autoclass command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import text_util from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil autoclass set (on|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil autoclass get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``set`` sub-command requires an additional sub-command, either ``on`` or ``off``, which enables or disables Autoclass for the specified bucket(s). """ _GET_DESCRIPTION = """ <B>GET</B> The ``get`` sub-command gets the current Autoclass configuration for a bucket. The returned configuration has the following fields: ``enabled``: a boolean field indicating whether the feature is on or off. ``toggleTime``: a timestamp indicating when the enabled field was set. """ _DESCRIPTION = """ The `Autoclass <https://cloud.google.com/storage/docs/autoclass>`_ feature automatically selects the best storage class for objects based on access patterns. This command has two sub-commands, ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _GCLOUD_FORMAT_STRING = ('--format=value[separator="' + shim_util.get_format_flag_newline() + '"](' + 'format("gs://{}:", name),' + ' format(" Enabled: {}",' + 'autoclass.enabled.yesno(True, False)),' + ' format(" Toggle Time: {}",' + 'autoclass.toggleTime))') class AutoclassCommand(Command): """Implements the gsutil autoclass command.""" command_spec = Command.CreateCommandSpec( 'autoclass', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='autoclass', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure Autoclass feature', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING, '--raw' ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--enable-autoclass', ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-enable-autoclass', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _get_autoclass(self, blr): """Gets the autoclass setting for a bucket.""" bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['autoclass'], provider=bucket_url.scheme) bucket = str(bucket_url).rstrip('/') if bucket_metadata.autoclass: enabled = getattr(bucket_metadata.autoclass, 'enabled', False) toggle_time = getattr(bucket_metadata.autoclass, 'toggleTime', None) else: enabled = False toggle_time = None print('{}:\n' ' Enabled: {}\n' ' Toggle Time: {}'.format(bucket, enabled, toggle_time)) def _set_autoclass(self, blr, setting_arg): """Turns autoclass on or off for a bucket.""" bucket_url = blr.storage_url autoclass_config = apitools_messages.Bucket.AutoclassValue() autoclass_config.enabled = (setting_arg == 'on') bucket_metadata = apitools_messages.Bucket(autoclass=autoclass_config) print('Setting Autoclass %s for %s' % (setting_arg, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['autoclass'], provider=bucket_url.scheme) return 0 def _autoclass(self): """Handles autoclass command on Cloud Storage buckets.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('autoclass only supports get|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._get_autoclass elif subcommand == 'set': subcommand_func = self._set_autoclass setting_arg = self.args.pop(0) text_util.InsistOnOrOff(setting_arg, 'Only on and off values allowed for set option') subcommand_args.append(setting_arg) if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be with the Cloud Storage ' 'JSON API.') % self.command_name))) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: if self.gsutil_api.GetApiSelector( bucket_listing_ref.storage_url.scheme) != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be used for GCS ' 'Buckets.') % self.command_name))) some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the autoclass command.""" action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) return self._autoclass() else: raise CommandException('Invalid subcommand "%s", use get|set instead.' % action_subcommand) commands/requesterpays.py 0000644 00000020435 15025013616 0011636 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of requesterpays configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil requesterpays set (on|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil requesterpays get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The "set" sub-command requires an additional sub-command, either "on" or "off", which, respectively, will enable or disable Requester Pays for the specified bucket. """ _GET_DESCRIPTION = """ <B>GET</B> The "get" sub-command gets the Requester Pays configuration for a bucket and displays whether or not it is enabled. """ _DESCRIPTION = """ The `Requester Pays <https://cloud.google.com/storage/docs/requester-pays>`_ feature enables you to configure a Google Cloud Storage bucket so that the requester pays all costs related to accessing the bucket and its objects. The gsutil requesterpays command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _GCLOUD_FORMAT_STRING = ('--format=value[separator=": "](' + 'name.sub("' + shim_util.get_format_flag_caret() + '", "gs://"),' + 'requester_pays.yesno("Enabled", "Disabled"))') class RequesterPaysCommand(Command): """Implementation of gsutil requesterpays command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'requesterpays', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ # ApiSelector.XML, # TODO: Uncomment once boto changes are added. ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),] }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='requesterpays', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Enable or disable requester pays for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--requester-pays' ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-requester-pays', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetRequesterPays(self): """Gets requesterpays configuration for a bucket.""" requesterpays_arg = self.args[0].lower() if requesterpays_arg not in ('on', 'off'): raise CommandException('Argument to "%s set" must be either <on|off>' % (self.command_name)) url_args = self.args[1:] if not url_args: self.RaiseWrongNumberOfArgumentsException() # Iterate over URLs, expanding wildcards and set the requesterpays # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True bucket_metadata = apitools_messages.Bucket( billing=apitools_messages.Bucket.BillingValue()) if requesterpays_arg == 'on': self.logger.info('Enabling requester pays for %s...', url) bucket_metadata.billing.requesterPays = True else: self.logger.info('Disabling requester pays for %s...', url) bucket_metadata.billing.requesterPays = False self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetRequesterPays(self): """Gets requesterpays configuration for one or more buckets.""" url_args = self.args # Iterate over URLs, expanding wildcards and getting the requesterpays # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['billing']) for blr in bucket_iter: some_matched = True if blr.root_object.billing and blr.root_object.billing.requesterPays: print('%s: Enabled' % blr.url_string.rstrip('/')) else: print('%s: Disabled' % blr.url_string.rstrip('/')) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def RunCommand(self): """Command entry point for the requesterpays command.""" action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._GetRequesterPays metrics.LogCommandParams(subcommands=[action_subcommand]) elif action_subcommand == 'set': func = self._SetRequesterPays requesterpays_arg = self.args[0].lower() if requesterpays_arg in ('on', 'off'): metrics.LogCommandParams( subcommands=[action_subcommand, requesterpays_arg]) else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help %s".') % (action_subcommand, self.command_name, self.command_name)) func() return 0 commands/lifecycle.py 0000644 00000021325 15025013616 0010660 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of lifecycle configuration command for GCS buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import LifecycleTranslation _GET_SYNOPSIS = """ gsutil lifecycle get gs://<bucket_name> """ _SET_SYNOPSIS = """ gsutil lifecycle set <config-json-file> gs://<bucket_name>... """ _SYNOPSIS = _GET_SYNOPSIS + _SET_SYNOPSIS.lstrip('\n') + '\n' _GET_DESCRIPTION = """ <B>GET</B> Gets the lifecycle management configuration for a given bucket. You can get the lifecycle management configuration for only one bucket at a time. To update the configuration, you can redirect the output of the ``get`` command into a file, edit the file, and then set it on the bucket using the ``set`` sub-command. """ _SET_DESCRIPTION = """ <B>SET</B> Sets the lifecycle management configuration on one or more buckets. The ``config-json-file`` specified on the command line should be a path to a local file containing the lifecycle configuration JSON document. """ _DESCRIPTION = """ You can use the ``lifecycle`` command to get or set lifecycle management policies for a given bucket. This command is supported for buckets only, not objects. For more information, see `Object Lifecycle Management <https://cloud.google.com/storage/docs/lifecycle>`_. The ``lifecycle`` command has two sub-commands: """ + _GET_DESCRIPTION + _SET_DESCRIPTION + """ <B>EXAMPLES</B> The following lifecycle management configuration JSON document specifies that all objects in this bucket that are more than 365 days old are deleted automatically: { "rule": [ { "action": {"type": "Delete"}, "condition": {"age": 365} } ] } The following empty lifecycle management configuration JSON document removes all lifecycle configuration for a bucket: {} """ _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class LifecycleCommand(Command): """Implementation of gsutil lifecycle command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'lifecycle', command_name_aliases=['lifecycleconfig'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=True, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ ApiSelector.JSON, ApiSelector.XML, ], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1),] }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='lifecycle', help_name_aliases=[ 'getlifecycle', 'setlifecycle', ], help_type='command_help', help_one_line_summary=('Get or set lifecycle configuration for a bucket'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) def get_gcloud_storage_args(self): if self.args[0] == 'set': gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--lifecycle-file={}'.format(self.args[1]), ] + self.args[2:], # Adds target bucket URLs. flag_map={}, ), }, flag_map={}, ) # Don't trigger unneeded translation now that complete command is above. self.args = self.args[:1] else: gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=lifecycle_config,empty=\' ' 'has no lifecycle configuration.\',empty_prefix_key=' 'storage_url]' ], flag_map={}, ), }, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def _SetLifecycleConfig(self): """Sets lifecycle configuration for a Google Cloud Storage bucket.""" lifecycle_arg = self.args[0] url_args = self.args[1:] # Disallow multi-provider 'lifecycle set' requests. if not UrlsAreForSingleProvider(url_args): raise CommandException('"%s" command spanning providers not allowed.' % self.command_name) # Open, read and parse file containing JSON document. lifecycle_file = open(lifecycle_arg, 'r') lifecycle_txt = lifecycle_file.read() lifecycle_file.close() # Iterate over URLs, expanding wildcards and setting the lifecycle on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['lifecycle']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting lifecycle configuration on %s...', blr) if url.scheme == 's3': self.gsutil_api.XmlPassThroughSetLifecycle(lifecycle_txt, url, provider=url.scheme) else: lifecycle = LifecycleTranslation.JsonLifecycleToMessage(lifecycle_txt) bucket_metadata = apitools_messages.Bucket(lifecycle=lifecycle) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _GetLifecycleConfig(self): """Gets lifecycle configuration for a Google Cloud Storage bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['lifecycle']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetLifecycle( bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.lifecycle and bucket_metadata.lifecycle.rule: sys.stdout.write( LifecycleTranslation.JsonLifecycleFromMessage( bucket_metadata.lifecycle)) else: sys.stdout.write('%s has no lifecycle configuration.\n' % bucket_url) return 0 def RunCommand(self): """Command entry point for the lifecycle command.""" subcommand = self.args.pop(0) if subcommand == 'get': metrics.LogCommandParams(subcommands=[subcommand]) return self._GetLifecycleConfig() elif subcommand == 'set': metrics.LogCommandParams(subcommands=[subcommand]) return self._SetLifecycleConfig() else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (subcommand, self.command_name)) commands/update.py 0000644 00000043353 15025013616 0010210 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of update command for updating gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import shutil import signal import stat import sys import tarfile import tempfile import textwrap from six.moves import input import gslib from gslib.command import Command from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.metrics import CheckAndMaybePromptForAnalyticsEnabling from gslib.sig_handling import RegisterSignalHandler from gslib.utils import system_util from gslib.utils.boto_util import GetConfigFilePaths from gslib.utils.boto_util import CERTIFICATE_VALIDATION_ENABLED from gslib.utils.constants import RELEASE_NOTES_URL from gslib.utils.text_util import CompareVersions from gslib.utils.update_util import DisallowUpdateIfDataInGsutilDir from gslib.utils.update_util import LookUpGsutilVersion from gslib.utils.update_util import GsutilPubTarball _SYNOPSIS = """ gsutil update [-f] [-n] [url] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> NOTE: This command is not available if you're using a gsutil installation from a package manager or the Cloud SDK. When using the Cloud SDK, use ``gcloud components update``. The gsutil update command downloads the latest gsutil release, checks its version, and offers to let you update to it if it differs from the version you're currently running. Once you say "Y" to the prompt of whether to install the update, the gsutil update command locates where the running copy of gsutil is installed, unpacks the new version into an adjacent directory, moves the previous version aside, moves the new version to where the previous version was installed, and removes the moved-aside old version. Because of this, users are cautioned not to store data in the gsutil directory, since that data will be lost when you update gsutil. (Some users change directories into the gsutil directory to run the command. We advise against doing that, for this reason.) Note also that the gsutil update command will refuse to run if it finds user data in the gsutil directory. By default gsutil update will retrieve the new code from %s, but you can optionally specify a URL to use instead. This is primarily used for distributing pre-release versions of the code to a small group of early test users. NOTE: gsutil periodically checks whether a more recent software update is available. By default this check is performed every 30 days; you can change (or disable) this check by editing the software_update_check_period variable in the .boto config file. Note also that gsutil will only check for software updates if stdin, stdout, and stderr are all connected to a TTY, to avoid interfering with cron jobs, streaming transfers, and other cases where gsutil input or output are redirected from/to files or pipes. Software update periodic checks are also disabled by the gsutil -q option (see 'gsutil help options') <B>OPTIONS</B> -f Forces the update command to offer to let you update, even if you have the most current copy already. This can be useful if you have a corrupted local copy. -n Causes update command to run without prompting [Y/n] whether to continue if an update is available. """ % GsutilPubTarball()) class UpdateCommand(Command): """Implementation of gsutil update command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'update', command_name_aliases=['refresh'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=1, supported_sub_args='fn', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='update', help_name_aliases=['refresh'], help_type='command_help', help_one_line_summary='Update to the latest gsutil release', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def _ExplainIfSudoNeeded(self, tf, dirs_to_remove, old_cwd): """Explains what to do if sudo needed to update gsutil software. Happens if gsutil was previously installed by a different user (typically if someone originally installed in a shared file system location, using sudo). Args: tf: Opened TarFile. dirs_to_remove: List of directories to remove. old_cwd: Path to the working directory we should chdir back to if sudo is needed. It's possible that we've chdir'd to a temp directory that's been deleted, which can cause odd behavior (e.g. OSErrors when opening the metrics subprocess). If this is not truthy, we won't attempt to chdir back to this value. Raises: CommandException: if errors encountered. """ # If running under Windows or Cygwin we don't need (or have) sudo. if system_util.IS_CYGWIN or system_util.IS_WINDOWS: return user_id = os.getuid() if os.stat(gslib.GSUTIL_DIR).st_uid == user_id: return # Won't fail - this command runs after main startup code that insists on # having a config file. config_file_list = GetConfigFilePaths() config_files = ' '.join(config_file_list) self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) # Pick current protection of each boto config file for command that restores # protection (rather than fixing at 600) to support use cases like how GCE # installs a service account with an /etc/boto.cfg file protected to 644. chmod_cmds = [] for config_file in config_file_list: mode = oct(stat.S_IMODE((os.stat(config_file)[stat.ST_MODE]))) chmod_cmds.append('\n\tsudo chmod %s %s' % (mode, config_file)) raise CommandException('\n'.join( textwrap.wrap( 'Since it was installed by a different user previously, you will need ' 'to update using the following commands. You will be prompted for your ' 'password, and the install will run as "root". If you\'re unsure what ' 'this means please ask your system administrator for help:')) + ( '\n\tsudo chmod 0644 %s\n\tsudo env BOTO_CONFIG="%s" %s update' '%s') % (config_files, config_files, self.gsutil_path, ' '.join(chmod_cmds)), informational=True) # This list is checked during gsutil update by doing a lowercased # slash-left-stripped check. For example "/Dev" would match the "dev" entry. unsafe_update_dirs = [ 'applications', 'auto', 'bin', 'boot', 'desktop', 'dev', 'documents and settings', 'etc', 'export', 'home', 'kernel', 'lib', 'lib32', 'library', 'lost+found', 'mach_kernel', 'media', 'mnt', 'net', 'null', 'network', 'opt', 'private', 'proc', 'program files', 'python', 'root', 'sbin', 'scripts', 'srv', 'sys', 'system', 'tmp', 'users', 'usr', 'var', 'volumes', 'win', 'win32', 'windows', 'winnt', ] def _EnsureDirsSafeForUpdate(self, dirs): """Raises Exception if any of dirs is known to be unsafe for gsutil update. This provides a fail-safe check to ensure we don't try to overwrite or delete any important directories. (That shouldn't happen given the way we construct tmp dirs, etc., but since the gsutil update cleanup uses shutil.rmtree() it's prudent to add extra checks.) Args: dirs: List of directories to check. Raises: CommandException: If unsafe directory encountered. """ for d in dirs: if not d: d = 'null' if d.lstrip(os.sep).lower() in self.unsafe_update_dirs: raise CommandException('EnsureDirsSafeForUpdate: encountered unsafe ' 'directory (%s); aborting update' % d) def _CleanUpUpdateCommand(self, tf, dirs_to_remove, old_cwd): """Cleans up temp files etc. from running update command. Args: tf: Opened TarFile, or None if none currently open. dirs_to_remove: List of directories to remove. old_cwd: Path to the working directory we should chdir back to. It's possible that we've chdir'd to a temp directory that's been deleted, which can cause odd behavior (e.g. OSErrors when opening the metrics subprocess). If this is not truthy, we won't attempt to chdir back to this value. """ if tf: tf.close() self._EnsureDirsSafeForUpdate(dirs_to_remove) for directory in dirs_to_remove: try: shutil.rmtree(directory) except OSError: # Ignore errors while attempting to remove old dirs under Windows. They # happen because of Windows exclusive file locking, and the update # actually succeeds but just leaves the old versions around in the # user's temp dir. if not system_util.IS_WINDOWS: raise if old_cwd: try: os.chdir(old_cwd) except OSError: pass def RunCommand(self): """Command entry point for the update command.""" if gslib.IS_PACKAGE_INSTALL: raise CommandException( 'The update command is only available for gsutil installed from a ' 'tarball. If you installed gsutil via another method, use the same ' 'method to update it.') if system_util.InvokedViaCloudSdk(): raise CommandException( 'The update command is disabled for Cloud SDK installs. Please run ' '"gcloud components update" to update it. Note: the Cloud SDK ' 'incorporates updates to the underlying tools approximately every 2 ' 'weeks, so if you are attempting to update to a recently created ' 'release / pre-release of gsutil it may not yet be available via ' 'the Cloud SDK.') https_validate_certificates = CERTIFICATE_VALIDATION_ENABLED if not https_validate_certificates: raise CommandException( 'Your boto configuration has https_validate_certificates = False.\n' 'The update command cannot be run this way, for security reasons.') DisallowUpdateIfDataInGsutilDir() force_update = False no_prompt = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-f': force_update = True if o == '-n': no_prompt = True dirs_to_remove = [] tmp_dir = tempfile.mkdtemp() dirs_to_remove.append(tmp_dir) old_cwd = os.getcwd() os.chdir(tmp_dir) if not no_prompt: self.logger.info('Checking for software update...') if self.args: update_from_url_str = self.args[0] if not update_from_url_str.endswith('.tar.gz'): raise CommandException( 'The update command only works with tar.gz files.') for i, result in enumerate(self.WildcardIterator(update_from_url_str)): if i > 0: raise CommandException( 'Invalid update URL. Must name a single .tar.gz file.') storage_url = result.storage_url if storage_url.IsFileUrl() and not storage_url.IsDirectory(): if not force_update: raise CommandException( ('"update" command does not support "file://" URLs without the ' '-f option.')) elif not (storage_url.IsCloudUrl() and storage_url.IsObject()): raise CommandException( 'Invalid update object URL. Must name a single .tar.gz file.') else: update_from_url_str = GsutilPubTarball() # Try to retrieve version info from tarball metadata; failing that; download # the tarball and extract the VERSION file. The version lookup will fail # when running the update system test, because it retrieves the tarball from # a temp file rather than a cloud URL (files lack the version metadata). tarball_version = LookUpGsutilVersion(self.gsutil_api, update_from_url_str) if tarball_version: tf = None else: tf = self._FetchAndOpenGsutilTarball(update_from_url_str) tf.extractall() with open(os.path.join('gsutil', 'VERSION'), 'r') as ver_file: tarball_version = ver_file.read().strip() if not force_update and gslib.VERSION == tarball_version: self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) if self.args: raise CommandException('You already have %s installed.' % update_from_url_str, informational=True) else: raise CommandException( 'You already have the latest gsutil release ' 'installed.', informational=True) if not no_prompt: CheckAndMaybePromptForAnalyticsEnabling() (_, major) = CompareVersions(tarball_version, gslib.VERSION) if major: print(('\n'.join( textwrap.wrap( 'This command will update to the "%s" version of gsutil at %s. ' 'NOTE: This a major new version, so it is strongly recommended ' 'that you review the release note details at %s before updating to ' 'this version, especially if you use gsutil in scripts.' % (tarball_version, gslib.GSUTIL_DIR, RELEASE_NOTES_URL))))) else: print(('This command will update to the "%s" version of\ngsutil at %s' % (tarball_version, gslib.GSUTIL_DIR))) self._ExplainIfSudoNeeded(tf, dirs_to_remove, old_cwd) if no_prompt: answer = 'y' else: answer = input('Proceed? [y/N] ') if not answer or answer.lower()[0] != 'y': self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) raise CommandException('Not running update.', informational=True) if not tf: tf = self._FetchAndOpenGsutilTarball(update_from_url_str) # Ignore keyboard interrupts during the update to reduce the chance someone # hitting ^C leaves gsutil in a broken state. RegisterSignalHandler(signal.SIGINT, signal.SIG_IGN) # gslib.GSUTIL_DIR lists the path where the code should end up (like # /usr/local/gsutil), which is one level down from the relative path in the # tarball (since the latter creates files in ./gsutil). So, we need to # extract at the parent directory level. gsutil_bin_parent_dir = os.path.normpath( os.path.join(gslib.GSUTIL_DIR, '..')) # Extract tarball to a temporary directory in a sibling to GSUTIL_DIR. old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) dirs_to_remove.append(old_dir) dirs_to_remove.append(new_dir) self._EnsureDirsSafeForUpdate(dirs_to_remove) try: tf.extractall(path=new_dir) except Exception as e: self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) raise CommandException('Update failed: %s.' % e) # For enterprise mode (shared/central) installation, users with # different user/group than the installation user/group must be # able to run gsutil so we need to do some permissions adjustments # here. Since enterprise mode is not not supported for Windows # users, we can skip this step when running on Windows, which # avoids the problem that Windows has no find or xargs command. if not system_util.IS_WINDOWS: # Make all files and dirs in updated area owner-RW and world-R, and make # all directories owner-RWX and world-RX. for dirname, subdirs, filenames in os.walk(new_dir): for filename in filenames: fd = os.open(os.path.join(dirname, filename), os.O_RDONLY) os.fchmod(fd, stat.S_IWRITE | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) os.close(fd) for subdir in subdirs: fd = os.open(os.path.join(dirname, subdir), os.O_RDONLY) os.fchmod( fd, stat.S_IRWXU | stat.S_IXGRP | stat.S_IXOTH | stat.S_IRGRP | stat.S_IROTH) os.close(fd) # Make main gsutil script owner-RWX and world-RX. fd = os.open(os.path.join(new_dir, 'gsutil', 'gsutil'), os.O_RDONLY) os.fchmod( fd, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) os.close(fd) # Move old installation aside and new into place. os.rename(gslib.GSUTIL_DIR, os.path.join(old_dir, 'old')) os.rename(os.path.join(new_dir, 'gsutil'), gslib.GSUTIL_DIR) self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) RegisterSignalHandler(signal.SIGINT, signal.SIG_DFL) self.logger.info('Update complete.') return 0 def _FetchAndOpenGsutilTarball(self, update_from_url_str): self.command_runner.RunNamedCommand( 'cp', [update_from_url_str, 'file://gsutil.tar.gz'], self.headers, self.debug, skip_update_check=True, ) # Note: tf is closed in _CleanUpUpdateCommand. tf = tarfile.open('gsutil.tar.gz') tf.errorlevel = 1 # So fatal tarball unpack errors raise exceptions. return tf commands/acl.py 0000644 00000057443 15025013616 0007472 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of acl command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from apitools.base.py import encoding from gslib import metrics from gslib import gcs_json_api from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import SetAclExceptionHandler from gslib.command import SetAclFuncWrapper from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.storage_url import RaiseErrorIfUrlsAreMixOfBucketsAndObjects from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import acl_helper from gslib.utils.constants import NO_MAX from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _SET_SYNOPSIS = """ gsutil acl set [-f] [-r] [-a] (<file-path>|<predefined-acl>) url... """ _GET_SYNOPSIS = """ gsutil acl get url """ _CH_SYNOPSIS = """ gsutil acl ch [-f] [-r] <grant>... url... where each <grant> is one of the following forms: -u <id>|<email>:<permission> -g <id>|<email>|<domain>|All|AllAuth:<permission> -p (viewers|editors|owners)-<project number>:<permission> -d <id>|<email>|<domain>|All|AllAuth|(viewers|editors|owners)-<project number> """ _GET_DESCRIPTION = """ <B>GET</B> The ``acl get`` command gets the ACL text for a bucket or object, which you can save and edit for the acl set command. """ _SET_DESCRIPTION = """ <B>SET</B> The ``acl set`` command allows you to set an Access Control List on one or more buckets and objects. As part of the command, you must specify either a predefined ACL or the path to a file that contains ACL text. The simplest way to use the ``acl set`` command is to specify one of the predefined ACLs, e.g.,: gsutil acl set private gs://example-bucket/example-object If you want to make an object or bucket publicly readable or writable, it is recommended to use ``acl ch``, to avoid accidentally removing OWNER permissions. See the ``acl ch`` section for details. See `Predefined ACLs <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ for a list of predefined ACLs. If you want to define more fine-grained control over your data, you can retrieve an ACL using the ``acl get`` command, save the output to a file, edit the file, and then use the ``acl set`` command to set that ACL on the buckets and/or objects. For example: gsutil acl get gs://bucket/file.txt > acl.txt Make changes to acl.txt such as adding an additional grant, then: gsutil acl set acl.txt gs://cats/file.txt Note that you can set an ACL on multiple buckets or objects at once. For example, to set ACLs on all .jpg files found in a bucket: gsutil acl set acl.txt gs://bucket/**.jpg If you have a large number of ACLs to update you might want to use the gsutil -m option, to perform a parallel (multi-threaded/multi-processing) update: gsutil -m acl set acl.txt gs://bucket/**.jpg Note that multi-threading/multi-processing is only done when the named URLs refer to objects, which happens either if you name specific objects or if you enumerate objects by using an object wildcard or specifying the acl -r flag. <B>SET OPTIONS</B> The "set" sub-command has the following options -R, -r Performs "acl set" request recursively, to all objects under the specified URL. -a Performs "acl set" request on all object versions. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. If some of the ACLs couldn't be set, gsutil's exit status will be non-zero even if this flag is set. This option is implicitly set when running "gsutil -m acl...". """ _CH_DESCRIPTION = """ <B>CH</B> The "acl ch" (or "acl change") command updates access control lists, similar in spirit to the Linux chmod command. You can specify multiple access grant additions and deletions in a single command run; all changes will be made atomically to each object in turn. For example, if the command requests deleting one grant and adding a different grant, the ACLs being updated will never be left in an intermediate state where one grant has been deleted but the second grant not yet added. Each change specifies a user or group grant to add or delete, and for grant additions, one of R, W, O (for the permission to be granted). A more formal description is provided in a later section; below we provide examples. <B>CH EXAMPLES</B> Examples for "ch" sub-command: Grant anyone on the internet READ access to the object example-object: gsutil acl ch -u allUsers:R gs://example-bucket/example-object NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see "gsutil help setmeta". Grant the user john.doe@example.com READ access to all objects in example-bucket that begin with folder/: gsutil acl ch -r -u john.doe@example.com:R gs://example-bucket/folder/ Grant the group admins@example.com OWNER access to all jpg files in example-bucket: gsutil acl ch -g admins@example.com:O gs://example-bucket/**.jpg Grant the owners of project example-project WRITE access to the bucket example-bucket: gsutil acl ch -p owners-example-project:W gs://example-bucket NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access to a project's viewers/editors respectively. Remove access to the bucket example-bucket for the viewers of project number 12345: gsutil acl ch -d viewers-12345 gs://example-bucket NOTE: You cannot remove the project owners group from ACLs of gs:// buckets in the given project. Attempts to do so will appear to succeed, but the service will add the project owners group into the new set of ACLs before applying it. Note that removing a project requires you to reference the project by its number (which you can see with the acl get command) as opposed to its project ID string. Grant the service account foo@developer.gserviceaccount.com WRITE access to the bucket example-bucket: gsutil acl ch -u foo@developer.gserviceaccount.com:W gs://example-bucket Grant all users from the `G Suite <https://www.google.com/work/apps/business/>`_ domain my-domain.org READ access to the bucket gcs.my-domain.org: gsutil acl ch -g my-domain.org:R gs://gcs.my-domain.org Remove any current access by john.doe@example.com from the bucket example-bucket: gsutil acl ch -d john.doe@example.com gs://example-bucket If you have a large number of objects to update, enabling multi-threading with the gsutil -m flag can significantly improve performance. The following command adds OWNER for admin@example.org using multi-threading: gsutil -m acl ch -r -u admin@example.org:O gs://example-bucket Grant READ access to everyone from my-domain.org and to all authenticated users, and grant OWNER to admin@mydomain.org, for the buckets my-bucket and my-other-bucket, with multi-threading enabled: gsutil -m acl ch -r -g my-domain.org:R -g AllAuth:R \\ -u admin@mydomain.org:O gs://my-bucket/ gs://my-other-bucket <B>CH ROLES</B> You may specify the following roles with either their shorthand or their full name: R: READ W: WRITE O: OWNER For more information on these roles and the access they grant, see the permissions section of the `Access Control Lists page <https://cloud.google.com/storage/docs/access-control/lists#permissions>`_. <B>CH ENTITIES</B> There are four different entity types: Users, Groups, All Authenticated Users, and All Users. Users are added with -u and a plain ID or email address, as in "-u john-doe@gmail.com:r". Note: Service Accounts are considered to be users. Groups are like users, but specified with the -g flag, as in "-g power-users@example.com:O". Groups may also be specified as a full domain, as in "-g my-company.com:r". allAuthenticatedUsers and allUsers are specified directly, as in "-g allUsers:R" or "-g allAuthenticatedUsers:O". These are case insensitive, and may be shortened to "all" and "allauth", respectively. Removing roles is specified with the -d flag and an ID, email address, domain, or one of allUsers or allAuthenticatedUsers. Many entities' roles can be specified on the same command line, allowing bundled changes to be executed in a single run. This will reduce the number of requests made to the server. <B>CH OPTIONS</B> The "ch" sub-command has the following options -d Remove all roles associated with the matching entity. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. With this option the gsutil exit status will be 0 even if some ACLs couldn't be changed. -g Add or modify a group entity's role. -p Add or modify a project viewers/editors/owners role. -R, -r Performs acl ch request recursively, to all objects under the specified URL. -u Add or modify a user entity's role. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = (""" The acl command has three sub-commands: """ + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION, _CH_DESCRIPTION])) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) def _ApplyExceptionHandler(cls, exception): cls.logger.error('Encountered a problem: %s', exception) cls.everything_set_okay = False def _ApplyAclChangesWrapper(cls, url_or_expansion_result, thread_state=None): cls.ApplyAclChanges(url_or_expansion_result, thread_state=thread_state) class AclCommand(Command): """Implementation of gsutil acl command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'acl', command_name_aliases=['getacl', 'setacl', 'chacl'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='afRrg:u:d:p:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeFileURLOrCannedACLArgument(), CommandArgument.MakeZeroOrMoreCloudURLsArgument() ], 'get': [CommandArgument.MakeNCloudURLsArgument(1)], 'ch': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='acl', help_name_aliases=['getacl', 'setacl', 'chmod', 'chacl'], help_type='command_help', help_one_line_summary='Get, set, or change bucket and/or object ACLs', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text }, ) def _get_shim_command_group(self): object_or_bucket_urls = [StorageUrlFromString(url) for url in self.args] recurse = False for (flag_key, _) in self.sub_opts: if flag_key in ('-r', '-R'): recurse = True break RaiseErrorIfUrlsAreMixOfBucketsAndObjects(object_or_bucket_urls, recurse) if object_or_bucket_urls[0].IsBucket() and not recurse: return 'buckets' else: return 'objects' def get_gcloud_storage_args(self): sub_command = self.args.pop(0) if sub_command == 'get': if StorageUrlFromString(self.args[0]).IsObject(): command_group = 'objects' else: command_group = 'buckets' gcloud_storage_map = GcloudStorageMap(gcloud_command=[ 'storage', command_group, 'describe', '--format=multi(acl:format=json)' ], flag_map={}) elif sub_command == 'set': # Flags must be at the start of self.args to get parsed. self.ParseSubOpts() acl_file_or_predefined_acl = self.args.pop(0) if os.path.isfile(acl_file_or_predefined_acl): acl_flag = '--acl-file=' + acl_file_or_predefined_acl else: if acl_file_or_predefined_acl in ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION): predefined_acl = ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION[ acl_file_or_predefined_acl]) else: predefined_acl = acl_file_or_predefined_acl acl_flag = '--predefined-acl=' + predefined_acl command_group = self._get_shim_command_group() gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', command_group, 'update'] + [acl_flag], flag_map={ '-a': GcloudStorageFlag('--all-versions'), '-f': GcloudStorageFlag('--continue-on-error'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), }) elif sub_command == 'ch': self.ParseSubOpts() self.sub_opts = acl_helper.translate_sub_opts_for_shim(self.sub_opts) command_group = self._get_shim_command_group() gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', command_group, 'update'], flag_map={ '-g': GcloudStorageFlag('--add-acl-grant'), '-p': GcloudStorageFlag('--add-acl-grant'), '-u': GcloudStorageFlag('--add-acl-grant'), '-d': GcloudStorageFlag('--remove-acl-grant'), '-a': GcloudStorageFlag('--all-versions'), '-f': GcloudStorageFlag('--continue-on-error'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), }) return super().get_gcloud_storage_args(gcloud_storage_map) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if (self.args[0].lower() == 'set') or (self.command_alias_used == 'setacl'): return 1 else: return 0 def _SetAcl(self): """Parses options and sets ACLs on the specified buckets/objects.""" self.continue_on_error = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-a': self.all_versions = True elif o == '-f': self.continue_on_error = True elif o == '-r' or o == '-R': self.recursion_requested = True else: self.RaiseInvalidArgumentException() try: self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler) except AccessDeniedException as unused_e: self._WarnServiceAccounts() raise if not self.everything_set_okay: raise CommandException('ACLs for some objects could not be set.') def _ChAcl(self): """Parses options and changes ACLs on the specified buckets/objects.""" self.parse_versions = True self.changes = [] self.continue_on_error = False if self.sub_opts: for o, a in self.sub_opts: if o == '-f': self.continue_on_error = True elif o == '-g': if 'gserviceaccount.com' in a: raise CommandException( 'Service accounts are considered users, not groups; please use ' '"gsutil acl ch -u" instead of "gsutil acl ch -g"') self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.GROUP)) elif o == '-p': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.PROJECT)) elif o == '-u': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.USER)) elif o == '-d': self.changes.append(acl_helper.AclDel(a)) elif o == '-r' or o == '-R': self.recursion_requested = True else: self.RaiseInvalidArgumentException() if not self.changes: raise CommandException('Please specify at least one access change ' 'with the -g, -u, or -d flags') if (not UrlsAreForSingleProvider(self.args) or StorageUrlFromString(self.args[0]).scheme != 'gs'): raise CommandException( 'The "{0}" command can only be used with gs:// URLs'.format( self.command_name)) self.everything_set_okay = True self.ApplyAclFunc(_ApplyAclChangesWrapper, _ApplyExceptionHandler, self.args, object_fields=['acl', 'generation', 'metageneration']) if not self.everything_set_okay: raise CommandException('ACLs for some objects could not be set.') def _RaiseForAccessDenied(self, url): self._WarnServiceAccounts() raise CommandException('Failed to set acl for %s. Please ensure you have ' 'OWNER-role access to this resource.' % url) @Retry(ServiceException, tries=3, timeout_secs=1) def ApplyAclChanges(self, name_expansion_result, thread_state=None): """Applies the changes in self.changes to the provided URL. Args: name_expansion_result: NameExpansionResult describing the target object. thread_state: If present, gsutil Cloud API instance to apply the changes. """ if thread_state: gsutil_api = thread_state else: gsutil_api = self.gsutil_api url = name_expansion_result.expanded_storage_url if url.IsBucket(): bucket = gsutil_api.GetBucket(url.bucket_name, provider=url.scheme, fields=['acl', 'metageneration']) current_acl = bucket.acl elif url.IsObject(): gcs_object = encoding.JsonToMessage(apitools_messages.Object, name_expansion_result.expanded_result) current_acl = gcs_object.acl if not current_acl: self._RaiseForAccessDenied(url) if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: self.logger.info('No changes to %s', url) return try: if url.IsBucket(): preconditions = Preconditions(meta_gen_match=bucket.metageneration) bucket_metadata = apitools_messages.Bucket(acl=current_acl) gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) else: # Object preconditions = Preconditions(gen_match=gcs_object.generation, meta_gen_match=gcs_object.metageneration) object_metadata = apitools_messages.Object(acl=current_acl) try: gsutil_api.PatchObjectMetadata(url.bucket_name, url.object_name, object_metadata, preconditions=preconditions, provider=url.scheme, generation=url.generation, fields=['id']) except PreconditionException as e: # Special retry case where we want to do an additional step, the read # of the read-modify-write cycle, to fetch the correct object # metadata before reattempting ACL changes. self._RefetchObjectMetadataAndApplyAclChanges(url, gsutil_api) self.logger.info('Updated ACL on %s', url) except BadRequestException as e: # Don't retry on bad requests, e.g. invalid email address. raise CommandException('Received bad request from server: %s' % str(e)) except AccessDeniedException: self._RaiseForAccessDenied(url) except PreconditionException as e: # For objects, retry attempts should have already been handled. if url.IsObject(): raise CommandException(str(e)) # For buckets, raise PreconditionException and continue to next retry. raise e @Retry(PreconditionException, tries=3, timeout_secs=1) def _RefetchObjectMetadataAndApplyAclChanges(self, url, gsutil_api): """Reattempts object ACL changes after a PreconditionException.""" gcs_object = gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, provider=url.scheme, fields=['acl', 'generation', 'metageneration']) current_acl = gcs_object.acl if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: self.logger.info('No changes to %s', url) return object_metadata = apitools_messages.Object(acl=current_acl) preconditions = Preconditions(gen_match=gcs_object.generation, meta_gen_match=gcs_object.metageneration) gsutil_api.PatchObjectMetadata(url.bucket_name, url.object_name, object_metadata, preconditions=preconditions, provider=url.scheme, generation=gcs_object.generation, fields=['id']) def _ApplyAclChangesAndReturnChangeCount(self, storage_url, acl_message): modification_count = 0 for change in self.changes: modification_count += change.Execute(storage_url, acl_message, 'acl', self.logger) return modification_count def RunCommand(self): """Command entry point for the acl command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) self.def_acl = False if action_subcommand == 'get': metrics.LogCommandParams(subcommands=[action_subcommand]) self.GetAndPrintAcl(self.args[0]) elif action_subcommand == 'set': metrics.LogCommandParams(subcommands=[action_subcommand]) self._SetAcl() elif action_subcommand in ('ch', 'change'): metrics.LogCommandParams(subcommands=[action_subcommand]) self._ChAcl() else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help acl".') % (action_subcommand, self.command_name)) return 0 commands/mb.py 0000644 00000041153 15025013616 0007320 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of mb command for creating cloud storage buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import textwrap from gslib.cloud_api import AccessDeniedException, BadRequestException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.commands.rpo import VALID_RPO_VALUES from gslib.commands.rpo import VALID_RPO_VALUES_STRING from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import InvalidUrlError from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.retention_util import RetentionInSeconds from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAscii from gslib.utils.text_util import InsistOnOrOff from gslib.utils.text_util import NormalizeStorageClass from gslib.utils.encryption_helper import ValidateCMEK _SYNOPSIS = """ gsutil mb [-b (on|off)] [-c <class>] [-k <key>] [-l <location>] [-p <project>] [--autoclass] [--retention <time>] [--pap <setting>] [--placement <region1>,<region2>] [--rpo {}] gs://<bucket_name>... """.format(VALID_RPO_VALUES_STRING) _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Create one or more new buckets. Google Cloud Storage has a single namespace, so you are not allowed to create a bucket with a name already in use by another user. You can, however, carve out parts of the bucket name space corresponding to your company's domain name (see "gsutil help naming"). If you don't specify a project ID or project number using the -p option, the buckets are created using the default project ID specified in your `gsutil configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_. The -l option specifies the location for the buckets. Once a bucket is created in a given location, it cannot be moved to a different location. Instead, you need to create a new bucket, move the data over, and then delete the original bucket. <B>BUCKET STORAGE CLASSES</B> You can specify one of the `storage classes <https://cloud.google.com/storage/docs/storage-classes>`_ for a bucket with the -c option. Example: gsutil mb -c nearline gs://some-bucket See online documentation for `pricing <https://cloud.google.com/storage/pricing>`_ and `SLA <https://cloud.google.com/storage/sla>`_ details. If you don't specify a -c option, the bucket is created with the default storage class Standard Storage. <B>BUCKET LOCATIONS</B> You can specify one of the `available locations <https://cloud.google.com/storage/docs/locations>`_ for a bucket with the -l option. Examples: gsutil mb -l asia gs://some-bucket gsutil mb -c standard -l us-east1 gs://some-bucket If you don't specify a -l option, the bucket is created in the default location (US). <B>Retention Policy</B> You can specify retention period in one of the following formats: --retention <number>s Specifies retention period of <number> seconds for objects in this bucket. --retention <number>d Specifies retention period of <number> days for objects in this bucket. --retention <number>m Specifies retention period of <number> months for objects in this bucket. --retention <number>y Specifies retention period of <number> years for objects in this bucket. Examples: gsutil mb --retention 1y gs://some-bucket gsutil mb --retention 36m gs://some-bucket If you don't specify a --retention option, the bucket is created with no retention policy. <B>OPTIONS</B> --autoclass Enables the Autoclass feature that automatically sets object storage classes. -b <on|off> Specifies the uniform bucket-level access setting. When "on", ACLs assigned to objects in the bucket are not evaluated. Consequently, only IAM policies grant access to objects in these buckets. Default is "off". -c class Specifies the default storage class. Default is ``Standard``. See `Available storage classes <https://cloud.google.com/storage/docs/storage-classes#classes>`_ for a list of possible values. -k <key> Set the default KMS key using the full path to the key, which has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -l location Can be any supported location. See https://cloud.google.com/storage/docs/locations for a discussion of this distinction. Default is US. Locations are case insensitive. -p project Specifies the project ID or project number to create the bucket under. -s class Same as -c. --retention time Specifies the retention policy. Default is no retention policy. This can only be set on gs:// buckets and requires using the JSON API. For more details about retention policy see "gsutil help retention" --pap setting Specifies the public access prevention setting. Valid values are "enforced" or "inherited". When "enforced", objects in this bucket cannot be made publicly accessible. Default is "inherited". --placement reg1,reg2 Two regions that form the custom dual-region. Only regions within the same continent are or will ever be valid. Invalid location pairs (such as mixed-continent, or with unsupported regions) will return an error. --rpo setting Specifies the `replication setting <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_. This flag is not valid for single-region buckets, and multi-region buckets only accept a value of DEFAULT. Valid values for dual region buckets are {rpo_values}. If unspecified, DEFAULT is applied for dual-region and multi-region buckets. """.format(rpo_values=VALID_RPO_VALUES_STRING)) # Regex to disallow buckets violating charset or not [3..255] chars total. BUCKET_NAME_RE = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9\._-]{1,253}[a-zA-Z0-9]$') # Regex to disallow buckets with individual DNS labels longer than 63. TOO_LONG_DNS_NAME_COMP = re.compile(r'[-_a-z0-9]{64}') _RETENTION_FLAG = '--retention' IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue BucketPolicyOnlyValue = IamConfigurationValue.BucketPolicyOnlyValue class MbCommand(Command): """Implementation of gsutil mb command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'mb', command_name_aliases=['makebucket', 'createbucket', 'md', 'mkdir'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='b:c:l:p:s:k:', supported_private_args=[ 'autoclass', 'retention=', 'pap=', 'placement=', 'rpo=' ], file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='mb', help_name_aliases=[ 'createbucket', 'makebucket', 'md', 'mkdir', 'location', 'dra', 'dras', 'reduced_availability', 'durable_reduced_availability', 'rr', 'reduced_redundancy', 'standard', 'storage class', 'nearline', 'nl', ], help_type='command_help', help_one_line_summary='Make buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'create'], flag_map={ '-b': GcloudStorageFlag({ 'on': '--uniform-bucket-level-access', 'off': None, }), '-c': GcloudStorageFlag('--default-storage-class'), '-k': GcloudStorageFlag('--default-encryption-key'), '-l': GcloudStorageFlag('--location'), '-p': GcloudStorageFlag('--project'), '--pap': GcloudStorageFlag({ 'enforced': '--public-access-prevention', 'inherited': None, }), '--placement': GcloudStorageFlag('--placement'), _RETENTION_FLAG: GcloudStorageFlag('--retention-period'), '--rpo': GcloudStorageFlag('--recovery-point-objective') }, ) def get_gcloud_storage_args(self): retention_arg_idx = 0 while retention_arg_idx < len(self.sub_opts): if self.sub_opts[retention_arg_idx][0] == _RETENTION_FLAG: break retention_arg_idx += 1 if retention_arg_idx < len(self.sub_opts): # Convert retention time to seconds, which gcloud knows how to handle. self.sub_opts[retention_arg_idx] = ( _RETENTION_FLAG, str(RetentionInSeconds(self.sub_opts[retention_arg_idx][1])) + 's') return super().get_gcloud_storage_args(MbCommand.gcloud_storage_map) def RunCommand(self): """Command entry point for the mb command.""" autoclass = False bucket_policy_only = None kms_key = None location = None storage_class = None seconds = None placements = None public_access_prevention = None rpo = None json_only_flags_in_command = [] if self.sub_opts: for o, a in self.sub_opts: if o == '--autoclass': autoclass = True json_only_flags_in_command.append(o) elif o == '-k': kms_key = a ValidateCMEK(kms_key) json_only_flags_in_command.append(o) elif o == '-l': location = a elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii(a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-c' or o == '-s': storage_class = NormalizeStorageClass(a) elif o == _RETENTION_FLAG: seconds = RetentionInSeconds(a) elif o == '--rpo': rpo = a.strip() if rpo not in VALID_RPO_VALUES: raise CommandException( 'Invalid value for --rpo. Must be one of: {},' ' provided: {}'.format(VALID_RPO_VALUES_STRING, a)) json_only_flags_in_command.append(o) elif o == '-b': InsistOnOrOff(a, 'Only on and off values allowed for -b option') bucket_policy_only = (a == 'on') json_only_flags_in_command.append(o) elif o == '--pap': public_access_prevention = a json_only_flags_in_command.append(o) elif o == '--placement': placements = a.split(',') if len(placements) != 2: raise CommandException( 'Please specify two regions separated by comma without space.' ' Specified: {}'.format(a)) json_only_flags_in_command.append(o) bucket_metadata = apitools_messages.Bucket(location=location, rpo=rpo, storageClass=storage_class) if autoclass: bucket_metadata.autoclass = apitools_messages.Bucket.AutoclassValue( enabled=autoclass) if bucket_policy_only or public_access_prevention: bucket_metadata.iamConfiguration = IamConfigurationValue() iam_config = bucket_metadata.iamConfiguration if bucket_policy_only: iam_config.bucketPolicyOnly = BucketPolicyOnlyValue() iam_config.bucketPolicyOnly.enabled = bucket_policy_only if public_access_prevention: iam_config.publicAccessPrevention = public_access_prevention if kms_key: encryption = apitools_messages.Bucket.EncryptionValue() encryption.defaultKmsKeyName = kms_key bucket_metadata.encryption = encryption if placements: placement_config = apitools_messages.Bucket.CustomPlacementConfigValue() placement_config.dataLocations = placements bucket_metadata.customPlacementConfig = placement_config for bucket_url_str in self.args: bucket_url = StorageUrlFromString(bucket_url_str) if seconds is not None: if bucket_url.scheme != 'gs': raise CommandException('Retention policy can only be specified for ' 'GCS buckets.') retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=seconds)) bucket_metadata.retentionPolicy = retention_policy if json_only_flags_in_command and self.gsutil_api.GetApiSelector( bucket_url.scheme) != ApiSelector.JSON: raise CommandException('The {} option(s) can only be used for GCS' ' Buckets with the JSON API'.format( ', '.join(json_only_flags_in_command))) if not bucket_url.IsBucket(): raise CommandException('The mb command requires a URL that specifies a ' 'bucket.\n"%s" is not valid.' % bucket_url) if (not BUCKET_NAME_RE.match(bucket_url.bucket_name) or TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)): raise InvalidUrlError('Invalid bucket name in URL "%s"' % bucket_url.bucket_name) self.logger.info('Creating %s...', bucket_url) # Pass storage_class param only if this is a GCS bucket. (In S3 the # storage class is specified on the key object.) try: self.gsutil_api.CreateBucket(bucket_url.bucket_name, project_id=self.project_id, metadata=bucket_metadata, provider=bucket_url.scheme) except AccessDeniedException as e: message = e.reason if 'key' in message: # This will print the error reason and append the following as a # suggested next step: # # To authorize, run: # gsutil kms authorize \ # -k <kms_key> \ # -p <project_id> message += ' To authorize, run:\n gsutil kms authorize' message += ' \\\n -k %s' % kms_key if (self.project_id): message += ' \\\n -p %s' % self.project_id raise CommandException(message) else: raise except BadRequestException as e: if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and bucket_url.scheme == 'gs'): bucket_name = bucket_url.bucket_name final_comp = bucket_name[bucket_name.rfind('.') + 1:] raise CommandException('\n'.join( textwrap.wrap( 'Buckets with "." in the name must be valid DNS names. The bucket' ' you are attempting to create (%s) is not a valid DNS name,' ' because the final component (%s) is not currently a valid part' ' of the top-level DNS tree.' % (bucket_name, final_comp)))) else: raise return 0 commands/retention.py 0000644 00000073414 15025013616 0010736 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Retention Policy configuration command for buckets.""" from __future__ import absolute_import import time from apitools.base.py import encoding from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import Preconditions from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import MetadataMessage from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import NO_MAX from gslib.utils.parallelism_framework_util import PutToQueueWithTimeout from gslib.utils.retention_util import ConfirmLockRequest from gslib.utils.retention_util import ReleaseEventHoldFuncWrapper from gslib.utils.retention_util import ReleaseTempHoldFuncWrapper from gslib.utils.retention_util import RetentionInSeconds from gslib.utils.retention_util import RetentionPolicyToString from gslib.utils.retention_util import SetEventHoldFuncWrapper from gslib.utils.retention_util import SetTempHoldFuncWrapper from gslib.utils.retention_util import UpdateObjectMetadataExceptionHandler from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import PreconditionsFromHeaders _SET_SYNOPSIS = """ gsutil retention set <retention_period> gs://<bucket_name>... """ _CLEAR_SYNOPSIS = """ gsutil retention clear gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil retention get gs://<bucket_name>... """ _LOCK_SYNOPSIS = """ gsutil retention lock gs://<bucket_name>... """ _EVENT_DEFAULT_SYNOPSIS = """ gsutil retention event-default (set|release) gs://<bucket_name>... """ _EVENT_SYNOPSIS = """ gsutil retention event (set|release) gs://<bucket_name>/<object_name>... """ _TEMP_SYNOPSIS = """ gsutil retention temp (set|release) gs://<bucket_name>/<object_name>... """ _SET_DESCRIPTION = """ <B>SET</B> You can configure a data retention policy for a Cloud Storage bucket that governs how long objects in the bucket must be retained. You can also lock the data retention policy, permanently preventing the policy from being reduced or removed. For more information, see `Retention policies and Bucket Lock <https://cloud.google.com/storage/docs/bucket-lock>`_. The ``gsutil retention set`` command allows you to set or update the retention policy on one or more buckets. To remove an unlocked retention policy from one or more buckets, use the ``gsutil retention clear`` command. The ``set`` sub-command can set a retention policy with the following formats: <B>SET FORMATS</B> Formats for the ``set`` subcommand include: <number>s Specifies retention period of <number> seconds for objects in this bucket. <number>d Specifies retention period of <number> days for objects in this bucket. <number>m Specifies retention period of <number> months for objects in this bucket. <number>y Specifies retention period of <number> years for objects in this bucket. GCS JSON API accepts retention periods as number of seconds. Durations provided in terms of days, months or years are converted to their rough equivalent values in seconds, using the following conversions: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Retention periods must be greater than 0 and less than 100 years. Retention durations must be in only one form (seconds, days, months, or years), and not a combination of them. Note that while it is possible to specify retention durations shorter than a day (using seconds), enforcement of such retention periods is not guaranteed. Such durations may only be used for testing purposes. <B>EXAMPLES</B> Setting a retention policy with a duration of 1 year on a bucket: gsutil retention set 1y gs://my-bucket Setting a retention policy with a duration of 36 months on a bucket: gsutil retention set 36m gs://some-bucket You can also provide a precondition on a bucket's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks a bucket's metageneration before setting the retention policy on the bucket: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention set 1y gs://my-bucket """ _CLEAR_DESCRIPTION = """ <B>CLEAR</B> The ``gsutil retention clear`` command removes an unlocked retention policy from one or more buckets. You cannot remove or reduce the duration of a locked retention policy. <B>EXAMPLES</B> Clearing an unlocked retention policy from a bucket: gsutil retention clear gs://my-bucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``gsutil retention get`` command retrieves the retention policy for a given bucket and displays a human-readable representation of the configuration. """ _LOCK_DESCRIPTION = """ <B>LOCK</B> The ``gsutil retention lock`` command PERMANENTLY locks an unlocked retention policy on one or more buckets. CAUTION: A locked retention policy cannot be removed from a bucket or reduced in duration. Once locked, deleting the bucket is the only way to "remove" a retention policy. """ _EVENT_DEFAULT_DESCRIPTION = """ <B>EVENT-DEFAULT</B> The ``gsutil retention event-default`` command sets the default value for an event-based hold on one or more buckets. By setting the default event-based hold on a bucket, newly-created objects inherit that value as their event-based hold (it is not applied retroactively). <B>EXAMPLES</B> Setting the default event-based hold on a bucket: gsutil retention event-default set gs://my-bucket Releasing the default event-based hold on a bucket: gsutil retention event-default release gs://my-bucket You can also provide a precondition on a bucket's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks a bucket's metageneration before setting the default event-based hold on the bucket: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention event-default set gs://my-bucket """ _EVENT_DESCRIPTION = """ <B>EVENT</B> The ``gsutil retention event`` command enables or disables an event-based hold on an object. <B>EXAMPLES</B> Setting the event-based hold on an object: gsutil retention event set gs://my-bucket/my-object Releasing the event-based hold on an object: gsutil retention event release gs://my-bucket/my-object You can also provide a precondition on an object's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks an object's metageneration before setting the event-based hold on the object: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention event set gs://my-bucket/my-object If you want to set or release an event-based hold on a large number of objects, then you might want to use the top-level '-m' option to perform a parallel update. For example, the following command sets an event-based hold on objects ending with .jpg in parallel, in the root folder: gsutil -m retention event set gs://my-bucket/*.jpg """ _TEMP_DESCRIPTION = """ <B>TEMP</B> The ``gsutil retention temp`` command enables or disables a temporary hold on an object. <B>EXAMPLES</B> Setting the temporary hold on an object: gsutil retention temp set gs://my-bucket/my-object Releasing the temporary hold on an object: gsutil retention temp release gs://my-bucket/my-object You can also provide a precondition on an object's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks an object's metageneration before setting the temporary hold on the object: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention temp set gs://my-bucket/my-object If you want to set or release a temporary hold on a large number of objects, then you might want to use the top-level '-m' option to perform a parallel update. For example, the following command sets a temporary hold on objects ending with .jpg in parallel, in the root folder: gsutil -m retention temp set gs://bucket/*.jpg """ _SYNOPSIS = (_SET_SYNOPSIS + _CLEAR_SYNOPSIS + _GET_SYNOPSIS + _LOCK_SYNOPSIS + _EVENT_DEFAULT_SYNOPSIS + _EVENT_SYNOPSIS + _TEMP_SYNOPSIS) _DESCRIPTION = (_SET_DESCRIPTION + _CLEAR_DESCRIPTION + _GET_DESCRIPTION + _LOCK_DESCRIPTION + _EVENT_DEFAULT_DESCRIPTION + _EVENT_DESCRIPTION + _TEMP_DESCRIPTION) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _clear_help_text = CreateHelpText(_CLEAR_SYNOPSIS, _CLEAR_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _lock_help_text = CreateHelpText(_LOCK_SYNOPSIS, _LOCK_DESCRIPTION) _event_default_help_text = CreateHelpText(_EVENT_DEFAULT_SYNOPSIS, _EVENT_DEFAULT_DESCRIPTION) _event_help_text = CreateHelpText(_EVENT_SYNOPSIS, _EVENT_DESCRIPTION) _temp_help_text = CreateHelpText(_TEMP_SYNOPSIS, _TEMP_DESCRIPTION) class RetentionCommand(Command): """Implementation of gsutil retention command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'retention', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'clear': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'lock': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'event-default': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, 'event': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, 'temp': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='retention', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Provides utilities to interact with Retention Policy feature.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'clear': _clear_help_text, 'lock': _lock_help_text, 'event-default': _event_default_help_text, 'event': _event_help_text, 'temp': _temp_help_text }, ) def get_gcloud_storage_args(self): if self.args[0] == 'set': gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--retention-period={}s'.format( RetentionInSeconds(self.args[1])) ] + self.args[2:], # Adds target bucket URLs. flag_map={}), }, flag_map={}) # Don't trigger unneeded translation now that complete command is above. self.args = self.args[:1] else: gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'clear': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--clear-retention-period' ], flag_map={}, ), 'event': GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--event-based-hold' ], flag_map={}, ), 'release': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--no-event-based-hold' ], flag_map={}, ), }, flag_map={}, ), 'event-default': GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--default-event-based-hold' ], flag_map={}, ), 'release': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-default-event-based-hold' ], flag_map={}, ), }, flag_map={}, ), 'get': GcloudStorageMap(gcloud_command=[ 'storage', 'buckets', 'describe', '--format=yaml(retentionPolicy)', '--raw' ], flag_map={}), 'lock': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--lock-retention-period' ], flag_map={}, ), 'temp': GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--temporary-hold' ], flag_map={}, ), 'release': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--no-temporary-hold' ], flag_map={}, ), }, flag_map={}, ), }, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def RunCommand(self): """Command entry point for the retention command.""" # If the only credential type the user supplies in their boto file is HMAC, # GetApiSelector logic will force us to use the XML API, which bucket lock # does not support at the moment. if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException(('The {} command can only be used with the GCS ' 'JSON API. If you have only supplied hmac ' 'credentials in your boto file, please instead ' 'supply a credential type that can be used with ' 'the JSON API.').format(self.command_name)) self.preconditions = PreconditionsFromHeaders(self.headers) action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) if action_subcommand == 'set': func = self._SetRetention elif action_subcommand == 'clear': func = self._ClearRetention elif action_subcommand == 'get': func = self._GetRetention elif action_subcommand == 'lock': func = self._LockRetention elif action_subcommand == 'event-default': func = self._DefaultEventHold elif action_subcommand == 'event': func = self._EventHold elif action_subcommand == 'temp': func = self._TempHold else: raise CommandException( ('Invalid subcommand "{}" for the {} command.\n' 'See "gsutil help retention".').format(action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) return func() def BucketUpdateFunc(self, url_args, bucket_metadata_update, fields, log_msg_template): preconditions = Preconditions( meta_gen_match=self.preconditions.meta_gen_match) # Iterate over URLs, expanding wildcards and setting the new bucket metadata # on each bucket. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info(log_msg_template, blr) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata_update, preconditions=preconditions, provider=url.scheme, fields=fields) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def ObjectUpdateMetadataFunc(self, patch_obj_metadata, log_template, name_expansion_result, thread_state=None): """Updates metadata on an object using PatchObjectMetadata. Args: patch_obj_metadata: Metadata changes that should be applied to the existing object. log_template: The log template that should be printed for each object. name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) exp_src_url = name_expansion_result.expanded_storage_url self.logger.info(log_template, exp_src_url) cloud_obj_metadata = encoding.JsonToMessage( apitools_messages.Object, name_expansion_result.expanded_result) preconditions = Preconditions( gen_match=self.preconditions.gen_match, meta_gen_match=self.preconditions.meta_gen_match) if preconditions.gen_match is None: preconditions.gen_match = cloud_obj_metadata.generation if preconditions.meta_gen_match is None: preconditions.meta_gen_match = cloud_obj_metadata.metageneration gsutil_api.PatchObjectMetadata(exp_src_url.bucket_name, exp_src_url.object_name, patch_obj_metadata, generation=exp_src_url.generation, preconditions=preconditions, provider=exp_src_url.scheme, fields=['id']) PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) def _GetObjectNameExpansionIterator(self, url_args): return NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, url_args, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.parallel_operations, bucket_listing_fields=['generation', 'metageneration']) def _GetSeekAheadNameExpansionIterator(self, url_args): return SeekAheadNameExpansionIterator(self.command_name, self.debug, self.GetSeekAheadGsutilApi(), url_args, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) def _SetRetention(self): """Set retention retention_period on one or more buckets.""" seconds = RetentionInSeconds(self.args[0]) retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=seconds)) log_msg_template = 'Setting Retention Policy on %s...' bucket_metadata_update = apitools_messages.Bucket( retentionPolicy=retention_policy) url_args = self.args[1:] self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'retentionPolicy'], log_msg_template=log_msg_template) return 0 def _ClearRetention(self): """Clear retention retention_period on one or more buckets.""" retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=None)) log_msg_template = 'Clearing Retention Policy on %s...' bucket_metadata_update = apitools_messages.Bucket( retentionPolicy=retention_policy) url_args = self.args self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'retentionPolicy'], log_msg_template=log_msg_template) return 0 def _GetRetention(self): """Get Retention Policy for a single bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['retentionPolicy']) print(RetentionPolicyToString(bucket_metadata.retentionPolicy, bucket_url)) return 0 def _LockRetention(self): """Lock Retention Policy on one or more buckets.""" url_args = self.args # Iterate over URLs, expanding wildcards and setting the Retention Policy # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True # Get bucket metadata to provide a precondition. bucket_metadata = self.gsutil_api.GetBucket( url.bucket_name, provider=url.scheme, fields=['id', 'metageneration', 'retentionPolicy']) if (not (bucket_metadata.retentionPolicy and bucket_metadata.retentionPolicy.retentionPeriod)): # TODO: implement '-c' flag to continue_on_error raise CommandException( 'Bucket "{}" does not have an Unlocked Retention Policy.'.format( url.bucket_name)) elif bucket_metadata.retentionPolicy.isLocked is True: self.logger.error('Retention Policy on "%s" is already locked.', blr) elif ConfirmLockRequest(url.bucket_name, bucket_metadata.retentionPolicy): self.logger.info('Locking Retention Policy on %s...', blr) self.gsutil_api.LockRetentionPolicy(url.bucket_name, bucket_metadata.metageneration, provider=url.scheme) else: self.logger.error( ' Abort Locking Retention Policy on {}'.format(blr)) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _DefaultEventHold(self): """Sets default value for Event-Based Hold on one or more buckets.""" hold = None if self.args: if self.args[0].lower() == 'set': hold = True elif self.args[0].lower() == 'release': hold = False else: raise CommandException( ('Invalid subcommand "{}" for the "retention event-default"' ' command.\nSee "gsutil help retention event".').format( self.sub_opts)) verb = 'Setting' if hold else 'Releasing' log_msg_template = '{} default Event-Based Hold on %s...'.format(verb) bucket_metadata_update = apitools_messages.Bucket( defaultEventBasedHold=hold) url_args = self.args[1:] self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'defaultEventBasedHold'], log_msg_template=log_msg_template) return 0 def _EventHold(self): """Sets or unsets Event-Based Hold on one or more objects.""" sub_command_name = 'event' sub_command_full_name = 'Event-Based' hold = self._ProcessHoldArgs(sub_command_name) url_args = self.args[1:] obj_metadata_update_wrapper = (SetEventHoldFuncWrapper if hold else ReleaseEventHoldFuncWrapper) self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name) return 0 def _TempHold(self): """Sets or unsets Temporary Hold on one or more objects.""" sub_command_name = 'temp' sub_command_full_name = 'Temporary' hold = self._ProcessHoldArgs(sub_command_name) url_args = self.args[1:] obj_metadata_update_wrapper = (SetTempHoldFuncWrapper if hold else ReleaseTempHoldFuncWrapper) self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name) return 0 def _ProcessHoldArgs(self, sub_command_name): """Processes command args for Temporary and Event-Based Hold sub-command. Args: sub_command_name: The name of the subcommand: "temp" / "event" Returns: Returns a boolean value indicating whether to set (True) or release (False)the Hold. """ hold = None if self.args[0].lower() == 'set': hold = True elif self.args[0].lower() == 'release': hold = False else: raise CommandException( ('Invalid subcommand "{}" for the "retention {}" command.\n' 'See "gsutil help retention {}".').format(self.args[0], sub_command_name, sub_command_name)) return hold def _SetHold(self, obj_metadata_update_wrapper, url_args, sub_command_full_name): """Common logic to set or unset Event-Based/Temporary Hold on objects. Args: obj_metadata_update_wrapper: The function for updating related fields in Object metadata. url_args: List of object URIs. sub_command_full_name: The full name for sub-command: "Temporary" / "Event-Based" """ if len(url_args) == 1 and not self.recursion_requested: url = StorageUrlFromString(url_args[0]) if not (url.IsCloudUrl() and url.IsObject()): raise CommandException('URL ({}) must name an object'.format( url_args[0])) name_expansion_iterator = self._GetObjectNameExpansionIterator(url_args) seek_ahead_iterator = self._GetSeekAheadNameExpansionIterator(url_args) # Used to track if any objects' metadata failed to be set. self.everything_set_okay = True try: # TODO: implement '-c' flag to continue_on_error # Perform requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(obj_metadata_update_wrapper, name_expansion_iterator, UpdateObjectMetadataExceptionHandler, fail_on_error=True, seek_ahead_iterator=seek_ahead_iterator) except AccessDeniedException as e: if e.status == 403: self._WarnServiceAccounts() raise if not self.everything_set_okay: raise CommandException( '{} Hold for some objects could not be set.'.format( sub_command_full_name)) commands/cat.py 0000644 00000012671 15025013616 0007474 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # Copyright 2011, Nexenta Systems Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like cat command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import six from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.utils import cat_helper from gslib.utils import constants from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap if six.PY3: long = int _SYNOPSIS = """ gsutil cat [-h] url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The cat command outputs the contents of one or more URLs to stdout. While the cat command does not compute a checksum, it is otherwise equivalent to doing: gsutil cp url... - (The final '-' causes gsutil to stream the output to stdout.) WARNING: The gsutil cat command does not compute a checksum of the downloaded data. Therefore, we recommend that users either perform their own validation of the output of gsutil cat or use gsutil cp or rsync (both of which perform integrity checking automatically). <B>OPTIONS</B> -h Prints short header for each object. For example: gsutil cat -h gs://bucket/meeting_notes/2012_Feb/*.txt This would print a header with the object name before the contents of each text object that matched the wildcard. -r range Causes gsutil to output just the specified byte range of the object. Ranges can be of these forms: start-end (e.g., -r 256-5939) start- (e.g., -r 256-) -numbytes (e.g., -r -5) where offsets start at 0, start-end means to return bytes start through end (inclusive), start- means to return bytes start through the end of the object, and -numbytes means to return the last numbytes of the object. For example: gsutil cat -r 256-939 gs://bucket/object returns bytes 256 through 939, while: gsutil cat -r -5 gs://bucket/object returns the final 5 bytes of the object. """) class CatCommand(Command): """Implementation of gsutil cat command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'cat', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='hr:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='cat', help_name_aliases=[], help_type='command_help', help_one_line_summary='Concatenate object content to stdout', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'cat'], flag_map={ '-h': GcloudStorageFlag('-d'), '-r': GcloudStorageFlag('-r'), }, ) # Command entry point. def RunCommand(self): """Command entry point for the cat command.""" show_header = False request_range = None start_byte = 0 end_byte = None if self.sub_opts: for o, a in self.sub_opts: if o == '-h': show_header = True elif o == '-r': request_range = a.strip() # This if statement ensures the full object is returned # instead of throwing a CommandException. if request_range == '-': continue range_matcher = re.compile( '^(?P<start>[0-9]+)-(?P<end>[0-9]*)$|^(?P<endslice>-[0-9]+)$') range_match = range_matcher.match(request_range) if not range_match: raise CommandException('Invalid range (%s)' % request_range) if range_match.group('start'): start_byte = long(range_match.group('start')) if range_match.group('end'): end_byte = long(range_match.group('end')) if range_match.group('endslice'): start_byte = long(range_match.group('endslice')) else: self.RaiseInvalidArgumentException() return cat_helper.CatHelper(self).CatUrlStrings(self.args, show_header=show_header, start_byte=start_byte, end_byte=end_byte) commands/label.py 0000644 00000037534 15025013616 0010011 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of label command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import codecs import json import os import six from gslib import metrics from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import shim_util from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import LabelTranslation _SET_SYNOPSIS = """ gsutil label set <label-json-file> gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil label get gs://<bucket_name> """ _CH_SYNOPSIS = """ gsutil label ch <label_modifier>... gs://<bucket_name>... where each <label_modifier> is one of the following forms: -l <key>:<value> -d <key> """ _GET_DESCRIPTION = """ <B>GET</B> The "label get" command gets the `labels <https://cloud.google.com/storage/docs/tags-and-labels#bucket-labels>`_ applied to a bucket, which you can save and edit for use with the "label set" command. """ _SET_DESCRIPTION = """ <B>SET</B> The "label set" command allows you to set the labels on one or more buckets. You can retrieve a bucket's labels using the "label get" command, save the output to a file, edit the file, and then use the "label set" command to apply those labels to the specified bucket(s). For example: gsutil label get gs://bucket > labels.json Make changes to labels.json, such as adding an additional label, then: gsutil label set labels.json gs://example-bucket Note that you can set these labels on multiple buckets at once: gsutil label set labels.json gs://bucket-foo gs://bucket-bar """ _CH_DESCRIPTION = """ <B>CH</B> The "label ch" command updates a bucket's label configuration, applying the label changes specified by the -l and -d flags. You can specify multiple label changes in a single command run; all changes will be made atomically to each bucket. <B>CH EXAMPLES</B> Examples for "ch" sub-command: Add the label "key-foo:value-bar" to the bucket "example-bucket": gsutil label ch -l key-foo:value-bar gs://example-bucket Change the above label to have a new value: gsutil label ch -l key-foo:other-value gs://example-bucket Add a new label and delete the old one from above: gsutil label ch -l new-key:new-value -d key-foo gs://example-bucket <B>CH OPTIONS</B> The "ch" sub-command has the following options -l Add or update a label with the specified key and value. -d Remove the label with the specified key. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ Gets, sets, or changes the label configuration (also called the tagging configuration by other storage providers) of one or more buckets. An example label JSON document looks like the following: { "your_label_key": "your_label_value", "your_other_label_key": "your_other_label_value" } The label command has three sub-commands: """ + _GET_DESCRIPTION + _SET_DESCRIPTION + _CH_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) class LabelCommand(Command): """Implementation of gsutil label command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'label', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='l:d:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'ch': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='label', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Get, set, or change the label configuration of a bucket.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text, }, ) gcloud_storage_map = GcloudStorageMap(gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=labels,empty=\' has no label ' 'configuration.\',empty_prefix_key=storage_url,indent=2]' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update', '--labels-file'], flag_map={}, ), 'ch': GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update'], flag_map={ '-d': GcloudStorageFlag( '--remove-labels', repeat_type=shim_util.RepeatFlagType.LIST), '-l': GcloudStorageFlag( '--update-labels', repeat_type=shim_util.RepeatFlagType.DICT), }, ), }, flag_map={}) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 # Filename comes before bucket arg(s). return 1 def _SetLabel(self): """Parses options and sets labels on the specified buckets.""" # At this point, "set" has been popped off the front of self.args. if len(self.args) < 2: self.RaiseWrongNumberOfArgumentsException() label_filename = self.args[0] if not os.path.isfile(label_filename): raise CommandException('Could not find the file "%s".' % label_filename) with codecs.open(label_filename, 'r', UTF8) as label_file: label_text = label_file.read() @Retry(PreconditionException, tries=3, timeout_secs=1) def _SetLabelForBucket(blr): url = blr.storage_url self.logger.info('Setting label configuration on %s...', blr) if url.scheme == 's3': # Uses only XML. self.gsutil_api.XmlPassThroughSetTagging(label_text, url, provider=url.scheme) else: # Must be a 'gs://' bucket. labels_message = None # When performing a read-modify-write cycle, include metageneration to # avoid race conditions (supported for GS buckets only). metageneration = None new_label_json = json.loads(label_text) if (self.gsutil_api.GetApiSelector(url.scheme) == ApiSelector.JSON): # Perform a read-modify-write so that we can specify which # existing labels need to be deleted. _, bucket_metadata = self.GetSingleBucketUrlFromArg( url.url_string, bucket_fields=['labels', 'metageneration']) metageneration = bucket_metadata.metageneration label_json = {} if bucket_metadata.labels: label_json = json.loads( LabelTranslation.JsonFromMessage(bucket_metadata.labels)) # Set all old keys' values to None; this will delete each key that # is not included in the new set of labels. merged_labels = dict( (key, None) for key, _ in six.iteritems(label_json)) merged_labels.update(new_label_json) labels_message = LabelTranslation.DictToMessage(merged_labels) else: # ApiSelector.XML # No need to read-modify-write with the XML API. labels_message = LabelTranslation.DictToMessage(new_label_json) preconditions = Preconditions(meta_gen_match=metageneration) bucket_metadata = apitools_messages.Bucket(labels=labels_message) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) some_matched = False url_args = self.args[1:] for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for bucket_listing_ref in bucket_iter: some_matched = True _SetLabelForBucket(bucket_listing_ref) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _ChLabel(self): """Parses options and changes labels on the specified buckets.""" self.label_changes = {} self.num_deletions = 0 if self.sub_opts: for o, a in self.sub_opts: if o == '-l': label_split = a.split(':') if len(label_split) != 2: raise CommandException( 'Found incorrectly formatted option for "gsutil label ch": ' '"%s". To add a label, please use the form <key>:<value>.' % a) self.label_changes[label_split[0]] = label_split[1] elif o == '-d': # Ensure only the key is supplied; stop if key:value was given. val_split = a.split(':') if len(val_split) != 1: raise CommandException( 'Found incorrectly formatted option for "gsutil label ch": ' '"%s". To delete a label, provide only its key.' % a) self.label_changes[a] = None self.num_deletions += 1 else: self.RaiseInvalidArgumentException() if not self.label_changes: raise CommandException( 'Please specify at least one label change with the -l or -d flags.') @Retry(PreconditionException, tries=3, timeout_secs=1) def _ChLabelForBucket(blr): url = blr.storage_url self.logger.info('Setting label configuration on %s...', blr) labels_message = None # When performing a read-modify-write cycle, include metageneration to # avoid race conditions (supported for GS buckets only). metageneration = None if (self.gsutil_api.GetApiSelector(url.scheme) == ApiSelector.JSON): # The JSON API's PATCH semantics allow us to skip read-modify-write, # with the exception of one edge case - attempting to delete a # nonexistent label returns an error iff no labels previously existed corrected_changes = self.label_changes if self.num_deletions: (_, bucket_metadata) = self.GetSingleBucketUrlFromArg( url.url_string, bucket_fields=['labels', 'metageneration']) if not bucket_metadata.labels: metageneration = bucket_metadata.metageneration # Remove each change that would try to delete a nonexistent key. corrected_changes = dict( (k, v) for k, v in six.iteritems(self.label_changes) if v) labels_message = LabelTranslation.DictToMessage(corrected_changes) else: # ApiSelector.XML # Perform a read-modify-write cycle so that we can specify which # existing labels need to be deleted. (_, bucket_metadata) = self.GetSingleBucketUrlFromArg( url.url_string, bucket_fields=['labels', 'metageneration']) metageneration = bucket_metadata.metageneration label_json = {} if bucket_metadata.labels: label_json = json.loads( LabelTranslation.JsonFromMessage(bucket_metadata.labels)) # Modify label_json such that all specified labels are added # (overwriting old labels if necessary) and all specified deletions # are removed from label_json if already present. for key, value in six.iteritems(self.label_changes): if not value and key in label_json: del label_json[key] else: label_json[key] = value labels_message = LabelTranslation.DictToMessage(label_json) preconditions = Preconditions(meta_gen_match=metageneration) bucket_metadata = apitools_messages.Bucket(labels=labels_message) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True _ChLabelForBucket(bucket_listing_ref) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetAndPrintLabel(self, bucket_arg): """Gets and prints the labels for a cloud bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( bucket_arg, bucket_fields=['labels']) if bucket_url.scheme == 's3': print((self.gsutil_api.XmlPassThroughGetTagging( bucket_url, provider=bucket_url.scheme))) else: if bucket_metadata.labels: print((LabelTranslation.JsonFromMessage(bucket_metadata.labels, pretty_print=True))) else: print(('%s has no label configuration.' % bucket_url)) def RunCommand(self): """Command entry point for the label command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) if action_subcommand == 'get': metrics.LogCommandParams(subcommands=[action_subcommand]) self._GetAndPrintLabel(self.args[0]) elif action_subcommand == 'set': metrics.LogCommandParams(subcommands=[action_subcommand]) self._SetLabel() elif action_subcommand == 'ch': metrics.LogCommandParams(subcommands=[action_subcommand]) self._ChLabel() else: raise CommandException( 'Invalid subcommand "%s" for the %s command.\nSee "gsutil help %s".' % (action_subcommand, self.command_name, self.command_name)) return 0 commands/ls.py 0000644 00000065317 15025013616 0007350 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like ls command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import six from gslib.cloud_api import NotFoundException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.constants import S3_DELETE_MARKER_GUID from gslib.utils.constants import UTF8 from gslib.utils.ls_helper import ENCRYPTED_FIELDS from gslib.utils.ls_helper import LsHelper from gslib.utils.ls_helper import PrintFullInfoAboutObject from gslib.utils.ls_helper import UNENCRYPTED_FULL_LISTING_FIELDS from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAscii from gslib.utils import text_util from gslib.utils.translation_helper import AclTranslation from gslib.utils.translation_helper import LabelTranslation from gslib.utils.unit_util import MakeHumanReadable # Regex that assists with converting JSON timestamp to ls-style output. # This excludes timestamp fractional seconds, for example: # 2013-07-03 20:32:53.048000+00:00 JSON_TIMESTAMP_RE = re.compile(r'([^\s]*)\s([^\.\+]*).*') _SYNOPSIS = """ gsutil ls [-a] [-b] [-d] [-l] [-L] [-r] [-p proj_id] url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Retrieves a list of providers, buckets, or objects matching the criteria, ordered in the list lexicographically by name. <B>LISTING PROVIDERS, BUCKETS, SUBDIRECTORIES, AND OBJECTS</B> If you run ``gsutil ls`` without URLs, it lists all of the Google Cloud Storage buckets under your default project ID (or all of the Cloud Storage buckets under the project you specify with the ``-p`` flag): gsutil ls If you specify one or more provider URLs, ``gsutil ls`` lists buckets at each listed provider: gsutil ls gs:// gsutil currently supports ``gs://`` and ``s3://`` as valid providers If you specify bucket URLs, or use `URI wildcards <https://cloud.google.com/storage/docs/wildcards>`_ to capture a set of buckets, ``gsutil ls`` lists objects at the top level of each bucket, along with the names of each subdirectory. For example: gsutil ls gs://bucket might produce output like: gs://bucket/obj1.htm gs://bucket/obj2.htm gs://bucket/images1/ gs://bucket/images2/ The "/" at the end of the last 2 URLs tells you they are subdirectories, which you can list using: gsutil ls gs://bucket/images* If you specify object URLs, ``gsutil ls`` lists the specified objects. For example: gsutil ls gs://bucket/*.txt lists all files whose name matches the above wildcard at the top level of the bucket. For more details, see `URI wildcards <https://cloud.google.com/storage/docs/wildcards>`_. <B>DIRECTORY BY DIRECTORY, FLAT, and RECURSIVE LISTINGS</B> Listing a bucket or subdirectory (as illustrated near the end of the previous section) only shows the objects and names of subdirectories it contains. You can list all objects in a bucket by using the -r option. For example: gsutil ls -r gs://bucket lists the top-level objects and buckets, then the objects and buckets under gs://bucket/images1, then those under gs://bucket/images2, etc. If you want to see all objects in the bucket in one "flat" listing use the recursive ("**") wildcard, like: gsutil ls -r gs://bucket/** or, for a flat listing of a subdirectory: gsutil ls -r gs://bucket/dir/** If you want to see only the subdirectory itself, use the -d option: gsutil ls -d gs://bucket/dir <B>LISTING OBJECT DETAILS</B> If you specify the -l option, gsutil outputs additional information about each matching provider, bucket, subdirectory, or object. For example: gsutil ls -l gs://bucket/*.html gs://bucket/*.txt prints the object size, creation time stamp, and name of each matching object, along with the total count and sum of sizes of all matching objects: 2276224 2020-03-02T19:25:17Z gs://bucket/obj1.html 3914624 2020-03-02T19:30:27Z gs://bucket/obj2.html 131 2020-03-02T19:37:45Z gs://bucket/obj3.txt TOTAL: 3 objects, 6190979 bytes (5.9 MiB) Note that the total listed in parentheses above is in mebibytes (or gibibytes, tebibytes, etc.), which corresponds to the unit of billing measurement for Google Cloud Storage. You can get a listing of all the objects in the top-level bucket directory (along with the total count and sum of sizes) using a command like: gsutil ls -l gs://bucket To print additional detail about objects and buckets use the gsutil ls -L option. For example: gsutil ls -L gs://bucket/obj1 prints something like: gs://bucket/obj1: Creation time: Fri, 26 May 2017 22:55:44 GMT Update time: Tue, 18 Jul 2017 12:31:18 GMT Storage class: STANDARD Content-Length: 60183 Content-Type: image/jpeg Hash (crc32c): zlUhtg== Hash (md5): Bv86IAzFzrD1Z2io/c7yqA== ETag: 5ca67960a586723b7344afffc81 Generation: 1378862725952000 Metageneration: 1 ACL: [ { "entity": "project-owners-867484910061", "projectTeam": { "projectNumber": "867484910061", "team": "owners" }, "role": "OWNER" }, { "email": "jane@gmail.com", "entity": "user-jane@gmail.com", "role": "OWNER" } ] TOTAL: 1 objects, 60183 bytes (58.77 KiB) Note that results may contain additional fields, such as custom metadata or a storage class update time, if they are applicable to the object. Also note that some fields, such as update time, are not available with the (non-default) XML API. See also "gsutil help acl" for getting a more readable version of the ACL. <B>LISTING BUCKET DETAILS</B> If you want to see information about the bucket itself, use the -b option. For example: gsutil ls -L -b gs://bucket prints something like: gs://bucket/ : Storage class: STANDARD Location constraint: US Versioning enabled: False Logging configuration: None Website configuration: None CORS configuration: None Lifecycle configuration: None Requester Pays enabled: True Labels: None Default KMS key: None Time created: Thu, 14 Jan 2016 19:25:17 GMT Time updated: Thu, 08 Jun 2017 21:17:59 GMT Metageneration: 1 Bucket Policy Only enabled: False ACL: [ { "entity": "project-owners-867489160491", "projectTeam": { "projectNumber": "867489160491", "team": "owners" }, "role": "OWNER" } ] Default ACL: [ { "entity": "project-owners-867489160491", "projectTeam": { "projectNumber": "867489160491", "team": "owners" }, "role": "OWNER" } ] Note that some fields above (time created, time updated, metageneration) are not available with the (non-default) XML API. <B>OPTIONS</B> -l Prints long listing (owner, length). -L Prints even more detail than -l. Note: If you use this option with the (non-default) XML API it generates an additional request per object being listed, which makes the -L option run much more slowly and cost more than the default JSON API. -d List matching subdirectory names instead of contents, and do not recurse into matching subdirectories even if the -R option is specified. -b Prints info about the bucket when used with a bucket URL. -h When used with -l, prints object sizes in human readable format (e.g., 1 KiB, 234 MiB, 2 GiB, etc.) -p proj_id Specifies the project ID or project number to use for listing buckets. -R, -r Requests a recursive listing, performing at least one listing operation per subdirectory. If you have a large number of subdirectories and do not require recursive-style output ordering, you may be able to instead use wildcards to perform a flat listing, e.g. ``gsutil ls gs://mybucket/**``, which generally performs fewer listing operations. -a Includes non-current object versions / generations in the listing (only useful with a versioning-enabled bucket). If combined with -l option also prints metageneration for each listed object. -e Include ETag in long listing (-l) output. """) class ListingStyle(object): """Enum class for specifying listing style.""" SHORT = 'SHORT' LONG = 'LONG' LONG_LONG = 'LONG_LONG' class LsCommand(Command): """Implementation of gsutil ls command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'ls', command_name_aliases=[ 'dir', 'list', ], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='aebdlLhp:rR', file_url_ok=False, provider_url_ok=True, urls_start_arg=0, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='ls', help_name_aliases=[ 'dir', 'list', ], help_type='command_help', help_one_line_summary='List providers, buckets, or objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) # TODO(b/206151616) Add mappings for remaining flags. gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'ls', '--fetch-encrypted-object-hashes'], flag_map={ '-r': GcloudStorageFlag('-r'), '-R': GcloudStorageFlag('-r'), '-l': GcloudStorageFlag('-l'), '-L': GcloudStorageFlag('-L'), '-b': GcloudStorageFlag('-b'), '-e': GcloudStorageFlag('-e'), '-a': GcloudStorageFlag('-a'), '-h': GcloudStorageFlag('--readable-sizes'), '-p': GcloudStorageFlag('--project'), }, ) def _PrintBucketInfo(self, bucket_blr, listing_style): """Print listing info for given bucket. Args: bucket_blr: BucketListingReference for the bucket being listed listing_style: ListingStyle enum describing type of output desired. Returns: Tuple (total objects, total bytes) in the bucket. """ if (listing_style == ListingStyle.SHORT or listing_style == ListingStyle.LONG): text_util.print_to_fd(bucket_blr) return # listing_style == ListingStyle.LONG_LONG: # We're guaranteed by the caller that the root object is populated. bucket = bucket_blr.root_object location_constraint = bucket.location storage_class = bucket.storageClass fields = { 'bucket': bucket_blr.url_string, 'storage_class': storage_class, 'location_constraint': location_constraint, 'acl': AclTranslation.JsonFromMessage(bucket.acl), 'default_acl': AclTranslation.JsonFromMessage(bucket.defaultObjectAcl), 'versioning': bucket.versioning and bucket.versioning.enabled, 'website_config': 'Present' if bucket.website else 'None', 'logging_config': 'Present' if bucket.logging else 'None', 'cors_config': 'Present' if bucket.cors else 'None', 'lifecycle_config': 'Present' if bucket.lifecycle else 'None', 'requester_pays': bucket.billing and bucket.billing.requesterPays } if bucket.retentionPolicy: fields['retention_policy'] = 'Present' if bucket.labels: fields['labels'] = LabelTranslation.JsonFromMessage(bucket.labels, pretty_print=True) else: fields['labels'] = 'None' if bucket.encryption and bucket.encryption.defaultKmsKeyName: fields['default_kms_key'] = bucket.encryption.defaultKmsKeyName else: fields['default_kms_key'] = 'None' fields['encryption_config'] = 'Present' if bucket.encryption else 'None' # Fields not available in all APIs (e.g. the XML API) if bucket.autoclass and bucket.autoclass.enabled: fields['autoclass_enabled_date'] = ( bucket.autoclass.toggleTime.strftime('%a, %d %b %Y')) if bucket.locationType: fields['location_type'] = bucket.locationType if bucket.customPlacementConfig: fields['custom_placement_locations'] = ( bucket.customPlacementConfig.dataLocations) if bucket.metageneration: fields['metageneration'] = bucket.metageneration if bucket.timeCreated: fields['time_created'] = bucket.timeCreated.strftime( '%a, %d %b %Y %H:%M:%S GMT') if bucket.updated: fields['updated'] = bucket.updated.strftime('%a, %d %b %Y %H:%M:%S GMT') if bucket.defaultEventBasedHold: fields['default_eventbased_hold'] = bucket.defaultEventBasedHold if bucket.iamConfiguration: if bucket.iamConfiguration.bucketPolicyOnly: enabled = bucket.iamConfiguration.bucketPolicyOnly.enabled fields['bucket_policy_only_enabled'] = enabled if bucket.iamConfiguration.publicAccessPrevention: fields[ 'public_access_prevention'] = bucket.iamConfiguration.publicAccessPrevention if bucket.rpo: fields['rpo'] = bucket.rpo if bucket.satisfiesPZS: fields['satisfies_pzs'] = bucket.satisfiesPZS # For field values that are multiline, add indenting to make it look # prettier. for key in fields: previous_value = fields[key] if (not isinstance(previous_value, six.string_types) or '\n' not in previous_value): continue new_value = previous_value.replace('\n', '\n\t ') # Start multiline values on a new line if they aren't already. if not new_value.startswith('\n'): new_value = '\n\t ' + new_value fields[key] = new_value # Only display certain properties if the given API returned them (JSON API # returns many fields that the XML API does not). autoclass_line = '' location_type_line = '' custom_placement_locations_line = '' metageneration_line = '' time_created_line = '' time_updated_line = '' default_eventbased_hold_line = '' retention_policy_line = '' bucket_policy_only_enabled_line = '' public_access_prevention_line = '' rpo_line = '' satisifies_pzs_line = '' if 'autoclass_enabled_date' in fields: autoclass_line = '\tAutoclass:\t\t\tEnabled on {autoclass_enabled_date}\n' if 'location_type' in fields: location_type_line = '\tLocation type:\t\t\t{location_type}\n' if 'custom_placement_locations' in fields: custom_placement_locations_line = ( '\tPlacement locations:\t\t{custom_placement_locations}\n') if 'metageneration' in fields: metageneration_line = '\tMetageneration:\t\t\t{metageneration}\n' if 'time_created' in fields: time_created_line = '\tTime created:\t\t\t{time_created}\n' if 'updated' in fields: time_updated_line = '\tTime updated:\t\t\t{updated}\n' if 'default_eventbased_hold' in fields: default_eventbased_hold_line = ( '\tDefault Event-Based Hold:\t{default_eventbased_hold}\n') if 'retention_policy' in fields: retention_policy_line = '\tRetention Policy:\t\t{retention_policy}\n' if 'bucket_policy_only_enabled' in fields: bucket_policy_only_enabled_line = ('\tBucket Policy Only enabled:\t' '{bucket_policy_only_enabled}\n') if 'public_access_prevention' in fields: public_access_prevention_line = ('\tPublic access prevention:\t' '{public_access_prevention}\n') if 'rpo' in fields: rpo_line = ('\tRPO:\t\t\t\t{rpo}\n') if 'satisfies_pzs' in fields: satisifies_pzs_line = '\tSatisfies PZS:\t\t\t{satisfies_pzs}\n' text_util.print_to_fd( ('{bucket} :\n' '\tStorage class:\t\t\t{storage_class}\n' + location_type_line + '\tLocation constraint:\t\t{location_constraint}\n' + custom_placement_locations_line + '\tVersioning enabled:\t\t{versioning}\n' '\tLogging configuration:\t\t{logging_config}\n' '\tWebsite configuration:\t\t{website_config}\n' '\tCORS configuration: \t\t{cors_config}\n' '\tLifecycle configuration:\t{lifecycle_config}\n' '\tRequester Pays enabled:\t\t{requester_pays}\n' + retention_policy_line + default_eventbased_hold_line + '\tLabels:\t\t\t\t{labels}\n' + '\tDefault KMS key:\t\t{default_kms_key}\n' + time_created_line + time_updated_line + metageneration_line + bucket_policy_only_enabled_line + autoclass_line + public_access_prevention_line + rpo_line + satisifies_pzs_line + '\tACL:\t\t\t\t{acl}\n' '\tDefault ACL:\t\t\t{default_acl}').format(**fields)) if bucket_blr.storage_url.scheme == 's3': text_util.print_to_fd( 'Note: this is an S3 bucket so configuration values may be ' 'blank. To retrieve bucket configuration values, use ' 'individual configuration commands such as gsutil acl get ' '<bucket>.') def _PrintLongListing(self, bucket_listing_ref): """Prints an object with ListingStyle.LONG.""" obj = bucket_listing_ref.root_object url_str = bucket_listing_ref.url_string if (obj.metadata and S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties): size_string = '0' num_bytes = 0 num_objs = 0 url_str += '<DeleteMarker>' else: size_string = (MakeHumanReadable(obj.size) if self.human_readable else str(obj.size)) num_bytes = obj.size num_objs = 1 timestamp = JSON_TIMESTAMP_RE.sub(r'\1T\2Z', str(obj.timeCreated)) printstr = '%(size)10s %(timestamp)s %(url)s' encoded_etag = None encoded_metagen = None if self.all_versions: printstr += ' metageneration=%(metageneration)s' encoded_metagen = str(obj.metageneration) if self.include_etag: printstr += ' etag=%(etag)s' encoded_etag = obj.etag format_args = { 'size': size_string, 'timestamp': timestamp, 'url': url_str, 'metageneration': encoded_metagen, 'etag': encoded_etag } text_util.print_to_fd(printstr % format_args) return (num_objs, num_bytes) def RunCommand(self): """Command entry point for the ls command.""" got_nomatch_errors = False got_bucket_nomatch_errors = False listing_style = ListingStyle.SHORT get_bucket_info = False self.recursion_requested = False self.all_versions = False self.include_etag = False self.human_readable = False self.list_subdir_contents = True if self.sub_opts: for o, a in self.sub_opts: if o == '-a': self.all_versions = True elif o == '-e': self.include_etag = True elif o == '-b': get_bucket_info = True elif o == '-h': self.human_readable = True elif o == '-l': listing_style = ListingStyle.LONG elif o == '-L': listing_style = ListingStyle.LONG_LONG elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii(a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-r' or o == '-R': self.recursion_requested = True elif o == '-d': self.list_subdir_contents = False if not self.args: # default to listing all gs buckets self.args = ['gs://'] total_objs = 0 total_bytes = 0 def MaybePrintBucketHeader(blr): if len(self.args) > 1: text_util.print_to_fd('%s:' % six.ensure_text(blr.url_string)) print_bucket_header = MaybePrintBucketHeader for url_str in self.args: storage_url = StorageUrlFromString(url_str) if storage_url.IsFileUrl(): raise CommandException('Only cloud URLs are supported for %s' % self.command_name) bucket_fields = None if (listing_style == ListingStyle.SHORT or listing_style == ListingStyle.LONG): bucket_fields = ['id'] elif listing_style == ListingStyle.LONG_LONG: bucket_fields = [ 'acl', 'autoclass', 'billing', 'cors', 'customPlacementConfig', 'defaultObjectAcl', 'encryption', 'iamConfiguration', 'labels', 'location', 'locationType', 'logging', 'lifecycle', 'metageneration', 'retentionPolicy', 'defaultEventBasedHold', 'rpo', 'satisfiesPZS', 'storageClass', 'timeCreated', 'updated', 'versioning', 'website', ] if storage_url.IsProvider(): # Provider URL: use bucket wildcard to list buckets. for blr in self.WildcardIterator( '%s://*' % storage_url.scheme).IterBuckets(bucket_fields=bucket_fields): self._PrintBucketInfo(blr, listing_style) elif storage_url.IsBucket() and get_bucket_info: # ls -b bucket listing request: List info about bucket(s). total_buckets = 0 for blr in self.WildcardIterator(url_str).IterBuckets( bucket_fields=bucket_fields): if not ContainsWildcard(url_str) and not blr.root_object: # Iterator does not make an HTTP call for non-wildcarded # listings with fields=='id'. Ensure the bucket exists by calling # GetBucket. self.gsutil_api.GetBucket(blr.storage_url.bucket_name, fields=['id'], provider=storage_url.scheme) self._PrintBucketInfo(blr, listing_style) total_buckets += 1 if not ContainsWildcard(url_str) and not total_buckets: got_bucket_nomatch_errors = True else: # URL names a bucket, object, or object subdir -> # list matching object(s) / subdirs. def _PrintPrefixLong(blr): text_util.print_to_fd('%-33s%s' % ('', six.ensure_text(blr.url_string))) if listing_style == ListingStyle.SHORT: # ls helper by default readies us for a short listing. listing_helper = LsHelper( self.WildcardIterator, self.logger, all_versions=self.all_versions, print_bucket_header_func=print_bucket_header, should_recurse=self.recursion_requested, list_subdir_contents=self.list_subdir_contents) elif listing_style == ListingStyle.LONG: bucket_listing_fields = [ 'name', 'size', 'timeCreated', 'updated', ] if self.all_versions: bucket_listing_fields.extend([ 'generation', 'metageneration', ]) if self.include_etag: bucket_listing_fields.append('etag') listing_helper = LsHelper( self.WildcardIterator, self.logger, print_object_func=self._PrintLongListing, print_dir_func=_PrintPrefixLong, print_bucket_header_func=print_bucket_header, all_versions=self.all_versions, should_recurse=self.recursion_requested, fields=bucket_listing_fields, list_subdir_contents=self.list_subdir_contents) elif listing_style == ListingStyle.LONG_LONG: # List all fields bucket_listing_fields = (UNENCRYPTED_FULL_LISTING_FIELDS + ENCRYPTED_FIELDS) listing_helper = LsHelper( self.WildcardIterator, self.logger, print_object_func=PrintFullInfoAboutObject, print_dir_func=_PrintPrefixLong, print_bucket_header_func=print_bucket_header, all_versions=self.all_versions, should_recurse=self.recursion_requested, fields=bucket_listing_fields, list_subdir_contents=self.list_subdir_contents) else: raise CommandException('Unknown listing style: %s' % listing_style) exp_dirs, exp_objs, exp_bytes = ( listing_helper.ExpandUrlAndPrint(storage_url)) if storage_url.IsObject() and exp_objs == 0 and exp_dirs == 0: got_nomatch_errors = True total_bytes += exp_bytes total_objs += exp_objs if total_objs and listing_style != ListingStyle.SHORT: text_util.print_to_fd( 'TOTAL: %d objects, %d bytes (%s)' % (total_objs, total_bytes, MakeHumanReadable(float(total_bytes)))) if got_nomatch_errors: raise CommandException('One or more URLs matched no objects.') if got_bucket_nomatch_errors: raise NotFoundException('One or more bucket URLs matched no buckets.') return 0 commands/defstorageclass.py 0000644 00000017352 15025013616 0012077 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Impl. of default bucket storage class command for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import NormalizeStorageClass from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil defstorageclass set <storage-class> gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil defstorageclass get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The "defstorageclass set" command sets the default `storage class <https://cloud.google.com/storage/docs/storage-classes>`_ for the specified bucket(s). If you specify a default storage class for a certain bucket, Google Cloud Storage applies the default storage class to all new objects uploaded to that bucket, except when the storage class is overridden by individual upload requests. Setting a default storage class on a bucket provides a convenient way to ensure newly uploaded objects have a specific storage class. If you don't set the bucket's default storage class, it will default to Standard. """ _GET_DESCRIPTION = """ <B>GET</B> Gets the default storage class for a bucket. """ _DESCRIPTION = """ The defstorageclass command has two sub-commands: """ + '\n'.join([_SET_DESCRIPTION + _GET_DESCRIPTION]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) # The url_string for buckets ends with a slash. # Substitute the last slash with a colon. _GCLOUD_FORMAT_STRING = ('--format=value[separator=\": \"](name.sub("' + shim_util.get_format_flag_caret() + '", "gs://"),' 'storageClass)') SHIM_GET_COMMAND_MAP = GcloudStorageMap( # Using a list because a string gets splitted up on space and the # format string below has a space. gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING, '--raw' ], flag_map={}, ) SHIM_SET_COMMAND_MAP = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update', '--default-storage-class'], flag_map={}, ) class DefStorageClassCommand(Command): """Implementation of gsutil defstorageclass command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'defstorageclass', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ # FreeTextArgument allows for using storage class abbreviations. CommandArgument.MakeFreeTextArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='defstorageclass', help_name_aliases=['defaultstorageclass'], help_type='command_help', help_one_line_summary='Get or set the default storage class on buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': SHIM_GET_COMMAND_MAP, 'set': SHIM_SET_COMMAND_MAP, }, flag_map={}, ) def _CheckIsGsUrl(self, url_str): if not url_str.startswith('gs://'): raise CommandException( '"%s" does not support the URL "%s". Did you mean to use a gs:// ' 'URL?' % (self.command_name, url_str)) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetDefStorageClass(self): """Sets the default storage class for a bucket.""" # At this point, "set" has been popped off the front of self.args. normalized_storage_class = NormalizeStorageClass(self.args[0]) url_args = self.args[1:] if not url_args: self.RaiseWrongNumberOfArgumentsException() some_matched = False for url_str in url_args: self._CheckIsGsUrl(url_str) # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: some_matched = True bucket_metadata = apitools_messages.Bucket() self.logger.info('Setting default storage class to "%s" for bucket %s' % (normalized_storage_class, blr.url_string.rstrip('/'))) bucket_metadata.storageClass = normalized_storage_class self.gsutil_api.PatchBucket(blr.storage_url.bucket_name, bucket_metadata, provider=blr.storage_url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetDefStorageClass(self): """Gets the default storage class for a bucket.""" # At this point, "get" has been popped off the front of self.args. url_args = self.args some_matched = False for url_str in url_args: self._CheckIsGsUrl(url_str) bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['storageClass']) for blr in bucket_iter: some_matched = True print('%s: %s' % (blr.url_string.rstrip('/'), blr.root_object.storageClass)) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def RunCommand(self): """Command entry point for the defstorageclass command.""" action_subcommand = self.args.pop(0) subcommand_args = [action_subcommand] if action_subcommand == 'get': func = self._GetDefStorageClass elif action_subcommand == 'set': func = self._SetDefStorageClass normalized_storage_class = NormalizeStorageClass(self.args[0]) subcommand_args.append(normalized_storage_class) else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help %s".') % (action_subcommand, self.command_name, self.command_name)) metrics.LogCommandParams(subcommands=subcommand_args) func() return 0 commands/rm.py 0000644 00000040001 15025013616 0007327 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like rm command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import time from gslib.cloud_api import BucketNotFoundException from gslib.cloud_api import NotEmptyException from gslib.cloud_api import NotFoundException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import DecrementFailureCount from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_PREFIX from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import StorageUrlFromString from gslib.thread_message import MetadataMessage from gslib.utils import constants from gslib.utils import parallelism_framework_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import StdinIterator from gslib.utils.translation_helper import PreconditionsFromHeaders _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout _SYNOPSIS = """ gsutil rm [-f] [-r] url... gsutil rm [-f] [-r] -I """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> NOTE: As part of verifying the existence of objects prior to deletion, ``gsutil rm`` makes ``GET`` requests to Cloud Storage for object metadata. These requests incur `network and operations charges <https://cloud.google.com/storage/pricing>`_. The gsutil rm command removes objects and/or buckets. For example, the following command removes the object ``kitten.png``: gsutil rm gs://bucket/kitten.png Use the -r option to specify recursive object deletion. For example, the following command removes gs://bucket/subdir and all objects and subdirectories under it: gsutil rm -r gs://bucket/subdir When working with versioning-enabled buckets, note that the -r option removes all object versions in the subdirectory. To remove only the live version of each object in the subdirectory, use the `** wildcard <https://cloud.google.com/storage/docs/wildcards>`_. The following command removes all versions of all objects in a bucket, and then deletes the bucket: gsutil rm -r gs://bucket To remove all objects and their versions from a bucket without deleting the bucket, use the ``-a`` option: gsutil rm -a gs://bucket/** If you have a large number of objects to remove, use the ``gsutil -m`` option, which enables multi-threading/multi-processing: gsutil -m rm -r gs://my_bucket/subdir You can pass a list of URLs (one per line) to remove on stdin instead of as command line arguments by using the -I option. This allows you to use gsutil in a pipeline to remove objects identified by a program, such as: some_program | gsutil -m rm -I The contents of stdin can name cloud URLs and wildcards of cloud URLs. Note that ``gsutil rm`` refuses to remove files from the local file system. For example, this fails: gsutil rm *.txt WARNING: Object removal cannot be undone. Cloud Storage is designed to give developers a high amount of flexibility and control over their data, and Google maintains strict controls over the processing and purging of deleted data. If you have concerns that your application software or your users may at some point erroneously delete or replace data, see `Options for controlling data lifecycles <https://cloud.google.com/storage/docs/control-data-lifecycles>`_ for ways to protect your data from accidental data deletion. <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when removing multiple objects. If some of the objects could not be removed, gsutil's exit status will be non-zero even if this flag is set. Execution will still halt if an inaccessible bucket is encountered. This option is implicitly set when running "gsutil -m rm ...". -I Causes gsutil to read the list of objects to remove from stdin. This allows you to run a program that generates the list of objects to remove. -R, -r The -R and -r options are synonymous. Causes bucket or bucket subdirectory contents (all objects and subdirectories that it contains) to be removed recursively. If used with a bucket-only URL (like gs://bucket), after deleting objects and subdirectories gsutil deletes the bucket. This option implies the -a option and deletes all object versions. If you only want to delete live object versions, use the `** wildcard <https://cloud.google.com/storage/docs/wildcards>`_ instead of -r. -a Delete all versions of an object. """) def _RemoveExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" if not cls.continue_on_error: cls.logger.error(str(e)) # TODO: Use shared state to track missing bucket names when we get a # BucketNotFoundException. Then improve bucket removal logic and exception # messages. if isinstance(e, BucketNotFoundException): cls.bucket_not_found_count += 1 cls.logger.error(str(e)) else: if _ExceptionMatchesBucketToDelete(cls.bucket_strings_to_delete, e): DecrementFailureCount() else: cls.op_failure_count += 1 # pylint: disable=unused-argument def _RemoveFoldersExceptionHandler(cls, e): """When removing folders, we don't mind if none exist.""" if ((isinstance(e, CommandException) and NO_URLS_MATCHED_PREFIX in e.reason) or isinstance(e, NotFoundException)): DecrementFailureCount() else: raise e def _RemoveFuncWrapper(cls, name_expansion_result, thread_state=None): cls.RemoveFunc(name_expansion_result, thread_state=thread_state) def _ExceptionMatchesBucketToDelete(bucket_strings_to_delete, e): """Returns True if the exception matches a bucket slated for deletion. A recursive delete call on an empty bucket will raise an exception when listing its objects, but if we plan to delete the bucket that shouldn't result in a user-visible error. Args: bucket_strings_to_delete: Buckets slated for recursive deletion. e: Exception to check. Returns: True if the exception was a no-URLs-matched exception and it matched one of bucket_strings_to_delete, None otherwise. """ if bucket_strings_to_delete: msg = NO_URLS_MATCHED_TARGET % '' if msg in str(e): parts = str(e).split(msg) return len(parts) == 2 and parts[1] in bucket_strings_to_delete class RmCommand(Command): """Implementation of gsutil rm command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rm', command_name_aliases=['del', 'delete', 'remove'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=constants.NO_MAX, supported_sub_args='afIrR', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rm', help_name_aliases=['del', 'delete', 'remove'], help_type='command_help', help_one_line_summary='Remove objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'rm'], flag_map={ '-r': GcloudStorageFlag('-r'), '-R': GcloudStorageFlag('-r'), '-a': GcloudStorageFlag('-a'), '-I': GcloudStorageFlag('-I'), '-f': GcloudStorageFlag('--continue-on-error'), }, ) def RunCommand(self): """Command entry point for the rm command.""" # self.recursion_requested is initialized in command.py (so it can be # checked in parent class for all commands). self.continue_on_error = self.parallel_operations self.read_args_from_stdin = False self.all_versions = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-a': self.all_versions = True elif o == '-f': self.continue_on_error = True elif o == '-I': self.read_args_from_stdin = True elif o == '-r' or o == '-R': self.recursion_requested = True self.all_versions = True if self.read_args_from_stdin: if self.args: raise CommandException('No arguments allowed with the -I flag.') url_strs = StdinIterator() else: if not self.args: raise CommandException('The rm command (without -I) expects at ' 'least one URL.') url_strs = self.args # Tracks number of object deletes that failed. self.op_failure_count = 0 # Tracks if any buckets were missing. self.bucket_not_found_count = 0 # Tracks buckets that are slated for recursive deletion. bucket_urls_to_delete = [] self.bucket_strings_to_delete = [] if self.recursion_requested: bucket_fields = ['id'] for url_str in url_strs: url = StorageUrlFromString(url_str) if url.IsBucket() or url.IsProvider(): for blr in self.WildcardIterator(url_str).IterBuckets( bucket_fields=bucket_fields): bucket_urls_to_delete.append(blr.storage_url) self.bucket_strings_to_delete.append(url_str) self.preconditions = PreconditionsFromHeaders(self.headers or {}) try: # Expand wildcards, dirs, buckets, and bucket subdirs in URLs. name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, url_strs, self.recursion_requested, project_id=self.project_id, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations) seek_ahead_iterator = None # Cannot seek ahead with stdin args, since we can only iterate them # once without buffering in memory. if not self.read_args_from_stdin: seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), url_strs, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) # Perform remove requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(_RemoveFuncWrapper, name_expansion_iterator, _RemoveExceptionHandler, fail_on_error=(not self.continue_on_error), shared_attrs=['op_failure_count', 'bucket_not_found_count'], seek_ahead_iterator=seek_ahead_iterator) # Assuming the bucket has versioning enabled, url's that don't map to # objects should throw an error even with all_versions, since the prior # round of deletes only sends objects to a history table. # This assumption that rm -a is only called for versioned buckets should be # corrected, but the fix is non-trivial. except CommandException as e: # Don't raise if there are buckets to delete -- it's valid to say: # gsutil rm -r gs://some_bucket # if the bucket is empty. if _ExceptionMatchesBucketToDelete(self.bucket_strings_to_delete, e): DecrementFailureCount() else: raise except ServiceException as e: if not self.continue_on_error: raise if self.bucket_not_found_count: raise CommandException('Encountered non-existent bucket during listing') if self.op_failure_count and not self.continue_on_error: raise CommandException('Some files could not be removed.') # If this was a gsutil rm -r command covering any bucket subdirs, # remove any dir_$folder$ objects (which are created by various web UI # tools to simulate folders). if self.recursion_requested: folder_object_wildcards = [] for url_str in url_strs: url = StorageUrlFromString(url_str) if url.IsObject(): folder_object_wildcards.append(url_str.rstrip('*') + '*_$folder$') if folder_object_wildcards: self.continue_on_error = True try: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, folder_object_wildcards, self.recursion_requested, project_id=self.project_id, all_versions=self.all_versions) # When we're removing folder objects, always continue on error self.Apply(_RemoveFuncWrapper, name_expansion_iterator, _RemoveFoldersExceptionHandler, fail_on_error=False) except CommandException as e: # Ignore exception from name expansion due to an absent folder file. if not e.reason.startswith(NO_URLS_MATCHED_PREFIX): raise # Now that all data has been deleted, delete any bucket URLs. for url in bucket_urls_to_delete: self.logger.info('Removing %s...', url) @Retry(NotEmptyException, tries=3, timeout_secs=1) def BucketDeleteWithRetry(): self.gsutil_api.DeleteBucket(url.bucket_name, provider=url.scheme) BucketDeleteWithRetry() if self.op_failure_count: plural_str = 's' if self.op_failure_count else '' raise CommandException('%d file%s/object%s could not be removed.' % (self.op_failure_count, plural_str, plural_str)) return 0 def RemoveFunc(self, name_expansion_result, thread_state=None): gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) exp_src_url = name_expansion_result.expanded_storage_url self.logger.info('Removing %s...', exp_src_url) try: gsutil_api.DeleteObject(exp_src_url.bucket_name, exp_src_url.object_name, preconditions=self.preconditions, generation=exp_src_url.generation, provider=exp_src_url.scheme) except NotFoundException as e: # DeleteObject will sometimes return a 504 (DEADLINE_EXCEEDED) when # the operation was in fact successful. When a retry is attempted in # these cases, it will fail with a (harmless) 404. The 404 is harmless # since it really just means the file was already deleted, which is # what we want anyway. Here we simply downgrade the message to info # rather than error and correct the command-level failure total. self.logger.info('Cannot find %s', exp_src_url) DecrementFailureCount() _PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) commands/logging.py 0000644 00000031674 15025013616 0010357 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of logging configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from apitools.base.py import encoding from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import text_util _SET_SYNOPSIS = """ gsutil logging set on -b <logging_bucket_name> [-o <log_object_prefix>] gs://<bucket_name>... gsutil logging set off gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil logging get gs://<bucket_name> """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + '\n' _SET_DESCRIPTION = """ <B>SET</B> The ``set`` sub-command has two sub-commands: <B>ON</B> The ``gsutil logging set on`` command enables usage and storage logging for the buckets named by the specified URLs, outputting log files to the bucket specified with the ``-b`` flag. Cloud Storage doesn't validate the existence of the output bucket, so users should ensure it already exists, and all URLs must name Cloud Storage buckets (e.g., ``gs://bucket``). The optional ``-o`` flag specifies the prefix for log object names. The default prefix is the bucket name. For example, the command: gsutil logging set on -b gs://my_logging_bucket -o UsageLog \\ gs://my_bucket1 gs://my_bucket2 causes all read and write activity to objects in ``gs://mybucket1`` and ``gs://mybucket2`` to be logged to objects prefixed with the name ``UsageLog``, with those log objects written to the bucket ``gs://my_logging_bucket``. In addition to enabling logging on your bucket(s), you also need to grant cloud-storage-analytics@google.com write access to the log bucket, using this command: gsutil acl ch -g cloud-storage-analytics@google.com:W gs://my_logging_bucket Note that log data may contain sensitive information, so you should make sure to set an appropriate default bucket ACL to protect that data. (See "gsutil help defacl".) <B>OFF</B> This command disables usage and storage logging for the buckets named by the specified URLs. All URLs must name Cloud Storage buckets (e.g., ``gs://bucket``). No logging data is removed from the log buckets when you disable logging, but Google Cloud Storage stops delivering new logs once you have run this command. """ _GET_DESCRIPTION = """ <B>GET</B> If logging is enabled for the specified bucket url, the server responds with a JSON document that looks something like this: { "logBucket": "my_logging_bucket", "logObjectPrefix": "UsageLog" } You can download log data from your log bucket using the gsutil cp command. """ _DESCRIPTION = """ Google Cloud Storage offers `usage logs and storage logs <https://cloud.google.com/storage/docs/access-logs>`_ in the form of CSV files that you can download and view. Usage logs provide information for all of the requests made on a specified bucket and are created hourly. Storage logs provide information about the storage consumption of that bucket for the last day and are created daily. Once set up, usage logs and storage logs are automatically created as new objects in a bucket that you specify. Usage logs and storage logs are subject to the same pricing as other objects stored in Cloud Storage. For a complete list of usage log fields and storage data fields, see `Usage and storage log format <https://cloud.google.com/storage/docs/access-logs#format>`_. The ``logging`` command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION + """ <B>OPTIONS</B> -b bucket_name Specifies the bucket that stores the generated logs. This flag is only available for the ``set on`` command and is required for that command. -o log_prefix Specifies a common prefix for the names of generated logs. This flag is only available for the ``set on`` command and is optional for that command. """ _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class LoggingCommand(Command): """Implementation of gsutil logging command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'logging', command_name_aliases=['disablelogging', 'enablelogging', 'getlogging'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='b:o:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='logging', help_name_aliases=[ 'loggingconfig', 'logs', 'log', 'getlogging', 'enablelogging', 'disablelogging', ], help_type='command_help', help_one_line_summary='Configure or retrieve logging on buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=logging_config,empty=\' has ' 'no logging configuration.\',empty_prefix_key=' 'storage_url]' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', ], flag_map={ '-b': GcloudStorageFlag('--log-bucket'), '-o': GcloudStorageFlag('--log-object-prefix'), }, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--clear-log-bucket', '--clear-log-object-prefix', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _Get(self): """Gets logging configuration for a bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['logging']) if bucket_url.scheme == 's3': text_util.print_to_fd(self.gsutil_api.XmlPassThroughGetLogging( bucket_url, provider=bucket_url.scheme), end='') else: if (bucket_metadata.logging and bucket_metadata.logging.logBucket and bucket_metadata.logging.logObjectPrefix): text_util.print_to_fd( str(encoding.MessageToJson(bucket_metadata.logging))) else: text_util.print_to_fd('%s has no logging configuration.' % bucket_url) return 0 def _Enable(self): """Enables logging configuration for a bucket.""" # Disallow multi-provider 'logging set on' calls, because the schemas # differ. if not UrlsAreForSingleProvider(self.args): raise CommandException('"logging set on" command spanning providers not ' 'allowed.') target_bucket_url = None target_prefix = None for opt, opt_arg in self.sub_opts: if opt == '-b': target_bucket_url = StorageUrlFromString(opt_arg) if opt == '-o': target_prefix = opt_arg if not target_bucket_url: raise CommandException('"logging set on" requires \'-b <log_bucket>\' ' 'option') if not target_bucket_url.IsBucket(): raise CommandException('-b option must specify a bucket URL.') # Iterate over URLs, expanding wildcards and setting logging on each. some_matched = False for url_str in self.args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Enabling logging on %s...', blr) logging = apitools_messages.Bucket.LoggingValue( logBucket=target_bucket_url.bucket_name, logObjectPrefix=target_prefix or url.bucket_name) bucket_metadata = apitools_messages.Bucket(logging=logging) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) return 0 def _Disable(self): """Disables logging configuration for a bucket.""" # Iterate over URLs, expanding wildcards, and disabling logging on each. some_matched = False for url_str in self.args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Disabling logging on %s...', blr) logging = apitools_messages.Bucket.LoggingValue() bucket_metadata = apitools_messages.Bucket(logging=logging) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) return 0 def RunCommand(self): """Command entry point for the logging command.""" # Parse the subcommand and alias for the new logging command. action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._Get metrics.LogCommandParams(subcommands=[action_subcommand]) elif action_subcommand == 'set': state_subcommand = self.args.pop(0) if not self.args: self.RaiseWrongNumberOfArgumentsException() if state_subcommand == 'on': func = self._Enable metrics.LogCommandParams( subcommands=[action_subcommand, state_subcommand]) elif state_subcommand == 'off': func = self._Disable metrics.LogCommandParams( subcommands=[action_subcommand, state_subcommand]) else: raise CommandException( ('Invalid subcommand "%s" for the "%s %s" command.\n' 'See "gsutil help logging".') % (state_subcommand, self.command_name, action_subcommand)) else: raise CommandException(('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help logging".') % (action_subcommand, self.command_name)) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) func() return 0 commands/compose.py 0000644 00000014442 15025013616 0010370 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of compose command for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from boto import config from gslib.bucket_listing_ref import BucketListingObject from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.encryption_helper import GetEncryptionKeyWrapper from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import PreconditionsFromHeaders MAX_COMPOSE_ARITY = 32 _SYNOPSIS = """ gsutil compose gs://bucket/source_obj1 [gs://bucket/source_obj2 ...] gs://bucket/composite_obj """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The compose command creates a new object whose content is the concatenation of a given sequence of source objects under the same bucket. gsutil uses the content type of the first source object to determine the destination object's content type and does not modify or delete the source objects as part of the compose command. For more information, see the `composite objects topic <https://cloud.google.com/storage/docs/composite-objects>`_. There is a limit (currently %d) to the number of components that can be composed in a single operation. """ % (MAX_COMPOSE_ARITY)) class ComposeCommand(Command): """Implementation of gsutil compose command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'compose', command_name_aliases=['concat'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=MAX_COMPOSE_ARITY + 1, supported_sub_args='', # Not files, just object names without gs:// prefix. file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='compose', help_name_aliases=['concat'], help_type='command_help', help_one_line_summary=( 'Concatenate a sequence of objects into a new composite object.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'objects', 'compose'], flag_map={}, ) def CheckProvider(self, url): if url.scheme != 'gs': raise CommandException( '"compose" called on URL with unsupported provider (%s).' % str(url)) # Command entry point. def RunCommand(self): """Command entry point for the compose command.""" target_url_str = self.args[-1] self.args = self.args[:-1] target_url = StorageUrlFromString(target_url_str) self.CheckProvider(target_url) if target_url.HasGeneration(): raise CommandException('A version-specific URL (%s) cannot be ' 'the destination for gsutil compose - abort.' % target_url) dst_obj_metadata = apitools_messages.Object(name=target_url.object_name, bucket=target_url.bucket_name) components = [] # Remember the first source object so we can get its content type. first_src_url = None for src_url_str in self.args: if ContainsWildcard(src_url_str): src_url_iter = self.WildcardIterator(src_url_str).IterObjects() else: src_url_iter = [BucketListingObject(StorageUrlFromString(src_url_str))] for blr in src_url_iter: src_url = blr.storage_url self.CheckProvider(src_url) if src_url.bucket_name != target_url.bucket_name: raise CommandException('GCS does not support inter-bucket composing.') if not first_src_url: first_src_url = src_url src_obj_metadata = ( apitools_messages.ComposeRequest.SourceObjectsValueListEntry( name=src_url.object_name)) if src_url.HasGeneration(): src_obj_metadata.generation = int(src_url.generation) components.append(src_obj_metadata) # Avoid expanding too many components, and sanity check each name # expansion result. if len(components) > MAX_COMPOSE_ARITY: raise CommandException('"compose" called with too many component ' 'objects. Limit is %d.' % MAX_COMPOSE_ARITY) if not components: raise CommandException('"compose" requires at least 1 component object.') first_src_obj_metadata = self.gsutil_api.GetObjectMetadata( first_src_url.bucket_name, first_src_url.object_name, provider=first_src_url.scheme, fields=['contentEncoding', 'contentType']) dst_obj_metadata.contentType = first_src_obj_metadata.contentType dst_obj_metadata.contentEncoding = first_src_obj_metadata.contentEncoding preconditions = PreconditionsFromHeaders(self.headers or {}) self.logger.info('Composing %s from %d component object(s).', target_url, len(components)) self.gsutil_api.ComposeObject( components, dst_obj_metadata, preconditions=preconditions, provider=target_url.scheme, encryption_tuple=GetEncryptionKeyWrapper(config)) commands/signurl.py 0000644 00000071613 15025013616 0010411 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of URL Signing workflow. see: https://cloud.google.com/storage/docs/access-control#Signed-URLs) """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import calendar import copy from datetime import datetime from datetime import timedelta from datetime import timezone import getpass import json import re import sys import six from six.moves import urllib from apitools.base.py.exceptions import HttpError from apitools.base.py.http_wrapper import MakeRequest from apitools.base.py.http_wrapper import Request from boto import config from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils import constants from gslib.utils.boto_util import GetNewHttp from gslib.utils.shim_util import GcloudStorageMap, GcloudStorageFlag from gslib.utils.signurl_helper import CreatePayload, GetFinalUrl, to_bytes try: # Check for openssl. # pylint: disable=C6204 from OpenSSL.crypto import FILETYPE_PEM from OpenSSL.crypto import load_privatekey from OpenSSL.crypto import sign from OpenSSL.crypto import PKey HAVE_OPENSSL = True except ImportError: load_privatekey = None sign = None HAVE_OPENSSL = False FILETYPE_PEM = None try: from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import padding from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.hazmat.primitives.serialization import pkcs12 from cryptography.hazmat._oid import NameOID HAVE_CRYPTO = True except ImportError: HAVE_CRYPTO = False _CRYPTO_IMPORT_ERROR = "pyca/cryptography is not available. Either install it, or please consider using the .json keyfile" _AUTO_DETECT_REGION = 'auto' _MAX_EXPIRATION_TIME = timedelta(days=7) _MAX_EXPIRATION_TIME_WITH_MINUS_U = timedelta(hours=12) _SYNOPSIS = """ gsutil signurl [-c <content_type>] [-d <duration>] [-m <http_method>] \\ [-p <password>] [-r <region>] [-b <project>] (-u | <private-key-file>) \\ (gs://<bucket_name> | gs://<bucket_name>/<object_name>)... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The signurl command will generate a signed URL that embeds authentication data so the URL can be used by someone who does not have a Google account. Please see the `Signed URLs documentation <https://cloud.google.com/storage/docs/access-control/signed-urls>`_ for background about signed URLs. Multiple gs:// URLs may be provided and may contain wildcards. A signed URL will be produced for each provided URL, authorized for the specified HTTP method and valid for the given duration. NOTE: Unlike the gsutil ls command, the signurl command does not support operations on sub-directories. For example, unless you have an object named ``some-directory/`` stored inside the bucket ``some-bucket``, the following command returns an error: ``gsutil signurl <private-key-file> gs://some-bucket/some-directory/`` The signurl command uses the private key for a service account (the '<private-key-file>' argument) to generate the cryptographic signature for the generated URL. The private key file must be in PKCS12 or JSON format. If the private key is encrypted the signed URL command will prompt for the passphrase used to protect the private key file (default 'notasecret'). For more information regarding generating a private key for use with the signurl command please see the `Authentication documentation. <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`_ If you used `service account credentials <https://cloud.google.com/storage/docs/gsutil/addlhelp/CredentialTypesSupportingVariousUseCases#supported-credential-types_1>`_ for authentication, you can replace the <private-key-file> argument with the -u or --use-service-account option to use the system-managed private key directly. This avoids the need to store a private key file locally, but prior to using this flag you must `configure <https://cloud.google.com/sdk/gcloud/reference/auth/activate-service-account>`_ ``gcloud`` to use your service account credentials. <B>OPTIONS</B> -b <project> Allows you to specify a user project that will be billed for requests that use the signed URL. This is useful for generating presigned links for buckets that use requester pays. Note that it's not valid to specify both the ``-b`` and ``--use-service-account`` options together. -c Specifies the content type for which the signed URL is valid for. -d Specifies the duration that the signed URL should be valid for, default duration is 1 hour. Times may be specified with no suffix (default hours), or with s = seconds, m = minutes, h = hours, d = days. This option may be specified multiple times, in which case the duration the link remains valid is the sum of all the duration options. The max duration allowed is 7 days when ``private-key-file`` is used. The max duration allowed is 12 hours when -u option is used. This limitation exists because the system-managed key used to sign the URL may not remain valid after 12 hours. -m Specifies the HTTP method to be authorized for use with the signed URL, default is GET. You may also specify RESUMABLE to create a signed resumable upload start URL. When using a signed URL to start a resumable upload session, you will need to specify the 'x-goog-resumable:start' header in the request or else signature validation will fail. -p Specify the private key password instead of prompting. -r <region> Specifies the `region <https://cloud.google.com/storage/docs/locations>`_ in which the resources for which you are creating signed URLs are stored. Default value is 'auto' which will cause gsutil to fetch the region for the resource. When auto-detecting the region, the current gsutil user's credentials, not the credentials from the private-key-file, are used to fetch the bucket's metadata. This option must be specified and not 'auto' when generating a signed URL to create a bucket. -u Use service account credentials instead of a private key file to sign the URL. You can also use the ``--use-service-account`` option, which is equivalent to ``-u``. Note that both options have a maximum allowed duration of 12 hours for a valid link. <B>USAGE</B> Create a signed URL for downloading an object valid for 10 minutes: gsutil signurl -d 10m <private-key-file> gs://<bucket>/<object> Create a signed URL, valid for one hour, for uploading a plain text file via HTTP PUT: gsutil signurl -m PUT -d 1h -c text/plain <private-key-file> \\ gs://<bucket>/<obj> To construct a signed URL that allows anyone in possession of the URL to PUT to the specified bucket for one day, creating an object of Content-Type image/jpg, run: gsutil signurl -m PUT -d 1d -c image/jpg <private-key-file> \\ gs://<bucket>/<obj> To construct a signed URL that allows anyone in possession of the URL to POST a resumable upload to the specified bucket for one day, creating an object of Content-Type image/jpg, run: gsutil signurl -m RESUMABLE -d 1d -c image/jpg <private-key-file> \\ gs://bucket/<obj> """) def _NowUTC(): """Returns the current utc time as a datetime object.""" return datetime.now(tz=timezone.utc).replace(tzinfo=None) def _DurationToTimeDelta(duration): r"""Parses the given duration and returns an equivalent timedelta.""" match = re.match(r'^(\d+)([dDhHmMsS])?$', duration) if not match: raise CommandException('Unable to parse duration string') duration, modifier = match.groups('h') duration = int(duration) modifier = modifier.lower() if modifier == 'd': ret = timedelta(days=duration) elif modifier == 'h': ret = timedelta(hours=duration) elif modifier == 'm': ret = timedelta(minutes=duration) elif modifier == 's': ret = timedelta(seconds=duration) return ret def _GenSignedUrl(key, api, use_service_account, provider, client_id, method, duration, gcs_path, logger, region, content_type=None, billing_project=None, string_to_sign_debug=False, generation=None): """Construct a string to sign with the provided key. Args: key: The private key to use for signing the URL. api: The CloudApiDelegator instance use_service_account: If True, use the service account credentials instead of using the key file to sign the URL provider: Cloud storage provider to connect to. If not present, class-wide default is used. client_id: Client ID signing this URL. method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. gcs_path: String path to the bucket of object for signing, in the form 'bucket' or 'bucket/object'. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. content_type: Optional Content-Type for the signed URL. HTTP requests using the URL must match this Content-Type. billing_project: Specify a user project to be billed for the request. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. generation: If not None, specifies a version of an object for signing. Returns: The complete URL (string). """ gs_host = config.get('Credentials', 'gs_host', 'storage.googleapis.com') signed_headers = {'host': gs_host} if method == 'RESUMABLE': method = 'POST' signed_headers['x-goog-resumable'] = 'start' if not content_type: logger.warn('Warning: no Content-Type header was specified with the -c ' 'flag, so uploads to the resulting Signed URL must not ' 'specify a Content-Type.') if content_type: signed_headers['content-type'] = content_type if use_service_account: final_url = api.SignUrl(provider=provider, method=method, duration=duration, path=gcs_path, generation=generation, logger=logger, region=region, signed_headers=signed_headers, string_to_sign_debug=string_to_sign_debug) else: if six.PY2: digest = b'RSA-SHA256' else: # Your IDE may complain about this due to a bad docstring in pyOpenSsl: # https://github.com/pyca/pyopenssl/issues/741 digest = 'RSA-SHA256' string_to_sign, canonical_query_string = CreatePayload( client_id=client_id, method=method, duration=duration, path=gcs_path, generation=generation, logger=logger, region=region, signed_headers=signed_headers, billing_project=billing_project, string_to_sign_debug=string_to_sign_debug) if isinstance(key, PKey): raw_signature = sign(key, string_to_sign, digest) else: if not HAVE_CRYPTO: raise CommandException(_CRYPTO_IMPORT_ERROR) raw_signature = key.sign(to_bytes(string_to_sign), padding.PKCS1v15(), hashes.SHA256()) final_url = GetFinalUrl(raw_signature, gs_host, gcs_path, canonical_query_string) return final_url def _ReadKeystore(key_string, passwd): key_string = to_bytes(key_string) passwd = to_bytes(passwd) # Cryptography package is not bundled with gsutil, Keeping it at top would throw error. if not HAVE_CRYPTO: raise CommandException(_CRYPTO_IMPORT_ERROR) try: key, cert, add = pkcs12.load_key_and_certificates(key_string, passwd) client_email = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value return key, client_email except: raise CommandException('Unable to load the keyfile, Invalid password or PKCS12 data.') def _ReadJSONKeystore(ks_contents, passwd=None): """Read the client email and private key from a JSON keystore. Assumes this keystore was downloaded from the Cloud Platform Console. By default, JSON keystore private keys from the Cloud Platform Console aren't encrypted so the passwd is optional as load_privatekey will prompt for the PEM passphrase if the key is encrypted. Arguments: ks_contents: JSON formatted string representing the keystore contents. Must be a valid JSON string and contain the fields 'private_key' and 'client_email'. passwd: Passphrase for encrypted private keys. Returns: key: Parsed private key from the keystore. client_email: The email address for the service account. Raises: ValueError: If unable to parse ks_contents or keystore is missing required fields. """ # ensuring that json.loads receives unicode in Python 3 and bytes in Python 2 # Previous to Python 3.6, there was no automatic conversion and str was req. ks = json.loads(six.ensure_str(ks_contents)) if 'client_email' not in ks or 'private_key' not in ks: raise ValueError('JSON keystore doesn\'t contain required fields') client_email = ks['client_email'] if passwd: key = load_privatekey(FILETYPE_PEM, ks['private_key'], passwd) else: key = load_privatekey(FILETYPE_PEM, ks['private_key']) return key, client_email class UrlSignCommand(Command): """Implementation of gsutil url_sign command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'signurl', command_name_aliases=['signedurl', 'queryauth'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='m:d:b:c:p:r:u', supported_private_args=['use-service-account'], file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreFileURLsArgument(), CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='signurl', help_name_aliases=[ 'signedurl', 'queryauth', ], help_type='command_help', help_one_line_summary='Create a signed URL', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): # TODO (b/287273841): Replace `copy` with a better pattern for # translating flag values. original_args = copy.deepcopy(self.args) original_sub_opts = copy.deepcopy(self.sub_opts) gcloud_command = [ 'storage', 'sign-url', '--format=csv[separator="\\t"](resource:label="URL",' ' http_verb:label="HTTP Method",' ' expiration:label="Expiration",' ' signed_url:label="Signed URL")', '--private-key-file=' + self.args[0] ] self.args = self.args[1:] duration_arg_idx = None http_verb_arg_idx = None content_type_arg_idx = None billing_project_arg_idx = None for i, (flag, _) in enumerate(self.sub_opts): if flag == '-d': duration_arg_idx = i elif flag == '-m': http_verb_arg_idx = i elif flag == '-c': content_type_arg_idx = i elif flag == '-b': billing_project_arg_idx = i if duration_arg_idx is not None: # Convert duration to seconds, which gcloud can handle. seconds = str( int( _DurationToTimeDelta( self.sub_opts[duration_arg_idx][1]).total_seconds())) + 's' self.sub_opts[duration_arg_idx] = ('-d', seconds) if http_verb_arg_idx is not None: if self.sub_opts[http_verb_arg_idx][1] == 'RESUMABLE': self.sub_opts[http_verb_arg_idx] = ('-m', 'POST') gcloud_command += ['--headers=x-goog-resumable=start'] if content_type_arg_idx is not None: content_type_value = self.sub_opts[content_type_arg_idx][1] self.sub_opts[content_type_arg_idx] = ('-c', 'content-type=' + content_type_value) if billing_project_arg_idx is not None: project_value = self.sub_opts[billing_project_arg_idx][1] self.sub_opts[billing_project_arg_idx] = ('-b', 'userProject=' + project_value) fully_translated_command = super().get_gcloud_storage_args( GcloudStorageMap(gcloud_command=gcloud_command, flag_map={ '-m': GcloudStorageFlag('--http-verb'), '-d': GcloudStorageFlag('--duration'), '-b': GcloudStorageFlag('--query-params'), '-c': GcloudStorageFlag('--headers'), '-r': GcloudStorageFlag('--region'), '-p': GcloudStorageFlag('--private-key-password'), })) # Ensures dry run mode works correctly, as flag translation requires # mutating command state. # TODO(b/287273841): Refactor so that there's a better pattern for # translating flag values. self.args = original_args self.sub_opts = original_sub_opts return fully_translated_command def _ParseAndCheckSubOpts(self): # Default argument values delta = None method = 'GET' content_type = '' passwd = None region = _AUTO_DETECT_REGION use_service_account = False billing_project = None for o, v in self.sub_opts: # TODO(PY3-ONLY): Delete this if block. if six.PY2: v = v.decode(sys.stdin.encoding or constants.UTF8) if o == '-d': if delta is not None: delta += _DurationToTimeDelta(v) else: delta = _DurationToTimeDelta(v) elif o == '-m': method = v elif o == '-c': content_type = v elif o == '-p': passwd = v elif o == '-r': region = v elif o == '-u' or o == '--use-service-account': use_service_account = True elif o == '-b': billing_project = v else: self.RaiseInvalidArgumentException() if delta is None: delta = timedelta(hours=1) else: if use_service_account and delta > _MAX_EXPIRATION_TIME_WITH_MINUS_U: # This restriction comes from the IAM SignBlob API. The SignBlob # API uses a system-managed key which can guarantee validation only # up to 12 hours. b/156160482#comment4 raise CommandException( 'Max valid duration allowed is %s when -u flag is used. For longer' ' duration, consider using the private-key-file instead of the -u' ' option.' % _MAX_EXPIRATION_TIME_WITH_MINUS_U) elif delta > _MAX_EXPIRATION_TIME: raise CommandException('Max valid duration allowed is ' '%s' % _MAX_EXPIRATION_TIME) if method not in ['GET', 'PUT', 'DELETE', 'HEAD', 'RESUMABLE']: raise CommandException('HTTP method must be one of' '[GET|HEAD|PUT|DELETE|RESUMABLE]') if not use_service_account and len(self.args) < 2: raise CommandException( 'The command requires a key file argument and one or more ' 'URL arguments if the --use-service-account flag is missing. ' 'Run `gsutil help signurl` for more info') if use_service_account and billing_project: raise CommandException( 'Specifying both the -b and --use-service-account options together is' 'invalid.') return method, delta, content_type, passwd, region, use_service_account, billing_project def _ProbeObjectAccessWithClient(self, key, use_service_account, provider, client_email, gcs_path, generation, logger, region, billing_project): """Performs a head request against a signed URL to check for read access.""" # Choose a reasonable time in the future; if the user's system clock is # 60 or more seconds behind the server's this will generate an error. signed_url = _GenSignedUrl(key=key, api=self.gsutil_api, use_service_account=use_service_account, provider=provider, client_id=client_email, method='HEAD', duration=timedelta(seconds=60), gcs_path=gcs_path, generation=generation, logger=logger, region=region, billing_project=billing_project, string_to_sign_debug=True) try: h = GetNewHttp() req = Request(signed_url, 'HEAD') response = MakeRequest(h, req) if response.status_code not in [200, 403, 404]: raise HttpError.FromResponse(response) return response.status_code except HttpError as http_error: if http_error.has_attr('response'): error_response = http_error.response error_string = ('Unexpected HTTP response code %s while querying ' 'object readability. Is your system clock accurate?' % error_response.status_code) if error_response.content: error_string += ' Content: %s' % error_response.content else: error_string = ('Expected an HTTP response code of ' '200 while querying object readability, but received ' 'an error: %s' % http_error) raise CommandException(error_string) def _EnumerateStorageUrls(self, in_urls): ret = [] for url_str in in_urls: if ContainsWildcard(url_str): ret.extend([blr.storage_url for blr in self.WildcardIterator(url_str)]) else: ret.append(StorageUrlFromString(url_str)) return ret def RunCommand(self): """Command entry point for signurl command.""" if not HAVE_OPENSSL: raise CommandException( 'The signurl command requires the pyopenssl library (try pip ' 'install pyopenssl or easy_install pyopenssl)') method, delta, content_type, passwd, region, use_service_account, billing_project = ( self._ParseAndCheckSubOpts()) arg_start_index = 0 if use_service_account else 1 storage_urls = self._EnumerateStorageUrls(self.args[arg_start_index:]) region_cache = {} key = None if not use_service_account: try: key, client_email = _ReadJSONKeystore( open(self.args[0], 'rb').read(), passwd) except ValueError: # Ignore and try parsing as a pkcs12. if not passwd: passwd = getpass.getpass('Keystore password:') try: key, client_email = _ReadKeystore( open(self.args[0], 'rb').read(), passwd) except ValueError: raise CommandException('Unable to parse private key from {0}'.format( self.args[0])) else: client_email = self.gsutil_api.GetServiceAccountId(provider='gs') print('URL\tHTTP Method\tExpiration\tSigned URL') for url in storage_urls: if url.scheme != 'gs': raise CommandException('Can only create signed urls from gs:// urls') if url.IsBucket(): if region == _AUTO_DETECT_REGION: raise CommandException('Generating signed URLs for creating buckets' ' requires a region be specified via the -r ' 'option. Run `gsutil help signurl` for more ' 'information about the \'-r\' option.') gcs_path = url.bucket_name if method == 'RESUMABLE': raise CommandException('Resumable signed URLs require an object ' 'name.') else: # Need to URL encode the object name as Google Cloud Storage does when # computing the string to sign when checking the signature. gcs_path = '{0}/{1}'.format( url.bucket_name, urllib.parse.quote(url.object_name.encode(constants.UTF8), safe=b'/~')) if region == _AUTO_DETECT_REGION: if url.bucket_name in region_cache: bucket_region = region_cache[url.bucket_name] else: try: _, bucket = self.GetSingleBucketUrlFromArg( 'gs://{}'.format(url.bucket_name), bucket_fields=['location']) except Exception as e: raise CommandException( '{}: Failed to auto-detect location for bucket \'{}\'. Please ' 'ensure you have storage.buckets.get permission on the bucket ' 'or specify the bucket\'s location using the \'-r\' option.'. format(e.__class__.__name__, url.bucket_name)) bucket_region = bucket.location.lower() region_cache[url.bucket_name] = bucket_region else: bucket_region = region final_url = _GenSignedUrl(key=key, api=self.gsutil_api, use_service_account=use_service_account, provider=url.scheme, client_id=client_email, method=method, duration=delta, gcs_path=gcs_path, generation=url.generation, logger=self.logger, region=bucket_region, content_type=content_type, billing_project=billing_project, string_to_sign_debug=True) expiration = calendar.timegm((datetime.now(tz=timezone.utc).replace(tzinfo=None) + delta).utctimetuple()) expiration_dt = datetime.fromtimestamp(expiration) time_str = expiration_dt.strftime('%Y-%m-%d %H:%M:%S') # TODO(PY3-ONLY): Delete this if block. if six.PY2: time_str = time_str.decode(constants.UTF8) url_info_str = '{0}\t{1}\t{2}\t{3}'.format(url.url_string, method, time_str, final_url) # TODO(PY3-ONLY): Delete this if block. if six.PY2: url_info_str = url_info_str.encode(constants.UTF8) print(url_info_str) response_code = self._ProbeObjectAccessWithClient( key, use_service_account, url.scheme, client_email, gcs_path, url.generation, self.logger, bucket_region, billing_project) if response_code == 404: if url.IsBucket() and method != 'PUT': raise CommandException( 'Bucket {0} does not exist. Please create a bucket with ' 'that name before a creating signed URL to access it.'.format( url)) else: if method != 'PUT' and method != 'RESUMABLE': raise CommandException( 'Object {0} does not exist. Please create/upload an object ' 'with that name before a creating signed URL to access it.'. format(url)) elif response_code == 403: self.logger.warn( '%s does not have permissions on %s, using this link will likely ' 'result in a 403 error until at least READ permissions are granted', client_email or 'The account', url) return 0 commands/rsync.py 0000644 00000237010 15025013616 0010057 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like rsync command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import errno import heapq import io from itertools import islice import logging import os import re import tempfile import textwrap import time import traceback import sys import six from six.moves import urllib from boto import config import crcmod from gslib.bucket_listing_ref import BucketListingObject from gslib.cloud_api import NotFoundException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import DummyArgChecker from gslib.commands.cp import ShimTranslatePredefinedAclSubOptForCopy from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.metrics import LogPerformanceSummaryParams from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.seek_ahead_thread import SeekAheadResult from gslib.sig_handling import GetCaughtSignals from gslib.sig_handling import RegisterSignalHandler from gslib.storage_url import GenerationFromUrlAndString from gslib.storage_url import IsCloudSubdirPlaceholder from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import constants from gslib.utils import copy_helper from gslib.utils import parallelism_framework_util from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.copy_helper import CreateCopyHelperOpts from gslib.utils.copy_helper import GetSourceFieldsNeededForCopy from gslib.utils.copy_helper import GZIP_ALL_FILES from gslib.utils.copy_helper import SkipUnsupportedObjectError from gslib.utils.hashing_helper import CalculateB64EncodedCrc32cFromContents from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents from gslib.utils.hashing_helper import SLOW_CRCMOD_RSYNC_WARNING from gslib.utils.hashing_helper import SLOW_CRCMOD_WARNING from gslib.utils.metadata_util import CreateCustomMetadata from gslib.utils.metadata_util import GetValueFromObjectCustomMetadata from gslib.utils.metadata_util import ObjectIsGzipEncoded from gslib.utils.posix_util import ATIME_ATTR from gslib.utils.posix_util import ConvertDatetimeToPOSIX from gslib.utils.posix_util import ConvertModeToBase8 from gslib.utils.posix_util import DeserializeFileAttributesFromObjectMetadata from gslib.utils.posix_util import GID_ATTR from gslib.utils.posix_util import InitializePreservePosixData from gslib.utils.posix_util import MODE_ATTR from gslib.utils.posix_util import MTIME_ATTR from gslib.utils.posix_util import NA_ID from gslib.utils.posix_util import NA_MODE from gslib.utils.posix_util import NA_TIME from gslib.utils.posix_util import NeedsPOSIXAttributeUpdate from gslib.utils.posix_util import ParseAndSetPOSIXAttributes from gslib.utils.posix_util import POSIXAttributes from gslib.utils.posix_util import SerializeFileAttributesToObjectMetadata from gslib.utils.posix_util import UID_ATTR from gslib.utils.posix_util import ValidateFilePermissionAccess from gslib.utils.posix_util import WarnFutureTimestamp from gslib.utils.posix_util import WarnInvalidValue from gslib.utils.posix_util import WarnNegativeAttribute from gslib.utils.rsync_util import DiffAction from gslib.utils.rsync_util import RsyncDiffToApply from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import IS_WINDOWS from gslib.utils.translation_helper import CopyCustomMetadata from gslib.utils.unit_util import CalculateThroughput from gslib.utils.unit_util import SECONDS_PER_DAY from gslib.utils.unit_util import TEN_MIB from gslib.wildcard_iterator import CreateWildcardIterator if six.PY3: long = int _SYNOPSIS = """ gsutil rsync [OPTION]... src_url dst_url """ # pylint: disable=anomalous-backslash-in-string _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil rsync command makes the contents under dst_url the same as the contents under src_url, by copying any missing files/objects (or those whose data has changed), and (if the -d option is specified) deleting any extra files/objects. src_url must specify a directory, bucket, or bucket subdirectory. For example, to sync the contents of the local directory "data" to the bucket gs://mybucket/data, you could do: gsutil rsync data gs://mybucket/data To recurse into directories use the -r option: gsutil rsync -r data gs://mybucket/data If you have a large number of objects to synchronize you might want to use the gsutil -m option (see "gsutil help options"), to perform parallel (multi-threaded/multi-processing) synchronization: gsutil -m rsync -r data gs://mybucket/data The -m option typically will provide a large performance boost if either the source or destination (or both) is a cloud URL. If both source and destination are file URLs the -m option will typically thrash the disk and slow synchronization down. Note 1: Shells (like bash, zsh) sometimes attempt to expand wildcards in ways that can be surprising. Also, attempting to copy files whose names contain wildcard characters can result in problems. For more details about these issues see `Wildcard behavior considerations <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. Note 2: If you are synchronizing a large amount of data between clouds you might consider setting up a `Google Compute Engine <https://cloud.google.com/products/compute-engine>`_ account and running gsutil there. Since cross-provider gsutil data transfers flow through the machine where gsutil is running, doing this can make your transfer run significantly faster than running gsutil on your local workstation. Note 3: rsync does not copy empty directory trees, since Cloud Storage uses a `flat namespace <https://cloud.google.com/storage/docs/folders>`_. <B>Using -d Option (with caution!) to mirror source and destination.</B> The rsync -d option is very useful and commonly used, because it provides a means of making the contents of a destination bucket or directory match those of a source bucket or directory. This is done by copying all data from the source to the destination and deleting all other data in the destination that is not in the source. Please exercise caution when you use this option: It's possible to delete large amounts of data accidentally if, for example, you erroneously reverse source and destination. To make the local directory my-data the same as the contents of gs://mybucket/data and delete objects in the local directory that are not in gs://mybucket/data: gsutil rsync -d -r gs://mybucket/data my-data To make the contents of gs://mybucket2 the same as gs://mybucket1 and delete objects in gs://mybucket2 that are not in gs://mybucket1: gsutil rsync -d -r gs://mybucket1 gs://mybucket2 You can also mirror data across local directories. This example will copy all objects from dir1 into dir2 and delete all objects in dir2 which are not in dir1: gsutil rsync -d -r dir1 dir2 To mirror your content across clouds: gsutil rsync -d -r gs://my-gs-bucket s3://my-s3-bucket Change detection works if the other Cloud provider is using md5 or CRC32. AWS multipart upload has an incompatible checksum. As mentioned above, using -d can be dangerous because of how quickly data can be deleted. For example, if you meant to synchronize a local directory from a bucket in the cloud but instead run the command: gsutil -m rsync -r -d ./your-dir gs://your-bucket and your-dir is currently empty, you will quickly delete all of the objects in gs://your-bucket. You can also cause large amounts of data to be lost quickly by specifying a subdirectory of the destination as the source of an rsync. For example, the command: gsutil -m rsync -r -d gs://your-bucket/data gs://your-bucket would cause most or all of the objects in gs://your-bucket to be deleted (some objects may survive if there are any with names that sort lower than "data" under gs://your-bucket/data). In addition to paying careful attention to the source and destination you specify with the rsync command, there are two more safety measures you can take when using gsutil rsync -d: 1. Try running the command with the rsync -n option first, to see what it would do without actually performing the operations. For example, if you run the command: gsutil -m rsync -r -d -n gs://your-bucket/data gs://your-bucket it will be immediately evident that running that command without the -n option would cause many objects to be deleted. 2. Enable object versioning in your bucket, which allows you to restore objects if you accidentally delete them. For more details see `Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_. <B>BE CAREFUL WHEN SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SYMLINKS, DEVICES, ETC.)</B> Running gsutil rsync over a directory containing operating system-specific file types (symbolic links, device files, sockets, named pipes, etc.) can cause various problems. For example, running a command like: gsutil rsync -r ./dir gs://my-bucket will cause gsutil to follow any symbolic links in ./dir, creating objects in my-bucket containing the data from the files to which the symlinks point. This can cause various problems: * If you use gsutil rsync as a simple way to backup a directory to a bucket, restoring from that bucket will result in files where the symlinks used to be. At best this is wasteful of space, and at worst it can result in outdated data or broken applications -- depending on what is consuming the symlinks. * If you use gsutil rsync over directories containing broken symlinks, gsutil rsync will abort (unless you pass the -e option). * gsutil rsync skips symlinks that point to directories. Since gsutil rsync is intended to support data operations (like moving a data set to the cloud for computational processing) and it needs to be compatible both in the cloud and across common operating systems, there are no plans for gsutil rsync to support operating system-specific file types like symlinks. We recommend that users do one of the following: * Don't use gsutil rsync over directories containing symlinks or other OS- specific file types. * Use the -e option to exclude symlinks or the -x option to exclude OS-specific file types by name. * Use a tool (such as tar) that preserves symlinks and other OS-specific file types, packaging up directories containing such files before uploading to the cloud. <B>EVENTUAL CONSISTENCY WITH NON-GOOGLE CLOUD PROVIDERS</B> While Google Cloud Storage is strongly consistent, some cloud providers only support eventual consistency. You may encounter scenarios where rsync synchronizes using stale listing data when working with these other cloud providers. For example, if you run rsync immediately after uploading an object to an eventually consistent cloud provider, the added object may not yet appear in the provider's listing. Consequently, rsync will miss adding the object to the destination. If this happens you can rerun the rsync operation again later (after the object listing has "caught up"). <B>FAILURE HANDLING</B> The rsync command retries failures when it is useful to do so, but if enough failures happen during a particular copy or delete operation, or if a failure isn't retryable, the overall command fails. If the -C option is provided, the command instead skips failing objects and moves on. At the end of the synchronization run, if any failures were not successfully retried, the rsync command reports the count of failures and exits with non-zero status. At this point you can run the rsync command again, and gsutil attempts any remaining needed copy and/or delete operations. For more details about gsutil's retry handling, see `Retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_. <B>CHANGE DETECTION ALGORITHM</B> To determine if a file or object has changed, gsutil rsync first checks whether the file modification time (mtime) of both the source and destination is available. If mtime is available at both source and destination, and the destination mtime is different than the source, or if the source and destination file size differ, gsutil rsync will update the destination. If the source is a cloud bucket and the destination is a local file system, and if mtime is not available for the source, gsutil rsync will use the time created for the cloud object as a substitute for mtime. Otherwise, if mtime is not available for either the source or the destination, gsutil rsync will fall back to using checksums. If the source and destination are both cloud buckets with checksums available, gsutil rsync will use these hashes instead of mtime. However, gsutil rsync will still update mtime at the destination if it is not present. If the source and destination have matching checksums and only the source has an mtime, gsutil rsync will copy the mtime to the destination. If neither mtime nor checksums are available, gsutil rsync will resort to comparing file sizes. Checksums will not be available when comparing composite Google Cloud Storage objects with objects at a cloud provider that does not support CRC32C (which is the only checksum available for composite objects). See 'gsutil help compose' for details about composite objects. <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> If both the source and destination URL are cloud URLs from the same provider, gsutil copies data "in the cloud" (i.e., without downloading to and uploading from the machine where you run gsutil). In addition to the performance and cost advantages of doing this, copying in the cloud preserves metadata (like Content-Type and Cache-Control). In contrast, when you download data from the cloud it ends up in a file, which has no associated metadata, other than file modification time (mtime). Thus, unless you have some way to hold on to or re-create that metadata, synchronizing a bucket to a directory in the local file system will not retain the metadata other than mtime. Note that by default, the gsutil rsync command does not copy the ACLs of objects being synchronized and instead will use the default bucket ACL (see "gsutil help defacl"). You can override this behavior with the -p option. See the `Options section <https://cloud.google.com/storage/docs/gsutil/commands/rsync#options>`_ to learn how. <B>LIMITATIONS</B> 1. The gsutil rsync command will only allow non-negative file modification times to be used in its comparisons. This means gsutil rsync will resort to using checksums for any file with a timestamp before 1970-01-01 UTC. 2. The gsutil rsync command considers only the live object version in the source and destination buckets when deciding what to copy / delete. If versioning is enabled in the destination bucket then gsutil rsync's replacing or deleting objects will end up creating versions, but the command doesn't try to make any noncurrent versions match in the source and destination buckets. 3. The gsutil rsync command does not support copying special file types such as sockets, device files, named pipes, or any other non-standard files intended to represent an operating system resource. If you run gsutil rsync on a source directory that includes such files (for example, copying the root directory on Linux that includes /dev ), you should use the -x flag to exclude these files. Otherwise, gsutil rsync may fail or hang. 4. The gsutil rsync command copies changed files in their entirety and does not employ the `rsync delta-transfer algorithm <https://rsync.samba.org/tech_report/>`_ to transfer portions of a changed file. This is because Cloud Storage objects are immutable and no facility exists to read partial object checksums or perform partial replacements. <B>OPTIONS</B> -a predef-acl Sets the specified predefined ACL on uploaded objects. See "gsutil help acls" for further details. Note that rsync will decide whether or not to perform a copy based only on object size and modification time, not current ACL state. Also see the -p option below. -c Causes the rsync command to compute and compare checksums (instead of comparing mtime) for files if the size of source and destination match. This option increases local disk I/O and run time if either src_url or dst_url are on the local file system. -C If an error occurs, continue to attempt to copy the remaining files. If errors occurred, gsutil's exit status will be non-zero even if this flag is set. This option is implicitly set when running "gsutil -m rsync...". NOTE: -C only applies to the actual copying operation. If an error occurs while iterating over the files in the local directory (e.g., invalid Unicode file name) gsutil will print an error message and abort. -d Delete extra files under dst_url not found under src_url. By default extra files are not deleted. NOTE: this option can delete data quickly if you specify the wrong source/destination combination. See the help section above, "BE CAREFUL WHEN USING -d OPTION!". -e Exclude symlinks. When specified, symbolic links will be ignored. Note that gsutil does not follow directory symlinks, regardless of whether -e is specified. -i Skip copying any files that already exist at the destination, regardless of their modification time. -j <ext,...> Applies gzip transport encoding to any file upload whose extension matches the -j extension list. This is useful when uploading files with compressible content (such as .js, .css, or .html files) because it saves network bandwidth while also leaving the data uncompressed in Google Cloud Storage. When you specify the -j option, files being uploaded are compressed in-memory and on-the-wire only. Both the local files and Cloud Storage objects remain uncompressed. The uploaded objects retain the Content-Type and name of the original files. Note that if you want to use the top-level -m option to parallelize copies along with the -j/-J options, your performance may be bottlenecked by the "max_upload_compression_buffer_size" boto config option, which is set to 2 GiB by default. This compression buffer size can be changed to a higher limit, e.g.: gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \\ -m rsync -j html,txt /local/source/dir gs://bucket/path -J Applies gzip transport encoding to file uploads. This option works like the -j option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If you use this option and some of the source files don't compress well (e.g., that's often true of binary data), this option may result in longer uploads. -n Causes rsync to run in "dry run" mode, i.e., just outputting what would be copied or deleted without actually doing any copying/deleting. -p Causes ACLs to be preserved when objects are copied. Note that rsync will decide whether or not to perform a copy based only on object size and modification time, not current ACL state. Thus, if the source and destination differ in size or modification time and you run gsutil rsync -p, the file will be copied and ACL preserved. However, if the source and destination don't differ in size or checksum but have different ACLs, running gsutil rsync -p will have no effect. Note that this option has performance and cost implications when using the XML API, as it requires separate HTTP calls for interacting with ACLs. The performance issue can be mitigated to some degree by using gsutil -m rsync to cause parallel synchronization. Also, this option only works if you have OWNER access to all of the objects that are copied. You can avoid the additional performance and cost of using rsync -p if you want all objects in the destination bucket to end up with the same ACL by setting a default object ACL on that bucket instead of using rsync -p. See 'gsutil help defacl'. -P Causes POSIX attributes to be preserved when objects are copied. With this feature enabled, gsutil rsync will copy fields provided by stat. These are the user ID of the owner, the group ID of the owning group, the mode (permissions) of the file, and the access/modification timestamps of the file. For downloads, these attributes will only be set if the source objects were uploaded with this flag enabled. On Windows, this flag will only set and restore access time and modification time. This is because Windows doesn't have a notion of POSIX uid/gid/mode. -R, -r The -R and -r options are synonymous. Causes directories, buckets, and bucket subdirectories to be synchronized recursively. If you neglect to use this option gsutil will make only the top-level directory in the source and destination URLs match, skipping any sub-directories. -u When a file/object is present in both the source and destination, if mtime is available for both, do not perform the copy if the destination mtime is newer. -U Skip objects with unsupported object types instead of failing. Unsupported object types are Amazon S3 Objects in the GLACIER storage class. -x pattern Causes files/objects matching pattern to be excluded, i.e., any matching files/objects are not copied or deleted. Note that the pattern is a `Python regular expression <https://docs.python.org/3/howto/regex.html>`_, not a wildcard (so, matching any string ending in "abc" would be specified using ".*abc$" rather than "*abc"). Note also that the exclude path is always relative (similar to Unix rsync or tar exclude options). For example, if you run the command: gsutil rsync -x "data.[/\\].*\\.txt$" dir gs://my-bucket it skips the file dir/data1/a.txt. You can use regex alternation to specify multiple exclusions, for example: gsutil rsync -x ".*\\.txt$|.*\\.jpg$" dir gs://my-bucket skips all .txt and .jpg files in dir. NOTE: When using the Windows cmd.exe command line interpreter, use ``^`` as an escape character instead of ``\\`` and escape the ``|`` character. When using Windows PowerShell, use ``'`` instead of ``"`` and surround the ``|`` character with ``"``. -y pattern Similar to the -x option, but the command will first skip directories/prefixes using the provided pattern and then exclude files/objects using the same pattern. This is usually much faster, but won't work as intended with negative lookahead patterns. For example, if you run the command: gsutil rsync -y "^(?!.*\\.txt$).*" dir gs://my-bucket This would first exclude all subdirectories unless they end in .txt before excluding all files except those ending in .txt. Running the same command with the -x option would result in all .txt files being included, regardless of whether they appear in subdirectories that end in .txt. """) # pylint: enable=anomalous-backslash-in-string _NA = '-' _OUTPUT_BUFFER_SIZE = 64 * 1024 _PROGRESS_REPORT_LISTING_COUNT = 10000 # Tracks files we need to clean up at end or if interrupted. Because some # files are passed to rsync's diff iterators, it is difficult to manage when # they should be closed, especially in the event that we receive a signal to # exit. Every time such a file is opened, its file object should be appended # to this list. _tmp_files = [] # pylint: disable=unused-argument def _HandleSignals(signal_num, cur_stack_frame): """Called when rsync command is killed with SIGINT, SIGQUIT or SIGTERM.""" CleanUpTempFiles() def CleanUpTempFiles(): """Cleans up temp files. This function allows the main (RunCommand) function to clean up at end of operation, or if gsutil rsync is interrupted (e.g., via ^C). This is necessary because tempfile.NamedTemporaryFile doesn't allow the created file to be re-opened in read mode on Windows, so we have to use tempfile.mkstemp, which doesn't automatically delete temp files. """ # First pass: Close all the files. Wrapped iterators result in open file # objects for the same file, and Windows does not allow removing the file # at a given path until all its open file handles have been closed. for fileobj in _tmp_files: # Windows requires temp files to be closed before unlinking. if not fileobj.closed: fileobj.close() # Second pass: Remove each file, skipping duplicates that have already been # removed. for fileobj in _tmp_files: if os.path.isfile(fileobj.name): try: os.unlink(fileobj.name) except Exception as e: # pylint: disable=broad-except logging.debug( 'Failed to close and delete temp file "%s". Got an error:\n%s', fileobj.name, e) def _DiffToApplyArgChecker(command_instance, diff_to_apply): """Arg checker that skips symlinks if -e flag specified.""" if (diff_to_apply.diff_action == DiffAction.REMOVE or not command_instance.exclude_symlinks): # No src URL is populated for REMOVE actions. return True exp_src_url = StorageUrlFromString(diff_to_apply.src_url_str) if exp_src_url.IsFileUrl() and os.path.islink(exp_src_url.object_name): command_instance.logger.info('Skipping symbolic link %s...', exp_src_url) return False return True def _ComputeNeededFileChecksums(logger, src_url_str, src_size, src_crc32c, src_md5, dst_url_str, dst_size, dst_crc32c, dst_md5): """Computes any file checksums needed by _CompareObjects. Args: logger: logging.logger for outputting log messages. src_url_str: Source URL string. src_size: Source size src_crc32c: Source CRC32c. src_md5: Source MD5. dst_url_str: Destination URL string. dst_size: Destination size dst_crc32c: Destination CRC32c. dst_md5: Destination MD5. Returns: (src_crc32c, src_md5, dst_crc32c, dst_md5) """ src_url = StorageUrlFromString(src_url_str) dst_url = StorageUrlFromString(dst_url_str) if src_url.IsFileUrl(): if dst_crc32c != _NA or dst_url.IsFileUrl(): if src_size > TEN_MIB: logger.info('Computing CRC32C for %s...', src_url_str) with open(src_url.object_name, 'rb') as fp: src_crc32c = CalculateB64EncodedCrc32cFromContents(fp) elif dst_md5 != _NA or dst_url.IsFileUrl(): if dst_size > TEN_MIB: logger.info('Computing MD5 for %s...', src_url_str) with open(src_url.object_name, 'rb') as fp: src_md5 = CalculateB64EncodedMd5FromContents(fp) if dst_url.IsFileUrl(): if src_crc32c != _NA: if src_size > TEN_MIB: logger.info('Computing CRC32C for %s...', dst_url_str) with open(dst_url.object_name, 'rb') as fp: dst_crc32c = CalculateB64EncodedCrc32cFromContents(fp) elif src_md5 != _NA: if dst_size > TEN_MIB: logger.info('Computing MD5 for %s...', dst_url_str) with open(dst_url.object_name, 'rb') as fp: dst_md5 = CalculateB64EncodedMd5FromContents(fp) return (src_crc32c, src_md5, dst_crc32c, dst_md5) def _ListUrlRootFunc(cls, args_tuple, thread_state=None): """Worker function for listing files/objects under to be sync'd. Outputs sorted list to out_file_name, formatted per _BuildTmpOutputLine. We sort the listed URLs because we don't want to depend on consistent sort order across file systems and cloud providers. Args: cls: Command instance. args_tuple: (base_url_str, out_file_name, desc), where base_url_str is top-level URL string to list; out_filename is name of file to which sorted output should be written; desc is 'source' or 'destination'. thread_state: gsutil Cloud API instance to use. """ gsutil_api = GetCloudApiInstance(cls, thread_state=thread_state) (base_url_str, out_filename, desc) = args_tuple # We sort while iterating over base_url_str, allowing parallelism of batched # sorting with collecting the listing. out_file = io.open(out_filename, mode='w', encoding=constants.UTF8) try: _BatchSort(_FieldedListingIterator(cls, gsutil_api, base_url_str, desc), out_file) except Exception as e: # pylint: disable=broad-except # Abandon rsync if an exception percolates up to this layer - retryable # exceptions are handled in the lower layers, so we got a non-retryable # exception (like 404 bucket not found) and proceeding would either be # futile or could result in data loss - for example: # gsutil rsync -d gs://non-existent-bucket ./localdir # would delete files from localdir. cls.logger.error('Caught non-retryable exception while listing %s: %s' % (base_url_str, e)) # Also print the full stack trace in debugging mode. This makes debugging # a bit easier. cls.logger.debug(traceback.format_exc()) cls.non_retryable_listing_failures = 1 out_file.close() def _LocalDirIterator(base_url): """A generator that yields a BLR for each file in a local directory. We use this function instead of WildcardIterator for listing a local directory without recursion, because the glob.globi implementation called by WildcardIterator skips "dot" files (which we don't want to do when synchronizing to or from a local directory). Args: base_url: URL for the directory over which to iterate. Yields: BucketListingObject for each file in the directory. """ for filename in os.listdir(base_url.object_name): filename = os.path.join(base_url.object_name, filename) if os.path.isfile(filename): yield BucketListingObject(StorageUrlFromString(filename), None) def _FieldedListingIterator(cls, gsutil_api, base_url_str, desc): """Iterator over base_url_str formatting output per _BuildTmpOutputLine. Args: cls: Command instance. gsutil_api: gsutil Cloud API instance to use for bucket listing. base_url_str: The top-level URL string over which to iterate. desc: 'source' or 'destination'. Yields: Output line formatted per _BuildTmpOutputLine. """ base_url = StorageUrlFromString(base_url_str) if base_url.scheme == 'file' and not cls.recursion_requested: iterator = _LocalDirIterator(base_url) else: if cls.recursion_requested: wildcard = '%s/**' % base_url_str.rstrip('/\\') else: wildcard = '%s/*' % base_url_str.rstrip('/\\') fields = [ 'crc32c', 'md5Hash', 'name', 'size', 'timeCreated', 'metadata/%s' % MTIME_ATTR, ] if cls.preserve_posix_attrs: fields.extend([ 'metadata/%s' % ATIME_ATTR, 'metadata/%s' % MODE_ATTR, 'metadata/%s' % GID_ATTR, 'metadata/%s' % UID_ATTR, ]) exclude_tuple = ( base_url, cls.exclude_dirs, cls.exclude_pattern) if cls.exclude_pattern is not None else None iterator = CreateWildcardIterator( wildcard, gsutil_api, project_id=cls.project_id, exclude_tuple=exclude_tuple, ignore_symlinks=cls.exclude_symlinks, logger=cls.logger).IterObjects( # Request just the needed fields, to reduce bandwidth usage. bucket_listing_fields=fields) i = 0 for blr in iterator: # Various GUI tools (like the GCS web console) create placeholder objects # ending with '/' when the user creates an empty directory. Normally these # tools should delete those placeholders once objects have been written # "under" the directory, but sometimes the placeholders are left around. # We need to filter them out here, otherwise if the user tries to rsync # from GCS to a local directory it will result in a directory/file # conflict (e.g., trying to download an object called "mydata/" where the # local directory "mydata" exists). url = blr.storage_url if IsCloudSubdirPlaceholder(url, blr=blr): # We used to output the message 'Skipping cloud sub-directory placeholder # object...' but we no longer do so because it caused customer confusion. continue if (cls.exclude_symlinks and url.IsFileUrl() and os.path.islink(url.object_name)): continue if cls.exclude_pattern: # The wildcard_iterator may optionally use the exclude pattern to exclude # directories while this section excludes individual files. str_to_check = url.url_string[len(base_url.url_string):] if str_to_check.startswith(url.delim): str_to_check = str_to_check[1:] if cls.exclude_pattern.match(str_to_check): continue i += 1 if i % _PROGRESS_REPORT_LISTING_COUNT == 0: cls.logger.info('At %s listing %d...', desc, i) yield _BuildTmpOutputLine(blr) def _BuildTmpOutputLine(blr): """Builds line to output to temp file for given BucketListingRef. Args: blr: The BucketListingRef. Returns: The output line, formatted as _EncodeUrl(URL)<sp>size<sp>time_created<sp>atime<sp>mtime<sp>mode<sp>uid<sp> gid<sp>crc32c<sp>md5 where md5 will only be present for cloud URLs that aren't composite objects. A missing field is populated with '-', or -1 in the case of atime/mtime/time_created. """ atime = NA_TIME crc32c = _NA gid = NA_ID md5 = _NA mode = NA_MODE mtime = NA_TIME time_created = NA_TIME uid = NA_ID url = blr.storage_url if url.IsFileUrl(): mode, _, _, _, uid, gid, size, atime, mtime, _ = os.stat(url.object_name) # atime/mtime can be a float, so it needs to be converted to a long. atime = long(atime) mtime = long(mtime) mode = ConvertModeToBase8(mode) # Don't use atime / mtime with times older than 1970-01-01 UTC. if atime < 0: atime = NA_TIME if mtime < 0: mtime = NA_TIME elif url.IsCloudUrl(): size = blr.root_object.size if blr.root_object.metadata is not None: found_m, mtime_str = GetValueFromObjectCustomMetadata( blr.root_object, MTIME_ATTR, NA_TIME) try: # The mtime value can be changed in the online console, this performs a # sanity check and sets the mtime to NA_TIME if it fails. mtime = long(mtime_str) if found_m and mtime <= NA_TIME: WarnNegativeAttribute('mtime', url.url_string) if mtime > long(time.time()) + SECONDS_PER_DAY: WarnFutureTimestamp('mtime', url.url_string) except ValueError: # Since mtime is a string, catch the case where it can't be cast as a # long. WarnInvalidValue('mtime', url.url_string) mtime = NA_TIME posix_attrs = DeserializeFileAttributesFromObjectMetadata( blr.root_object, url.url_string) mode = posix_attrs.mode.permissions atime = posix_attrs.atime uid = posix_attrs.uid gid = posix_attrs.gid # Sanitize the timestamp returned, and put it in UTC format. For more # information see the UTC class in gslib/util.py. time_created = ConvertDatetimeToPOSIX(blr.root_object.timeCreated) crc32c = blr.root_object.crc32c or _NA md5 = blr.root_object.md5Hash or _NA else: raise CommandException('Got unexpected URL type (%s)' % url.scheme) attrs = [ _EncodeUrl(url.url_string), # binary str in py2 / unicode str py 3 size, # int time_created, # int atime, # long mtime, # long mode, # int uid, # int gid, # int crc32c, # unicode md5, # unicode ] attrs = [six.ensure_text(str(i)) for i in attrs] return ' '.join(attrs) + '\n' def _EncodeUrl(url_string): """Encodes url_str with quote plus encoding and UTF8 character encoding. We use this for all URL encodings. Args: url_string (unicode): String URL to encode. Returns: (str) A string encoded using urllib's `quote_plus()` method. """ # N.B.: `quote_plus()` raises an error for unicode characters like è if you # don't pass it the language-appropriate string type. If you pass it `unicode` # in Python 2 or `bytes` in Python 3, it leads to surprising behavior for text # containing unicode chars. url_string = six.ensure_str(url_string) return urllib.parse.quote_plus(url_string, safe=b'~') def _DecodeUrl(enc_url_string): """Inverts encoding from `_EncodeUrl()`. Args: enc_url_string (str): String containing UTF-8-decodable characters that were encoded using urllib's `quote_plus()`. Returns: (unicode) A decoded URL. """ url = urllib.parse.unquote_plus(enc_url_string) if six.PY2: url = url.decode(constants.UTF8) return url # pylint: disable=bare-except def _BatchSort(in_iter, out_file): """Sorts input lines from in_iter and outputs to out_file. Sorts in batches as input arrives, so input file does not need to be loaded into memory all at once. Derived from Python Recipe 466302: Sorting big files the Python 2.4 way by Nicolas Lehuen. Sorted format is per _BuildTmpOutputLine. We're sorting on the entire line when we could just sort on the first record (URL); but the sort order is identical either way. Args: in_iter: Input iterator. out_file: Output file. """ # Note: If chunk_files gets very large we can run out of open FDs. See .boto # file comments about rsync_buffer_lines. If increasing rsync_buffer_lines # doesn't suffice (e.g., for someone synchronizing with a really large # bucket), an option would be to make gsutil merge in passes, never # opening all chunk files simultaneously. buffer_size = config.getint('GSUtil', 'rsync_buffer_lines', 32000) chunk_files = [] try: while True: current_chunk = sorted(islice(in_iter, buffer_size)) if not current_chunk: break output_chunk = io.open('%s-%06i' % (out_file.name, len(chunk_files)), mode='w+', encoding=constants.UTF8) chunk_files.append(output_chunk) output_chunk.write(six.text_type(''.join(current_chunk))) output_chunk.flush() output_chunk.seek(0) out_file.writelines(heapq.merge(*chunk_files)) except IOError as e: if e.errno == errno.EMFILE: raise CommandException('\n'.join( textwrap.wrap( 'Synchronization failed because too many open file handles were ' 'needed while building synchronization state. Please see the ' 'comments about rsync_buffer_lines in your .boto config file for a ' 'possible way to address this problem.'))) raise finally: for chunk_file in chunk_files: try: chunk_file.close() os.remove(chunk_file.name) except Exception as e: # pylint: disable=broad-except logging.debug( 'Failed to remove rsync chunk file "%s". Got an error:\n%s', chunk_file.name, e) class _DiffIterator(object): """Iterator yielding sequence of RsyncDiffToApply objects.""" def __init__(self, command_obj, base_src_url, base_dst_url): self.command_obj = command_obj self.compute_file_checksums = command_obj.compute_file_checksums self.delete_extras = command_obj.delete_extras self.recursion_requested = command_obj.recursion_requested self.logger = self.command_obj.logger self.base_src_url = base_src_url self.base_dst_url = base_dst_url self.preserve_posix = command_obj.preserve_posix_attrs self.skip_old_files = command_obj.skip_old_files self.ignore_existing = command_obj.ignore_existing self.logger.info('Building synchronization state...') # Files to track src and dst state should be created in the system's # preferred temp directory so that they are eventually cleaned up if our # cleanup callback is interrupted. temp_src_file = tempfile.NamedTemporaryFile(prefix='gsutil-rsync-src-', delete=False) temp_dst_file = tempfile.NamedTemporaryFile(prefix='gsutil-rsync-dst-', delete=False) self.sorted_list_src_file_name = temp_src_file.name self.sorted_list_dst_file_name = temp_dst_file.name _tmp_files.append(temp_src_file) _tmp_files.append(temp_dst_file) # Close the files, but don't delete them. Because Windows does not allow # a temporary file to be reopened until it's been closed, we close the # files before proceeding. This allows each step below to open the file at # the specified path, perform I/O, and close it so that the next step may # do the same thing. temp_src_file.close() temp_dst_file.close() # Build sorted lists of src and dst URLs in parallel. To do this, pass # args to _ListUrlRootFunc as tuple (base_url_str, out_filename, desc) # where base_url_str is the starting URL string for listing. args_iter = iter([ ( self.base_src_url.url_string, self.sorted_list_src_file_name, 'source', ), ( self.base_dst_url.url_string, self.sorted_list_dst_file_name, 'destination', ), ]) # Contains error message from non-retryable listing failure. command_obj.non_retryable_listing_failures = 0 shared_attrs = ['non_retryable_listing_failures'] command_obj.Apply( _ListUrlRootFunc, args_iter, _RootListingExceptionHandler, shared_attrs, arg_checker=DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SPEED, fail_on_error=True) if command_obj.non_retryable_listing_failures: raise CommandException('Caught non-retryable exception - aborting rsync') # Note that while this leaves 2 open file handles, we track these in a # global list to be closed (if not closed in the calling scope) and deleted # at exit time. self.sorted_list_src_file = open(self.sorted_list_src_file_name, 'r') self.sorted_list_dst_file = open(self.sorted_list_dst_file_name, 'r') _tmp_files.append(self.sorted_list_src_file) _tmp_files.append(self.sorted_list_dst_file) if (base_src_url.IsCloudUrl() and base_dst_url.IsFileUrl() and self.preserve_posix): self.sorted_src_urls_it = PluralityCheckableIterator( iter(self.sorted_list_src_file)) self._ValidateObjectAccess() # Reset our file pointers to the beginning. self.sorted_list_src_file.seek(0) # Wrap iterators in PluralityCheckableIterator so we can check emptiness. self.sorted_src_urls_it = PluralityCheckableIterator( iter(self.sorted_list_src_file)) self.sorted_dst_urls_it = PluralityCheckableIterator( iter(self.sorted_list_dst_file)) def _ValidateObjectAccess(self): """Validates that the user won't lose access to the files if copied. Iterates over the src file list to check if access will be maintained. If at any point we would orphan a file, a list of errors is compiled and logged with an exception raised to the user. """ errors = collections.deque() for src_url in self.sorted_src_urls_it: src_url_str, _, _, _, _, src_mode, src_uid, src_gid, _, _ = ( self._ParseTmpFileLine(src_url)) valid, err = ValidateFilePermissionAccess(src_url_str, uid=src_uid, gid=src_gid, mode=src_mode) if not valid: errors.append(err) if errors: for err in errors: self.logger.critical(err) raise CommandException('This sync will orphan file(s), please fix their ' 'permissions before trying again.') def _ParseTmpFileLine(self, line): """Parses output from _BuildTmpOutputLine. Parses into tuple: (URL, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) where crc32c and/or md5 can be _NA and atime/mtime/time_created can be NA_TIME. Args: line: The line to parse. Returns: Parsed tuple: (url, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) """ (encoded_url, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) = line.rsplit(None, 9) return ( _DecodeUrl(encoded_url), int(size), long(time_created), long(atime), long(mtime), int(mode), int(uid), int(gid), crc32c, md5.strip(), ) def _WarnIfMissingCloudHash(self, url_str, crc32c, md5): """Warns if given url_str is a cloud URL and is missing both crc32c and md5. Args: url_str: Destination URL string. crc32c: Destination CRC32c. md5: Destination MD5. Returns: True if issued warning. """ # One known way this can currently happen is when rsync'ing objects larger # than 5 GB from S3 (for which the etag is not an MD5). if (StorageUrlFromString(url_str).IsCloudUrl() and crc32c == _NA and md5 == _NA): self.logger.warn( 'Found no hashes to validate %s. Integrity cannot be assured without ' 'hashes.', url_str) return True return False def _CompareObjects( self, src_url_str, src_size, src_mtime, src_crc32c, src_md5, dst_url_str, dst_size, dst_mtime, dst_crc32c, dst_md5, ): """Returns whether src should replace dst object, and if mtime is present. Uses mtime, size, or whatever checksums are available. Args: src_url_str: Source URL string. src_size: Source size. src_mtime: Source modification time. src_crc32c: Source CRC32c. src_md5: Source MD5. dst_url_str: Destination URL string. dst_size: Destination size. dst_mtime: Destination modification time. dst_crc32c: Destination CRC32c. dst_md5: Destination MD5. Returns: A 3-tuple indicating if src should replace dst, and if src and dst have mtime. """ # Note: This function is called from __iter__, which is called from the # Command.Apply driver. Thus, all checksum computation will be run in a # single thread, which is good (having multiple threads concurrently # computing checksums would thrash the disk). # # Comparison Hierarchy: # 1. mtime # 2. md5/crc32c hashes (if available) # 3. size has_src_mtime = src_mtime > NA_TIME has_dst_mtime = dst_mtime > NA_TIME use_hashes = (self.compute_file_checksums or (StorageUrlFromString(src_url_str).IsCloudUrl() and StorageUrlFromString(dst_url_str).IsCloudUrl())) if self.ignore_existing: return False, has_src_mtime, has_dst_mtime if (self.skip_old_files and has_src_mtime and has_dst_mtime and src_mtime < dst_mtime): return False, has_src_mtime, has_dst_mtime if not use_hashes and has_src_mtime and has_dst_mtime: return (src_mtime != dst_mtime or src_size != dst_size, has_src_mtime, has_dst_mtime) if src_size != dst_size: return True, has_src_mtime, has_dst_mtime src_crc32c, src_md5, dst_crc32c, dst_md5 = _ComputeNeededFileChecksums( self.logger, src_url_str, src_size, src_crc32c, src_md5, dst_url_str, dst_size, dst_crc32c, dst_md5, ) if src_md5 != _NA and dst_md5 != _NA: self.logger.debug('Comparing md5 for %s and %s', src_url_str, dst_url_str) return src_md5 != dst_md5, has_src_mtime, has_dst_mtime if src_crc32c != _NA and dst_crc32c != _NA: self.logger.debug('Comparing crc32c for %s and %s', src_url_str, dst_url_str) return src_crc32c != dst_crc32c, has_src_mtime, has_dst_mtime if not self._WarnIfMissingCloudHash(src_url_str, src_crc32c, src_md5): self._WarnIfMissingCloudHash(dst_url_str, dst_crc32c, dst_md5) # Without checksums or mtime to compare we depend only on basic size # comparison. return False, has_src_mtime, has_dst_mtime def __iter__(self): """Iterates over src/dst URLs and produces a RsyncDiffToApply sequence. Yields: The RsyncDiffToApply. """ # Strip trailing slashes, if any, so we compute tail length against # consistent position regardless of whether trailing slashes were included # or not in URL. base_src_url_len = len(self.base_src_url.url_string.rstrip('/\\')) base_dst_url_len = len(self.base_dst_url.url_string.rstrip('/\\')) out_of_src_items = False src_url_str = dst_url_str = None # Invariant: After each yield, the URLs in src_url_str, dst_url_str, # self.sorted_src_urls_it, and self.sorted_dst_urls_it are not yet # processed. Each time we encounter None in src_url_str or dst_url_str we # populate from the respective iterator, and we reset one or the other value # to None after yielding an action that disposes of that URL. while True: if src_url_str is None: if self.sorted_src_urls_it.IsEmpty(): out_of_src_items = True else: (src_url_str, src_size, src_time_created, src_atime, src_mtime, src_mode, src_uid, src_gid, src_crc32c, src_md5) = (self._ParseTmpFileLine(next(self.sorted_src_urls_it))) posix_attrs = POSIXAttributes(atime=src_atime, mtime=src_mtime, uid=src_uid, gid=src_gid, mode=src_mode) # Skip past base URL and normalize slashes so we can compare across # clouds/file systems (including Windows). src_url_str_to_check = _EncodeUrl( src_url_str[base_src_url_len:].replace('\\', '/')) dst_url_str_would_copy_to = copy_helper.ConstructDstUrl( src_url=self.base_src_url, exp_src_url=StorageUrlFromString(src_url_str), src_url_names_container=True, have_multiple_srcs=True, has_multiple_top_level_srcs=False, exp_dst_url=self.base_dst_url, have_existing_dest_subdir=False, recursion_requested=self.recursion_requested).url_string if dst_url_str is None: if not self.sorted_dst_urls_it.IsEmpty(): # We don't need time created at the destination. (dst_url_str, dst_size, _, dst_atime, dst_mtime, dst_mode, dst_uid, dst_gid, dst_crc32c, dst_md5) = self._ParseTmpFileLine(next(self.sorted_dst_urls_it)) # Skip past base URL and normalize slashes so we can compare across # clouds/file systems (including Windows). dst_url_str_to_check = _EncodeUrl( dst_url_str[base_dst_url_len:].replace('\\', '/')) # Only break once we've attempted to populate {str,dst}_url_to_check and # we know we're out of src objects. if out_of_src_items: break # We're guaranteed to have a value for src_url_str_to_check here, but may # be out of dst objects. if (dst_url_str is None or src_url_str_to_check < dst_url_str_to_check): # There's no dst object corresponding to src object, so copy src to dst. yield RsyncDiffToApply(src_url_str, dst_url_str_would_copy_to, posix_attrs, DiffAction.COPY, src_size) src_url_str = None elif src_url_str_to_check > dst_url_str_to_check: # dst object without a corresponding src object, so remove dst if -d # option was specified. if self.delete_extras: yield RsyncDiffToApply(None, dst_url_str, POSIXAttributes(), DiffAction.REMOVE, None) dst_url_str = None else: # There is a dst object corresponding to src object, so check if objects # match. if (StorageUrlFromString(src_url_str).IsCloudUrl() and StorageUrlFromString(dst_url_str).IsFileUrl() and src_mtime == NA_TIME): src_mtime = src_time_created should_replace, has_src_mtime, has_dst_mtime = (self._CompareObjects( src_url_str, src_size, src_mtime, src_crc32c, src_md5, dst_url_str, dst_size, dst_mtime, dst_crc32c, dst_md5)) if should_replace: yield RsyncDiffToApply(src_url_str, dst_url_str, posix_attrs, DiffAction.COPY, src_size) elif self.preserve_posix: posix_attrs, needs_update = NeedsPOSIXAttributeUpdate( src_atime, dst_atime, src_mtime, dst_mtime, src_uid, dst_uid, src_gid, dst_gid, src_mode, dst_mode) if needs_update: yield RsyncDiffToApply(src_url_str, dst_url_str, posix_attrs, DiffAction.POSIX_SRC_TO_DST, src_size) elif has_src_mtime and not has_dst_mtime: # File/object at destination matches source but is missing mtime # attribute at destination. yield RsyncDiffToApply(src_url_str, dst_url_str, posix_attrs, DiffAction.MTIME_SRC_TO_DST, src_size) # else: we don't need to copy the file from src to dst since they're # the same files. # Advance to the next two objects. src_url_str = None dst_url_str = None if not self.delete_extras: return # If -d option was specified any files/objects left in dst iteration should # be removed. if dst_url_str: yield RsyncDiffToApply(None, dst_url_str, POSIXAttributes(), DiffAction.REMOVE, None) for line in self.sorted_dst_urls_it: (dst_url_str, _, _, _, _, _, _, _, _, _) = self._ParseTmpFileLine(line) yield RsyncDiffToApply(None, dst_url_str, POSIXAttributes(), DiffAction.REMOVE, None) class _SeekAheadDiffIterator(object): """Wraps _AvoidChecksumAndListingDiffIterator and yields SeekAheadResults.""" def __init__(self, cloned_diff_iterator): self.cloned_diff_iterator = cloned_diff_iterator def __iter__(self): for diff_to_apply in self.cloned_diff_iterator: bytes_to_copy = diff_to_apply.copy_size or 0 if (diff_to_apply.diff_action == DiffAction.MTIME_SRC_TO_DST or diff_to_apply.diff_action == DiffAction.POSIX_SRC_TO_DST): # Assume MTIME_SRC_TO_DST and POSIX_SRC_TO_DST are metadata-only # copies. However, if the user does not have OWNER permission on # an object, the data must be re-sent, and this function will # underestimate the amount of bytes that rsync must copy. bytes_to_copy = 0 yield SeekAheadResult(data_bytes=bytes_to_copy) class _AvoidChecksumAndListingDiffIterator(_DiffIterator): """Iterator initialized from an existing _DiffIterator. This iterator yields RsyncDiffToApply objects used to estimate the total work that will be performed by the DiffIterator, while avoiding expensive computation. """ # pylint: disable=super-init-not-called def __init__(self, initialized_diff_iterator): # Intentionally don't call the _DiffIterator constructor. This class # reuses the initialized_diff_iterator values to avoid unnecessary # computation, and inherits the __iter__ function. # We're providing an estimate, so avoid computing checksums even though # that may cause our estimate to be off. self.compute_file_checksums = False self.delete_extras = initialized_diff_iterator.delete_extras self.recursion_requested = initialized_diff_iterator.delete_extras # TODO: Add a test that mocks the appropriate values in RsyncFunc and # ensure that running this iterator succeeds. self.preserve_posix = False # This iterator shouldn't output any log messages. self.logger = logging.getLogger('dummy') self.base_src_url = initialized_diff_iterator.base_src_url self.base_dst_url = initialized_diff_iterator.base_dst_url self.skip_old_files = initialized_diff_iterator.skip_old_files self.ignore_existing = initialized_diff_iterator.ignore_existing # Note that while this leaves 2 open file handles, we track these in a # global list to be closed (if not closed in the calling scope) and deleted # at exit time. self.sorted_list_src_file = open( initialized_diff_iterator.sorted_list_src_file_name, 'r') self.sorted_list_dst_file = open( initialized_diff_iterator.sorted_list_dst_file_name, 'r') _tmp_files.append(self.sorted_list_src_file) _tmp_files.append(self.sorted_list_dst_file) # Wrap iterators in PluralityCheckableIterator so we can check emptiness. self.sorted_src_urls_it = PluralityCheckableIterator( iter(self.sorted_list_src_file)) self.sorted_dst_urls_it = PluralityCheckableIterator( iter(self.sorted_list_dst_file)) # pylint: enable=super-init-not-called def _RsyncFunc(cls, diff_to_apply, thread_state=None): """Worker function for performing the actual copy and remove operations.""" gsutil_api = GetCloudApiInstance(cls, thread_state=thread_state) dst_url_str = diff_to_apply.dst_url_str dst_url = StorageUrlFromString(dst_url_str) posix_attrs = diff_to_apply.src_posix_attrs if diff_to_apply.diff_action == DiffAction.REMOVE: if cls.dryrun: cls.logger.info('Would remove %s', dst_url) else: cls.logger.info('Removing %s', dst_url) if dst_url.IsFileUrl(): try: os.unlink(dst_url.object_name) except FileNotFoundError: # Missing file errors occur occasionally with .gstmp files # and can be ignored for deletes. cls.logger.debug('%s was already removed', dst_url) pass else: try: gsutil_api.DeleteObject(dst_url.bucket_name, dst_url.object_name, generation=dst_url.generation, provider=dst_url.scheme) except NotFoundException: # If the object happened to be deleted by an external process, this # is fine because it moves us closer to the desired state. pass elif diff_to_apply.diff_action == DiffAction.COPY: src_url_str = diff_to_apply.src_url_str src_url = StorageUrlFromString(src_url_str) if cls.dryrun: if src_url.IsFileUrl(): # Try to open the local file to detect errors that would occur in # non-dry-run mode. try: with open(src_url.object_name, 'rb') as _: pass except Exception as e: # pylint: disable=broad-except cls.logger.info('Could not open %s' % src_url.object_name) raise cls.logger.info('Would copy %s to %s', src_url, dst_url) else: try: src_obj_metadata = None if src_url.IsCloudUrl(): src_generation = GenerationFromUrlAndString(src_url, src_url.generation) src_obj_metadata = gsutil_api.GetObjectMetadata( src_url.bucket_name, src_url.object_name, generation=src_generation, provider=src_url.scheme, fields=cls.source_metadata_fields) if ObjectIsGzipEncoded(src_obj_metadata): cls.logger.info( '%s has a compressed content-encoding, so it will be ' 'decompressed upon download; future executions of gsutil rsync ' 'with this source object will always download it. If you wish ' 'to synchronize such an object efficiently, compress the ' 'source objects in place before synchronizing them, rather ' 'than (for example) using gsutil cp -Z to compress them ' 'on-the-fly (which results in compressed content-encoding).' % src_url) else: # src_url.IsFileUrl() src_obj_metadata = apitools_messages.Object() # getmtime can return a float, so it needs to be converted to long. if posix_attrs.mtime > long(time.time()) + SECONDS_PER_DAY: WarnFutureTimestamp('mtime', src_url.url_string) if src_url.IsFifo() or src_url.IsStream(): type_text = 'Streams' if src_url.IsStream() else 'Named pipes' cls.logger.warn( 'WARNING: %s are not supported by gsutil rsync and ' 'will likely fail. Use the -x option to exclude %s by name.', type_text, src_url.url_string) if src_obj_metadata.metadata: custom_metadata = src_obj_metadata.metadata else: custom_metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) SerializeFileAttributesToObjectMetadata( posix_attrs, custom_metadata, preserve_posix=cls.preserve_posix_attrs) tmp_obj_metadata = apitools_messages.Object() tmp_obj_metadata.metadata = custom_metadata CopyCustomMetadata(tmp_obj_metadata, src_obj_metadata, override=True) copy_result = copy_helper.PerformCopy( cls.logger, src_url, dst_url, gsutil_api, cls, _RsyncExceptionHandler, src_obj_metadata=src_obj_metadata, headers=cls.headers, is_rsync=True, gzip_encoded=cls.gzip_encoded, gzip_exts=cls.gzip_exts, preserve_posix=cls.preserve_posix_attrs) if copy_result is not None: (_, bytes_transferred, _, _) = copy_result with cls.stats_lock: cls.total_bytes_transferred += bytes_transferred except SkipUnsupportedObjectError as e: cls.logger.info('Skipping item %s with unsupported object type %s', src_url, e.unsupported_type) elif diff_to_apply.diff_action == DiffAction.MTIME_SRC_TO_DST: # If the destination is an object in a bucket, this will not blow away other # metadata. This behavior is unlike if the file/object actually needed to be # copied from the source to the destination. dst_url = StorageUrlFromString(diff_to_apply.dst_url_str) if cls.dryrun: cls.logger.info('Would set mtime for %s', dst_url) else: cls.logger.info('Copying mtime from src to dst for %s', dst_url.url_string) mtime = posix_attrs.mtime obj_metadata = apitools_messages.Object() obj_metadata.metadata = CreateCustomMetadata({MTIME_ATTR: mtime}) if dst_url.IsCloudUrl(): dst_url = StorageUrlFromString(diff_to_apply.dst_url_str) dst_generation = GenerationFromUrlAndString(dst_url, dst_url.generation) try: # Assume we have permission, and can patch the object. gsutil_api.PatchObjectMetadata(dst_url.bucket_name, dst_url.object_name, obj_metadata, provider=dst_url.scheme, generation=dst_url.generation) except ServiceException as err: cls.logger.debug('Error while trying to patch: %s', err) # We don't have permission to patch apparently, so it must be copied. cls.logger.info( 'Copying whole file/object for %s instead of patching' ' because you don\'t have patch permission on the ' 'object.', dst_url.url_string) _RsyncFunc(cls, RsyncDiffToApply(diff_to_apply.src_url_str, diff_to_apply.dst_url_str, posix_attrs, DiffAction.COPY, diff_to_apply.copy_size), thread_state=thread_state) else: ParseAndSetPOSIXAttributes(dst_url.object_name, obj_metadata, preserve_posix=cls.preserve_posix_attrs) elif diff_to_apply.diff_action == DiffAction.POSIX_SRC_TO_DST: # If the destination is an object in a bucket, this will not blow away other # metadata. This behavior is unlike if the file/object actually needed to be # copied from the source to the destination. dst_url = StorageUrlFromString(diff_to_apply.dst_url_str) if cls.dryrun: cls.logger.info('Would set POSIX attributes for %s', dst_url) else: cls.logger.info('Copying POSIX attributes from src to dst for %s', dst_url.url_string) obj_metadata = apitools_messages.Object() obj_metadata.metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) SerializeFileAttributesToObjectMetadata(posix_attrs, obj_metadata.metadata, preserve_posix=True) if dst_url.IsCloudUrl(): dst_generation = GenerationFromUrlAndString(dst_url, dst_url.generation) dst_obj_metadata = gsutil_api.GetObjectMetadata( dst_url.bucket_name, dst_url.object_name, generation=dst_generation, provider=dst_url.scheme, fields=['acl']) try: # Assume we have ownership, and can patch the object. gsutil_api.PatchObjectMetadata(dst_url.bucket_name, dst_url.object_name, obj_metadata, provider=dst_url.scheme, generation=dst_url.generation) except ServiceException as err: cls.logger.debug('Error while trying to patch: %s', err) # Apparently we don't have object ownership, so it must be copied. cls.logger.info( 'Copying whole file/object for %s instead of patching' ' because you don\'t have patch permission on the ' 'object.', dst_url.url_string) _RsyncFunc(cls, RsyncDiffToApply(diff_to_apply.src_url_str, diff_to_apply.dst_url_str, posix_attrs, DiffAction.COPY, diff_to_apply.copy_size), thread_state=thread_state) else: raise CommandException('Got unexpected DiffAction (%d)' % diff_to_apply.diff_action) def _RootListingExceptionHandler(cls, e): """Simple exception handler for exceptions during listing URLs to sync.""" cls.logger.error(str(e)) def _RsyncExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) cls.op_failure_count += 1 cls.logger.debug('\n\nEncountered exception while syncing:\n%s\n', traceback.format_exc()) class RsyncCommand(Command): """Implementation of gsutil rsync command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rsync', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=2, supported_sub_args='a:cCdenpPriRuUx:y:j:J', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeNCloudOrFileURLsArgument(2)]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rsync', help_name_aliases=['sync', 'synchronize'], help_type='command_help', help_one_line_summary='Synchronize content of two buckets/directories', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): ShimTranslatePredefinedAclSubOptForCopy(self.sub_opts) gcloud_command = ['storage', 'rsync'] flag_keys = [flag for flag, _ in self.sub_opts] if '-e' not in flag_keys: gcloud_command += ['--no-ignore-symlinks'] self.logger.warn( 'By default, gsutil copies file symlinks, but, by default, this' ' command (run via the gcloud storage shim) does not copy any' ' symlinks.') if '-P' in flag_keys: _, (source_path, destination_path) = self.ParseSubOpts( should_update_sub_opts_and_args=False) if (StorageUrlFromString(source_path).IsCloudUrl() and StorageUrlFromString(destination_path).IsFileUrl()): self.logger.warn( 'For preserving POSIX with rsync downloads, gsutil aborts if a' ' single download will result in invalid destination POSIX.' ' However, this command (run via the gcloud storage shim) will' ' skip invalid copies and still perform valid copies.') gcloud_storage_map = GcloudStorageMap( gcloud_command=gcloud_command, flag_map={ '-a': GcloudStorageFlag('--predefined-acl'), '-c': GcloudStorageFlag('--checksums-only'), '-C': GcloudStorageFlag('--continue-on-error'), '-d': GcloudStorageFlag('--delete-unmatched-destination-objects'), '-e': GcloudStorageFlag('--ignore-symlinks'), '-i': GcloudStorageFlag('--no-clobber'), '-J': GcloudStorageFlag('--gzip-in-flight-all'), '-j': GcloudStorageFlag('--gzip-in-flight'), '-n': GcloudStorageFlag('--dry-run'), '-P': GcloudStorageFlag('--preserve-posix'), '-p': GcloudStorageFlag('--preserve-acl'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), '-U': GcloudStorageFlag('--skip-unsupported'), '-u': GcloudStorageFlag('--skip-if-dest-has-newer-mtime'), '-x': GcloudStorageFlag('--exclude'), }, ) return super().get_gcloud_storage_args(gcloud_storage_map) def _InsistContainer(self, url_str, treat_nonexistent_object_as_subdir): """Sanity checks that URL names an existing container. Args: url_str: URL string to check. treat_nonexistent_object_as_subdir: indicates if should treat a non-existent object as a subdir. Returns: URL for checked string. Raises: CommandException if url_str doesn't name an existing container. """ (url, have_existing_container) = copy_helper.ExpandUrlToSingleBlr( url_str, self.gsutil_api, self.project_id, treat_nonexistent_object_as_subdir, logger=self.logger) if not have_existing_container: raise CommandException( 'arg (%s) does not name a directory, bucket, or bucket subdir.\n' 'If there is an object with the same path, please add a trailing\n' 'slash to specify the directory.' % url_str) return url def RunCommand(self): """Command entry point for the rsync command.""" self._ParseOpts() self.total_bytes_transferred = 0 # Use a lock to ensure accurate statistics in the face of # multi-threading/multi-processing. self.stats_lock = parallelism_framework_util.CreateLock() if not UsingCrcmodExtension(): if self.compute_file_checksums: self.logger.warn(SLOW_CRCMOD_WARNING) else: self.logger.warn(SLOW_CRCMOD_RSYNC_WARNING) src_url = self._InsistContainer(self.args[0], False) dst_url = self._InsistContainer(self.args[1], True) is_daisy_chain = (src_url.IsCloudUrl() and dst_url.IsCloudUrl() and src_url.scheme != dst_url.scheme) LogPerformanceSummaryParams(has_file_src=src_url.IsFileUrl(), has_cloud_src=src_url.IsCloudUrl(), has_file_dst=dst_url.IsFileUrl(), has_cloud_dst=dst_url.IsCloudUrl(), is_daisy_chain=is_daisy_chain, uses_fan=self.parallel_operations, provider_types=[src_url.scheme, dst_url.scheme]) self.source_metadata_fields = GetSourceFieldsNeededForCopy( dst_url.IsCloudUrl(), self.skip_unsupported_objects, self.preserve_acl, is_rsync=True, preserve_posix=self.preserve_posix_attrs) # Tracks if any copy or rm operations failed. self.op_failure_count = 0 # Tuple of attributes to share/manage across multiple processes in # parallel (-m) mode. shared_attrs = ('op_failure_count', 'total_bytes_transferred') for signal_num in GetCaughtSignals(): RegisterSignalHandler(signal_num, _HandleSignals) process_count, thread_count = self._GetProcessAndThreadCount( process_count=None, thread_count=None, parallel_operations_override=self.ParallelOverrideReason.SPEED, print_macos_warning=False) copy_helper.TriggerReauthForDestinationProviderIfNecessary( dst_url, self.gsutil_api, worker_count=process_count * thread_count, ) # Perform sync requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. diff_iterator = _DiffIterator(self, src_url, dst_url) # For estimation purposes, create a SeekAheadIterator based on the # source and destination files generated when creating the diff iterator. # This iteration should avoid expensive operations like file checksumming. seek_ahead_iterator = _SeekAheadDiffIterator( _AvoidChecksumAndListingDiffIterator(diff_iterator)) self.logger.info('Starting synchronization...') start_time = time.time() try: self.Apply(_RsyncFunc, diff_iterator, _RsyncExceptionHandler, shared_attrs, arg_checker=_DiffToApplyArgChecker, fail_on_error=True, seek_ahead_iterator=seek_ahead_iterator) finally: CleanUpTempFiles() end_time = time.time() self.total_elapsed_time = end_time - start_time self.total_bytes_per_second = CalculateThroughput( self.total_bytes_transferred, self.total_elapsed_time) LogPerformanceSummaryParams( avg_throughput=self.total_bytes_per_second, total_elapsed_time=self.total_elapsed_time, total_bytes_transferred=self.total_bytes_transferred) if self.op_failure_count: plural_str = 's' if self.op_failure_count else '' raise CommandException('%d file%s/object%s could not be copied/removed.' % (self.op_failure_count, plural_str, plural_str)) def _ParseOpts(self): # exclude_symlinks is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.exclude_symlinks = False # continue_on_error is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.continue_on_error = False self.delete_extras = False self.preserve_acl = False self.preserve_posix_attrs = False self.compute_file_checksums = False self.dryrun = False self.exclude_dirs = False self.exclude_pattern = None self.skip_old_files = False self.ignore_existing = False self.skip_unsupported_objects = False # self.recursion_requested is initialized in command.py (so it can be # checked in parent class for all commands). canned_acl = None # canned_acl is handled by a helper function in parent # Command class, so save in Command state rather than CopyHelperOpts. self.canned = None # Files matching these extensions should be compressed. # The gzip_encoded flag marks if the files should be compressed during # the upload. gzip_encoded = False gzip_arg_exts = None gzip_arg_all = None if self.sub_opts: for o, a in self.sub_opts: if o == '-a': canned_acl = a self.canned = True if o == '-c': self.compute_file_checksums = True # Note: In gsutil cp command this is specified using -c but here we use # -C so we can use -c for checksum arg (to be consistent with Unix rsync # command options). elif o == '-C': self.continue_on_error = True elif o == '-d': self.delete_extras = True elif o == '-e': self.exclude_symlinks = True elif o == '-j': gzip_encoded = True gzip_arg_exts = [x.strip() for x in a.split(',')] elif o == '-J': gzip_encoded = True gzip_arg_all = GZIP_ALL_FILES elif o == '-n': self.dryrun = True elif o == '-p': self.preserve_acl = True elif o == '-P': self.preserve_posix_attrs = True if not IS_WINDOWS: InitializePreservePosixData() elif o == '-r' or o == '-R': self.recursion_requested = True elif o == '-u': self.skip_old_files = True elif o == '-i': self.ignore_existing = True elif o == '-U': self.skip_unsupported_objects = True elif o == '-x' or o == '-y': if o == '-y': self.exclude_dirs = True if not a: raise CommandException('Invalid blank exclude filter') try: self.exclude_pattern = re.compile(a) except re.error: raise CommandException('Invalid exclude filter (%s)' % a) if self.preserve_acl and canned_acl: raise CommandException( 'Specifying both the -p and -a options together is invalid.') if gzip_arg_exts and gzip_arg_all: raise CommandException( 'Specifying both the -j and -J options together is invalid.') self.gzip_encoded = gzip_encoded self.gzip_exts = gzip_arg_exts or gzip_arg_all return CreateCopyHelperOpts( canned_acl=canned_acl, preserve_acl=self.preserve_acl, skip_unsupported_objects=self.skip_unsupported_objects) commands/kms.py 0000644 00000051363 15025013616 0007520 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the kms command to gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import getopt import textwrap from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.kms_api import KmsApi from gslib.project_id import PopulateProjectId from gslib.third_party.kms_apitools.cloudkms_v1_messages import Binding from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import text_util from gslib.utils.constants import NO_MAX from gslib.utils.encryption_helper import ValidateCMEK from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _AUTHORIZE_SYNOPSIS = """ gsutil kms authorize [-p <proj_id>] -k <kms_key> """ _ENCRYPTION_SYNOPSIS = """ gsutil kms encryption [(-d|[-k <kms_key>])] [-w] gs://<bucket_name>... """ _SERVICEACCOUNT_SYNOPSIS = """ gsutil kms serviceaccount [-p <proj_id>] """ _SYNOPSIS = (_AUTHORIZE_SYNOPSIS + _ENCRYPTION_SYNOPSIS.lstrip('\n') + _SERVICEACCOUNT_SYNOPSIS.lstrip('\n') + '\n') # pylint: disable=line-too-long _AUTHORIZE_DESCRIPTION = """ <B>AUTHORIZE</B> The authorize sub-command checks that the default (or supplied) project has a Cloud Storage service agent created for it, and if not, it creates one. It then adds appropriate encrypt/decrypt permissions to Cloud KMS resources such that the service agent can write and read Cloud KMS-encrypted objects in buckets associated with the service agent's project. <B>AUTHORIZE EXAMPLES</B> Authorize "my-project" to use a Cloud KMS key: gsutil kms authorize -p my-project \\ -k projects/key-project/locations/us-east1/keyRings/key-ring/cryptoKeys/my-key <B>AUTHORIZE OPTIONS</B> -k <key> The path to the KMS key to use. The path has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -p <project> The ID or number of the project being authorized to use the Cloud KMS key. If this flag is not included, your default project is authorized. """ _ENCRYPTION_DESCRIPTION = """ <B>ENCRYPTION</B> The encryption sub-command is used to set, display, or clear a bucket's default KMS key, which is used to encrypt newly-written objects if no other key is specified. <B>ENCRYPTION EXAMPLES</B> Set the default KMS key for my-bucket: gsutil kms encryption \\ -k projects/key-project/locations/us-east1/keyRings/key-ring/cryptoKeys/my-key \\ gs://my-bucket Show the default KMS key for my-bucket, if one is set: gsutil kms encryption gs://my-bucket Clear the default KMS key so newly-written objects are not encrypted using it: gsutil kms encryption -d gs://my-bucket Once you clear the default KMS key, newly-written objects are encrypted with Google-managed encryption keys by default. <B>ENCRYPTION OPTIONS</B> -k <key> Set the default KMS key for my-bucket using the full path to the key, which has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -w (used with -k key) Display a warning rather than failing if gsutil is unable to verify that the specified key contains the correct IAM bindings for encryption/decryption. This is useful for users that do not have getIamPolicy permission but know that the key has the correct IAM policy for encryption in the user's project. -d Clear the default KMS key. """ # pylint: enable=line-too-long _SERVICEACCOUNT_DESCRIPTION = """ <B>SERVICEACCOUNT</B> The serviceaccount sub-command displays the Cloud Storage service agent that is used to perform Cloud KMS operations against your default project (or a supplied project). <B>SERVICEACCOUNT EXAMPLES</B> Show the service account for my-project: gsutil kms serviceaccount -p my-project <B>SERVICEACCOUNT OPTIONS</B> -p <project> The ID or number of the project whose Cloud Storage service agent is being requested. If this flag is not included, your default project is used. """ _DESCRIPTION = """ The kms command is used to configure Google Cloud Storage and Cloud KMS resources to support encryption of Cloud Storage objects with `Cloud KMS keys <https://cloud.google.com/storage/docs/encryption/customer-managed-keys>`_. The kms command has three sub-commands that deal with configuring Cloud Storage's integration with Cloud KMS: ``authorize``, ``encryption``, and ``serviceaccount``. Before using this command, read the `prerequisites <https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys#prereqs>`_. for using Cloud KMS with Cloud Storage. """ + (_AUTHORIZE_DESCRIPTION + _ENCRYPTION_DESCRIPTION + _SERVICEACCOUNT_DESCRIPTION) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _authorize_help_text = CreateHelpText(_AUTHORIZE_SYNOPSIS, _AUTHORIZE_DESCRIPTION) _encryption_help_text = CreateHelpText(_ENCRYPTION_SYNOPSIS, _ENCRYPTION_DESCRIPTION) _serviceaccount_help_text = CreateHelpText(_SERVICEACCOUNT_SYNOPSIS, _SERVICEACCOUNT_DESCRIPTION) _AUTHORIZE_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'service-agent'], flag_map={ '-p': GcloudStorageFlag('--project'), '-k': GcloudStorageFlag('--authorize-cmek'), }) _SERVICEACCOUNT_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'service-agent'], flag_map={ '-p': GcloudStorageFlag('--project'), }) class KmsCommand(Command): """Implements of gsutil kms command.""" command_spec = Command.CreateCommandSpec( 'kms', usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='dk:p:w', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'authorize': [], 'encryption': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'serviceaccount': [], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='kms', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure Cloud KMS encryption', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'authorize': _authorize_help_text, 'encryption': _encryption_help_text, 'serviceaccount': _serviceaccount_help_text }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'authorize': _AUTHORIZE_COMMAND, 'serviceaccount': _SERVICEACCOUNT_COMMAND, # "encryption" subcommand handled in get_gcloud_storage_args. }, flag_map={}) def get_gcloud_storage_args(self): if self.args[0] == 'encryption': gcloud_storage_map = GcloudStorageMap(gcloud_command={ 'encryption': GcloudStorageMap( gcloud_command=['storage', 'buckets'], flag_map={ '-d': GcloudStorageFlag('--clear-default-encryption-key'), '-k': GcloudStorageFlag('--default-encryption-key'), '-w': GcloudStorageFlag(''), }), }, flag_map={}) if '-d' in self.args or '-k' in self.args: gcloud_storage_map.gcloud_command['encryption'].gcloud_command += [ 'update' ] else: gcloud_storage_map.gcloud_command['encryption'].gcloud_command += [ 'describe', ('--format=value[separator=\": \"](name, encryption' '.defaultKmsKeyName.yesno(no="No default encryption key."))'), '--raw' ] else: gcloud_storage_map = KmsCommand.gcloud_storage_map return super().get_gcloud_storage_args(gcloud_storage_map) def _GatherSubOptions(self, subcommand_name): self.CheckArguments() self.clear_kms_key = False self.kms_key = None self.warn_on_key_authorize_failure = False if self.sub_opts: for o, a in self.sub_opts: if o == '-p': self.project_id = a elif o == '-k': self.kms_key = a ValidateCMEK(self.kms_key) elif o == '-d': self.clear_kms_key = True elif o == '-w': self.warn_on_key_authorize_failure = True if self.warn_on_key_authorize_failure and ( self.subcommand_name != 'encryption' or not self.kms_key): raise CommandException('\n'.join( textwrap.wrap( 'The "-w" option should only be specified for the "encryption" ' 'subcommand and must be used with the "-k" option.'))) # Determine the project (used in the serviceaccount and authorize # subcommands), either from the "-p" option's value or the default specified # in the user's Boto config file. if not self.project_id: self.project_id = PopulateProjectId(None) @Retry(ServiceException, tries=3, timeout_secs=1) def _AuthorizeProject(self, project_id, kms_key): """Authorizes a project's service account to be used with a KMS key. Authorizes the Cloud Storage-owned service account for project_id to be used with kms_key. Args: project_id: (str) Project id string (not number). kms_key: (str) Fully qualified resource name for the KMS key. Returns: (str, bool) A 2-tuple consisting of: 1) The email address for the service account associated with the project, which is authorized to encrypt/decrypt with the specified key. 2) A bool value - True if we had to grant the service account permission to encrypt/decrypt with the given key; False if the required permission was already present. """ # Request the Cloud Storage-owned service account for project_id, creating # it if it does not exist. service_account = self.gsutil_api.GetProjectServiceAccount( project_id, provider='gs').email_address kms_api = KmsApi(logger=self.logger) self.logger.debug('Getting IAM policy for %s', kms_key) try: policy = kms_api.GetKeyIamPolicy(kms_key) self.logger.debug('Current policy is %s', policy) # Check if the required binding is already present; if not, add it and # update the key's IAM policy. added_new_binding = False binding = Binding(role='roles/cloudkms.cryptoKeyEncrypterDecrypter', members=['serviceAccount:%s' % service_account]) if binding not in policy.bindings: policy.bindings.append(binding) kms_api.SetKeyIamPolicy(kms_key, policy) added_new_binding = True return (service_account, added_new_binding) except AccessDeniedException: if self.warn_on_key_authorize_failure: text_util.print_to_fd('\n'.join( textwrap.wrap( 'Warning: Check that your Cloud Platform project\'s service ' 'account has the "cloudkms.cryptoKeyEncrypterDecrypter" role ' 'for the specified key. Without this role, you may not be ' 'able to encrypt or decrypt objects using the key which will ' 'prevent you from uploading or downloading objects.'))) return (service_account, False) else: raise def _Authorize(self): self._GatherSubOptions('authorize') if not self.kms_key: raise CommandException('%s %s requires a key to be specified with -k' % (self.command_name, self.subcommand_name)) _, newly_authorized = self._AuthorizeProject(self.project_id, self.kms_key) if newly_authorized: print('Authorized project %s to encrypt and decrypt with key:\n%s' % (self.project_id, self.kms_key)) else: print('Project %s was already authorized to encrypt and decrypt with ' 'key:\n%s.' % (self.project_id, self.kms_key)) return 0 def _EncryptionClearKey(self, bucket_metadata, bucket_url): """Clears the defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. """ bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue() print('Clearing default encryption key for %s...' % str(bucket_url).rstrip('/')) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['encryption'], provider=bucket_url.scheme) def _EncryptionSetKey(self, bucket_metadata, bucket_url, svc_acct_for_project_num): """Sets defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. svc_acct_for_project_num: (Dict[int, str]) Mapping of project numbers to their corresponding service account. """ bucket_project_number = bucket_metadata.projectNumber try: # newly_authorized will always be False if the project number is in our # cache dict, since we've already called _AuthorizeProject on it. service_account, newly_authorized = ( svc_acct_for_project_num[bucket_project_number], False) except KeyError: service_account, newly_authorized = self._AuthorizeProject( bucket_project_number, self.kms_key) svc_acct_for_project_num[bucket_project_number] = service_account if newly_authorized: text_util.print_to_fd('Authorized service account %s to use key:\n%s' % (service_account, self.kms_key)) bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue( defaultKmsKeyName=self.kms_key) print('Setting default KMS key for bucket %s...' % str(bucket_url).rstrip('/')) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['encryption'], provider=bucket_url.scheme) def _Encryption(self): self._GatherSubOptions('encryption') # For each project, we should only make one API call to look up its # associated Cloud Storage-owned service account; subsequent lookups can be # pulled from this cache dict. svc_acct_for_project_num = {} def _EncryptionForBucket(blr): """Set, clear, or get the defaultKmsKeyName for a bucket.""" bucket_url = blr.storage_url if bucket_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) # Determine the project from the provided bucket. bucket_metadata = self.gsutil_api.GetBucket( bucket_url.bucket_name, fields=['encryption', 'projectNumber'], provider=bucket_url.scheme) # "-d" flag was specified, so clear the default KMS key and return. if self.clear_kms_key: self._EncryptionClearKey(bucket_metadata, bucket_url) return 0 # "-k" flag was specified, so set the default KMS key and return. if self.kms_key: self._EncryptionSetKey(bucket_metadata, bucket_url, svc_acct_for_project_num) return 0 # Neither "-d" nor "-k" was specified, so emit the default KMS key and # return. bucket_url_string = str(bucket_url).rstrip('/') if (bucket_metadata.encryption and bucket_metadata.encryption.defaultKmsKeyName): print('Default encryption key for %s:\n%s' % (bucket_url_string, bucket_metadata.encryption.defaultKmsKeyName)) else: print('Bucket %s has no default encryption key' % bucket_url_string) return 0 # Iterate over bucket args, performing the specified encryption operation # for each. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True _EncryptionForBucket(bucket_listing_ref) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _ServiceAccount(self): self.CheckArguments() if not self.args: self.args = ['gs://'] if self.sub_opts: for o, a in self.sub_opts: if o == '-p': self.project_id = a if not self.project_id: self.project_id = PopulateProjectId(None) # Request the service account for that project; this might create the # service account if it doesn't already exist. self.logger.debug('Checking service account for project %s', self.project_id) service_account = self.gsutil_api.GetProjectServiceAccount( self.project_id, provider='gs').email_address print(service_account) return 0 def _RunSubCommand(self, func): try: self.sub_opts, self.args = getopt.getopt( self.args, self.command_spec.supported_sub_args) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) return func(self) except getopt.GetoptError: self.RaiseInvalidArgumentException() def RunCommand(self): """Command entry point for the kms command.""" # If the only credential type the user supplies in their boto file is hmac, # GetApiSelector logic will force us to use the XML API. As the XML API does # not support all the operations needed for kms subcommands, fail early. if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the GCS JSON API. If you ' 'have only supplied hmac credentials in your boto file, please ' 'instead supply a credential type that can be used with the JSON ' 'API.' % self.command_name))) def RunCommand(self): """Command entry point for the kms command.""" # If the only credential type the user supplies in their boto file is hmac, # GetApiSelector logic will force us to use the XML API. As the XML API does # not support all the operations needed for kms subcommands, fail early. if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the GCS JSON API, which ' 'cannot use HMAC credentials. Please supply a credential ' 'type that is compatible with the JSON API (e.g. OAuth2) in your ' 'boto config file.' % self.command_name))) method_for_subcommand = { 'authorize': KmsCommand._Authorize, 'encryption': KmsCommand._Encryption, 'serviceaccount': KmsCommand._ServiceAccount } self.subcommand_name = self.args.pop(0) if self.subcommand_name in method_for_subcommand: metrics.LogCommandParams(subcommands=[self.subcommand_name]) return self._RunSubCommand(method_for_subcommand[self.subcommand_name]) else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (self.subcommand_name, self.command_name)) commands/hash.py 0000644 00000023504 15025013616 0007645 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of hash command for calculating hashes of local files.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import time import crcmod import six from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.progress_callback import FileProgressCallbackHandler from gslib.progress_callback import ProgressCallbackWithTimeout from gslib.storage_url import StorageUrlFromString from gslib.thread_message import FileMessage from gslib.thread_message import FinalMessage from gslib.utils import boto_util from gslib.utils import constants from gslib.utils import hashing_helper from gslib.utils import parallelism_framework_util from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout _SYNOPSIS = """ gsutil hash [-c] [-h] [-m] filename... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Calculate hashes on local files, which can be used to compare with ``gsutil ls -L`` output. If a specific hash option is not provided, this command calculates all gsutil-supported hashes for the files. Note that gsutil automatically performs hash validation when uploading or downloading files, so this command is only needed if you want to write a script that separately checks the hash. If you calculate a CRC32c hash for files without a precompiled crcmod installation, hashing will be very slow. See "gsutil help crcmod" for details. <B>OPTIONS</B> -c Calculate a CRC32c hash for the specified files. -h Output hashes in hex format. By default, gsutil uses base64. -m Calculate a MD5 hash for the specified files. """) _GCLOUD_FORMAT_STRING = ( '--format=' + 'value[separator="",terminator=""](' + 'digest_format.sub("' + shim_util.get_format_flag_caret() + '", "Hashes ["),' + 'url.sub("' + shim_util.get_format_flag_caret() + '", "] for ").sub("$", ":^\\^n"),' + 'md5_hash.yesno(yes="\tHash (md5):\t\t", no=""),' + 'md5_hash.yesno(no=""),' + 'md5_hash.yesno(yes="NEWLINE", no="")' + '.sub("NEWLINE", "' + shim_util.get_format_flag_newline() + '"),' + 'crc32c_hash.yesno(yes="\tHash (crc32c):\t\t", no=""),' + 'crc32c_hash.yesno(no=""),' + 'crc32c_hash.yesno(yes="NEWLINE", no="")' + '.sub("NEWLINE", "' + shim_util.get_format_flag_newline() + '")' + ')') class HashCommand(Command): """Implementation of gsutil hash command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'hash', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='chm', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreFileURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='hash', help_name_aliases=['checksum'], help_type='command_help', help_one_line_summary='Calculate file hashes', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'hash', _GCLOUD_FORMAT_STRING, ], flag_map={ '-h': GcloudStorageFlag('--hex'), '-c': None, '-m': None, }, ) args_set = set(self.args + [flag for flag, _ in self.sub_opts]) if '-c' in args_set and '-m' not in args_set: gcloud_storage_map.gcloud_command += ['--skip-md5'] elif '-m' in args_set and '-c' not in args_set: gcloud_storage_map.gcloud_command += ['--skip-crc32c'] return super().get_gcloud_storage_args(gcloud_storage_map) @classmethod def _ParseOpts(cls, sub_opts, logger): """Returns behavior variables based on input options. Args: sub_opts: getopt sub-arguments for the command. logger: logging.Logger for the command. Returns: Tuple of calc_crc32c: Boolean, if True, command should calculate a CRC32c checksum. calc_md5: Boolean, if True, command should calculate an MD5 hash. format_func: Function used for formatting the hash in the desired format. cloud_format_func: Function used for formatting the hash in the desired format. output_format: String describing the hash output format. """ calc_crc32c = False calc_md5 = False format_func = ( lambda digest: hashing_helper.Base64EncodeHash(digest.hexdigest())) cloud_format_func = lambda digest: digest found_hash_option = False output_format = 'base64' if sub_opts: for o, unused_a in sub_opts: if o == '-c': calc_crc32c = True found_hash_option = True elif o == '-h': output_format = 'hex' format_func = lambda digest: digest.hexdigest() cloud_format_func = lambda digest: ( hashing_helper.Base64ToHexHash(digest).decode('ascii') ) # yapf: disable elif o == '-m': calc_md5 = True found_hash_option = True if not found_hash_option: calc_crc32c = True calc_md5 = True if calc_crc32c and not boto_util.UsingCrcmodExtension(): logger.warn(hashing_helper.SLOW_CRCMOD_WARNING) return calc_crc32c, calc_md5, format_func, cloud_format_func, output_format def _GetHashClassesFromArgs(self, calc_crc32c, calc_md5): """Constructs the dictionary of hashes to compute based on the arguments. Args: calc_crc32c: If True, CRC32c should be included. calc_md5: If True, MD5 should be included. Returns: Dictionary of {string: hash digester}, where string the name of the digester algorithm. """ hash_dict = {} if calc_crc32c: hash_dict['crc32c'] = crcmod.predefined.Crc('crc-32c') if calc_md5: hash_dict['md5'] = hashing_helper.GetMd5() return hash_dict def RunCommand(self): """Command entry point for the hash command.""" (calc_crc32c, calc_md5, format_func, cloud_format_func, output_format) = (self._ParseOpts(self.sub_opts, self.logger)) matched_one = False for url_str in self.args: for file_ref in self.WildcardIterator(url_str).IterObjects( bucket_listing_fields=[ 'crc32c', 'customerEncryption', 'md5Hash', 'size', ]): matched_one = True url = StorageUrlFromString(url_str) file_name = file_ref.storage_url.object_name if StorageUrlFromString(url_str).IsFileUrl(): file_size = os.path.getsize(file_name) self.gsutil_api.status_queue.put( FileMessage(url, None, time.time(), size=file_size, finished=False, message_type=FileMessage.FILE_HASH)) callback_processor = ProgressCallbackWithTimeout( file_size, FileProgressCallbackHandler(self.gsutil_api.status_queue, src_url=StorageUrlFromString(url_str), operation_name='Hashing').call) hash_dict = self._GetHashClassesFromArgs(calc_crc32c, calc_md5) with open(file_name, 'rb') as fp: hashing_helper.CalculateHashesFromContents( fp, hash_dict, callback_processor=callback_processor) self.gsutil_api.status_queue.put( FileMessage(url, None, time.time(), size=file_size, finished=True, message_type=FileMessage.FILE_HASH)) else: hash_dict = {} obj_metadata = file_ref.root_object file_size = obj_metadata.size md5_present = obj_metadata.md5Hash is not None crc32c_present = obj_metadata.crc32c is not None if not md5_present and not crc32c_present: logging.getLogger().warn('No hashes present for %s', url_str) continue if md5_present: hash_dict['md5'] = obj_metadata.md5Hash if crc32c_present: hash_dict['crc32c'] = obj_metadata.crc32c print('Hashes [%s] for %s:' % (output_format, file_name)) for name, digest in six.iteritems(hash_dict): print('\tHash (%s):\t\t%s' % (name, (format_func(digest) if url.IsFileUrl() else cloud_format_func(digest)))) if not matched_one: raise CommandException('No files matched') _PutToQueueWithTimeout(self.gsutil_api.status_queue, FinalMessage(time.time())) return 0 commands/iam.py 0000644 00000104005 15025013616 0007464 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of IAM policy management command for GCS.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import itertools import json import os import re import subprocess import textwrap import six from six.moves import zip from apitools.base.protorpclite import protojson from apitools.base.protorpclite.messages import DecodeError from boto import config from gslib.cloud_api import ArgumentException from gslib.cloud_api import PreconditionException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import GetFailureCount from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import IamChOnResourceWithConditionsException from gslib.help_provider import CreateHelpText from gslib.metrics import LogCommandParams from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.storage_url import GetSchemeFromUrlString from gslib.storage_url import IsKnownUrlScheme from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreMixOfBucketsAndObjects from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import shim_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import IAM_POLICY_VERSION from gslib.utils.constants import NO_MAX from gslib.utils import iam_helper from gslib.utils.iam_helper import BindingStringToTuple from gslib.utils.iam_helper import BindingsTuple from gslib.utils.iam_helper import DeserializeBindingsTuple from gslib.utils.iam_helper import IsEqualBindings from gslib.utils.iam_helper import PatchBindings from gslib.utils.iam_helper import SerializeBindingsTuple from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.shim_util import GcloudStorageFlag _SET_SYNOPSIS = """ gsutil iam set [-afRr] [-e <etag>] file url ... """ _GET_SYNOPSIS = """ gsutil iam get url """ # Note that the commands below are put in quotation marks instead of backticks; # this is done because the whole ch synopsis gets rendered in one <pre> tag in # the web docs, and having literal double-backticks looks weird. _CH_SYNOPSIS = """ gsutil iam ch [-fRr] binding ... url where each binding is of the form: [-d] ("user"|"serviceAccount"|"domain"|"group"):id:role[,...] [-d] ("allUsers"|"allAuthenticatedUsers"):role[,...] -d ("user"|"serviceAccount"|"domain"|"group"):id -d ("allUsers"|"allAuthenticatedUsers") NOTE: The "iam ch" command does not support changing Cloud IAM policies with bindings that contain conditions. As such, "iam ch" cannot be used to add conditions to a policy or to change the policy of a resource that already contains conditions. See additional details below. NOTE: The "gsutil iam" command does not allow you to add convenience values (projectOwner, projectEditor, projectViewer), but you can remove existing ones. """ _GET_DESCRIPTION = """ <B>GET</B> The ``iam get`` command gets the Cloud IAM policy for a bucket or object, which you can save and edit for use with the ``iam set`` command. The following examples save the bucket or object's Cloud IAM policy to a text file: gsutil iam get gs://example > bucket_iam.txt gsutil iam get gs://example/important.txt > object_iam.txt The Cloud IAM policy returned by ``iam get`` includes an etag. The etag is used in the precondition check for ``iam set`` unless you override it using ``iam set -e``. """ _SET_DESCRIPTION = """ <B>SET</B> The ``iam set`` command sets a Cloud IAM policy on one or more buckets or objects, replacing the existing policy on those buckets or objects. For an example of the correct formatting for a Cloud IAM policy, see the output of the ``iam get`` command. You can use the ``iam ch`` command to edit an existing policy, even in the presence of concurrent updates. You can also edit the policy concurrently using the ``-e`` flag to override the Cloud IAM policy's etag. Specifying ``-e`` with an empty string (i.e. ``gsutil iam set -e '' ...``) instructs gsutil to skip the precondition check when setting the Cloud IAM policy. When you set a Cloud IAM policy on a large number of objects, you should use the gsutil ``-m`` option for concurrent processing. The following command applies ``iam.txt`` to all objects in the ``dogs`` bucket: gsutil -m iam set -r iam.txt gs://dogs Note that only object-level operations are parallelized; setting a Cloud IAM policy on a large number of buckets with the ``-m`` flag does not improve performance. <B>SET OPTIONS</B> The ``set`` sub-command has the following options: -R, -r Performs ``iam set`` recursively on all objects under the specified bucket. This flag can only be set if the policy exclusively uses ``roles/storage.legacyObjectReader`` or ``roles/storage.legacyObjectOwner``. This flag cannot be used if the bucket is configured for uniform bucket-level access. -a Performs ``iam set`` on all object versions. -e <etag> Performs the precondition check on each object with the specified etag before setting the policy. You can retrieve the policy's etag using ``iam get``. -f The default gsutil error-handling mode is fail-fast. This flag changes the request to fail-silent mode. This option is implicitly set when you use the gsutil ``-m`` option. """ _CH_DESCRIPTION = """ <B>CH</B> The ``iam ch`` command incrementally updates Cloud IAM policies. You can specify multiple access grants or removals in a single command. The access changes are applied as a batch to each url in the order in which they appear in the command line arguments. Each access change specifies a principal and a role that is either granted or revoked. You can use gsutil ``-m`` to handle object-level operations in parallel. NOTE: The ``iam ch`` command cannot be used to change the Cloud IAM policy of a resource that contains conditions in its policy bindings. Attempts to do so result in an error. To change the Cloud IAM policy of such a resource, you can perform a read-modify-write operation by saving the policy to a file using ``iam get``, editing the file, and setting the updated policy using ``iam set``. <B>CH EXAMPLES</B> Examples for the ``ch`` sub-command: To grant a single role to a single principal for some targets: gsutil iam ch user:john.doe@example.com:objectCreator gs://ex-bucket To make a bucket's objects publicly readable: gsutil iam ch allUsers:objectViewer gs://ex-bucket To grant multiple bindings to a bucket: gsutil iam ch user:john.doe@example.com:objectCreator \\ domain:www.my-domain.org:objectViewer gs://ex-bucket To specify more than one role for a particular principal: gsutil iam ch user:john.doe@example.com:objectCreator,objectViewer \\ gs://ex-bucket To specify a custom role for a particular principal: gsutil iam ch user:john.doe@example.com:roles/customRoleName gs://ex-bucket To apply a grant and simultaneously remove a binding to a bucket: gsutil iam ch -d group:readers@example.com:legacyBucketReader \\ group:viewers@example.com:objectViewer gs://ex-bucket To remove a user from all roles on a bucket: gsutil iam ch -d user:john.doe@example.com gs://ex-bucket <B>CH OPTIONS</B> The ``ch`` sub-command has the following options: -d Removes roles granted to the specified principal. -R, -r Performs ``iam ch`` recursively to all objects under the specified bucket. This flag can only be set if the policy exclusively uses ``roles/storage.legacyObjectReader`` or ``roles/storage.legacyObjectOwner``. This flag cannot be used if the bucket is configured for uniform bucket-level access. -f The default gsutil error-handling mode is fail-fast. This flag changes the request to fail-silent mode. This is implicitly set when you invoke the gsutil ``-m`` option. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ Cloud Identity and Access Management (Cloud IAM) allows you to control who has access to the resources in your Google Cloud project. For more information, see `Cloud Identity and Access Management <https://cloud.google.com/storage/docs/access-control/iam>`_. The iam command has three sub-commands: """ + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION, _CH_DESCRIPTION]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) STORAGE_URI_REGEX = re.compile(r'[a-z]+://.+') IAM_CH_CONDITIONS_WORKAROUND_MSG = ( 'To change the IAM policy of a resource that has bindings containing ' 'conditions, perform a read-modify-write operation using "iam get" and ' '"iam set".') def _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(urls, recursion_requested): if UrlsAreMixOfBucketsAndObjects(urls) and not recursion_requested: raise CommandException('Cannot operate on a mix of buckets and objects.') def _PatchIamWrapper(cls, iter_result, thread_state): (serialized_bindings_tuples, expansion_result) = iter_result return cls.PatchIamHelper( expansion_result.expanded_storage_url, # Deserialize the JSON object passed from Command.Apply. [DeserializeBindingsTuple(t) for t in serialized_bindings_tuples], thread_state=thread_state) def _SetIamWrapper(cls, iter_result, thread_state): (serialized_policy, expansion_result) = iter_result return cls.SetIamHelper( expansion_result.expanded_storage_url, # Deserialize the JSON object passed from Command.Apply. protojson.decode_message(apitools_messages.Policy, serialized_policy), thread_state=thread_state) def _SetIamExceptionHandler(cls, e): cls.logger.error(str(e)) def _PatchIamExceptionHandler(cls, e): cls.logger.error(str(e)) class IamCommand(Command): """Implementation of gsutil iam command.""" command_spec = Command.CreateCommandSpec( 'iam', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='afRrd:e:', file_url_ok=True, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], 'ch': [ CommandArgument.MakeOneOrMoreBindingsArgument(), CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], }, ) help_spec = Command.HelpSpec( help_name='iam', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Get, set, or change bucket and/or object IAM permissions.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text, }, ) def _get_resource_type(self, url_patterns): if self.recursion_requested or url_patterns[0].IsObject(): return 'objects' else: return 'buckets' def get_gcloud_storage_args(self): self.sub_command = self.args.pop(0) if self.sub_command == 'get': if StorageUrlFromString(self.args[0]).IsObject(): command_group = 'objects' else: command_group = 'buckets' gcloud_storage_map = GcloudStorageMap(gcloud_command=[ 'storage', command_group, 'get-iam-policy', '--format=json' ], flag_map={}) elif self.sub_command == 'set': gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', None, 'set-iam-policy', '--format=json'], flag_map={ '-a': GcloudStorageFlag('--all-versions'), '-e': GcloudStorageFlag('--etag'), # Hack to make gcloud recognize empty string value. '_empty_etag': GcloudStorageFlag('--etag='), '-f': GcloudStorageFlag('--continue-on-error'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), }) # Flags must be at the start of self.args to get parsed. self.ParseSubOpts() # Flags moved to self.sub_opts, leaving [POLICY-FILE, URLS...]. url_strings = self.args[1:] url_objects = list(map(StorageUrlFromString, url_strings)) recurse = False for i, (flag_key, flag_value) in enumerate(self.sub_opts): if flag_key in ('-r', '-R'): recurse = True elif flag_key == '-e' and flag_value == '': # Allow for empty string values. self.sub_opts[i] = ('_empty_etag', '') _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(url_objects, recurse) if recurse or url_objects[0].IsObject(): gcloud_storage_map.gcloud_command[1] = 'objects' else: gcloud_storage_map.gcloud_command[1] = 'buckets' # Gsutil takes policy file then URLs, and gcloud storage does the reverse. self.args = url_strings + self.args[:1] elif self.sub_command == 'ch': if shim_util.HIDDEN_SHIM_MODE( config.get('GSUtil', 'hidden_shim_mode', 'none')) is shim_util.HIDDEN_SHIM_MODE.DRY_RUN: self.logger.warning( 'The shim maps iam ch commands to several gcloud storage commands,' ' which cannot be determined without running gcloud storage.') return [] # Translation occurs in self.run_gcloud_storage. return super().get_gcloud_storage_args(gcloud_storage_map) def _RaiseIfInvalidUrl(self, url): if url.IsFileUrl(): error_msg = 'Invalid Cloud URL "%s".' % url.object_name if set(url.object_name).issubset(set('-Rrf')): error_msg += ( ' This resource handle looks like a flag, which must appear' ' before all bindings. See "gsutil help iam ch" for more details.') raise CommandException(error_msg) def _GetSettingsAndDiffs(self): self.continue_on_error = False self.recursion_requested = False patch_bindings_tuples = [] if self.sub_opts: for o, a in self.sub_opts: if o in ['-r', '-R']: self.recursion_requested = True elif o == '-f': self.continue_on_error = True elif o == '-d': patch_bindings_tuples.append(BindingStringToTuple(False, a)) url_pattern_strings = [] # N.B.: self.sub_opts stops taking in options at the first non-flagged # token. The rest of the tokens are sent to self.args. Thus, in order to # handle input of the form "-d <binding> <binding> <url>", we will have to # parse self.args for a mix of both bindings and CloudUrls. We are not # expecting to come across the -r, -f flags here. it = iter(self.args) for token in it: if (STORAGE_URI_REGEX.match(token) and IsKnownUrlScheme(GetSchemeFromUrlString(token))): url_pattern_strings.append(token) break if token == '-d': try: patch_bindings_tuples.append(BindingStringToTuple(False, next(it))) except StopIteration: raise CommandException( 'A -d flag is missing an argument specifying bindings to remove.') else: patch_bindings_tuples.append(BindingStringToTuple(True, token)) if not patch_bindings_tuples: raise CommandException('Must specify at least one binding.') # All following arguments are urls. for token in it: url_pattern_strings.append(token) url_objects = list(map(StorageUrlFromString, url_pattern_strings)) _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(url_objects, self.recursion_requested) return patch_bindings_tuples, url_objects def _run_ch_subprocess(self, args, stdin=None): _, command = self._get_full_gcloud_storage_execution_information(args) process = subprocess.run( command, env=self._get_shim_command_environment_variables(), input=stdin, stderr=subprocess.PIPE, stdout=subprocess.PIPE, text=True, ) if process.returncode != 0: if self.continue_on_error: self.logger.error(process.stderr) else: raise CommandException(process.stderr) return process def run_gcloud_storage(self): if self.sub_command != 'ch': return super().run_gcloud_storage() self.ParseSubOpts() bindings_tuples, patterns = self._GetSettingsAndDiffs() resource_type = self._get_resource_type(patterns) list_settings = [] if self.recursion_requested: list_settings.append('-r') return_code = 0 for url_pattern in patterns: self._RaiseIfInvalidUrl(url_pattern) if resource_type == 'objects': ls_process = self._run_ch_subprocess(['storage', 'ls', '--json'] + list_settings + [str(url_pattern)]) if ls_process.returncode: return_code = 1 continue ls_output = json.loads(ls_process.stdout) urls = [ resource['url'] for resource in ls_output if resource['type'] == 'cloud_object' ] else: urls = [str(url_pattern)] for url in urls: get_process = self._run_ch_subprocess( ['storage', resource_type, 'get-iam-policy', url, '--format=json']) if get_process.returncode: return_code = 1 continue policy = json.loads(get_process.stdout) bindings = iam_helper.BindingsDictToUpdateDict(policy['bindings']) for (is_grant, diff) in bindings_tuples: diff_dict = iam_helper.BindingsDictToUpdateDict(diff) bindings = PatchBindings(bindings, diff_dict, is_grant) policy['bindings'] = [{ 'role': r, 'members': sorted(list(m)) } for r, m in sorted(six.iteritems(bindings))] set_process = self._run_ch_subprocess( ['storage', resource_type, 'set-iam-policy', url, '-'], stdin=json.dumps(policy, sort_keys=True)) if set_process.returncode: return_code = 1 return return_code def GetIamHelper(self, storage_url, thread_state=None): """Gets an IAM policy for a single, resolved bucket / object URL. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the global -m flag is specified. Will use self.gsutil_api if thread_state is set to None. Returns: Policy instance. """ gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) if storage_url.IsBucket(): policy = gsutil_api.GetBucketIamPolicy( storage_url.bucket_name, provider=storage_url.scheme, fields=['bindings', 'etag'], ) else: policy = gsutil_api.GetObjectIamPolicy( storage_url.bucket_name, storage_url.object_name, generation=storage_url.generation, provider=storage_url.scheme, fields=['bindings', 'etag'], ) return policy def _GetIam(self, thread_state=None): """Gets IAM policy for single bucket or object.""" pattern = self.args[0] matches = PluralityCheckableIterator( self.WildcardIterator(pattern).IterAll(bucket_listing_fields=['name'])) if matches.IsEmpty(): raise CommandException('%s matched no URLs' % pattern) if matches.HasPlurality(): raise CommandException( '%s matched more than one URL, which is not allowed by the %s ' 'command' % (pattern, self.command_name)) storage_url = StorageUrlFromString(list(matches)[0].url_string) policy = self.GetIamHelper(storage_url, thread_state=thread_state) policy_json = json.loads(protojson.encode_message(policy)) policy_str = json.dumps( policy_json, sort_keys=True, separators=(',', ': '), indent=2, ) print(policy_str) def _SetIamHelperInternal(self, storage_url, policy, thread_state=None): """Sets IAM policy for a single, resolved bucket / object URL. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. policy: A Policy object to set on the bucket / object. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the -m flag is specified. Will use self.gsutil_api if thread_state is set to None. Raises: ServiceException passed from the API call if an HTTP error was returned. """ # SetIamHelper may be called by a command.WorkerThread. In the # single-threaded case, WorkerThread will not pass the CloudApiDelegator # instance to thread_state. GetCloudInstance is called to resolve the # edge case. gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) if storage_url.IsBucket(): gsutil_api.SetBucketIamPolicy(storage_url.bucket_name, policy, provider=storage_url.scheme) else: gsutil_api.SetObjectIamPolicy(storage_url.bucket_name, storage_url.object_name, policy, generation=storage_url.generation, provider=storage_url.scheme) def SetIamHelper(self, storage_url, policy, thread_state=None): """Handles the potential exception raised by the internal set function.""" try: self._SetIamHelperInternal(storage_url, policy, thread_state=thread_state) except ServiceException: if self.continue_on_error: self.everything_set_okay = False else: raise def PatchIamHelper(self, storage_url, bindings_tuples, thread_state=None): """Patches an IAM policy for a single, resolved bucket / object URL. The patch is applied by altering the policy from an IAM get request, and setting the new IAM with the specified etag. Because concurrent IAM set requests may alter the etag, we may need to retry this operation several times before success. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. bindings_tuples: A list of BindingsTuple instances. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the -m flag is specified. Will use self.gsutil_api if thread_state is set to None. """ try: self._PatchIamHelperInternal(storage_url, bindings_tuples, thread_state=thread_state) except ServiceException: if self.continue_on_error: self.everything_set_okay = False else: raise except IamChOnResourceWithConditionsException as e: if self.continue_on_error: self.everything_set_okay = False self.tried_ch_on_resource_with_conditions = True self.logger.debug(e.message) else: raise CommandException(e.message) @Retry(PreconditionException, tries=3, timeout_secs=1.0) def _PatchIamHelperInternal(self, storage_url, bindings_tuples, thread_state=None): policy = self.GetIamHelper(storage_url, thread_state=thread_state) (etag, bindings) = (policy.etag, policy.bindings) # If any of the bindings have conditions present, raise an exception. # See the docstring for the IamChOnResourceWithConditionsException class # for more details on why we raise this exception. for binding in bindings: if binding.condition: message = 'Could not patch IAM policy for %s.' % storage_url message += '\n' message += '\n'.join( textwrap.wrap( 'The resource had conditions present in its IAM policy bindings, ' 'which is not supported by "iam ch". %s' % IAM_CH_CONDITIONS_WORKAROUND_MSG)) raise IamChOnResourceWithConditionsException(message) # Create a backup which is untainted by any references to the original # bindings. orig_bindings = list(bindings) for (is_grant, diff) in bindings_tuples: bindings_dict = iam_helper.BindingsMessageToUpdateDict(bindings) diff_dict = iam_helper.BindingsMessageToUpdateDict(diff) new_bindings_dict = PatchBindings(bindings_dict, diff_dict, is_grant) bindings = [ apitools_messages.Policy.BindingsValueListEntry(role=r, members=list(m)) for (r, m) in six.iteritems(new_bindings_dict) ] if IsEqualBindings(bindings, orig_bindings): self.logger.info('No changes made to %s', storage_url) return policy = apitools_messages.Policy(bindings=bindings, etag=etag) # We explicitly wish for etag mismatches to raise an error and allow this # function to error out, so we are bypassing the exception handling offered # by IamCommand.SetIamHelper in lieu of our own handling (@Retry). self._SetIamHelperInternal(storage_url, policy, thread_state=thread_state) def _PatchIam(self): raw_bindings_tuples, url_patterns = self._GetSettingsAndDiffs() patch_bindings_tuples = [] for is_grant, bindings in raw_bindings_tuples: bindings_messages = [] for binding in bindings: bindings_message = apitools_messages.Policy.BindingsValueListEntry( members=binding['members'], role=binding['role']) bindings_messages.append(bindings_message) patch_bindings_tuples.append( BindingsTuple(is_grant=is_grant, bindings=bindings_messages)) self.everything_set_okay = True self.tried_ch_on_resource_with_conditions = False threaded_wildcards = [] for surl in url_patterns: try: if surl.IsBucket(): if self.recursion_requested: surl.object = '*' threaded_wildcards.append(surl.url_string) else: self.PatchIamHelper(surl, patch_bindings_tuples) else: threaded_wildcards.append(surl.url_string) except AttributeError: self._RaiseIfInvalidUrl(surl) if threaded_wildcards: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, threaded_wildcards, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=['name']) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), threaded_wildcards, self.recursion_requested, all_versions=self.all_versions) serialized_bindings_tuples_it = itertools.repeat( [SerializeBindingsTuple(t) for t in patch_bindings_tuples]) self.Apply(_PatchIamWrapper, zip(serialized_bindings_tuples_it, name_expansion_iterator), _PatchIamExceptionHandler, fail_on_error=not self.continue_on_error, seek_ahead_iterator=seek_ahead_iterator) self.everything_set_okay &= not GetFailureCount() > 0 # TODO: Add an error counter for files and objects. if not self.everything_set_okay: msg = 'Some IAM policies could not be patched.' if self.tried_ch_on_resource_with_conditions: msg += '\n' msg += '\n'.join( textwrap.wrap( 'Some resources had conditions present in their IAM policy ' 'bindings, which is not supported by "iam ch". %s' % (IAM_CH_CONDITIONS_WORKAROUND_MSG))) raise CommandException(msg) # TODO(iam-beta): Add an optional flag to specify etag and edit the policy # accordingly to be passed into the helper functions. def _SetIam(self): """Set IAM policy for given wildcards on the command line.""" self.continue_on_error = False self.recursion_requested = False self.all_versions = False force_etag = False etag = '' if self.sub_opts: for o, arg in self.sub_opts: if o in ['-r', '-R']: self.recursion_requested = True elif o == '-f': self.continue_on_error = True elif o == '-a': self.all_versions = True elif o == '-e': etag = str(arg) force_etag = True else: self.RaiseInvalidArgumentException() file_url = self.args[0] patterns = self.args[1:] # Load the IAM policy file and raise error if the file is invalid JSON or # does not exist. try: with open(file_url, 'r') as fp: policy = json.loads(fp.read()) except IOError: raise ArgumentException('Specified IAM policy file "%s" does not exist.' % file_url) except ValueError as e: self.logger.debug('Invalid IAM policy file, ValueError:\n%s', e) raise ArgumentException('Invalid IAM policy file "%s".' % file_url) bindings = policy.get('bindings', []) if not force_etag: etag = policy.get('etag', '') policy_json = json.dumps({ 'bindings': bindings, 'etag': etag, 'version': IAM_POLICY_VERSION }) try: policy = protojson.decode_message(apitools_messages.Policy, policy_json) except DecodeError: raise ArgumentException('Invalid IAM policy file "%s" or etag "%s".' % (file_url, etag)) self.everything_set_okay = True # This list of wildcard strings will be handled by NameExpansionIterator. threaded_wildcards = [] surls = list(map(StorageUrlFromString, patterns)) _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(surls, self.recursion_requested) for surl in surls: print(surl.url_string) if surl.IsBucket(): if self.recursion_requested: surl.object_name = '*' threaded_wildcards.append(surl.url_string) else: self.SetIamHelper(surl, policy) else: threaded_wildcards.append(surl.url_string) # N.B.: If threaded_wildcards contains a non-existent bucket # (e.g. ["gs://non-existent", "gs://existent"]), NameExpansionIterator # will raise an exception in iter.next. This halts all iteration, even # when -f is set. This behavior is also evident in acl set. This behavior # also appears for any exception that will be raised when iterating over # wildcard expansions (access denied if bucket cannot be listed, etc.). if threaded_wildcards: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, threaded_wildcards, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=['name']) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), threaded_wildcards, self.recursion_requested, all_versions=self.all_versions) policy_it = itertools.repeat(protojson.encode_message(policy)) self.Apply(_SetIamWrapper, zip(policy_it, name_expansion_iterator), _SetIamExceptionHandler, fail_on_error=not self.continue_on_error, seek_ahead_iterator=seek_ahead_iterator) self.everything_set_okay &= not GetFailureCount() > 0 # TODO: Add an error counter for files and objects. if not self.everything_set_okay: raise CommandException('Some IAM policies could not be set.') def RunCommand(self): """Command entry point for the acl command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. LogCommandParams(sub_opts=self.sub_opts) self.def_acl = False if action_subcommand == 'get': LogCommandParams(subcommands=[action_subcommand]) self._GetIam() elif action_subcommand == 'set': LogCommandParams(subcommands=[action_subcommand]) self._SetIam() elif action_subcommand == 'ch': LogCommandParams(subcommands=[action_subcommand]) self._PatchIam() else: raise CommandException('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help iam".' % (action_subcommand, self.command_name)) return 0 commands/help.py 0000644 00000022271 15025013616 0007652 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of gsutil help command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import itertools import os import pkgutil import re import six from subprocess import PIPE from subprocess import Popen import gslib.addlhelp from gslib.command import Command from gslib.command import OLD_ALIAS_MAP import gslib.commands from gslib.exception import CommandException from gslib.help_provider import HelpProvider from gslib.help_provider import MAX_HELP_NAME_LEN from gslib.utils import constants from gslib.utils.system_util import IS_WINDOWS from gslib.utils.system_util import IsRunningInteractively from gslib.utils.system_util import GetTermLines from gslib.utils import text_util _SYNOPSIS = """ gsutil help [command or topic] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The following command provides a summary of gsutil commands and additional topics on which help is available: gsutil help The following command provides help about the specified command or topic: gsutil help [command or topic] The following command provides help about the specified sub-command: gsutil help [command] [sub-command] For example, running the following provides help about the "set" sub-command of the "acl" command: gsutil help acl set If you set the PAGER environment variable to the path to a pager program (such as /bin/less on Linux), long help sections are piped through the specified pager. """) top_level_usage_string = ( 'Usage: gsutil [-D] [-DD] [-h header]... [-i service_account] ' '[-m] [-o section:flag=value]... [-q] [-u user_project] [command [opts...] args...]' ) class HelpCommand(Command): """Implementation of gsutil help command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'help', command_name_aliases=['?', 'man'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=2, supported_sub_args='', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='help', help_name_aliases=['?'], help_type='command_help', help_one_line_summary='Get help about commands and topics', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def RunCommand(self): """Command entry point for the help command.""" (help_type_map, help_name_map) = self._LoadHelpMaps() output = [] if not self.args: output.append('%s\nAvailable commands:\n' % top_level_usage_string) format_str = ' %-' + str(MAX_HELP_NAME_LEN) + 's%s\n' for help_prov in sorted(help_type_map['command_help'], key=lambda hp: hp.help_spec.help_name): output.append(format_str % (help_prov.help_spec.help_name, help_prov.help_spec.help_one_line_summary)) output.append('\nAdditional help topics:\n') for help_prov in sorted(help_type_map['additional_help'], key=lambda hp: hp.help_spec.help_name): output.append(format_str % (help_prov.help_spec.help_name, help_prov.help_spec.help_one_line_summary)) output.append('\nUse gsutil help <command or topic> for detailed help.') else: invalid_subcommand = False arg = self.args[0] if arg not in help_name_map: output.append('No help available for "%s"' % arg) else: help_prov = help_name_map[arg] help_name = None if len(self.args) > 1: # We also have a subcommand argument. subcommand_map = help_prov.help_spec.subcommand_help_text if subcommand_map and self.args[1] in subcommand_map: help_name = arg + ' ' + self.args[1] help_text = subcommand_map[self.args[1]] else: invalid_subcommand = True if not subcommand_map: output.append( ('The "%s" command has no subcommands. You can ask for the ' 'full help by running:\n\n\tgsutil help %s\n') % (arg, arg)) else: subcommand_examples = [] for subcommand in subcommand_map: subcommand_examples.append('\tgsutil help %s %s' % (arg, subcommand)) output.append( ('Subcommand "%s" does not exist for command "%s".\n' 'You can either ask for the full help about the command by ' 'running:\n\n\tgsutil help %s\n\n' 'Or you can ask for help about one of the subcommands:\n\n%s' ) % (self.args[1], arg, arg, '\n'.join(subcommand_examples))) if not invalid_subcommand: if not help_name: # No subcommand or invalid subcommand. help_name = help_prov.help_spec.help_name help_text = help_prov.help_spec.help_text output.append('<B>NAME</B>\n') output.append(' %s - %s\n' % (help_name, help_prov.help_spec.help_one_line_summary)) output.append('\n\n') output.append(help_text.strip('\n')) new_alias = OLD_ALIAS_MAP.get(arg, [None])[0] if new_alias: deprecation_warning = """ The "%s" alias is deprecated, and will eventually be removed completely. Please use the "%s" command instead.""" % (arg, new_alias) output.append('\n\n\n<B>DEPRECATION WARNING</B>\n') output.append(deprecation_warning) self._OutputHelp(''.join(output)) return 0 def _OutputHelp(self, help_str): """Outputs simply formatted string. This function paginates if the string is too long, PAGER is defined, and the output is a tty. Args: help_str: String to format. """ # Remove <B> and </B> tags and replace them with ANSI control codes if # writing to a compatible tty. if IS_WINDOWS or not IsRunningInteractively(): help_str = re.sub('<B>', '', help_str) help_str = re.sub('</B>', '', help_str) text_util.print_to_fd(help_str) return help_str = re.sub('<B>', '\033[1m', help_str) help_str = re.sub('</B>', '\033[0;0m', help_str) num_lines = len(help_str.split('\n')) if 'PAGER' in os.environ and num_lines >= GetTermLines(): # Use -r option for less to make bolding work right. pager = os.environ['PAGER'].split(' ') if pager[0].endswith('less'): pager.append('-r') try: if six.PY2: input_for_pager = help_str.encode(constants.UTF8) else: input_for_pager = help_str Popen(pager, stdin=PIPE, universal_newlines=True).communicate(input=input_for_pager) except OSError as e: raise CommandException('Unable to open pager (%s): %s' % (' '.join(pager), e)) else: text_util.print_to_fd(help_str) def _LoadHelpMaps(self): """Returns tuple of help type and help name. help type is a dict with key: help type value: list of HelpProviders help name is a dict with key: help command name or alias value: HelpProvider Returns: (help type, help name) """ # Import all gslib.commands submodules. for _, module_name, _ in pkgutil.iter_modules(gslib.commands.__path__): __import__('gslib.commands.%s' % module_name) # Import all gslib.addlhelp submodules. for _, module_name, _ in pkgutil.iter_modules(gslib.addlhelp.__path__): __import__('gslib.addlhelp.%s' % module_name) help_type_map = {} help_name_map = {} for s in gslib.help_provider.ALL_HELP_TYPES: help_type_map[s] = [] # Only include HelpProvider subclasses in the dict. for help_prov in itertools.chain(HelpProvider.__subclasses__(), Command.__subclasses__()): if help_prov is Command: # Skip the Command base class itself; we just want its subclasses, # where the help command text lives (in addition to non-Command # HelpProviders, like naming.py). continue gslib.help_provider.SanityCheck(help_prov, help_name_map) help_name_map[help_prov.help_spec.help_name] = help_prov for help_name_aliases in help_prov.help_spec.help_name_aliases: help_name_map[help_name_aliases] = help_prov help_type_map[help_prov.help_spec.help_type].append(help_prov) return (help_type_map, help_name_map) commands/setmeta.py 0000644 00000037563 15025013616 0010376 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of setmeta command for setting cloud object metadata.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import time from apitools.base.py import encoding from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import MetadataMessage from gslib.utils import constants from gslib.utils import parallelism_framework_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.metadata_util import IsCustomMetadataHeader from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAsciiHeader from gslib.utils.text_util import InsistAsciiHeaderValue from gslib.utils.translation_helper import CopyObjectMetadata from gslib.utils.translation_helper import ObjectMetadataFromHeaders from gslib.utils.translation_helper import PreconditionsFromHeaders _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout _SYNOPSIS = """ gsutil setmeta -h [header:value|header] ... url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil setmeta command allows you to set or remove the metadata on one or more objects. It takes one or more header arguments followed by one or more URLs, where each header argument is in one of two forms: - If you specify ``header:value``, it sets the provided value for the given header on all applicable objects. - If you specify ``header`` (with no value), it removes the given header from all applicable objects. For example, the following command sets the ``Content-Type`` and ``Cache-Control`` headers while also removing the ``Content-Disposition`` header on the specified objects: gsutil setmeta -h "Content-Type:text/html" \\ -h "Cache-Control:public, max-age=3600" \\ -h "Content-Disposition" gs://bucket/*.html If you have a large number of objects to update you might want to use the gsutil -m option, to perform a parallel (multi-threaded/multi-processing) update: gsutil -m setmeta -h "Content-Type:text/html" \\ -h "Cache-Control:public, max-age=3600" \\ -h "Content-Disposition" gs://bucket/*.html You can also use the setmeta command to set custom metadata on an object: gsutil setmeta -h "x-goog-meta-icecreamflavor:vanilla" gs://bucket/object Custom metadata is always prefixed in gsutil with ``x-goog-meta-``. This distinguishes it from standard request headers. Other tools that send and receive object metadata by using the request body do not use this prefix. While gsutil supports custom metadata with arbitrary Unicode values, note that when setting metadata using the XML API, which sends metadata as HTTP headers, Unicode characters are encoded using UTF-8, then url-encoded to ASCII. For example: gsutil setmeta -h "x-goog-meta-foo: ã" gs://bucket/object stores the custom metadata key-value pair of ``foo`` and ``%C3%A3``. Subsequently, running ``ls -L`` using the JSON API to list the object's metadata prints ``%C3%A3``, while ``ls -L`` using the XML API url-decodes this value automatically, printing the character ``ã``. The setmeta command reads each object's current generation and metageneration and uses those as preconditions unless they are otherwise specified by top-level arguments. For example, the following command sets the custom metadata ``icecreamflavor:vanilla`` if the current live object has a metageneration of 2: gsutil -h "x-goog-if-metageneration-match:2" setmeta -h "x-goog-meta-icecreamflavor:vanilla" See `Object metadata <https://cloud.google.com/storage/docs/metadata>`_ for more information about object metadata. <B>OPTIONS</B> -h Specifies a header:value to be added, or header to be removed, from each named object. """) # Setmeta assumes a header-like model which doesn't line up with the JSON way # of doing things. This list comes from functionality that was supported by # gsutil3 at the time gsutil4 was released. SETTABLE_FIELDS = [ 'cache-control', 'content-disposition', 'content-encoding', 'content-language', 'content-type', 'custom-time' ] _GCLOUD_OBJECTS_UPDATE_COMMAND = ['storage', 'objects', 'update'] def _SetMetadataExceptionHandler(cls, e): """Exception handler that maintains state about post-completion status.""" cls.logger.error(e) cls.everything_set_okay = False def _SetMetadataFuncWrapper(cls, name_expansion_result, thread_state=None): cls.SetMetadataFunc(name_expansion_result, thread_state=thread_state) class SetMetaCommand(Command): """Implementation of gsutil setmeta command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'setmeta', command_name_aliases=['setheader'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='h:rR', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='setmeta', help_name_aliases=['setheader'], help_type='command_help', help_one_line_summary='Set metadata on already uploaded objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=_GCLOUD_OBJECTS_UPDATE_COMMAND, flag_map={}, ) def get_gcloud_storage_args(self): recursive_flag = [] for o, _ in self.sub_opts: if o in ('-r', '-R'): recursive_flag = ['--recursive'] break clear_set, set_dict = self._ParseMetadataHeaders( self._GetHeaderStringsFromSubOpts()) # Handle translation through "gcloud_command" instead of "flag_map". self.sub_opts = [] clear_flags = self._translate_headers( {clear_key: None for clear_key in clear_set}, unset=True) set_flags = self._translate_headers(set_dict, unset=False) command = (_GCLOUD_OBJECTS_UPDATE_COMMAND + recursive_flag + clear_flags + set_flags) gcloud_storage_map = GcloudStorageMap( gcloud_command=command, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def RunCommand(self): """Command entry point for the setmeta command.""" metadata_minus, metadata_plus = self._ParseMetadataHeaders( self._GetHeaderStringsFromSubOpts()) self.metadata_change = metadata_plus for header in metadata_minus: self.metadata_change[header] = '' if not self.metadata_change: raise CommandException( 'gsutil setmeta requires one or more headers to be provided with the' ' -h flag. See "gsutil help setmeta" for more information.') if len(self.args) == 1 and not self.recursion_requested: url = StorageUrlFromString(self.args[0]) if not (url.IsCloudUrl() and url.IsObject()): raise CommandException('URL (%s) must name an object' % self.args[0]) # Used to track if any objects' metadata failed to be set. self.everything_set_okay = True self.preconditions = PreconditionsFromHeaders(self.headers) name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, self.args, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.parallel_operations, bucket_listing_fields=['generation', 'metadata', 'metageneration']) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), self.args, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) try: # Perform requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(_SetMetadataFuncWrapper, name_expansion_iterator, _SetMetadataExceptionHandler, fail_on_error=True, seek_ahead_iterator=seek_ahead_iterator) except AccessDeniedException as e: if e.status == 403: self._WarnServiceAccounts() raise if not self.everything_set_okay: raise CommandException('Metadata for some objects could not be set.') return 0 @Retry(PreconditionException, tries=3, timeout_secs=1) def SetMetadataFunc(self, name_expansion_result, thread_state=None): """Sets metadata on an object. Args: name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) exp_src_url = name_expansion_result.expanded_storage_url self.logger.info('Setting metadata on %s...', exp_src_url) cloud_obj_metadata = encoding.JsonToMessage( apitools_messages.Object, name_expansion_result.expanded_result) preconditions = Preconditions( gen_match=self.preconditions.gen_match, meta_gen_match=self.preconditions.meta_gen_match) if preconditions.gen_match is None: preconditions.gen_match = cloud_obj_metadata.generation if preconditions.meta_gen_match is None: preconditions.meta_gen_match = cloud_obj_metadata.metageneration # Patch handles the patch semantics for most metadata, but we need to # merge the custom metadata field manually. patch_obj_metadata = ObjectMetadataFromHeaders(self.metadata_change) api = gsutil_api.GetApiSelector(provider=exp_src_url.scheme) # For XML we only want to patch through custom metadata that has # changed. For JSON we need to build the complete set. if api == ApiSelector.XML: pass elif api == ApiSelector.JSON: CopyObjectMetadata(patch_obj_metadata, cloud_obj_metadata, override=True) patch_obj_metadata = cloud_obj_metadata # Patch body does not need the object generation and metageneration. patch_obj_metadata.generation = None patch_obj_metadata.metageneration = None gsutil_api.PatchObjectMetadata(exp_src_url.bucket_name, exp_src_url.object_name, patch_obj_metadata, generation=exp_src_url.generation, preconditions=preconditions, provider=exp_src_url.scheme, fields=['id']) _PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) def _GetHeaderStringsFromSubOpts(self): """Gets header values from after the "setmeta" part of the command. Example: $ gsutil -h not:parsed setmeta is:parsed gs://bucket/object -> ["is:parsed"] Returns: List[str]: Headers without the "-h" but not yet split on colons. Raises: CommandException Found canned ACL. """ if not self.sub_opts: return [] headers = [] for o, a in self.sub_opts: if o == '-h': if 'x-goog-acl' in a or 'x-amz-acl' in a: raise CommandException( 'gsutil setmeta no longer allows canned ACLs. Use gsutil acl ' 'set ... to set canned ACLs.') headers.append(a) return headers def _ParseMetadataHeaders(self, headers): """Validates and parses metadata changes from the headers argument. Args: headers: Header dict to validate and parse. Returns: (metadata_plus, metadata_minus): Tuple of header sets to add and remove. """ metadata_minus = set() cust_metadata_minus = set() metadata_plus = {} cust_metadata_plus = {} # Build a count of the keys encountered from each plus and minus arg so we # can check for dupe field specs. num_metadata_plus_elems = 0 num_cust_metadata_plus_elems = 0 num_metadata_minus_elems = 0 num_cust_metadata_minus_elems = 0 for md_arg in headers: # Use partition rather than split, as we should treat all characters past # the initial : as part of the header's value. parts = md_arg.partition(':') (header, _, value) = parts InsistAsciiHeader(header) # Translate headers to lowercase to match the casing assumed by our # sanity-checking operations. lowercase_header = header.lower() # This check is overly simple; it would be stronger to check, for each # URL argument, whether the header starts with the provider # metadata_prefix, but here we just parse the spec once, before # processing any of the URLs. This means we will not detect if the user # tries to set an x-goog-meta- field on an another provider's object, # for example. is_custom_meta = IsCustomMetadataHeader(lowercase_header) if not is_custom_meta and lowercase_header not in SETTABLE_FIELDS: raise CommandException( 'Invalid or disallowed header (%s).\nOnly these fields (plus ' 'x-goog-meta-* fields) can be set or unset:\n%s' % (header, sorted(list(SETTABLE_FIELDS)))) if value: if is_custom_meta: # Allow non-ASCII data for custom metadata fields. cust_metadata_plus[header] = value num_cust_metadata_plus_elems += 1 else: # Don't unicode encode other fields because that would perturb their # content (e.g., adding %2F's into the middle of a Cache-Control # value). InsistAsciiHeaderValue(header, value) value = str(value) metadata_plus[lowercase_header] = value num_metadata_plus_elems += 1 else: if is_custom_meta: cust_metadata_minus.add(header) num_cust_metadata_minus_elems += 1 else: metadata_minus.add(lowercase_header) num_metadata_minus_elems += 1 if (num_metadata_plus_elems != len(metadata_plus) or num_cust_metadata_plus_elems != len(cust_metadata_plus) or num_metadata_minus_elems != len(metadata_minus) or num_cust_metadata_minus_elems != len(cust_metadata_minus) or metadata_minus.intersection(set(metadata_plus.keys()))): raise CommandException('Each header must appear at most once.') metadata_plus.update(cust_metadata_plus) metadata_minus.update(cust_metadata_minus) return (metadata_minus, metadata_plus) commands/mv.py 0000644 00000014555 15025013616 0007352 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like mv command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.commands.cp import CP_AND_MV_SHIM_FLAG_MAP from gslib.commands.cp import CP_SUB_ARGS from gslib.commands.cp import ShimTranslatePredefinedAclSubOptForCopy from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap _SYNOPSIS = """ gsutil mv [-p] src_url dst_url gsutil mv [-p] src_url... dst_url gsutil mv [-p] -I dst_url """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The ``gsutil mv`` command allows you to move data between your local file system and the cloud, move data within the cloud, and move data between cloud storage providers. For example, to move all objects from a bucket to a local directory you could use: gsutil mv gs://my_bucket/* dir Similarly, to move all objects from a local directory to a bucket you could use: gsutil mv ./dir gs://my_bucket <B>RENAMING GROUPS OF OBJECTS</B> You can use the ``gsutil mv`` command to rename all objects with a given prefix to have a new prefix. For example, the following command renames all objects under gs://my_bucket/oldprefix to be under gs://my_bucket/newprefix, otherwise preserving the naming structure: gsutil mv gs://my_bucket/oldprefix gs://my_bucket/newprefix If you do a rename as specified above and you want to preserve ACLs, you should use the ``-p`` option (see OPTIONS). If you have a large number of files to move you might want to use the ``gsutil -m`` option, to perform a multi-threaded/multi-processing move: gsutil -m mv gs://my_bucket/oldprefix gs://my_bucket/newprefix <B>NON-ATOMIC OPERATION</B> Unlike the case with many file systems, the gsutil mv command does not perform a single atomic operation. Rather, it performs a copy from source to destination followed by removing the source for each object. A consequence of this is that, in addition to normal network and operation charges, if you move a Nearline Storage, Coldline Storage, or Archive Storage object, deletion and data retrieval charges apply. See the `documentation <https://cloud.google.com/storage/pricing>`_ for pricing details. <B>OPTIONS</B> All options that are available for the gsutil cp command are also available for the gsutil mv command (except for the -R flag, which is implied by the ``gsutil mv`` command). Please see the `Options section for cp <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_ for more information. """) class MvCommand(Command): """Implementation of gsutil mv command. Note that there is no atomic rename operation - this command is simply a shorthand for 'cp' followed by 'rm'. """ # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'mv', command_name_aliases=['move', 'ren', 'rename'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, # Flags for mv are passed through to cp. supported_sub_args=CP_SUB_ARGS, file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument() ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='mv', help_name_aliases=['move', 'rename'], help_type='command_help', help_one_line_summary='Move/rename objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): ShimTranslatePredefinedAclSubOptForCopy(self.sub_opts) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'mv'], flag_map=CP_AND_MV_SHIM_FLAG_MAP, ) return super().get_gcloud_storage_args(gcloud_storage_map) def RunCommand(self): """Command entry point for the mv command.""" # Check each source arg up, refusing to delete a bucket src URL (force users # to explicitly do that as a separate operation). for arg_to_check in self.args[0:-1]: url = StorageUrlFromString(arg_to_check) if url.IsCloudUrl() and (url.IsBucket() or url.IsProvider()): raise CommandException('You cannot move a source bucket using the mv ' 'command. If you meant to move\nall objects in ' 'the bucket, you can use a command like:\n' '\tgsutil mv %s/* %s' % (arg_to_check, self.args[-1])) # Insert command-line opts in front of args so they'll be picked up by cp # and rm commands (e.g., for -p option). Use undocumented (internal # use-only) cp -M option, which causes each original object to be deleted # after successfully copying to its destination, and also causes naming # behavior consistent with Unix mv naming behavior (see comments in # ConstructDstUrl). unparsed_args = ['-M'] if self.recursion_requested: unparsed_args.append('-R') unparsed_args.extend(self.unparsed_args) self.command_runner.RunNamedCommand( 'cp', args=unparsed_args, headers=self.headers, debug=self.debug, trace_token=self.trace_token, user_project=self.user_project, parallel_operations=self.parallel_operations) return 0 commands/defacl.py 0000644 00000035036 15025013616 0010143 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of default object acl command for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from gslib import gcs_json_api from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import Preconditions from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import SetAclExceptionHandler from gslib.command import SetAclFuncWrapper from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import acl_helper from gslib.utils.constants import NO_MAX from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import PRIVATE_DEFAULT_OBJ_ACL _SET_SYNOPSIS = """ gsutil defacl set (<file-path>|<predefined-acl>) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil defacl get gs://<bucket_name> """ _CH_SYNOPSIS = """ gsutil defacl ch [-f] -u|-g|-d|-p <grant>... gs://<bucket_name>... """ _SET_DESCRIPTION = """ <B>SET</B> The ``defacl set`` command sets default object ACLs for the specified buckets. If you specify a default object ACL for a certain bucket, Cloud Storage applies the default object ACL to all new objects uploaded to that bucket, unless an ACL for that object is separately specified during upload. Similar to the ``acl set`` command, the ``defacl set`` command specifies either a predefined ACL or the path to a file that contains ACL text. See "gsutil help acl" for examples of editing and setting ACLs via the acl command. See `Predefined ACLs <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ for a list of predefined ACLs. Setting a default object ACL on a bucket provides a convenient way to ensure newly uploaded objects have a specific ACL. If you don't set the bucket's default object ACL, it will default to project-private. If you then upload objects that need a different ACL, you will need to perform a separate ACL update operation for each object. Depending on how many objects require updates, this could be very time-consuming. """ _GET_DESCRIPTION = """ <B>GET</B> Gets the default ACL text for a bucket, which you can save and edit for use with the "defacl set" command. """ _CH_DESCRIPTION = """ <B>CH</B> The "defacl ch" (or "defacl change") command updates the default object access control list for a bucket. The syntax is shared with the "acl ch" command, so see the "CH" section of "gsutil help acl" for the full help description. <B>CH EXAMPLES</B> Grant anyone on the internet READ access by default to any object created in the bucket example-bucket: gsutil defacl ch -u AllUsers:R gs://example-bucket NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see "gsutil help setmeta". Add the user john.doe@example.com to the default object ACL on bucket example-bucket with READ access: gsutil defacl ch -u john.doe@example.com:READ gs://example-bucket Add the group admins@example.com to the default object ACL on bucket example-bucket with OWNER access: gsutil defacl ch -g admins@example.com:O gs://example-bucket Remove the group admins@example.com from the default object ACL on bucket example-bucket: gsutil defacl ch -d admins@example.com gs://example-bucket Add the owners of project example-project-123 to the default object ACL on bucket example-bucket with READ access: gsutil defacl ch -p owners-example-project-123:R gs://example-bucket NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access to a project's viewers/editors respectively. <B>CH OPTIONS</B> The "ch" sub-command has the following options -d Remove all roles associated with the matching entity. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. With this option the gsutil exit status will be 0 even if some ACLs couldn't be changed. -g Add or modify a group entity's role. -p Add or modify a project viewers/editors/owners role. -u Add or modify a user entity's role. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ The defacl command has three sub-commands: """ + '\n'.join([_SET_DESCRIPTION + _GET_DESCRIPTION + _CH_DESCRIPTION]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) class DefAclCommand(Command): """Implementation of gsutil defacl command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'defacl', command_name_aliases=['setdefacl', 'getdefacl', 'chdefacl'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='fg:u:d:p:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeFileURLOrCannedACLArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'ch': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='defacl', help_name_aliases=['default acl', 'setdefacl', 'getdefacl', 'chdefacl'], help_type='command_help', help_one_line_summary='Get, set, or change default ACL on buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text, }, ) def get_gcloud_storage_args(self): sub_command = self.args.pop(0) if sub_command == 'get': gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=multi(defaultObjectAcl:format=json)', '--raw' ], flag_map={}, ) elif sub_command == 'set': acl_file_or_predefined_acl = self.args.pop(0) if (os.path.isfile(acl_file_or_predefined_acl)): gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--default-object-acl-file=' + acl_file_or_predefined_acl ], flag_map={}, ) else: if acl_file_or_predefined_acl in ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION): predefined_acl = ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION[ acl_file_or_predefined_acl]) else: predefined_acl = acl_file_or_predefined_acl gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--predefined-default-object-acl=' + predefined_acl ], flag_map={}, ) elif sub_command == 'ch': self.ParseSubOpts() self.sub_opts = acl_helper.translate_sub_opts_for_shim(self.sub_opts) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update'], flag_map={ '-g': GcloudStorageFlag('--add-default-object-acl-grant'), '-p': GcloudStorageFlag('--add-default-object-acl-grant'), '-u': GcloudStorageFlag('--add-default-object-acl-grant'), '-d': GcloudStorageFlag('--remove-default-object-acl-grant'), '-f': GcloudStorageFlag('--continue-on-error'), }) return super().get_gcloud_storage_args(gcloud_storage_map) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if (self.args[0].lower() == 'set' or self.command_alias_used == 'setdefacl'): return 1 else: return 0 def _SetDefAcl(self): if not StorageUrlFromString(self.args[-1]).IsBucket(): raise CommandException('URL must name a bucket for the %s command' % self.command_name) try: self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler) except AccessDeniedException: self._WarnServiceAccounts() raise def _GetDefAcl(self): if not StorageUrlFromString(self.args[0]).IsBucket(): raise CommandException('URL must name a bucket for the %s command' % self.command_name) self.GetAndPrintAcl(self.args[0]) def _ChDefAcl(self): """Parses options and changes default object ACLs on specified buckets.""" self.parse_versions = True self.changes = [] if self.sub_opts: for o, a in self.sub_opts: if o == '-g': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.GROUP)) if o == '-u': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.USER)) if o == '-p': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.PROJECT)) if o == '-d': self.changes.append(acl_helper.AclDel(a)) if not self.changes: raise CommandException('Please specify at least one access change ' 'with the -g, -u, or -d flags') if (not UrlsAreForSingleProvider(self.args) or StorageUrlFromString(self.args[0]).scheme != 'gs'): raise CommandException( 'The "{0}" command can only be used with gs:// URLs'.format( self.command_name)) bucket_urls = set() for url_arg in self.args: for result in self.WildcardIterator(url_arg): if not result.storage_url.IsBucket(): raise CommandException( 'The defacl ch command can only be applied to buckets.') bucket_urls.add(result.storage_url) for storage_url in bucket_urls: self.ApplyAclChanges(storage_url) @Retry(ServiceException, tries=3, timeout_secs=1) def ApplyAclChanges(self, url): """Applies the changes in self.changes to the provided URL.""" bucket = self.gsutil_api.GetBucket( url.bucket_name, provider=url.scheme, fields=['defaultObjectAcl', 'metageneration']) # Default object ACLs can be blank if the ACL was set to private, or # if the user doesn't have permission. We warn about this with defacl get, # so just try the modification here and if the user doesn't have # permission they'll get an AccessDeniedException. current_acl = bucket.defaultObjectAcl if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: self.logger.info('No changes to %s', url) return if not current_acl: # Use a sentinel value to indicate a private (no entries) default # object ACL. current_acl.append(PRIVATE_DEFAULT_OBJ_ACL) try: preconditions = Preconditions(meta_gen_match=bucket.metageneration) bucket_metadata = apitools_messages.Bucket(defaultObjectAcl=current_acl) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) self.logger.info('Updated default ACL on %s', url) except BadRequestException as e: # Don't retry on bad requests, e.g. invalid email address. raise CommandException('Received bad request from server: %s' % str(e)) except AccessDeniedException: self._WarnServiceAccounts() raise CommandException('Failed to set acl for %s. Please ensure you have ' 'OWNER-role access to this resource.' % url) def _ApplyAclChangesAndReturnChangeCount(self, storage_url, defacl_message): modification_count = 0 for change in self.changes: modification_count += change.Execute(storage_url, defacl_message, 'defacl', self.logger) return modification_count def RunCommand(self): """Command entry point for the defacl command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) self.def_acl = True self.continue_on_error = False if action_subcommand == 'get': func = self._GetDefAcl elif action_subcommand == 'set': func = self._SetDefAcl elif action_subcommand in ('ch', 'change'): func = self._ChDefAcl else: raise CommandException(('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help defacl".') % (action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) func() return 0 commands/rpo.py 0000644 00000017543 15025013616 0007530 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the rpo command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap VALID_RPO_VALUES = ('ASYNC_TURBO', 'DEFAULT') VALID_RPO_VALUES_STRING = '({})'.format('|'.join(VALID_RPO_VALUES)) _SET_SYNOPSIS = """ gsutil rpo set {} gs://<bucket_name>... """.format(VALID_RPO_VALUES_STRING) _GET_SYNOPSIS = """ gsutil rpo get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``rpo set`` command configures turbo replication for dual-region Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use turbo replication: gsutil rpo set ASYNC_TURBO gs://redbucket gs://bluebucket Configure your buckets to NOT use turbo replication: gsutil rpo set DEFAULT gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``rpo get`` command returns the replication setting for the specified Cloud Storage buckets. <B>GET EXAMPLES</B> Check if your buckets are using turbo replication: gsutil rpo get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The ``rpo`` command is used to retrieve or configure the `replication setting <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_ of dual-region Cloud Storage buckets. This command has two sub-commands: ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) GET_COMMAND = GcloudStorageMap(gcloud_command=[ 'storage', 'buckets', 'list', '--format=value[separator=": "](format("gs://{}", name),' 'rpo.yesno(no="None"))', '--raw' ], flag_map={}) SET_COMMAND = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--recovery-point-objective' ], flag_map={}, ) class RpoCommand(Command): """Implements the gsutil rpo command.""" command_spec = Command.CreateCommandSpec( 'rpo', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1)], 'set': [ CommandArgument('mode', choices=list(VALID_RPO_VALUES)), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rpo', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure replication', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GET_COMMAND, 'set': SET_COMMAND, }, flag_map={}, ) def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetRpo(self, blr): """Gets the rpo setting for a bucket.""" bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['rpo'], provider=bucket_url.scheme) rpo = bucket_metadata.rpo bucket = str(bucket_url).rstrip('/') print('%s: %s' % (bucket, rpo)) def _SetRpo(self, blr, rpo_value): """Sets the rpo setting for a bucket.""" bucket_url = blr.storage_url formatted_rpo_value = rpo_value if formatted_rpo_value not in VALID_RPO_VALUES: raise CommandException( 'Invalid value for rpo set.' ' Should be one of {}'.format(VALID_RPO_VALUES_STRING)) bucket_metadata = apitools_messages.Bucket(rpo=formatted_rpo_value) self.logger.info('Setting rpo %s for %s' % (formatted_rpo_value, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['rpo'], provider=bucket_url.scheme) return 0 def _Rpo(self): """Handles rpo command on Cloud Storage buckets.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('rpo only supports get|set') subcommand_func = None subcommand_args = [] if subcommand == 'get': subcommand_func = self._GetRpo elif subcommand == 'set': subcommand_func = self._SetRpo setting_arg = self.args.pop(0) subcommand_args.append(setting_arg) # TODO: Remove this as rpo should work for XML as well. if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be with the Cloud Storage ' 'JSON API.') % self.command_name))) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: # TODO: Make this more general when adding RPO support for the XML API. if self.gsutil_api.GetApiSelector( bucket_listing_ref.storage_url.scheme) != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be used for GCS ' 'buckets.') % self.command_name))) some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the rpo command.""" action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) return self._Rpo() else: raise CommandException('Invalid subcommand "%s", use get|set instead.' % action_subcommand) commands/version.py 0000644 00000016171 15025013616 0010411 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of gsutil version command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import platform import re import sys import six import boto import crcmod import gslib from gslib.command import Command from gslib.utils import system_util from gslib.utils.boto_util import GetFriendlyConfigFilePaths from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.constants import UTF8 from gslib.utils.hashing_helper import GetMd5 from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit _SYNOPSIS = """ gsutil version """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Prints information about the version of gsutil. <B>OPTIONS</B> -l Prints additional information, such as the version of Python being used, the version of the Boto library, a checksum of the code, the path to gsutil, and the path to gsutil's configuration file. """) class VersionCommand(Command): """Implementation of gsutil version command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'version', command_name_aliases=['ver'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=0, supported_sub_args='l', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='version', help_name_aliases=['ver'], help_type='command_help', help_one_line_summary='Print version info about gsutil', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def RunCommand(self): """Command entry point for the version command.""" long_form = False if self.sub_opts: for o, _ in self.sub_opts: if o == '-l': long_form = True config_paths = ', '.join(GetFriendlyConfigFilePaths()) shipped_checksum = gslib.CHECKSUM try: cur_checksum = self._ComputeCodeChecksum() except IOError: cur_checksum = 'MISSING FILES' if shipped_checksum == cur_checksum: checksum_ok_str = 'OK' else: checksum_ok_str = '!= %s' % shipped_checksum sys.stdout.write('gsutil version: %s\n' % gslib.VERSION) if long_form: long_form_output = ( 'checksum: {checksum} ({checksum_ok})\n' 'boto version: {boto_version}\n' 'python version: {python_version}\n' 'OS: {os_version}\n' 'multiprocessing available: {multiprocessing_available}\n' 'using cloud sdk: {cloud_sdk}\n' 'pass cloud sdk credentials to gsutil: {cloud_sdk_credentials}\n' 'config path(s): {config_paths}\n' 'gsutil path: {gsutil_path}\n' 'compiled crcmod: {compiled_crcmod}\n' 'installed via package manager: {is_package_install}\n' 'editable install: {is_editable_install}\n' 'shim enabled: {is_shim_enabled}\n') sys.stdout.write( long_form_output.format( checksum=cur_checksum, checksum_ok=checksum_ok_str, boto_version=boto.__version__, python_version=sys.version.replace('\n', ''), os_version='%s %s' % (platform.system(), platform.release()), multiprocessing_available=( CheckMultiprocessingAvailableAndInit().is_available), cloud_sdk=system_util.InvokedViaCloudSdk(), cloud_sdk_credentials=system_util.CloudSdkCredPassingEnabled(), config_paths=config_paths, gsutil_path=(GetCloudSdkGsutilWrapperScriptPath() or gslib.GSUTIL_PATH), compiled_crcmod=UsingCrcmodExtension(), is_package_install=gslib.IS_PACKAGE_INSTALL, is_editable_install=gslib.IS_EDITABLE_INSTALL, is_shim_enabled=boto.config.getbool('GSUtil', 'use_gcloud_storage', False))) return 0 def _ComputeCodeChecksum(self): """Computes a checksum of gsutil code. This checksum can be used to determine if users locally modified gsutil when requesting support. (It's fine for users to make local mods, but when users ask for support we ask them to run a stock version of gsutil so we can reduce possible variables.) Returns: MD5 checksum of gsutil code. """ if gslib.IS_PACKAGE_INSTALL: return 'PACKAGED_GSUTIL_INSTALLS_DO_NOT_HAVE_CHECKSUMS' m = GetMd5() # Checksum gsutil and all .py files under gslib directory. files_to_checksum = [gslib.GSUTIL_PATH] for root, _, files in os.walk(gslib.GSLIB_DIR): for filepath in files: if filepath.endswith('.py'): files_to_checksum.append(os.path.join(root, filepath)) # Sort to ensure consistent checksum build, no matter how os.walk # orders the list. for filepath in sorted(files_to_checksum): if six.PY2: f = open(filepath, 'rb') content = f.read() content = re.sub(r'(\r\n|\r|\n)', b'\n', content) m.update(content) f.close() else: f = open(filepath, 'r', encoding=UTF8) content = f.read() content = re.sub(r'(\r\n|\r|\n)', '\n', content) m.update(content.encode(UTF8)) f.close() return m.hexdigest() def GetCloudSdkGsutilWrapperScriptPath(): """If gsutil was invoked via the Cloud SDK, find its gsutil wrapper script. Returns: (str) The path to the Cloud SDK's gsutil wrapper script, or an empty string if gsutil was not invoked via the Cloud SDK or the wrapper script could not be found at its expected path. """ # If running via the Cloud SDK, the "real" gsutil script was probably invoked # indirectly through gcloud's gsutil wrapper script (the former is at # <sdk-root>/platform/gsutil/gsutil, while the latter is located at # <sdk-root>/bin/gsutil). We should print the wrapper script's path if it # exists and we were invoked via the Cloud SDK. gsutil_path = gslib.GSUTIL_PATH if system_util.InvokedViaCloudSdk(): platform_path_suffix = os.path.join('platform', 'gsutil', 'gsutil') if gsutil_path.endswith(platform_path_suffix): bin_path = os.path.join( gsutil_path[0:gsutil_path.rfind(platform_path_suffix)], 'bin', 'gsutil', ) if os.path.exists(bin_path): return bin_path return '' commands/bucketpolicyonly.py 0000644 00000021310 15025013616 0012312 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import getopt import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.commands import ubla from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.text_util import InsistOnOrOff _SET_SYNOPSIS = """ gsutil bucketpolicyonly set (on|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil bucketpolicyonly get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``bucketpolicyonly set`` command enables or disables the uniform bucket-level access feature on Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use uniform bucket-level access: gsutil bucketpolicyonly set on gs://redbucket gs://bluebucket Configure your buckets to NOT use uniform bucket-level access: gsutil bucketpolicyonly set off gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``bucketpolicyonly get`` command shows whether uniform bucket-level access is enabled for the specified Cloud Storage bucket. <B>GET EXAMPLES</B> Check if your buckets are using uniform bucket-level access: gsutil bucketpolicyonly get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The Bucket Policy Only feature is now known as `uniform bucket-level access <https://cloud.google.com/storage/docs/uniform-bucket-level-access>`_. The ``bucketpolicyonly`` command is still supported, but we recommend using the equivalent ``ubla`` `command <https://cloud.google.com/storage/docs/gsutil/commands/ubla>`_. The ``bucketpolicyonly`` command is used to retrieve or configure the uniform bucket-level access setting of Cloud Storage buckets. This command has two sub-commands, ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) # Aliases to make these more likely to fit on one line. IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue BucketPolicyOnlyValue = IamConfigurationValue.BucketPolicyOnlyValue class BucketPolicyOnlyCommand(Command): """Implements the gsutil bucketpolicyonly command.""" command_spec = Command.CreateCommandSpec( 'bucketpolicyonly', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='bucketpolicyonly', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure uniform bucket-level access', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) # The get subcommand works exactly like ubla except we want to replace # "Uniform bucket-level access" with the bpo text. gcloud_storage_map = ubla.UblaCommand.gcloud_storage_map format_flag = gcloud_storage_map.gcloud_command['get'].gcloud_command[3] gcloud_storage_map.gcloud_command['get'].gcloud_command[ 3] = format_flag.replace('Uniform bucket-level access', 'Bucket Policy Only') def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetBucketPolicyOnly(self, blr): """Gets the Bucket Policy Only setting for a bucket.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['iamConfiguration'], provider=bucket_url.scheme) iam_config = bucket_metadata.iamConfiguration bucket_policy_only = iam_config.bucketPolicyOnly fields = { 'bucket': str(bucket_url).rstrip('/'), 'enabled': bucket_policy_only.enabled } locked_time_line = '' if bucket_policy_only.lockedTime: fields['locked_time'] = bucket_policy_only.lockedTime locked_time_line = ' LockedTime: {locked_time}\n' if bucket_policy_only: print(('Bucket Policy Only setting for {bucket}:\n' ' Enabled: {enabled}\n' + locked_time_line).format(**fields)) def _SetBucketPolicyOnly(self, blr, setting_arg): """Sets the Bucket Policy Only setting for a bucket on or off.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url iam_config = IamConfigurationValue() iam_config.bucketPolicyOnly = BucketPolicyOnlyValue() iam_config.bucketPolicyOnly.enabled = (setting_arg == 'on') bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config) setting_verb = 'Enabling' if setting_arg == 'on' else 'Disabling' print('%s Bucket Policy Only for %s...' % (setting_verb, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['iamConfiguration'], provider=bucket_url.scheme) return 0 def _BucketPolicyOnly(self): """Handles bucketpolicyonly command on a Cloud Storage bucket.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('bucketpolicyonly only supports get|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._GetBucketPolicyOnly elif subcommand == 'set': subcommand_func = self._SetBucketPolicyOnly setting_arg = self.args.pop(0) InsistOnOrOff(setting_arg, 'Only on and off values allowed for set option') subcommand_args.append(setting_arg) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the bucketpolicyonly command.""" if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the Cloud Storage JSON API.' % self.command_name))) action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) self._BucketPolicyOnly() else: raise CommandException('Invalid subcommand "%s", use get|set instead.' % action_subcommand) commands/pap.py 0000644 00000022403 15025013616 0007477 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the pap command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil pap set (enforced|inherited) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil pap get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``pap set`` command configures public access prevention for Cloud Storage buckets. If you set a bucket to be ``inherited``, it uses public access prevention only if the bucket is subject to the `public access prevention <https://cloud.google.com/storage/docs/org-policy-constraints#public-access-prevention>`_ organization policy constraint. <B>SET EXAMPLES</B> Configure ``redbucket`` and ``bluebucket`` to use public access prevention: gsutil pap set enforced gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``pap get`` command returns public access prevention values for the specified Cloud Storage buckets. <B>GET EXAMPLES</B> Check if ``redbucket`` and ``bluebucket`` are using public access prevention: gsutil pap get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The ``pap`` command is used to retrieve or configure the `public access prevention <https://cloud.google.com/storage/docs/public-access-prevention>`_ setting of Cloud Storage buckets. This command has two sub-commands: ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) # Aliases to make these more likely to fit enforced one line. IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue _GCLOUD_LIST_FORMAT = ('--format=value[separator=": "]' + '(name.sub("' + shim_util.get_format_flag_caret() + '","gs://"),' + 'iamConfiguration.publicAccessPrevention.yesno(' + 'no="inherited"))') class PapCommand(Command): """Implements the gsutil pap command.""" command_spec = Command.CreateCommandSpec( 'pap', command_name_aliases=['publicaccessprevention'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['enforced', 'inherited']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='pap', help_name_aliases=['publicaccessprevention'], help_type='command_help', help_one_line_summary='Configure public access prevention', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_LIST_FORMAT, '--raw' ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'enforced': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--public-access-prevention', ], flag_map={}, ), 'inherited': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-public-access-prevention', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetPublicAccessPrevention(self, blr): """Gets the public access prevention setting for a bucket.""" bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['iamConfiguration'], provider=bucket_url.scheme) iam_config = bucket_metadata.iamConfiguration public_access_prevention = iam_config.publicAccessPrevention or 'inherited' bucket = str(bucket_url).rstrip('/') print('%s: %s' % (bucket, public_access_prevention)) def _SetPublicAccessPrevention(self, blr, setting_arg): """Sets the Public Access Prevention setting for a bucket enforced or inherited.""" bucket_url = blr.storage_url iam_config = IamConfigurationValue() iam_config.publicAccessPrevention = setting_arg bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config) print('Setting Public Access Prevention %s for %s' % (setting_arg, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['iamConfiguration'], provider=bucket_url.scheme) return 0 def _Pap(self): """Handles pap command on Cloud Storage buckets.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('pap only supports get|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._GetPublicAccessPrevention elif subcommand == 'set': subcommand_func = self._SetPublicAccessPrevention setting_arg = self.args.pop(0) subcommand_args.append(setting_arg) if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be with the Cloud Storage ' 'JSON API.') % self.command_name))) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: if self.gsutil_api.GetApiSelector( bucket_listing_ref.storage_url.scheme) != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be used for GCS ' 'Buckets.') % self.command_name))) some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the pap command.""" action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) self._Pap() else: raise CommandException('Invalid subcommand "%s", use get|set instead.' % action_subcommand) commands/hmac.py 0000644 00000041234 15025013616 0007632 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Implementation of HMAC key management command for GCS. NOTE: Any modification to this file or corresponding HMAC logic should be submitted in its own PR and release to avoid concurrency issues in testing. """ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.metrics import LogCommandParams from gslib.project_id import PopulateProjectId from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAscii from gslib.utils import shim_util _CREATE_SYNOPSIS = """ gsutil hmac create [-p <project>] <service_account_email> """ _DELETE_SYNOPSIS = """ gsutil hmac delete [-p <project>] <access_id> """ _GET_SYNOPSIS = """ gsutil hmac get [-p <project>] <access_id> """ _LIST_SYNOPSIS = """ gsutil hmac list [-a] [-l] [-p <project>] [-u <service_account_email>] """ _UPDATE_SYNOPSIS = """ gsutil hmac update -s (ACTIVE|INACTIVE) [-e <etag>] [-p <project>] <access_id> """ _CREATE_DESCRIPTION = """ <B>CREATE</B> The ``hmac create`` command creates an HMAC key for the specified service account: gsutil hmac create test.service.account@test_project.iam.gserviceaccount.com The secret key material is only available upon creation, so be sure to store the returned secret along with the access_id. <B>CREATE OPTIONS</B> The ``create`` sub-command has the following option -p <project> Specify the ID or number of the project in which to create a key. """ _DELETE_DESCRIPTION = """ <B>DELETE</B> The ``hmac delete`` command permanently deletes the specified HMAC key: gsutil hmac delete GOOG56JBMFZX6PMPTQ62VD2 Note that keys must be updated to be in the ``INACTIVE`` state before they can be deleted. <B>DELETE OPTIONS</B> The ``delete`` sub-command has the following option -p <project> Specify the ID or number of the project from which to delete a key. """ _GET_DESCRIPTION = """ <B>GET</B> The ``hmac get`` command retrieves the specified HMAC key's metadata: gsutil hmac get GOOG56JBMFZX6PMPTQ62VD2 Note that there is no option to retrieve a key's secret material after it has been created. <B>GET OPTIONS</B> The ``get`` sub-command has the following option -p <project> Specify the ID or number of the project from which to get a key. """ _LIST_DESCRIPTION = """ <B>LIST</B> The ``hmac list`` command lists the HMAC key metadata for keys in the specified project. If no project is specified in the command, the default project is used. <B>LIST OPTIONS</B> The ``list`` sub-command has the following options -a Show all keys, including recently deleted keys. -l Use long listing format. Shows each key's full metadata excluding the secret. -p <project> Specify the ID or number of the project from which to list keys. -u <service_account_email> Filter keys for a single service account. """ _UPDATE_DESCRIPTION = """ <B>UPDATE</B> The ``hmac update`` command sets the state of the specified key: gsutil hmac update -s INACTIVE -e M42da= GOOG56JBMFZX6PMPTQ62VD2 Valid state arguments are ``ACTIVE`` and ``INACTIVE``. To set a key to state ``DELETED``, use the ``hmac delete`` command on an ``INACTIVE`` key. If an etag is set in the command, it will only succeed if the provided etag matches the etag of the stored key. <B>UPDATE OPTIONS</B> The ``update`` sub-command has the following options -s <ACTIVE|INACTIVE> Sets the state of the specified key to either ``ACTIVE`` or ``INACTIVE``. -e <etag> If provided, the update will only be performed if the specified etag matches the etag of the stored key. -p <project> Specify the ID or number of the project in which to update a key. """ _SYNOPSIS = (_CREATE_SYNOPSIS + _DELETE_SYNOPSIS.lstrip('\n') + _GET_SYNOPSIS.lstrip('\n') + _LIST_SYNOPSIS.lstrip('\n') + _UPDATE_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ You can use the ``hmac`` command to interact with service account `HMAC keys <https://cloud.google.com/storage/docs/authentication/hmackeys>`_. The ``hmac`` command has five sub-commands: """ + '\n'.join([ _CREATE_DESCRIPTION, _DELETE_DESCRIPTION, _GET_DESCRIPTION, _LIST_DESCRIPTION, _UPDATE_DESCRIPTION, ]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _VALID_UPDATE_STATES = ['INACTIVE', 'ACTIVE'] _TIME_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' _create_help_text = CreateHelpText(_CREATE_SYNOPSIS, _CREATE_DESCRIPTION) _delete_help_text = CreateHelpText(_DELETE_SYNOPSIS, _DELETE_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _list_help_text = CreateHelpText(_LIST_SYNOPSIS, _LIST_DESCRIPTION) _update_help_text = CreateHelpText(_UPDATE_SYNOPSIS, _UPDATE_DESCRIPTION) def _AccessIdException(command_name, subcommand, synopsis): return CommandException( '%s %s requires an Access ID to be specified as the last argument.\n%s' % (command_name, subcommand, synopsis)) def _KeyMetadataOutput(metadata): """Format the key metadata for printing to the console.""" def FormatInfo(name, value, new_line=True): """Format the metadata name-value pair into two aligned columns.""" width = 22 info_str = '\t%-*s %s' % (width, name + ':', value) if new_line: info_str += '\n' return info_str message = 'Access ID %s:\n' % metadata.accessId message += FormatInfo('State', metadata.state) message += FormatInfo('Service Account', metadata.serviceAccountEmail) message += FormatInfo('Project', metadata.projectId) message += FormatInfo('Time Created', metadata.timeCreated.strftime(_TIME_FORMAT)) message += FormatInfo('Time Last Updated', metadata.updated.strftime(_TIME_FORMAT)) message += FormatInfo('Etag', metadata.etag, new_line=False) return message _CREATE_COMMAND_FORMAT = ('--format=value[separator="' + shim_util.get_format_flag_newline() + '"](' + 'format("Access ID: {}", metadata.accessId),' + 'format("Secret: {}", secret))') _DESCRIBE_COMMAND_FORMAT = ( '--format=value[separator="' + shim_util.get_format_flag_newline() + '"](format("Access ID {}:", accessId),' + 'format("\tState: {}", state),' + 'format("\tService Account: {}", serviceAccountEmail),' + 'format("\tProject: {}", projectId),' + 'format("\tTime Created: {}",' + ' timeCreated.date(format="%a %d %b %Y %H:%M:%S GMT")),' + 'format("\tTime Last Updated: {}",' + ' updated.date(format="%a %d %b %Y %H:%M:%S GMT")),' + 'format("\tEtag: {}", etag))') _LIST_COMMAND_SHORT_FORMAT = ( '--format=table[no-heading](format("{} ", accessId),' 'state:width=11, serviceAccountEmail)') _PROJECT_FLAG = GcloudStorageFlag('--project') CREATE_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'create', _CREATE_COMMAND_FORMAT], flag_map={ '-p': _PROJECT_FLAG, }) DELETE_COMMAND = GcloudStorageMap(gcloud_command=['storage', 'hmac', 'delete'], flag_map={ '-p': _PROJECT_FLAG, }) GET_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'describe', _DESCRIBE_COMMAND_FORMAT], flag_map={'-p': _PROJECT_FLAG}) LIST_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'list', _LIST_COMMAND_SHORT_FORMAT], flag_map={ '-a': GcloudStorageFlag('--all'), '-u': GcloudStorageFlag('--service-account'), '-p': _PROJECT_FLAG }) LIST_COMMAND_LONG_FORMAT = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'list', _DESCRIBE_COMMAND_FORMAT], flag_map={ '-a': GcloudStorageFlag('--all'), '-l': GcloudStorageFlag('--long'), '-u': GcloudStorageFlag('--service-account'), '-p': _PROJECT_FLAG }) UPDATE_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'update', _DESCRIBE_COMMAND_FORMAT], flag_map={ '-s': GcloudStorageFlag({ 'ACTIVE': '--activate', 'INACTIVE': '--deactivate', }), '-e': GcloudStorageFlag('--etag'), '-p': _PROJECT_FLAG }) class HmacCommand(Command): """Implementation of gsutil hmac command.""" command_spec = Command.CreateCommandSpec( 'hmac', min_args=1, max_args=8, supported_sub_args='ae:lp:s:u:', file_url_ok=True, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, usage_synopsis=_SYNOPSIS, argparse_arguments={ 'create': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'delete': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'get': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'list': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'update': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], }, ) help_spec = Command.HelpSpec( help_name='hmac', help_name_aliases=[], help_type='command_help', help_one_line_summary=('CRUD operations on service account HMAC keys.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'create': _create_help_text, 'delete': _delete_help_text, 'get': _get_help_text, 'list': _list_help_text, 'update': _update_help_text, }) def get_gcloud_storage_args(self): if self.args[0] == 'list' and '-l' in self.args: gcloud_storage_map = GcloudStorageMap( gcloud_command={'list': LIST_COMMAND_LONG_FORMAT}, flag_map={}, ) else: gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'create': CREATE_COMMAND, 'delete': DELETE_COMMAND, 'update': UPDATE_COMMAND, 'get': GET_COMMAND, 'list': LIST_COMMAND }, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def _CreateHmacKey(self, thread_state=None): """Creates HMAC key for a service account.""" if self.args: self.service_account_email = self.args[0] else: err_msg = ('%s %s requires a service account to be specified as the ' 'last argument.\n%s') raise CommandException( err_msg % (self.command_name, self.action_subcommand, _CREATE_SYNOPSIS)) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.CreateHmacKey(self.project_id, self.service_account_email, provider='gs') print('%-12s %s' % ('Access ID:', response.metadata.accessId)) print('%-12s %s' % ('Secret:', response.secret)) def _DeleteHmacKey(self, thread_state=None): """Deletes an HMAC key.""" if self.args: access_id = self.args[0] else: raise _AccessIdException(self.command_name, self.action_subcommand, _DELETE_SYNOPSIS) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) gsutil_api.DeleteHmacKey(self.project_id, access_id, provider='gs') def _GetHmacKey(self, thread_state=None): """Gets HMAC key from its Access Id.""" if self.args: access_id = self.args[0] else: raise _AccessIdException(self.command_name, self.action_subcommand, _GET_SYNOPSIS) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.GetHmacKey(self.project_id, access_id, provider='gs') print(_KeyMetadataOutput(response)) def _ListHmacKeys(self, thread_state=None): """Lists HMAC keys for a project or service account.""" if self.args: raise CommandException( '%s %s received unexpected arguments.\n%s' % (self.command_name, self.action_subcommand, _LIST_SYNOPSIS)) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.ListHmacKeys(self.project_id, self.service_account_email, self.show_all, provider='gs') short_list_format = '%s\t%-12s %s' if self.long_list: for item in response: print(_KeyMetadataOutput(item)) print() else: for item in response: print(short_list_format % (item.accessId, item.state, item.serviceAccountEmail)) def _UpdateHmacKey(self, thread_state=None): """Update an HMAC key's state.""" if not self.state: raise CommandException( 'A state flag must be supplied for %s %s\n%s' % (self.command_name, self.action_subcommand, _UPDATE_SYNOPSIS)) elif self.state not in _VALID_UPDATE_STATES: raise CommandException('The state flag value must be one of %s' % ', '.join(_VALID_UPDATE_STATES)) if self.args: access_id = self.args[0] else: raise _AccessIdException(self.command_name, self.action_subcommand, _UPDATE_SYNOPSIS) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.UpdateHmacKey(self.project_id, access_id, self.state, self.etag, provider='gs') print(_KeyMetadataOutput(response)) def RunCommand(self): """Command entry point for the hmac command.""" if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException( 'The "hmac" command can only be used with the GCS JSON API') self.action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. LogCommandParams(sub_opts=self.sub_opts) self.service_account_email = None self.state = None self.show_all = False self.long_list = False self.etag = None if self.sub_opts: for o, a in self.sub_opts: if o == '-u': self.service_account_email = a elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii(a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-s': self.state = a elif o == '-a': self.show_all = True elif o == '-l': self.long_list = True elif o == '-e': self.etag = a if not self.project_id: self.project_id = PopulateProjectId(None) method_for_arg = { 'create': self._CreateHmacKey, 'delete': self._DeleteHmacKey, 'get': self._GetHmacKey, 'list': self._ListHmacKeys, 'update': self._UpdateHmacKey, } if self.action_subcommand not in method_for_arg: raise CommandException('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help hmac".' % (self.action_subcommand, self.command_name)) LogCommandParams(subcommands=[self.action_subcommand]) method_for_arg[self.action_subcommand]() return 0 commands/rb.py 0000644 00000013326 15025013616 0007326 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of rb command for deleting cloud storage buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.cloud_api import NotEmptyException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _SYNOPSIS = """ gsutil rb [-f] gs://<bucket_name>... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Delete one or more buckets. Buckets must be empty before you can delete them. Be certain you want to delete a bucket before you do so, as once it is deleted the name becomes available and another user may create a bucket with that name. (But see also "DOMAIN NAMED BUCKETS" under "gsutil help naming" for help carving out parts of the bucket name space.) <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when removing buckets. If some buckets couldn't be removed, gsutil's exit status will be non-zero even if this flag is set. If no buckets could be removed, the command raises a "no matches" error. """) class RbCommand(Command): """Implementation of gsutil rb command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rb', command_name_aliases=[ 'deletebucket', 'removebucket', 'removebuckets', 'rmdir', ], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='f', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rb', help_name_aliases=[ 'deletebucket', 'removebucket', 'removebuckets', 'rmdir', ], help_type='command_help', help_one_line_summary='Remove buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'delete'], flag_map={ '-f': GcloudStorageFlag('--continue-on-error'), }, ) def RunCommand(self): """Command entry point for the rb command.""" self.continue_on_error = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-f': self.continue_on_error = True did_some_work = False some_failed = False for url_str in self.args: wildcard_url = StorageUrlFromString(url_str) if wildcard_url.IsObject(): raise CommandException('"rb" command requires a provider or ' 'bucket URL') # Wrap WildcardIterator call in try/except so we can avoid printing errors # with -f option if a non-existent URL listed, permission denial happens # while listing, etc. try: # Materialize iterator results into a list to catch exceptions. # Since this is listing buckets this list shouldn't be too large to fit # in memory at once. # Also, avoid listing all fields to avoid performing unnecessary bucket # metadata GETs. These would also be problematic when billing is # disabled, as deletes are allowed but GetBucket is not. blrs = list( self.WildcardIterator(url_str).IterBuckets(bucket_fields=['id'])) except: # pylint: disable=bare-except some_failed = True if self.continue_on_error: continue else: raise for blr in blrs: url = blr.storage_url self.logger.info('Removing %s...', url) try: self.gsutil_api.DeleteBucket(url.bucket_name, provider=url.scheme) except NotEmptyException as e: some_failed = True if self.continue_on_error: continue elif 'VersionedBucketNotEmpty' in e.reason: raise CommandException('Bucket is not empty. Note: this is a ' 'versioned bucket, so to delete all ' 'objects\nyou need to use:' '\n\tgsutil rm -r %s' % url) else: raise except: # pylint: disable=bare-except some_failed = True if not self.continue_on_error: raise did_some_work = True if not did_some_work: raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) return 1 if some_failed else 0 commands/stat.py 0000644 00000015203 15025013616 0007672 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like stat command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import sys from gslib.bucket_listing_ref import BucketListingObject from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import EncryptionException from gslib.cloud_api import NotFoundException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import InvalidUrlError from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.ls_helper import ENCRYPTED_FIELDS from gslib.utils.ls_helper import PrintFullInfoAboutObject from gslib.utils.ls_helper import UNENCRYPTED_FULL_LISTING_FIELDS from gslib.utils.shim_util import GcloudStorageMap _SYNOPSIS = """ gsutil stat url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The stat command outputs `object metadata <https://cloud.google.com/storage/docs/metadata>`_ for the specified object URLs. It is similar to running: gsutil ls -L gs://some-bucket/some-object but is more efficient because it avoids listing objects and gets the minimum necessary amount of object metadata. Moreover, because it avoids listing objects (which for some storage providers are eventually consistent) the ``gsutil stat`` command provides a strongly consistent way to check for the existence (and read the metadata) of an object. The ``gsutil stat`` command does, however, make listing object requests if you specify URLs using wildcards. If run with the ``gsutil -q`` option nothing is printed, e.g.: gsutil -q stat gs://some-bucket/some-object This behavior can be useful when writing scripts: even though nothing is printed from the command, it still has an exit status of 0 for an existing object and 1 for a non-existent object. NOTE: Unlike the gsutil ls command, the stat command does not support operations on sub-directories. For example, if you run the command: gsutil -q stat gs://some-bucket/some-subdir/ gsutil looks for information about an object called "some-subdir/" (with a trailing slash) inside the bucket "some-bucket", as opposed to operating on objects nested under gs://some-bucket/some-subdir/. Unless you actually have an object with that name, the operation fails. However, you can use the stat command on objects within subdirectories. For example, this command works as expected: gsutil -q stat gs://some-bucket/some-subdir/file.txt """) # TODO: Add ability to stat buckets. class StatCommand(Command): """Implementation of gsutil stat command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'stat', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='stat', help_name_aliases=[], help_type='command_help', help_one_line_summary='Display object status', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=[ # Formatting done in gcloud. 'storage', 'objects', 'list', '--fetch-encrypted-object-hashes', '--stat', ], flag_map={}, ) def RunCommand(self): """Command entry point for stat command.""" stat_fields = ENCRYPTED_FIELDS + UNENCRYPTED_FULL_LISTING_FIELDS found_nonmatching_arg = False for url_str in self.args: arg_matches = 0 url = StorageUrlFromString(url_str) if not url.IsObject(): raise CommandException('The stat command only works with object URLs') try: if ContainsWildcard(url_str): blr_iter = self.WildcardIterator(url_str).IterObjects( bucket_listing_fields=stat_fields) else: try: single_obj = self.gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, generation=url.generation, provider=url.scheme, fields=stat_fields) except EncryptionException: # Retry without requesting hashes. single_obj = self.gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, generation=url.generation, provider=url.scheme, fields=UNENCRYPTED_FULL_LISTING_FIELDS) blr_iter = [BucketListingObject(url, root_object=single_obj)] for blr in blr_iter: if blr.IsObject(): arg_matches += 1 # TODO: Request fewer fields if we're not printing the object. if logging.getLogger().isEnabledFor(logging.INFO): PrintFullInfoAboutObject(blr, incl_acl=False) except AccessDeniedException: if logging.getLogger().isEnabledFor(logging.INFO): sys.stderr.write('You aren\'t authorized to read %s - skipping' % url_str + '\n') except InvalidUrlError: raise except NotFoundException: pass if not arg_matches: if logging.getLogger().isEnabledFor(logging.INFO): sys.stderr.write(NO_URLS_MATCHED_TARGET % url_str + '\n') found_nonmatching_arg = True if found_nonmatching_arg: return 1 return 0 commands/cors.py 0000644 00000021054 15025013616 0007666 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of cors configuration command for GCS buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import CorsTranslation from gslib.utils.translation_helper import REMOVE_CORS_CONFIG _GET_SYNOPSIS = """ gsutil cors get gs://<bucket_name> """ _SET_SYNOPSIS = """ gsutil cors set <cors-json-file> gs://<bucket_name>... """ _GET_DESCRIPTION = """ <B>GET</B> Gets the CORS configuration for a single bucket. The output from ``cors get`` can be redirected into a file, edited and then updated using ``cors set``. """ _SET_DESCRIPTION = """ <B>SET</B> Sets the CORS configuration for one or more buckets. The ``cors-json-file`` specified on the command line should be a path to a local file containing a JSON-formatted CORS configuration, such as the example described above. """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + '\n\n' _DESCRIPTION = (""" Gets or sets the Cross-Origin Resource Sharing (CORS) configuration on one or more buckets. This command is supported for buckets only, not objects. An example CORS JSON file looks like the following: [ { "origin": ["http://origin1.example.com"], "responseHeader": ["Content-Type"], "method": ["GET"], "maxAgeSeconds": 3600 } ] The above CORS configuration explicitly allows cross-origin GET requests from http://origin1.example.com and may include the Content-Type response header. The preflight request may be cached for 1 hour. Note that requests to the authenticated browser download endpoint ``storage.cloud.google.com`` do not allow CORS requests. For more information about supported endpoints for CORS, see `Cloud Storage CORS support <https://cloud.google.com/storage/docs/cross-origin#server-side-support>`_. The following (empty) CORS JSON file removes any CORS configuration for a bucket: [] The cors command has two sub-commands: """ + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION]) + """ For more info about CORS generally, see https://www.w3.org/TR/cors/. For more info about CORS in Cloud Storage, see the `CORS concept page <https://cloud.google.com/storage/docs/cross-origin>`_. """) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class CorsCommand(Command): """Implementation of gsutil cors command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'cors', command_name_aliases=['getcors', 'setcors'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1),] }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='cors', help_name_aliases=[ 'getcors', 'setcors', 'cross-origin', ], help_type='command_help', help_one_line_summary=( 'Get or set a CORS configuration for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=cors_config,empty=\' has no ' 'CORS configuration.\',empty_prefix_key=storage_url]' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--cors-file' ], flag_map={}, ), }, flag_map={}, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetCors(self): """Sets CORS configuration on a Google Cloud Storage bucket.""" cors_arg = self.args[0] url_args = self.args[1:] # Disallow multi-provider 'cors set' requests. if not UrlsAreForSingleProvider(url_args): raise CommandException('"%s" command spanning providers not allowed.' % self.command_name) # Open, read and parse file containing JSON document. cors_file = open(cors_arg, 'r') cors_txt = cors_file.read() cors_file.close() self.api = self.gsutil_api.GetApiSelector( StorageUrlFromString(url_args[0]).scheme) # Iterate over URLs, expanding wildcards and setting the CORS on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting CORS on %s...', blr) if url.scheme == 's3': self.gsutil_api.XmlPassThroughSetCors(cors_txt, url, provider=url.scheme) else: cors = CorsTranslation.JsonCorsToMessageEntries(cors_txt) if not cors: cors = REMOVE_CORS_CONFIG bucket_metadata = apitools_messages.Bucket(cors=cors) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _GetCors(self): """Gets CORS configuration for a Google Cloud Storage bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['cors']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetCors(bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.cors: sys.stdout.write( CorsTranslation.MessageEntriesToJson(bucket_metadata.cors)) else: sys.stdout.write('%s has no CORS configuration.\n' % bucket_url) return 0 def RunCommand(self): """Command entry point for the cors command.""" action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._GetCors elif action_subcommand == 'set': func = self._SetCors else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help cors".') % (action_subcommand, self.command_name)) metrics.LogCommandParams(subcommands=[action_subcommand]) return func() commands/config.py 0000644 00000146223 15025013616 0010173 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of config command for creating a gsutil configuration file.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import datetime import json import multiprocessing import os import signal import socket import stat import sys import textwrap import time import webbrowser from six.moves import input from six.moves.http_client import ResponseNotReady import boto from boto.provider import Provider import gslib from gslib.command import Command from gslib.command import DEFAULT_TASK_ESTIMATION_THRESHOLD from gslib.commands.compose import MAX_COMPOSE_ARITY from gslib.cred_types import CredTypes from gslib.exception import AbortException from gslib.exception import CommandException from gslib.metrics import CheckAndMaybePromptForAnalyticsEnabling from gslib.sig_handling import RegisterSignalHandler from gslib.utils import constants from gslib.utils import system_util from gslib.utils.hashing_helper import CHECK_HASH_ALWAYS from gslib.utils.hashing_helper import CHECK_HASH_IF_FAST_ELSE_FAIL from gslib.utils.hashing_helper import CHECK_HASH_IF_FAST_ELSE_SKIP from gslib.utils.hashing_helper import CHECK_HASH_NEVER from gslib.utils.parallelism_framework_util import ShouldProhibitMultiprocessing from httplib2 import ServerNotFoundError from oauth2client.client import HAS_CRYPTO _SYNOPSIS = """ gsutil [-D] config (-a|-e) [-n] [-o <file>] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The ``gsutil config`` command generally applies to users who have legacy standalone installations of gsutil. If you installed gsutil via the Cloud SDK, ``gsutil config`` fails unless you are specifically using the ``-a`` flag or have configured gcloud to not pass its managed credentials to gsutil (via the command ``gcloud config set pass_credentials_to_gsutil false``). For all other use cases, Cloud SDK users should use the ``gcloud auth`` group of commands instead, which configures OAuth2 credentials that gcloud implicitly passes to gsutil at runtime. To check if you are using gsutil from the Cloud SDK or as a legacy standalone, use ``gsutil version -l`` and in the output look for "using cloud sdk". Important: The default behavior for the ``gsutil config`` command is to obtain user account credentials for authentication. However, user account credentials are no longer supported for standalone gsutil. For this reason, running the default ``gsutil config`` command fails, and using any of the following flags causes the command to fail: ``-b``, ``-f``, ``-r``, ``--reauth``, ``-s``, ``-w``. When using standalone gsutil, it's recommended that you use service account credentials via the ``-e`` flag. The ``gsutil config`` command obtains access credentials for Cloud Storage and writes a `boto/gsutil configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_ containing the obtained credentials along with a number of other configuration- controllable values. Unless specified otherwise (see OPTIONS), the configuration file is written to ~/.boto (i.e., the file .boto under the user's home directory). If the default file already exists, an attempt is made to rename the existing file to ~/.boto.bak; if that attempt fails the command exits. A different destination file can be specified with the ``-o`` option (see OPTIONS). Because the boto configuration file contains your credentials you should keep its file permissions set so no one but you has read access. (The file is created read-only when you run ``gsutil config``.) <B>CONFIGURING SERVICE ACCOUNT CREDENTIALS</B> Service accounts are useful for authenticating on behalf of a service or application (as opposed to a user). If you use gsutil as a legacy stand-alone tool, you configure credentials for service accounts using the ``-e`` option: gsutil config -e Note that if you use gsutil through the Cloud SDK, you instead activate your service account via the `gcloud auth activate-service-account <https://cloud.google.com/sdk/gcloud/reference/auth/activate-service-account>`_ command. When you run ``gsutil config -e``, you are prompted for the path to your private key file and, if not using a JSON key file, your service account email address and key file password. To get this data, follow the instructions on `Service Accounts <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`_. Using this information, gsutil populates the "gs_service_key_file" attribute in the boto configuration file. If not using a JSON key file, gsutil also populates the "gs_service_client_id" and "gs_service_key_file_password" attributes. Note that your service account is NOT considered an Owner for the purposes of API access (see "gsutil help creds" for more information about this). See https://developers.google.com/identity/protocols/OAuth2ServiceAccount for further information on service account authentication. If you want to use credentials based on access key and secret (the older authentication method before OAuth2 was supported), see the ``-a`` option in the OPTIONS section. If you wish to use gsutil with other providers (or to copy data back and forth between multiple providers) you can edit their credentials into the [Credentials] section after creating the initial boto configuration file. <B>OPTIONS</B> -a Prompt for Google Cloud Storage access key and secret (the older authentication method before OAuth2 was supported) instead of obtaining an OAuth2 token. -e Prompt for service account credentials. This option requires that ``-a`` is not set. -n Write the configuration file without authentication configured. This flag is mutually exclusive with all flags other than ``-o``. -o <file> Write the configuration to <file> instead of ~/.boto. Use ``-`` for stdout. """) GOOG_CLOUD_CONSOLE_URI = 'https://cloud.google.com/console#/project' CONFIG_PRELUDE_CONTENT = """ # This file contains credentials and other configuration information needed # by the boto library, used by gsutil. You can edit this file (e.g., to add # credentials) but be careful not to mis-edit any of the variable names (like # "gs_access_key_id") or remove important markers (like the "[Credentials]" and # "[Boto]" section delimiters). # """ # Default number of OS processes and Python threads for parallel operations. # On Linux systems we automatically scale the number of processes to match # the underlying CPU/core count. Given we'll be running multiple concurrent # processes on a typical multi-core Linux computer, to avoid being too # aggressive with resources, the default number of threads is reduced from # the previous value of 24 to 5. # # We also cap the maximum number of default processes at 32. Since each level # of recursion depth gets its own process pool, this means a maximum of # 64 processes with the current maximum recursion depth of 2. We limit this # number because testing with more than 200 processes showed fatal locking # exceptions in Python's multiprocessing.Manager. More processes are # probably not needed to saturate most networks. # # On Windows and Alpine Linux, Python multi-processing presents various # challenges so we retain compatibility with the established parallel mode # operation, i.e. one process and 24 threads. should_prohibit_multiprocessing, unused_os = ShouldProhibitMultiprocessing() if should_prohibit_multiprocessing: DEFAULT_PARALLEL_PROCESS_COUNT = 1 DEFAULT_PARALLEL_THREAD_COUNT = 24 else: DEFAULT_PARALLEL_PROCESS_COUNT = min(multiprocessing.cpu_count(), 32) DEFAULT_PARALLEL_THREAD_COUNT = 5 # TODO: Once compiled crcmod is being distributed by major Linux distributions # revert DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD value to '150M'. DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD = '0' DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE = '50M' DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD = '150M' DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE = '200M' DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS = 4 # Compressed transport encoded uploads buffer chunks of compressed data. When # running many uploads in parallel, compression may consume more memory than # available. This restricts the number of compressed transport encoded uploads # running in parallel such that they don't consume more memory than set here. DEFAULT_MAX_UPLOAD_COMPRESSION_BUFFER_SIZE = '2G' # gzip compression level. This is simply making the python stdlib default explicit. DEFAULT_GZIP_COMPRESSION_LEVEL = 9 CONFIG_BOTO_SECTION_CONTENT = """ [Boto] # http_socket_timeout specifies the timeout (in seconds) used to tell httplib # how long to wait for socket timeouts. The default is 70 seconds. Note that # this timeout only applies to httplib, not to httplib2 (which is used for # OAuth2 refresh/access token exchanges). #http_socket_timeout = 70 # The following two options control the use of a secure transport for requests # to S3 and Google Cloud Storage. It is highly recommended to set both options # to True in production environments, especially when using OAuth2 bearer token # authentication with Google Cloud Storage. # Set 'https_validate_certificates' to False to disable server certificate # checking. The default for this option in the boto library is currently # 'False' (to avoid breaking apps that depend on invalid certificates); it is # therefore strongly recommended to always set this option explicitly to True # in configuration files, to protect against "man-in-the-middle" attacks. https_validate_certificates = True # 'debug' controls the level of debug messages printed for the XML API only: # 0 for none, 1 for basic boto debug, 2 for all boto debug plus HTTP # requests/responses. #debug = <0, 1, or 2> # 'num_retries' controls the number of retry attempts made when errors occur # during data transfers. The default is 6. # Note 1: You can cause gsutil to retry failures effectively infinitely by # setting this value to a large number (like 10000). Doing that could be useful # in cases where your network connection occasionally fails and is down for an # extended period of time, because when it comes back up gsutil will continue # retrying. However, in general we recommend not setting the value above 10, # because otherwise gsutil could appear to "hang" due to excessive retries # (since unless you run gsutil -D you won't see any logged evidence that gsutil # is retrying). # Note 2: Don't set this value to 0, as it will cause boto to fail when reusing # HTTP connections. #num_retries = <integer value> # 'max_retry_delay' controls the max delay (in seconds) between retries. The # default value is 60, so the backoff sequence will be 1 seconds, 2 seconds, 4, # 8, 16, 32, and then 60 for all subsequent retries for a given HTTP request. # Note: At present this value only impacts the XML API and the JSON API uses a # fixed value of 60. #max_retry_delay = <integer value> """ CONFIG_GOOGLECOMPUTE_SECTION_CONTENT = """ [GoogleCompute] # 'service_account' specifies the a Google Compute Engine service account to # use for credentials. This value is intended for use only on Google Compute # Engine virtual machines and usually lives in /etc/boto.cfg. Most users # shouldn't need to edit this part of the config. #service_account = default """ CONFIG_INPUTLESS_GSUTIL_SECTION_CONTENT = """ [GSUtil] # 'resumable_threshold' specifies the smallest file size [bytes] for which # resumable Google Cloud Storage uploads are attempted. The default is 8388608 # (8 MiB). #resumable_threshold = %(resumable_threshold)d # 'rsync_buffer_lines' specifies the number of lines of bucket or directory # listings saved in each temp file during sorting. (The complete set is # split across temp files and separately sorted/merged, to avoid needing to # fit everything in memory at once.) If you are trying to synchronize very # large directories/buckets (e.g., containing millions or more objects), # having too small a value here can cause gsutil to run out of open file # handles. If that happens, you can try to increase the number of open file # handles your system allows (e.g., see 'man ulimit' on Linux; see also # http://docs.python.org/2/library/resource.html). If you can't do that (or # if you're already at the upper limit), increasing rsync_buffer_lines will # cause gsutil to use fewer file handles, but at the cost of more memory. With # rsync_buffer_lines set to 32000 and assuming a typical URL is 100 bytes # long, gsutil will require approximately 10 MiB of memory while building # the synchronization state, and will require approximately 60 open file # descriptors to build the synchronization state over all 1M source and 1M # destination URLs. Memory and file descriptors are only consumed while # building the state; once the state is built, it resides in two temp files that # are read and processed incrementally during the actual copy/delete # operations. #rsync_buffer_lines = 32000 # 'state_dir' specifies the base location where files that # need a static location are stored, such as pointers to credentials, # resumable transfer tracker files, and the last software update check. # By default these files are stored in ~/.gsutil #state_dir = <file_path> # gsutil periodically checks whether a new version of the gsutil software is # available. 'software_update_check_period' specifies the number of days # between such checks. The default is 30. Setting the value to 0 disables # periodic software update checks. #software_update_check_period = 30 # 'tab_completion_timeout' controls the timeout (in seconds) for tab # completions that involve remote requests (such as bucket or object names). # If tab completion does not succeed within this timeout, no tab completion # suggestions will be returned. # A value of 0 will disable completions that involve remote requests. #tab_completion_timeout = 5 # 'parallel_process_count' and 'parallel_thread_count' specify the number # of OS processes and Python threads, respectively, to use when executing # operations in parallel. The default settings should work well as configured, # however, to enhance performance for transfers involving large numbers of # files, you may experiment with hand tuning these values to optimize # performance for your particular system configuration. #parallel_process_count = %(parallel_process_count)d #parallel_thread_count = %(parallel_thread_count)d # 'parallel_composite_upload_threshold' specifies the maximum size of a file to # upload in a single stream. Files larger than this threshold will be # partitioned into component parts and uploaded in parallel and then composed # into a single object. # The number of components will be the smaller of # ceil(file_size / parallel_composite_upload_component_size) and # MAX_COMPOSE_ARITY. The current value of MAX_COMPOSE_ARITY is # %(max_compose_arity)d. # If 'parallel_composite_upload_threshold' is set to 0, then automatic parallel # uploads will never occur. # Setting an extremely low threshold is unadvisable. The vast majority of # environments will see degraded performance for thresholds below 80M, and it # is almost never advantageous to have a threshold below 20M. # 'parallel_composite_upload_component_size' specifies the ideal size of a # component in bytes, which will act as an upper bound to the size of the # components if ceil(file_size / parallel_composite_upload_component_size) is # less than MAX_COMPOSE_ARITY. # Values can be provided either in bytes or as human-readable values # (e.g., "150M" to represent 150 mebibytes) # # Note: At present parallel composite uploads are disabled by default, because # using composite objects requires a compiled crcmod (see "gsutil help crcmod"), # and for operating systems that don't already have this package installed this # makes gsutil harder to use. Google is actively working with a number of the # Linux distributions to get crcmod included with the stock distribution. Once # that is done we will re-enable parallel composite uploads by default in # gsutil. # # Note: Parallel composite uploads should not be used with NEARLINE, COLDLINE, # or ARCHIVE storage class buckets, as doing this incurs an early deletion # charge for each component object. # # Note: Parallel composite uploads are not enabled with Cloud KMS encrypted # objects as a source or destination, as composition with KMS objects is not yet # supported. #parallel_composite_upload_threshold = %(parallel_composite_upload_threshold)s #parallel_composite_upload_component_size = %(parallel_composite_upload_component_size)s # # 'parallel_composite_upload_bypass_kms_check' removes the object/bucket KMS checks # used to guard composition of KMS objects. #disable_parallel_composite_upload_kms_check = False # 'sliced_object_download_threshold' and # 'sliced_object_download_component_size' have analogous functionality to # their respective parallel_composite_upload config values. # 'sliced_object_download_max_components' specifies the maximum number of # slices to be used when performing a sliced object download. #sliced_object_download_threshold = %(sliced_object_download_threshold)s #sliced_object_download_component_size = %(sliced_object_download_component_size)s #sliced_object_download_max_components = %(sliced_object_download_max_components)s # Compressed transport encoded uploads buffer chunks of compressed data. When # running a composite upload and/or many uploads in parallel, compression may # consume more memory than available. This setting restricts the number of # compressed transport encoded uploads running in parallel such that they # don't consume more memory than set here. This is 2GiB by default. # Values can be provided either in bytes or as human-readable values # (e.g., "2G" to represent 2 gibibytes) #max_upload_compression_buffer_size = %(max_upload_compression_buffer_size)s # GZIP compression level, if using compression. Reducing this can have # a dramatic impact on compression speed with minor size increases. # This is a value from 0-9, with 9 being max compression. # A good level to try is 6, which is the default used by the gzip tool. #gzip_compression_level = %(gzip_compression_level)s # 'task_estimation_threshold' controls how many files or objects gsutil # processes before it attempts to estimate the total work that will be # performed by the command. Estimation makes extra directory listing or API # list calls and is performed only if multiple processes and/or threads are # used. Estimation can slightly increase cost due to extra # listing calls; to disable it entirely, set this value to 0. #task_estimation_threshold=%(task_estimation_threshold)s # 'use_magicfile' specifies if the 'file --mime <filename>' command should be # used to guess content types instead of the default filename extension-based # mechanism. Available on UNIX and macOS (and possibly on Windows, if you're # running Cygwin or some other package that provides implementations of # UNIX-like commands). When available and enabled use_magicfile should be more # robust because it analyzes file contents in addition to extensions. #use_magicfile = False # Service account emails for testing the hmac command. If these fields are not # populated with distinct service accounts the tests for the hmac command will # not be run. Primarily useful for tool developers. #test_hmac_service_account = #test_hmac_alt_service_account = #test_hmac_list_service_account = # Service account emails for testing impersonation credentials. If this field is # not populated with a service account the tests for service account # impersonation will not run. Primarily useful for tool developers. #test_impersonate_service_account = # 'content_language' specifies the ISO 639-1 language code of the content, to be # passed in the Content-Language header. By default no Content-Language is sent. # See the ISO 639-1 column of # http://www.loc.gov/standards/iso639-2/php/code_list.php for a list of # language codes. content_language = en # 'check_hashes' specifies how strictly to require integrity checking for # downloaded data. Legal values are: # '%(hash_fast_else_fail)s' - (default) Only integrity check if the digest # will run efficiently (using compiled code), else fail the download. # '%(hash_fast_else_skip)s' - Only integrity check if the server supplies a # hash and the local digest computation will run quickly, else skip the # check. # '%(hash_always)s' - Always check download integrity regardless of possible # performance costs. # '%(hash_never)s' - Don't perform download integrity checks. This setting is # not recommended except for special cases such as measuring download # performance excluding time for integrity checking. # This option exists to assist users who wish to download a GCS composite object # and are unable to install crcmod with the C-extension. CRC32c is the only # available integrity check for composite objects, and without the C-extension, # download performance can be significantly degraded by the digest computation. # This option is ignored for daisy-chain copies, which don't compute hashes but # instead (inexpensively) compare the cloud source and destination hashes. #check_hashes = if_fast_else_fail # 'encryption_key' specifies a single customer-supplied encryption key that # will be used for all data written to Google Cloud Storage. See # "gsutil help encryption" for more information # Encryption key: RFC 4648 section 4 base64-encoded AES256 string # Warning: If decrypt_key is specified without an encrypt_key, objects will be # decrypted when copied in the cloud. #encryption_key= # Each 'decryption_key' entry specifies a customer-supplied decryption key that # will be used to access and Google Cloud Storage objects encrypted with # the corresponding key. # Decryption keys: Up to 100 RFC 4648 section 4 base64-encoded AES256 strings # in ascending numerical order, starting with 1. #decryption_key1= #decryption_key2= #decryption_key3= # The ability to specify an alternative JSON API version is primarily for cloud # storage service developers. #json_api_version = v1 # Specifies the API to use when interacting with cloud storage providers. If the # gsutil command supports this API for the provider, it will be used instead of # the default API. Commands typically default to XML for S3 and JSON for GCS. # Note that if any encryption configuration options are set (see above), the # JSON API will be used for interacting with Google Cloud Storage buckets even # if XML is preferred, as gsutil does not currently support this functionality # when using the XML API. #prefer_api = json #prefer_api = xml # Disables the prompt asking for opt-in to data collection for analytics. #disable_analytics_prompt = True # The "test" command runs tests against regional buckets (unless you supply the # `-b` option). By default, the region used is us-central1, but you can change # the default region using this option. #test_cmd_regional_bucket_location = us-central1 # Tests for the "notification watchbucket" command require a notification URL. # If this option is not supplied, those tests will be skipped. #test_notification_url = https://yourdomain.url/notification-endpoint # Used in conjunction with --stet flag on cp command for end-to-end encryption. # STET binary path. If not specified, gsutil checks PATH for "stet". #stet_binary_path = <Path to binary "/usr/local/bin/stet"> # STET config path. If not specified, the STET binary will run with its default # settings. #stet_config_path = ~/.config/my_config.yaml # Adds an API call before parallel operations that triggers a reauth challenge. #trigger_reauth_challenge_for_parallel_operations = False """ % { 'hash_fast_else_fail': CHECK_HASH_IF_FAST_ELSE_FAIL, 'hash_fast_else_skip': CHECK_HASH_IF_FAST_ELSE_SKIP, 'hash_always': CHECK_HASH_ALWAYS, 'hash_never': CHECK_HASH_NEVER, 'resumable_threshold': constants.RESUMABLE_THRESHOLD_B, 'parallel_process_count': DEFAULT_PARALLEL_PROCESS_COUNT, 'parallel_thread_count': DEFAULT_PARALLEL_THREAD_COUNT, 'parallel_composite_upload_threshold': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD), 'parallel_composite_upload_component_size': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE), 'sliced_object_download_threshold': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD), 'sliced_object_download_component_size': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE), 'sliced_object_download_max_components': (DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS), 'max_compose_arity': MAX_COMPOSE_ARITY, 'task_estimation_threshold': DEFAULT_TASK_ESTIMATION_THRESHOLD, 'max_upload_compression_buffer_size': (DEFAULT_MAX_UPLOAD_COMPRESSION_BUFFER_SIZE), 'gzip_compression_level': DEFAULT_GZIP_COMPRESSION_LEVEL, } CONFIG_OAUTH2_CONFIG_CONTENT = """ [OAuth2] # This section specifies options used with OAuth2 authentication. # 'token_cache' specifies how the OAuth2 client should cache access tokens. # Valid values are: # 'in_memory': an in-memory cache is used. This is only useful if the boto # client instance (and with it the OAuth2 plugin instance) persists # across multiple requests. # 'file_system' : access tokens will be cached in the file system, in files # whose names include a key derived from the refresh token the access token # based on. # The default is 'file_system'. #token_cache = file_system #token_cache = in_memory # 'token_cache_path_pattern' specifies a path pattern for token cache files. # This option is only relevant if token_cache = file_system. # The value of this option should be a path, with place-holders '%(key)s' (which # will be replaced with a key derived from the refresh token the cached access # token was based on), and (optionally), %(uid)s (which will be replaced with # the UID of the current user, if available via os.getuid()). # Note that the config parser itself interpolates '%' placeholders, and hence # the above placeholders need to be escaped as '%%(key)s'. # The default value of this option is # token_cache_path_pattern = <tmpdir>/oauth2client-tokencache.%%(uid)s.%%(key)s # where <tmpdir> is the system-dependent default temp directory. # The following options specify the label and endpoint URIs for the OAUth2 # authorization provider being used. Primarily useful for tool developers. #provider_label = Google #provider_authorization_uri = https://accounts.google.com/o/oauth2/auth #provider_token_uri = https://oauth2.googleapis.com/token # 'oauth2_refresh_retries' controls the number of retry attempts made when # rate limiting errors occur for OAuth2 requests to retrieve an access token. # The default value is 6. #oauth2_refresh_retries = <integer value> # The following options specify the OAuth2 client identity and secret that is # used when requesting and using OAuth2 tokens. If not specified, a default # OAuth2 client for the gsutil tool is used; for uses of the boto library (with # OAuth2 authentication plugin) in other client software, it is recommended to # use a tool/client-specific OAuth2 client. For more information on OAuth2, see # http://code.google.com/apis/accounts/docs/OAuth2.html """ class ConfigCommand(Command): """Implementation of gsutil config command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'config', command_name_aliases=['cfg', 'conf', 'configure'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=0, supported_sub_args='abefhno:rs:w', supported_private_args=['reauth'], file_url_ok=False, provider_url_ok=False, urls_start_arg=0, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='config', help_name_aliases=['cfg', 'conf', 'configure', 'aws', 's3'], help_type='command_help', help_one_line_summary=( 'Obtain credentials and create configuration file'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def _OpenConfigFile(self, file_path): """Creates and opens a configuration file for writing. The file is created with mode 0600, and attempts to open existing files will fail (the latter is important to prevent symlink attacks). It is the caller's responsibility to close the file. Args: file_path: Path of the file to be created. Returns: A writable file object for the opened file. Raises: CommandException: if an error occurred when opening the file (including when the file already exists). """ flags = os.O_RDWR | os.O_CREAT | os.O_EXCL # Accommodate Windows; copied from python2.6/tempfile.py. if hasattr(os, 'O_NOINHERIT'): flags |= os.O_NOINHERIT try: fd = os.open(file_path, flags, 0o600) except (OSError, IOError) as e: raise CommandException('Failed to open %s for writing: %s' % (file_path, e)) return os.fdopen(fd, 'w') def _CheckPrivateKeyFilePermissions(self, file_path): """Checks that the file has reasonable permissions for a private key. In particular, check that the filename provided by the user is not world- or group-readable. If either of these are true, we issue a warning and offer to fix the permissions. Args: file_path: The name of the private key file. """ if system_util.IS_WINDOWS: # For Windows, this check doesn't work (it actually just checks whether # the file is read-only). Since Windows files have a complicated ACL # system, this check doesn't make much sense on Windows anyway, so we # just don't do it. return st = os.stat(file_path) if bool((stat.S_IRGRP | stat.S_IROTH) & st.st_mode): self.logger.warn( '\nYour private key file is readable by people other than yourself.\n' 'This is a security risk, since anyone with this information can use ' 'your service account.\n') fix_it = input('Would you like gsutil to change the file ' 'permissions for you? (y/N) ') if fix_it in ('y', 'Y'): try: os.chmod(file_path, 0o400) self.logger.info( '\nThe permissions on your file have been successfully ' 'modified.' '\nThe only access allowed is readability by the user ' '(permissions 0400 in chmod).') except Exception as _: # pylint: disable=broad-except self.logger.warn( '\nWe were unable to modify the permissions on your file.\n' 'If you would like to fix this yourself, consider running:\n' '"sudo chmod 400 </path/to/key>" for improved security.') else: self.logger.info( '\nYou have chosen to allow this file to be readable by others.\n' 'If you would like to fix this yourself, consider running:\n' '"sudo chmod 400 </path/to/key>" for improved security.') def _WriteConfigLineMaybeCommented(self, config_file, name, value, desc): """Writes proxy name/value pair or comment line to config file. Writes proxy name/value pair if value is not None. Otherwise writes comment line. Args: config_file: File object to which the resulting config file will be written. name: The config variable name. value: The value, or None. desc: Human readable description (for comment). """ if not value: name = '#%s' % name value = '<%s>' % desc config_file.write('%s = %s\n' % (name, value)) def _WriteProxyConfigFileSection(self, config_file): """Writes proxy section of configuration file. Args: config_file: File object to which the resulting config file will be written. """ config = boto.config config_file.write( '# To use a proxy, edit and uncomment the proxy and proxy_port lines.\n' '# If you need a user/password with this proxy, edit and uncomment\n' '# those lines as well. If your organization also disallows DNS\n' '# lookups by client machines, set proxy_rdns to True (the default).\n' '# If you have installed gsutil through the Cloud SDK and have \n' '# configured proxy settings in gcloud, those proxy settings will \n' '# override any other options (including those set here, along with \n' '# any settings in proxy-related environment variables). Otherwise, \n' '# if proxy_host and proxy_port are not specified in this file and\n' '# one of the OS environment variables http_proxy, https_proxy, or\n' '# HTTPS_PROXY is defined, gsutil will use the proxy server specified\n' '# in these environment variables, in order of precedence according\n' '# to how they are listed above.\n') self._WriteConfigLineMaybeCommented(config_file, 'proxy', config.get_value('Boto', 'proxy', None), 'proxy host') self._WriteConfigLineMaybeCommented( config_file, 'proxy_type', config.get_value('Boto', 'proxy_type', None), 'proxy type (socks4, socks5, http) | Defaults to http') self._WriteConfigLineMaybeCommented( config_file, 'proxy_port', config.get_value('Boto', 'proxy_port', None), 'proxy port') self._WriteConfigLineMaybeCommented( config_file, 'proxy_user', config.get_value('Boto', 'proxy_user', None), 'proxy user') self._WriteConfigLineMaybeCommented( config_file, 'proxy_pass', config.get_value('Boto', 'proxy_pass', None), 'proxy password') self._WriteConfigLineMaybeCommented( config_file, 'proxy_rdns', config.get_value('Boto', 'proxy_rdns', False), 'let proxy server perform DNS lookups (True,False); socks proxy not supported' ) # pylint: disable=dangerous-default-value,too-many-statements def _WriteBotoConfigFile(self, config_file, cred_type=CredTypes.OAUTH2_USER_ACCOUNT, configure_auth=True): """Creates a boto config file interactively. Needed credentials are obtained interactively, either by asking the user for access key and secret, or by walking the user through the OAuth2 approval flow. Args: config_file: File object to which the resulting config file will be written. cred_type: There are three options: - for HMAC, ask the user for access key and secret - for OAUTH2_USER_ACCOUNT, raise an error - for OAUTH2_SERVICE_ACCOUNT, prompt the user for OAuth2 for service account email address and private key file (and if the file is a .p12 file, the password for that file). configure_auth: Boolean, whether or not to configure authentication in the generated file. """ # Collect credentials provider_map = {'aws': 'aws', 'google': 'gs'} uri_map = {'aws': 's3', 'google': 'gs'} key_ids = {} sec_keys = {} service_account_key_is_json = False if configure_auth: if cred_type == CredTypes.OAUTH2_SERVICE_ACCOUNT: gs_service_key_file = input('What is the full path to your private ' 'key file? ') # JSON files have the email address built-in and don't require a # password. try: with open(gs_service_key_file, 'rb') as key_file_fp: json.loads(key_file_fp.read()) service_account_key_is_json = True except ValueError: if not HAS_CRYPTO: raise CommandException( 'Service account authentication via a .p12 file requires ' 'either\nPyOpenSSL or PyCrypto 2.6 or later. Please install ' 'either of these\nto proceed, use a JSON-format key file, or ' 'configure a different type of credentials.') if not service_account_key_is_json: gs_service_client_id = input('What is your service account email ' 'address? ') gs_service_key_file_password = input('\n'.join( textwrap.wrap( 'What is the password for your service key file [if you ' 'haven\'t set one explicitly, leave this line blank]?')) + ' ') self._CheckPrivateKeyFilePermissions(gs_service_key_file) elif cred_type == CredTypes.OAUTH2_USER_ACCOUNT: raise CommandException( 'The user account authentication flow no longer works as of ' 'February 1, 2023. Tokens generated before this date will ' 'continue to work. To authenticate with your user account, ' 'install gsutil via Cloud SDK and run "gcloud auth login"') elif cred_type == CredTypes.HMAC: got_creds = False for provider in provider_map: if provider == 'google': key_ids[provider] = input('What is your %s access key ID? ' % provider) sec_keys[provider] = input('What is your %s secret access ' 'key? ' % provider) got_creds = True if not key_ids[provider] or not sec_keys[provider]: raise CommandException( 'Incomplete credentials provided. Please try again.') if not got_creds: raise CommandException('No credentials provided. Please try again.') # Write the config file prelude. config_file.write(CONFIG_PRELUDE_CONTENT.lstrip()) config_file.write( '# This file was created by gsutil version %s at %s.\n' % (gslib.VERSION, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) config_file.write( '#\n# You can create additional configuration files by ' 'running\n# gsutil config [options] [-o <config-file>]\n\n\n') # Write the config file Credentials section. config_file.write('[Credentials]\n\n') if configure_auth: if cred_type == CredTypes.OAUTH2_SERVICE_ACCOUNT: config_file.write('# Google OAuth2 service account credentials ' '(for "gs://" URIs):\n') config_file.write('gs_service_key_file = %s\n' % gs_service_key_file) if not service_account_key_is_json: config_file.write('gs_service_client_id = %s\n' % gs_service_client_id) if not gs_service_key_file_password: config_file.write( '# If you would like to set your password, you can do so\n' '# using the following commands (replaced with your\n' '# information):\n' '# "openssl pkcs12 -in cert1.p12 -out temp_cert.pem"\n' '# "openssl pkcs12 -export -in temp_cert.pem -out cert2.p12"\n' '# "rm -f temp_cert.pem"\n' '# Your initial password is "notasecret" - for more\n' '# information, please see \n' '# http://www.openssl.org/docs/apps/pkcs12.html.\n') config_file.write('#gs_service_key_file_password =\n\n') else: config_file.write('gs_service_key_file_password = %s\n\n' % gs_service_key_file_password) else: config_file.write( '# To add Google OAuth2 credentials ("gs://" URIs), ' 'edit and uncomment the\n# following line:\n' '#gs_oauth2_refresh_token = <your OAuth2 refresh token>\n\n') else: if system_util.InvokedViaCloudSdk(): config_file.write( '# Google OAuth2 credentials are managed by the Cloud SDK and\n' '# do not need to be present in this file.\n') for provider in provider_map: key_prefix = provider_map[provider] uri_scheme = uri_map[provider] if provider in key_ids and provider in sec_keys: config_file.write('# %s credentials ("%s://" URIs):\n' % (provider, uri_scheme)) config_file.write('%s_access_key_id = %s\n' % (key_prefix, key_ids[provider])) config_file.write('%s_secret_access_key = %s\n' % (key_prefix, sec_keys[provider])) else: config_file.write( '# To add HMAC %s credentials for "%s://" URIs, edit and ' 'uncomment the\n# following two lines:\n' '#%s_access_key_id = <your %s access key ID>\n' '#%s_secret_access_key = <your %s secret access key>\n' % (provider, uri_scheme, key_prefix, provider, key_prefix, provider)) host_key = Provider.HostKeyMap[provider] config_file.write( '# The ability to specify an alternate storage host and port\n' '# is primarily for cloud storage service developers.\n' '# Setting a non-default gs_host only works if prefer_api=xml.\n' '#%s_host = <alternate storage host address>\n' '#%s_port = <alternate storage host port>\n' '# In some cases, (e.g. VPC requests) the "host" HTTP header should\n' '# be different than the host used in the request URL.\n' '#%s_host_header = <alternate storage host header>\n' % (host_key, host_key, host_key)) if host_key == 'gs': config_file.write( '#%s_json_host = <alternate JSON API storage host address>\n' '#%s_json_port = <alternate JSON API storage host port>\n' '#%s_json_host_header = <alternate JSON API storage host header>\n\n' % (host_key, host_key, host_key)) config_file.write( '# To impersonate a service account for "%s://" URIs over\n' '# JSON API, edit and uncomment the following line:\n' '#%s_impersonate_service_account = <service account email>\n\n') # Add device certificate mTLS fields. config_file.write( textwrap.dedent("""\ # This configuration setting enables or disables mutual TLS # authentication. The default value for this setting is "false". When # set to "true", gsutil uses the configured client certificate as # transport credential to access the APIs. The use of mTLS ensures that # the access originates from a trusted enterprise device. When enabled, # the client certificate is auto discovered using the endpoint # verification agent. When set to "true" but no client certificate or # key is found, users receive an error. #use_client_certificate = False # The command line to execute, which prints the # certificate, private key, or password to use in # conjunction with "use_client_certificate = True". #cert_provider_command = <Absolute path to command to run for # certification. Ex: "/scripts/gen_cert.sh"> """)) # Write the config file Boto section. config_file.write('%s\n' % CONFIG_BOTO_SECTION_CONTENT) self._WriteProxyConfigFileSection(config_file) # Write the Google Compute Engine section. config_file.write(CONFIG_GOOGLECOMPUTE_SECTION_CONTENT) # Write the config file GSUtil section that doesn't depend on user input. config_file.write(CONFIG_INPUTLESS_GSUTIL_SECTION_CONTENT) # Write the default API version. config_file.write(""" # 'default_api_version' specifies the default Google Cloud Storage XML API # version to use. If not set below gsutil defaults to API version 1. """) api_version = 2 if cred_type == CredTypes.HMAC: api_version = 1 config_file.write('default_api_version = %d\n' % api_version) # Write the config file GSUtil section that includes the default # project ID input from the user. if not system_util.InvokedViaCloudSdk(): default_project_id = input('What is your project-id? ').strip() project_id_section_prelude = """ # 'default_project_id' specifies the default Google Cloud Storage project ID to # use with the 'mb' and 'ls' commands. This default can be overridden by # specifying the -p option to the 'mb' and 'ls' commands. """ if not default_project_id: raise CommandException( 'No default project ID entered. The default project ID is needed ' 'by the\nls and mb commands; please try again.') config_file.write('%sdefault_project_id = %s\n\n\n' % (project_id_section_prelude, default_project_id)) CheckAndMaybePromptForAnalyticsEnabling() # Write the config file OAuth2 section that doesn't depend on user input. config_file.write(CONFIG_OAUTH2_CONFIG_CONTENT) config_file.write('#client_id = <OAuth2 client id>\n' '#client_secret = <OAuth2 client secret>\n') def RunCommand(self): """Command entry point for the config command.""" cred_type = CredTypes.OAUTH2_USER_ACCOUNT output_file_name = None has_a = False has_e = False configure_auth = True for opt, opt_arg in self.sub_opts: if opt == '-a': cred_type = CredTypes.HMAC has_a = True elif opt == '-e': cred_type = CredTypes.OAUTH2_SERVICE_ACCOUNT has_e = True elif opt == '-n': configure_auth = False elif opt == '-o': output_file_name = opt_arg else: self.RaiseInvalidArgumentException() if has_e and has_a: raise CommandException('Both -a and -e cannot be specified. Please see ' '"gsutil help config" for more information.') if not configure_auth and (has_a or has_e): raise CommandException( 'The -a and -e flags cannot be specified with the -n flag. Please ' 'see "gsutil help config" for more information.') # Don't allow users to configure Oauth2 (any option other than -a and -n) # when running in the Cloud SDK, unless they have the Cloud SDK configured # not to pass credentials to gsutil. if (system_util.InvokedViaCloudSdk() and system_util.CloudSdkCredPassingEnabled() and not has_a and configure_auth): raise CommandException('\n'.join([ 'OAuth2 is the preferred authentication mechanism with the Cloud ' 'SDK.', 'Run "gcloud auth login" to configure authentication, unless:', '\n'.join( textwrap.wrap( 'You don\'t want gsutil to use OAuth2 credentials from the Cloud ' 'SDK, but instead want to manage credentials with .boto files ' 'generated by running "gsutil config"; in which case run ' '"gcloud config set pass_credentials_to_gsutil false".', initial_indent='- ', subsequent_indent=' ')), '\n'.join( textwrap.wrap( 'You want to authenticate with an HMAC access key and secret, in ' 'which case run "gsutil config -a".', initial_indent='- ', subsequent_indent=' ')), ])) if system_util.InvokedViaCloudSdk() and has_a: sys.stderr.write('\n'.join( textwrap.wrap( 'This command will configure HMAC credentials, but gsutil will use ' 'OAuth2 credentials from the Cloud SDK by default. To make sure ' 'the HMAC credentials are used, run: "gcloud config set ' 'pass_credentials_to_gsutil false".')) + '\n\n') default_config_path_bak = None if not output_file_name: # Check to see if a default config file name is requested via # environment variable. If so, use it, otherwise use the hard-coded # default file. Then use the default config file name, if it doesn't # exist or can be moved out of the way without clobbering an existing # backup file. boto_config_from_env = os.environ.get('BOTO_CONFIG', None) if boto_config_from_env: default_config_path = boto_config_from_env else: default_config_path = os.path.expanduser(os.path.join('~', '.boto')) if not os.path.exists(default_config_path): output_file_name = default_config_path else: default_config_path_bak = default_config_path + '.bak' if os.path.exists(default_config_path_bak): raise CommandException('Cannot back up existing config ' 'file "%s": backup file exists ("%s").' % (default_config_path, default_config_path_bak)) else: try: sys.stderr.write( 'Backing up existing config file "%s" to "%s"...\n' % (default_config_path, default_config_path_bak)) os.rename(default_config_path, default_config_path_bak) except Exception as e: raise CommandException( 'Failed to back up existing config ' 'file ("%s" -> "%s"): %s.' % (default_config_path, default_config_path_bak, e)) output_file_name = default_config_path if output_file_name == '-': output_file = sys.stdout else: output_file = self._OpenConfigFile(output_file_name) sys.stderr.write('\n'.join( textwrap.wrap( 'This command will create a boto config file at %s containing your ' 'credentials, based on your responses to the following questions.' % output_file_name)) + '\n') # Catch ^C so we can restore the backup. RegisterSignalHandler(signal.SIGINT, _CleanupHandler) try: self._WriteBotoConfigFile(output_file, cred_type=cred_type, configure_auth=configure_auth) except Exception as e: user_aborted = isinstance(e, AbortException) if user_aborted: sys.stderr.write('\nCaught ^C; cleaning up\n') # If an error occurred during config file creation, remove the invalid # config file and restore the backup file. if output_file_name != '-': output_file.close() os.unlink(output_file_name) try: if default_config_path_bak: sys.stderr.write('Restoring previous backed up file (%s)\n' % default_config_path_bak) os.rename(default_config_path_bak, output_file_name) except Exception as e: # Raise the original exception so that we can see what actually went # wrong, rather than just finding out that we died before assigning # a value to default_config_path_bak. raise e raise if output_file_name != '-': output_file.close() if not boto.config.has_option('Boto', 'proxy'): sys.stderr.write('\n' + '\n'.join( textwrap.wrap( 'Boto config file "%s" created.\nIf you need to use a proxy to ' 'access the Internet please see the instructions in that file.' % output_file_name)) + '\n') return 0 def _CleanupHandler(unused_signalnum, unused_handler): raise AbortException('User interrupted config command') commands/rewrite.py 0000644 00000063314 15025013616 0010406 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of rewrite command (in-place cloud object transformation).""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys import textwrap import time from apitools.base.py import encoding from boto import config from gslib.cloud_api import EncryptionException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.progress_callback import FileProgressCallbackHandler from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import FileMessage from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 from gslib.utils.encryption_helper import CryptoKeyType from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.encryption_helper import GetEncryptionKeyWrapper from gslib.utils.encryption_helper import MAX_DECRYPTION_KEYS from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import StdinIterator from gslib.utils.text_util import ConvertRecursiveToFlatWildcard from gslib.utils.text_util import NormalizeStorageClass from gslib.utils import text_util from gslib.utils.translation_helper import PreconditionsFromHeaders MAX_PROGRESS_INDICATOR_COLUMNS = 65 _SYNOPSIS = """ gsutil rewrite -k [-O] [-f] [-r] [-s] url... gsutil rewrite -k [-O] [-f] [-r] [-s] -I gsutil rewrite -s [-k] [-O] [-f] [-r] url... gsutil rewrite -s [-k] [-O] [-f] [-r] -I """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil rewrite command rewrites cloud objects, applying the specified transformations to them. The transformation(s) are atomic for each affected object and applied based on the input transformation flags. Object metadata values are preserved unless altered by a transformation. At least one transformation flag, -k or -s, must be included in the command. The -k flag is supported to add, rotate, or remove encryption keys on objects. For example, the command: gsutil rewrite -k -r gs://bucket updates all objects in gs://bucket with the current encryption key from your boto config file, which may either be a base64-encoded CSEK or the fully-qualified name of a Cloud KMS key. The rewrite command acts only on live object versions, so specifying a URL with a generation number fails. If you want to rewrite a noncurrent version, first copy it to the live version, then rewrite it, for example: gsutil cp gs://bucket/object#123 gs://bucket/object gsutil rewrite -k gs://bucket/object You can use the -s option to specify a new storage class for objects. For example, the command: gsutil rewrite -s nearline gs://bucket/foo rewrites the object, changing its storage class to nearline. If you specify the -k option and you have an encryption key set in your boto configuration file, the rewrite command skips objects that are already encrypted with the specified key. For example, if you run: gsutil rewrite -k -r gs://bucket and gs://bucket contains objects encrypted with the key specified in your boto configuration file, gsutil skips rewriting those objects and only rewrites objects that are not encrypted with the specified key. This avoids the cost of performing redundant rewrite operations. If you specify the -k option and you do not have an encryption key set in your boto configuration file, gsutil always rewrites each object, without explicitly specifying an encryption key. This results in rewritten objects being encrypted with either the bucket's default KMS key (if one is set) or Google-managed encryption (no CSEK or CMEK). Gsutil does not attempt to determine whether the operation is redundant (and thus skippable) because gsutil cannot be sure how the object is encrypted after the rewrite. Note that if your goal is to encrypt objects with a bucket's default KMS key, you can avoid redundant rewrite costs by specifying the bucket's default KMS key in your boto configuration file; this allows gsutil to perform an accurate comparison of the objects' current and desired encryption configurations and skip rewrites for objects already encrypted with that key. If have an encryption key set in your boto configuration file and specify multiple transformations, gsutil only skips those that would not change the object's state. For example, if you run: gsutil rewrite -s nearline -k -r gs://bucket and gs://bucket contains objects that already match the encryption configuration but have a storage class of standard, the only transformation applied to those objects would be the change in storage class. You can pass a list of URLs (one per line) to rewrite on stdin instead of as command line arguments by using the -I option. This allows you to use gsutil in a pipeline to rewrite objects identified by a program, such as: some_program | gsutil -m rewrite -k -I The contents of stdin can name cloud URLs and wildcards of cloud URLs. The rewrite command requires OWNER permissions on each object to preserve object ACLs. You can bypass this by using the -O flag, which causes gsutil not to read the object's ACL and instead apply the default object ACL to the rewritten object: gsutil rewrite -k -O -r gs://bucket <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when rewriting multiple objects. If some of the objects could not be rewritten, gsutil's exit status is non-zero even if this flag is set. This option is implicitly set when running "gsutil -m rewrite ...". -I Causes gsutil to read the list of objects to rewrite from stdin. This allows you to run a program that generates the list of objects to rewrite. -k Rewrite objects with the current encryption key specified in your boto configuration file. The value for encryption_key may be either a base64-encoded CSEK or a fully-qualified KMS key name. If no value is specified for encryption_key, gsutil ignores this flag. Instead, rewritten objects are encrypted with the bucket's default KMS key, if one is set, or Google-managed encryption, if no default KMS key is set. -O When a bucket has uniform bucket-level access (UBLA) enabled, the -O flag is required and skips all ACL checks. When a bucket has UBLA disabled, the -O flag rewrites objects with the bucket's default object ACL instead of the existing object ACL. This is needed if you do not have OWNER permission on the object. -R, -r The -R and -r options are synonymous. Causes bucket or bucket subdirectory contents to be rewritten recursively. -s <class> Rewrite objects using the specified storage class. """) def _RewriteExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" if not cls.continue_on_error: cls.logger.error(str(e)) cls.op_failure_count += 1 def _RewriteFuncWrapper(cls, name_expansion_result, thread_state=None): cls.RewriteFunc(name_expansion_result, thread_state=thread_state) def GenerationCheckGenerator(url_strs): """Generator function that ensures generation-less (live) arguments.""" for url_str in url_strs: if StorageUrlFromString(url_str).generation is not None: raise CommandException('"rewrite" called on URL with generation (%s).' % url_str) yield url_str class _TransformTypes(object): """Enum class for valid transforms.""" CRYPTO_KEY = 'crypto_key' STORAGE_CLASS = 'storage_class' class RewriteCommand(Command): """Implementation of gsutil rewrite command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rewrite', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='fkIrROs:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rewrite', help_name_aliases=['rekey', 'rotate'], help_type='command_help', help_one_line_summary='Rewrite objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'objects', 'update'], flag_map={ '-I': GcloudStorageFlag('-I'), '-f': GcloudStorageFlag('--continue-on-error'), # Adding encryptions handled in shim_util.py. '-k': None if config.get('GSUtil', 'encryption_key', None) else GcloudStorageFlag('--clear-encryption-key'), '-O': GcloudStorageFlag('--no-preserve-acl'), '-r': GcloudStorageFlag('-r'), '-R': GcloudStorageFlag('-r'), '-s': GcloudStorageFlag('-s'), }, ) def CheckProvider(self, url): if url.scheme != 'gs': raise CommandException( '"rewrite" called on URL with unsupported provider: %s' % str(url)) def RunCommand(self): """Command entry point for the rewrite command.""" self.continue_on_error = self.parallel_operations self.csek_hash_to_keywrapper = {} self.dest_storage_class = None self.no_preserve_acl = False self.read_args_from_stdin = False self.supported_transformation_flags = ['-k', '-s'] self.transform_types = set() self.op_failure_count = 0 self.boto_file_encryption_keywrapper = GetEncryptionKeyWrapper(config) self.boto_file_encryption_sha256 = ( self.boto_file_encryption_keywrapper.crypto_key_sha256 if self.boto_file_encryption_keywrapper else None) if self.sub_opts: for o, a in self.sub_opts: if o == '-f': self.continue_on_error = True elif o == '-k': self.transform_types.add(_TransformTypes.CRYPTO_KEY) elif o == '-I': self.read_args_from_stdin = True elif o == '-O': self.no_preserve_acl = True elif o == '-r' or o == '-R': self.recursion_requested = True self.all_versions = True elif o == '-s': self.transform_types.add(_TransformTypes.STORAGE_CLASS) self.dest_storage_class = NormalizeStorageClass(a) if self.read_args_from_stdin: if self.args: raise CommandException('No arguments allowed with the -I flag.') url_strs = StdinIterator() else: if not self.args: raise CommandException('The rewrite command (without -I) expects at ' 'least one URL.') url_strs = self.args if not self.transform_types: raise CommandException( 'rewrite command requires at least one transformation flag. ' 'Currently supported transformation flags: %s' % self.supported_transformation_flags) self.preconditions = PreconditionsFromHeaders(self.headers or {}) url_strs_generator = GenerationCheckGenerator(url_strs) # Convert recursive flag to flat wildcard to avoid performing multiple # listings. if self.recursion_requested: url_strs_generator = ConvertRecursiveToFlatWildcard(url_strs_generator) # Expand the source argument(s). name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, url_strs_generator, self.recursion_requested, project_id=self.project_id, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=['name', 'size']) seek_ahead_iterator = None # Cannot seek ahead with stdin args, since we can only iterate them # once without buffering in memory. if not self.read_args_from_stdin: # Perform the same recursive-to-flat conversion on original url_strs so # that it is as true to the original iterator as possible. seek_ahead_url_strs = ConvertRecursiveToFlatWildcard(url_strs) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), seek_ahead_url_strs, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) # Rather than have each worker repeatedly calculate the sha256 hash for each # decryption_key in the boto config, do this once now and cache the results. for i in range(0, MAX_DECRYPTION_KEYS): key_number = i + 1 keywrapper = CryptoKeyWrapperFromKey( config.get('GSUtil', 'decryption_key%s' % str(key_number), None)) if keywrapper is None: # Stop at first attribute absence in lexicographical iteration. break if keywrapper.crypto_type == CryptoKeyType.CSEK: self.csek_hash_to_keywrapper[keywrapper.crypto_key_sha256] = keywrapper # Also include the encryption_key, since it should be used to decrypt and # then encrypt if the object's CSEK should remain the same. if self.boto_file_encryption_sha256 is not None: self.csek_hash_to_keywrapper[self.boto_file_encryption_sha256] = ( self.boto_file_encryption_keywrapper) if self.boto_file_encryption_keywrapper is None: msg = '\n'.join( textwrap.wrap( 'NOTE: No encryption_key was specified in the boto configuration ' 'file, so gsutil will not provide an encryption key in its rewrite ' 'API requests. This will decrypt the objects unless they are in ' 'buckets with a default KMS key set, in which case the service ' 'will automatically encrypt the rewritten objects with that key.') ) print('%s\n' % msg, file=sys.stderr) # Perform rewrite requests in parallel (-m) mode, if requested. self.Apply(_RewriteFuncWrapper, name_expansion_iterator, _RewriteExceptionHandler, fail_on_error=(not self.continue_on_error), shared_attrs=['op_failure_count'], seek_ahead_iterator=seek_ahead_iterator) if self.op_failure_count: plural_str = 's' if self.op_failure_count else '' raise CommandException('%d file%s/object%s could not be rewritten.' % (self.op_failure_count, plural_str, plural_str)) return 0 def RewriteFunc(self, name_expansion_result, thread_state=None): gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) transform_url = name_expansion_result.expanded_storage_url self.CheckProvider(transform_url) # Get all fields so that we can ensure that the target metadata is # specified correctly. src_metadata = gsutil_api.GetObjectMetadata( transform_url.bucket_name, transform_url.object_name, generation=transform_url.generation, provider=transform_url.scheme) if self.no_preserve_acl: # Leave ACL unchanged. src_metadata.acl = [] elif not src_metadata.acl: raise CommandException( 'No OWNER permission found for object %s. If your bucket has uniform ' 'bucket-level access (UBLA) enabled, include the -O option in your ' 'command to avoid this error. If your bucket does not use UBLA, you ' 'can use the -O option to apply the bucket\'s default object ACL ' 'when rewriting.' % transform_url) # Note: If other transform types are added, they must ensure that the # encryption key configuration matches the boto configuration, because # gsutil maintains an invariant that all objects it writes use the # encryption_key value (including decrypting if no key is present). # Store metadata about src encryption to make logic below easier to read. src_encryption_kms_key = (src_metadata.kmsKeyName if src_metadata.kmsKeyName else None) src_encryption_sha256 = None if (src_metadata.customerEncryption and src_metadata.customerEncryption.keySha256): src_encryption_sha256 = src_metadata.customerEncryption.keySha256 # In python3, hashes are bytes, use ascii since it should be ascii src_encryption_sha256 = src_encryption_sha256.encode('ascii') src_was_encrypted = (src_encryption_sha256 is not None or src_encryption_kms_key is not None) # Also store metadata about dest encryption. dest_encryption_kms_key = None if (self.boto_file_encryption_keywrapper is not None and self.boto_file_encryption_keywrapper.crypto_type == CryptoKeyType.CMEK): dest_encryption_kms_key = self.boto_file_encryption_keywrapper.crypto_key dest_encryption_sha256 = None if (self.boto_file_encryption_keywrapper is not None and self.boto_file_encryption_keywrapper.crypto_type == CryptoKeyType.CSEK): dest_encryption_sha256 = ( self.boto_file_encryption_keywrapper.crypto_key_sha256) should_encrypt_dest = self.boto_file_encryption_keywrapper is not None encryption_unchanged = (src_encryption_sha256 == dest_encryption_sha256 and src_encryption_kms_key == dest_encryption_kms_key) # Prevent accidental key rotation. if (_TransformTypes.CRYPTO_KEY not in self.transform_types and not encryption_unchanged): raise EncryptionException( 'The "-k" flag was not passed to the rewrite command, but the ' 'encryption_key value in your boto config file did not match the key ' 'used to encrypt the object "%s" (hash: %s). To encrypt the object ' 'using a different key, you must specify the "-k" flag.' % (transform_url, src_encryption_sha256)) # Determine if we can skip this rewrite operation (this should only be done # when ALL of the specified transformations are redundant). redundant_transforms = [] # STORAGE_CLASS transform is redundant if the target storage class matches # the existing storage class. if (_TransformTypes.STORAGE_CLASS in self.transform_types and self.dest_storage_class == NormalizeStorageClass( src_metadata.storageClass)): redundant_transforms.append('storage class') # CRYPTO_KEY transform is redundant if we're using the same encryption # key that was used to encrypt the source. However, if no encryption key was # specified, we should still perform the rewrite. This results in the # rewritten object either being encrypted with its bucket's default KMS key # or having no CSEK/CMEK encryption applied. While we could attempt fetching # the bucket's metadata and checking its default KMS key before performing # the rewrite (in the case where we appear to be transitioning from # no key to no key), that is vulnerable to the race condition where the # default KMS key is changed between when we check it and when we rewrite # the object. if (_TransformTypes.CRYPTO_KEY in self.transform_types and should_encrypt_dest and encryption_unchanged): redundant_transforms.append('encryption key') if len(redundant_transforms) == len(self.transform_types): self.logger.info('Skipping %s, all transformations were redundant: %s' % (transform_url, redundant_transforms)) return # First make a deep copy of the source metadata, then overwrite any # requested attributes (e.g. if a storage class change was specified). dest_metadata = encoding.PyValueToMessage( apitools_messages.Object, encoding.MessageToPyValue(src_metadata)) # Remove some unnecessary/invalid fields. dest_metadata.generation = None # Service has problems if we supply an ID, but it is responsible for # generating one, so it is not necessary to include it here. dest_metadata.id = None # Ensure we don't copy over the KMS key name or CSEK key info from the # source object; those should only come from the boto config's # encryption_key value. dest_metadata.customerEncryption = None dest_metadata.kmsKeyName = None # Both a storage class change and CMEK encryption should be set as part of # the dest object's metadata. CSEK encryption, if specified, is added to the # request later via headers obtained from the keywrapper value passed to # encryption_tuple. if _TransformTypes.STORAGE_CLASS in self.transform_types: dest_metadata.storageClass = self.dest_storage_class if dest_encryption_kms_key is not None: dest_metadata.kmsKeyName = dest_encryption_kms_key # Make sure we have the CSEK key necessary to decrypt. decryption_keywrapper = None if src_encryption_sha256 is not None: if src_encryption_sha256 in self.csek_hash_to_keywrapper: decryption_keywrapper = ( self.csek_hash_to_keywrapper[src_encryption_sha256]) else: raise EncryptionException( 'Missing decryption key with SHA256 hash %s. No decryption key ' 'matches object %s' % (src_encryption_sha256, transform_url)) operation_name = 'Rewriting' if _TransformTypes.CRYPTO_KEY in self.transform_types: if src_was_encrypted and should_encrypt_dest: if not encryption_unchanged: operation_name = 'Rotating' # Else, keep "Rewriting". This might occur when -k was specified and was # redundant, but we're performing the operation anyway because some # other transformation was not redundant. elif src_was_encrypted and not should_encrypt_dest: operation_name = 'Decrypting' elif not src_was_encrypted and should_encrypt_dest: operation_name = 'Encrypting' # TODO: Remove this call (used to verify tests) and make it processed by # the UIThread. sys.stderr.write( _ConstructAnnounceText(operation_name, transform_url.url_string)) sys.stderr.flush() # Message indicating beginning of operation. gsutil_api.status_queue.put( FileMessage(transform_url, None, time.time(), finished=False, size=src_metadata.size, message_type=FileMessage.FILE_REWRITE)) progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=transform_url, operation_name=operation_name).call gsutil_api.CopyObject(src_metadata, dest_metadata, src_generation=transform_url.generation, preconditions=self.preconditions, progress_callback=progress_callback, decryption_tuple=decryption_keywrapper, encryption_tuple=self.boto_file_encryption_keywrapper, provider=transform_url.scheme, fields=[]) # Message indicating end of operation. gsutil_api.status_queue.put( FileMessage(transform_url, None, time.time(), finished=True, size=src_metadata.size, message_type=FileMessage.FILE_REWRITE)) def _ConstructAnnounceText(operation_name, url_string): """Constructs announce text for ongoing operations on url_string. This truncates the text to a maximum of MAX_PROGRESS_INDICATOR_COLUMNS, and informs the rewrite-related operation ('Encrypting', 'Rotating', or 'Decrypting'). Args: operation_name: String describing the operation, i.e. 'Rotating' or 'Encrypting'. url_string: String describing the file/object being processed. Returns: Formatted announce text for outputting operation progress. """ # Operation name occupies 10 characters (enough for 'Encrypting'), plus a # space. The rest is used for url_string. If a longer operation name is # used, it will be truncated. We can revisit this size if we need to support # a longer operation, but want to make sure the terminal output is meaningful. justified_op_string = operation_name[:10].ljust(11) start_len = len(justified_op_string) end_len = len(': ') if (start_len + len(url_string) + end_len > MAX_PROGRESS_INDICATOR_COLUMNS): ellipsis_len = len('...') url_string = '...%s' % url_string[-(MAX_PROGRESS_INDICATOR_COLUMNS - start_len - end_len - ellipsis_len):] base_announce_text = '%s%s:' % (justified_op_string, url_string) format_str = '{0:%ds}' % MAX_PROGRESS_INDICATOR_COLUMNS return format_str.format(base_announce_text) commands/du.py 0000644 00000024753 15025013616 0007341 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like du command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import locale import sys import six from gslib.bucket_listing_ref import BucketListingObject from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils import ls_helper from gslib.utils.constants import NO_MAX from gslib.utils.constants import S3_DELETE_MARKER_GUID from gslib.utils.constants import UTF8 from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import print_to_fd from gslib.utils.unit_util import MakeHumanReadable from gslib.utils import text_util _SYNOPSIS = """ gsutil du url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The du command displays the amount of space in bytes used up by the objects in a bucket, subdirectory, or project. The syntax emulates the Linux ``du -b`` command, which reports the disk usage of files and subdirectories. For example, the following command reports the total space used by all objects and subdirectories under gs://your-bucket/dir: gsutil du -s -a gs://your-bucket/dir <B>OPTIONS</B> -0 Ends each output line with a 0 byte rather than a newline. You can use this to make the output machine-readable. -a Includes both live and noncurrent object versions. Also prints the generation and metageneration number for each listed object. If this flag is not specified, only live object versions are included. -c Includes a total size at the end of the output. -e Exclude a pattern from the report. Example: -e "*.o" excludes any object that ends in ".o". Can be specified multiple times. -h Prints object sizes in human-readable format. For example, ``1 KiB``, ``234 MiB``, or ``2GiB``. -s Displays only the total size for each argument, omitting the list of individual objects. -X Similar to ``-e``, but excludes patterns from the given file. The patterns to exclude should be listed one per line. <B>EXAMPLES</B> To list the size of each object in a bucket: gsutil du gs://bucketname To list the size of each object in the ``prefix`` subdirectory: gsutil du gs://bucketname/prefix/* To include the total number of bytes in human-readable form: gsutil du -ch gs://bucketname To see only the summary of the total number of (live) bytes in two given buckets: gsutil du -s gs://bucket1 gs://bucket2 To list the size of each object in a bucket with `Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_ enabled, including noncurrent objects: gsutil du -a gs://bucketname To list the size of each object in a bucket, except objects that end in ".bak", with each object printed ending in a null byte: gsutil du -e "*.bak" -0 gs://bucketname To list the size of each bucket in a project and the total size of the project: gsutil -o GSUtil:default_project_id=project-name du -shc """) class DuCommand(Command): """Implementation of gsutil du command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'du', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='0ace:hsX:', file_url_ok=False, provider_url_ok=True, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='du', help_name_aliases=[], help_type='command_help', help_one_line_summary='Display object size usage', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'du'], flag_map={ '-0': GcloudStorageFlag('--zero-terminator'), '-a': GcloudStorageFlag('--all-versions'), '-c': GcloudStorageFlag('--total'), '-e': GcloudStorageFlag('--exclude-name-pattern'), '-h': GcloudStorageFlag('--readable-sizes'), '-s': GcloudStorageFlag('--summarize'), '-X': GcloudStorageFlag('--exclude-name-pattern-file'), }, ) def _PrintSummaryLine(self, num_bytes, name): size_string = (MakeHumanReadable(num_bytes) if self.human_readable else six.text_type(num_bytes)) text_util.print_to_fd('{size:<11} {name}'.format( size=size_string, name=six.ensure_text(name)), end=self.line_ending) def _PrintInfoAboutBucketListingRef(self, bucket_listing_ref): """Print listing info for given bucket_listing_ref. Args: bucket_listing_ref: BucketListing being listed. Returns: Tuple (number of objects, object size) Raises: Exception: if calling bug encountered. """ obj = bucket_listing_ref.root_object url_str = bucket_listing_ref.url_string if (obj.metadata and S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties): size_string = '0' num_bytes = 0 num_objs = 0 url_str += '<DeleteMarker>' else: size_string = (MakeHumanReadable(obj.size) if self.human_readable else str(obj.size)) num_bytes = obj.size num_objs = 1 if not self.summary_only: url_detail = '{size:<11} {url}{ending}'.format( size=size_string, url=six.ensure_text(url_str), ending=six.ensure_text(self.line_ending)) print_to_fd(url_detail, file=sys.stdout, end='') return (num_objs, num_bytes) def RunCommand(self): """Command entry point for the du command.""" self.line_ending = '\n' self.all_versions = False self.produce_total = False self.human_readable = False self.summary_only = False self.exclude_patterns = [] if self.sub_opts: for o, a in self.sub_opts: if o == '-0': self.line_ending = '\0' elif o == '-a': self.all_versions = True elif o == '-c': self.produce_total = True elif o == '-e': self.exclude_patterns.append(a) elif o == '-h': self.human_readable = True elif o == '-s': self.summary_only = True elif o == '-X': if a == '-': f = sys.stdin f_close = False else: f = open(a, 'r') if six.PY2 else open(a, 'r', encoding=UTF8) f_close = True self.exclude_patterns = [six.ensure_text(line.strip()) for line in f] if f_close: f.close() if not self.args: # Default to listing all gs buckets. self.args = ['gs://'] total_bytes = 0 got_nomatch_errors = False def _PrintObjectLong(blr): return self._PrintInfoAboutBucketListingRef(blr) def _PrintNothing(unused_blr=None): pass def _PrintDirectory(num_bytes, blr): if not self.summary_only: self._PrintSummaryLine(num_bytes, blr.url_string.encode(UTF8)) for url_arg in self.args: top_level_storage_url = StorageUrlFromString(url_arg) if top_level_storage_url.IsFileUrl(): raise CommandException('Only cloud URLs are supported for %s' % self.command_name) bucket_listing_fields = ['size'] listing_helper = ls_helper.LsHelper( self.WildcardIterator, self.logger, print_object_func=_PrintObjectLong, print_dir_func=_PrintNothing, print_dir_header_func=_PrintNothing, print_dir_summary_func=_PrintDirectory, print_newline_func=_PrintNothing, all_versions=self.all_versions, should_recurse=True, exclude_patterns=self.exclude_patterns, fields=bucket_listing_fields) # LsHelper expands to objects and prefixes, so perform a top-level # expansion first. if top_level_storage_url.IsProvider(): # Provider URL: use bucket wildcard to iterate over all buckets. top_level_iter = self.WildcardIterator( '%s://*' % top_level_storage_url.scheme).IterBuckets(bucket_fields=['id']) elif top_level_storage_url.IsBucket(): top_level_iter = self.WildcardIterator( '%s://%s' % (top_level_storage_url.scheme, top_level_storage_url.bucket_name)).IterBuckets( bucket_fields=['id']) else: top_level_iter = [BucketListingObject(top_level_storage_url)] for blr in top_level_iter: storage_url = blr.storage_url if storage_url.IsBucket() and self.summary_only: storage_url = StorageUrlFromString( storage_url.CreatePrefixUrl(wildcard_suffix='**')) _, exp_objs, exp_bytes = listing_helper.ExpandUrlAndPrint(storage_url) if (storage_url.IsObject() and exp_objs == 0 and ContainsWildcard(url_arg) and not self.exclude_patterns): got_nomatch_errors = True total_bytes += exp_bytes if self.summary_only: self._PrintSummaryLine(exp_bytes, blr.url_string.rstrip('/').encode(UTF8)) if self.produce_total: self._PrintSummaryLine(total_bytes, 'total') if got_nomatch_errors: raise CommandException('One or more URLs matched no objects.') return 0 commands/web.py 0000644 00000023570 15025013616 0007502 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of website configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from apitools.base.py import encoding from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _SET_SYNOPSIS = """ gsutil web set [-m <main_page_suffix>] [-e <error_page>] gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil web get gs://<bucket_name> """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``gsutil web set`` command allows you to configure or disable the website configuration on your buckets. The ``set`` sub-command has the following options (omit both options to unset the configuration): <B>SET OPTIONS</B> -m <index.html> Specifies the object name to serve when a bucket listing is requested via a custom domain. -e <404.html> Specifies the error page to serve when a request is made for a non-existent object via a custom domain """ _GET_DESCRIPTION = """ <B>GET</B> The ``gsutil web get`` command retrieves the web semantics configuration for a bucket and displays a JSON representation of the configuration. In Google Cloud Storage, this would look like the following: { "notFoundPage": "404.html", "mainPageSuffix": "index.html" } """ _DESCRIPTION = """ Cloud Storage allows you to configure a bucket to behave like a static website when the bucket is accessed through a `custom domain <https://cloud.google.com/storage/docs/request-endpoints#custom-domains>`_. For example, if you set a ``main_page_suffix``, a subsequent GET bucket request through a custom domain serves the specified "main" page instead of performing the usual bucket listing. Similarly, if you set an ``error_page``, a subsequent GET object request through a custom domain for a non-existent object serves the specified error page instead of the standard Cloud Storage error. Requests to a bucket through other `endpoints <https://cloud.google.com/storage/docs/request-endpoints>`_ are unaffected by the bucket's website configuration. See `Static website examples and tips <https://cloud.google.com/storage/docs/static-website>`_ for additional examples and information. Notes: 1. Because the main page is only served when a bucket listing request is made via a custom domain endpoint, you can continue to use ``gsutil ls`` as you normally would for bucket listing. 2. The main_page_suffix applies to each subdirectory of the bucket. For example, with the main_page_suffix configured to be index.html, a GET request for ``http://www.example.com`` retrieves ``http://www.example.com/index.html`` but shows ``http://www.example.com`` as the URL to the requester, and a GET request for ``http://www.example.com/photos`` retrieves ``http://www.example.com/photos/index.html`` but shows ``http://www.example.com/photos`` as the URL to the requester. 3. There is just one 404.html page. For example, a GET request for ``http://www.example.com/photos/missing`` retrieves ``http://www.example.com/404.html``, not ``http://www.example.com/photos/404.html``. The web command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class WebCommand(Command): """Implementation of gsutil web command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'web', command_name_aliases=['setwebcfg', 'getwebcfg'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='m:e:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='web', help_name_aliases=['getwebcfg', 'setwebcfg'], help_type='command_help', help_one_line_summary=('Set a website configuration for a bucket'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=website_config,empty=\' has ' 'no website configuration.\',empty_prefix_key=' 'storage_url]' ], flag_map={}, supports_output_translation=True, ), # "set" subcommand handled in get_gcloud_storage_args. }, flag_map={}, ) def get_gcloud_storage_args(self): if self.args[0] == 'set': set_command_map = GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', ], flag_map={ '-e': GcloudStorageFlag('--web-error-page'), '-m': GcloudStorageFlag('--web-main-page-suffix'), }) }, flag_map={}, ) if not ('-e' in self.args or '-m' in self.args): set_command_map.gcloud_command['set'].gcloud_command += [ '--clear-web-error-page', '--clear-web-main-page-suffix' ] gcloud_storage_map = set_command_map else: gcloud_storage_map = WebCommand.gcloud_storage_map return super().get_gcloud_storage_args(gcloud_storage_map) def _GetWeb(self): """Gets website configuration for a bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['website']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetWebsite(bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.website and (bucket_metadata.website.mainPageSuffix or bucket_metadata.website.notFoundPage): sys.stdout.write( str(encoding.MessageToJson(bucket_metadata.website)) + '\n') else: sys.stdout.write('%s has no website configuration.\n' % bucket_url) return 0 def _SetWeb(self): """Sets website configuration for a bucket.""" main_page_suffix = None error_page = None if self.sub_opts: for o, a in self.sub_opts: if o == '-m': main_page_suffix = a elif o == '-e': error_page = a url_args = self.args website = apitools_messages.Bucket.WebsiteValue( mainPageSuffix=main_page_suffix, notFoundPage=error_page) # Iterate over URLs, expanding wildcards and setting the website # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting website configuration on %s...', blr) bucket_metadata = apitools_messages.Bucket(website=website) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the web command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) if action_subcommand == 'get': func = self._GetWeb elif action_subcommand == 'set': func = self._SetWeb else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help web".') % (action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) return func() commands/versioning.py 0000644 00000020535 15025013616 0011106 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of versioning configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil versioning set (on|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil versioning get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The "set" sub-command requires an additional sub-command, either "on" or "off", which, respectively, will enable or disable versioning for the specified bucket(s). """ _GET_DESCRIPTION = """ <B>GET</B> The "get" sub-command gets the versioning configuration for a bucket and displays whether or not it is enabled. """ _DESCRIPTION = """ The `Versioning Configuration <https://cloud.google.com/storage/docs/object-versioning>`_ feature enables you to configure a Google Cloud Storage bucket to keep old versions of objects. The gsutil versioning command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _GCLOUD_FORMAT_STRING = ('--format=value[separator=""](' + 'name.sub("' + shim_util.get_format_flag_caret() + '", "gs://").sub("$", ": "),' + 'versioning.enabled' + '.yesno("Enabled", "Suspended"))') _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class VersioningCommand(Command): """Implementation of gsutil versioning command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'versioning', command_name_aliases=['setversioning', 'getversioning'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], 'get': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='versioning', help_name_aliases=[ 'getversioning', 'setversioning', ], help_type='command_help', help_one_line_summary=( 'Enable or suspend versioning for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', '--raw', _GCLOUD_FORMAT_STRING ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--versioning', ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-versioning', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetVersioning(self): """Gets versioning configuration for a bucket.""" versioning_arg = self.args[0].lower() if versioning_arg not in ('on', 'off'): raise CommandException('Argument to "%s set" must be either <on|off>' % (self.command_name)) url_args = self.args[1:] if not url_args: self.RaiseWrongNumberOfArgumentsException() # Iterate over URLs, expanding wildcards and set the versioning # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True bucket_metadata = apitools_messages.Bucket( versioning=apitools_messages.Bucket.VersioningValue()) if versioning_arg == 'on': self.logger.info('Enabling versioning for %s...', url) bucket_metadata.versioning.enabled = True else: self.logger.info('Suspending versioning for %s...', url) bucket_metadata.versioning.enabled = False self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetVersioning(self): """Gets versioning configuration for one or more buckets.""" url_args = self.args # Iterate over URLs, expanding wildcards and getting the versioning # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['versioning']) for blr in bucket_iter: some_matched = True if blr.root_object.versioning and blr.root_object.versioning.enabled: print('%s: Enabled' % blr.url_string.rstrip('/')) else: print('%s: Suspended' % blr.url_string.rstrip('/')) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def RunCommand(self): """Command entry point for the versioning command.""" action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._GetVersioning metrics.LogCommandParams(subcommands=[action_subcommand]) elif action_subcommand == 'set': func = self._SetVersioning versioning_arg = self.args[0].lower() if versioning_arg in ('on', 'off'): metrics.LogCommandParams( subcommands=[action_subcommand, versioning_arg]) else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help %s".') % (action_subcommand, self.command_name, self.command_name)) func() return 0 commands/__init__.py 0000644 00000001435 15025013616 0010460 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Package marker file.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals commands/cp.py 0000644 00000162765 15025013616 0007341 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # Copyright 2011, Nexenta Systems Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like cp command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import errno import itertools import logging import os import time import traceback from apitools.base.py import encoding from gslib import gcs_json_api from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.metrics import LogPerformanceSummaryParams from gslib.name_expansion import CopyObjectsIterator from gslib.name_expansion import DestinationInfo from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import NameExpansionIteratorDestinationTuple from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import ContainsWildcard from gslib.storage_url import IsCloudSubdirPlaceholder from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import cat_helper from gslib.utils import copy_helper from gslib.utils import parallelism_framework_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import DEBUGLEVEL_DUMP_REQUESTS from gslib.utils.constants import NO_MAX from gslib.utils.copy_helper import CreateCopyHelperOpts from gslib.utils.copy_helper import GetSourceFieldsNeededForCopy from gslib.utils.copy_helper import GZIP_ALL_FILES from gslib.utils.copy_helper import ItemExistsError from gslib.utils.copy_helper import Manifest from gslib.utils.copy_helper import SkipUnsupportedObjectError from gslib.utils.posix_util import ConvertModeToBase8 from gslib.utils.posix_util import DeserializeFileAttributesFromObjectMetadata from gslib.utils.posix_util import InitializePreservePosixData from gslib.utils.posix_util import POSIXAttributes from gslib.utils.posix_util import SerializeFileAttributesToObjectMetadata from gslib.utils.posix_util import ValidateFilePermissionAccess from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import GetStreamFromFileUrl from gslib.utils.system_util import StdinIterator from gslib.utils.system_util import StdinIteratorCls from gslib.utils.text_util import NormalizeStorageClass from gslib.utils.text_util import RemoveCRLFFromString from gslib.utils.unit_util import CalculateThroughput from gslib.utils.unit_util import MakeHumanReadable _SYNOPSIS = """ gsutil cp [OPTION]... src_url dst_url gsutil cp [OPTION]... src_url... dst_url gsutil cp [OPTION]... -I dst_url """ _SYNOPSIS_TEXT = """ <B>SYNOPSIS</B> """ + _SYNOPSIS _DESCRIPTION_TEXT = """ <B>DESCRIPTION</B> The ``gsutil cp`` command allows you to copy data between your local file system and the cloud, within the cloud, and between cloud storage providers. For example, to upload all text files from the local directory to a bucket, you can run: gsutil cp *.txt gs://my-bucket You can also download data from a bucket. The following command downloads all text files from the top-level of a bucket to your current directory: gsutil cp gs://my-bucket/*.txt . You can use the ``-n`` option to prevent overwriting the content of existing files. The following example downloads text files from a bucket without clobbering the data in your directory: gsutil cp -n gs://my-bucket/*.txt . Use the ``-r`` option to copy an entire directory tree. For example, to upload the directory tree ``dir``: gsutil cp -r dir gs://my-bucket If you have a large number of files to transfer, you can perform a parallel multi-threaded/multi-processing copy using the top-level gsutil ``-m`` option (see "gsutil help options"): gsutil -m cp -r dir gs://my-bucket You can use the ``-I`` option with ``stdin`` to specify a list of URLs to copy, one per line. This allows you to use gsutil in a pipeline to upload or download objects as generated by a program: cat filelist | gsutil -m cp -I gs://my-bucket or: cat filelist | gsutil -m cp -I ./download_dir where the output of ``cat filelist`` is a list of files, cloud URLs, and wildcards of files and cloud URLs. NOTE: Shells like ``bash`` and ``zsh`` sometimes attempt to expand wildcards in ways that can be surprising. You may also encounter issues when attempting to copy files whose names contain wildcard characters. For more details about these issues, see `Wildcard behavior considerations <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. """ _NAME_CONSTRUCTION_TEXT = """ <B>HOW NAMES ARE CONSTRUCTED</B> The ``gsutil cp`` command attempts to name objects in ways that are consistent with the Linux ``cp`` command. This means that names are constructed depending on whether you're performing a recursive directory copy or copying individually-named objects, or whether you're copying to an existing or non-existent directory. When you perform recursive directory copies, object names are constructed to mirror the source directory structure starting at the point of recursive processing. For example, if ``dir1/dir2`` contains the file ``a/b/c``, then the following command creates the object ``gs://my-bucket/dir2/a/b/c``: gsutil cp -r dir1/dir2 gs://my-bucket In contrast, copying individually-named files results in objects named by the final path component of the source files. For example, assuming again that ``dir1/dir2`` contains ``a/b/c``, the following command creates the object ``gs://my-bucket/c``: gsutil cp dir1/dir2/** gs://my-bucket Note that in the above example, the '**' wildcard matches all names anywhere under ``dir``. The wildcard '*' matches names just one level deep. For more details, see `URI wildcards <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. The same rules apply for uploads and downloads: recursive copies of buckets and bucket subdirectories produce a mirrored filename structure, while copying individually or wildcard-named objects produce flatly-named files. In addition, the resulting names depend on whether the destination subdirectory exists. For example, if ``gs://my-bucket/subdir`` exists as a subdirectory, the following command creates the object ``gs://my-bucket/subdir/dir2/a/b/c``: gsutil cp -r dir1/dir2 gs://my-bucket/subdir In contrast, if ``gs://my-bucket/subdir`` does not exist, this same ``gsutil cp`` command creates the object ``gs://my-bucket/subdir/a/b/c``. NOTE: The `Google Cloud Platform Console <https://console.cloud.google.com>`_ creates folders by creating "placeholder" objects that end with a "/" character. gsutil skips these objects when downloading from the cloud to the local file system, because creating a file that ends with a "/" is not allowed on Linux and macOS. We recommend that you only create objects that end with "/" if you don't intend to download such objects using gsutil. """ _SUBDIRECTORIES_TEXT = """ <B>COPYING TO/FROM SUBDIRECTORIES; DISTRIBUTING TRANSFERS ACROSS MACHINES</B> You can use gsutil to copy to and from subdirectories by using a command like this: gsutil cp -r dir gs://my-bucket/data This causes ``dir`` and all of its files and nested subdirectories to be copied under the specified destination, resulting in objects with names like ``gs://my-bucket/data/dir/a/b/c``. Similarly, you can download from bucket subdirectories using the following command: gsutil cp -r gs://my-bucket/data dir This causes everything nested under ``gs://my-bucket/data`` to be downloaded into ``dir``, resulting in files with names like ``dir/data/a/b/c``. Copying subdirectories is useful if you want to add data to an existing bucket directory structure over time. It's also useful if you want to parallelize uploads and downloads across multiple machines (potentially reducing overall transfer time compared with running ``gsutil -m cp`` on one machine). For example, if your bucket contains this structure: gs://my-bucket/data/result_set_01/ gs://my-bucket/data/result_set_02/ ... gs://my-bucket/data/result_set_99/ you can perform concurrent downloads across 3 machines by running these commands on each machine, respectively: gsutil -m cp -r gs://my-bucket/data/result_set_[0-3]* dir gsutil -m cp -r gs://my-bucket/data/result_set_[4-6]* dir gsutil -m cp -r gs://my-bucket/data/result_set_[7-9]* dir Note that ``dir`` could be a local directory on each machine, or a directory mounted off of a shared file server. The performance of the latter depends on several factors, so we recommend experimenting to find out what works best for your computing environment. """ _COPY_IN_CLOUD_TEXT = """ <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> If both the source and destination URL are cloud URLs from the same provider, gsutil copies data "in the cloud" (without downloading to and uploading from the machine where you run gsutil). In addition to the performance and cost advantages of doing this, copying in the cloud preserves metadata such as ``Content-Type`` and ``Cache-Control``. In contrast, when you download data from the cloud, it ends up in a file with no associated metadata, unless you have some way to keep or re-create that metadata. Copies spanning locations and/or storage classes cause data to be rewritten in the cloud, which may take some time (but is still faster than downloading and re-uploading). Such operations can be resumed with the same command if they are interrupted, so long as the command parameters are identical. Note that by default, the gsutil ``cp`` command does not copy the object ACL to the new object, and instead uses the default bucket ACL (see "gsutil help defacl"). You can override this behavior with the ``-p`` option. When copying in the cloud, if the destination bucket has Object Versioning enabled, by default ``gsutil cp`` copies only live versions of the source object. For example, the following command causes only the single live version of ``gs://bucket1/obj`` to be copied to ``gs://bucket2``, even if there are noncurrent versions of ``gs://bucket1/obj``: gsutil cp gs://bucket1/obj gs://bucket2 To also copy noncurrent versions, use the ``-A`` flag: gsutil cp -A gs://bucket1/obj gs://bucket2 The top-level gsutil ``-m`` flag is not allowed when using the ``cp -A`` flag. """ _CHECKSUM_VALIDATION_TEXT = """ <B>CHECKSUM VALIDATION</B> gsutil automatically performs checksum validation for copies to and from Cloud Storage. For more information, see `Hashes and ETags <https://cloud.google.com/storage/docs/hashes-etags#cli>`_. """ _RETRY_HANDLING_TEXT = """ <B>RETRY HANDLING</B> The ``cp`` command retries when failures occur, but if enough failures happen during a particular copy or delete operation, or if a failure isn't retryable, the ``cp`` command skips that object and moves on. If any failures were not successfully retried by the end of the copy run, the ``cp`` command reports the number of failures and exits with a non-zero status. For details about gsutil's overall retry handling, see `Retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_. """ _RESUMABLE_TRANSFERS_TEXT = """ <B>RESUMABLE TRANSFERS</B> gsutil automatically resumes interrupted downloads and interrupted `resumable uploads <https://cloud.google.com/storage/docs/resumable-uploads#gsutil>`_, except when performing streaming transfers. In the case of an interrupted download, a partially downloaded temporary file is visible in the destination directory with the suffix ``_.gstmp`` in its name. Upon completion, the original file is deleted and replaced with the downloaded contents. Resumable transfers store state information in files under ~/.gsutil, named by the destination object or file. See "gsutil help prod" for details on using resumable transfers in production. """ _STREAMING_TRANSFERS_TEXT = """ <B>STREAMING TRANSFERS</B> Use '-' in place of src_url or dst_url to perform a `streaming transfer <https://cloud.google.com/storage/docs/streaming>`_. Streaming uploads using the `JSON API <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ are buffered in memory part-way back into the file and can thus sometimes resume in the event of network or service problems. gsutil does not support resuming streaming uploads using the XML API or resuming streaming downloads for either JSON or XML. If you have a large amount of data to transfer in these cases, we recommend that you write the data to a local file and copy that file rather than streaming it. """ _SLICED_OBJECT_DOWNLOADS_TEXT = """ <B>SLICED OBJECT DOWNLOADS</B> gsutil can automatically use ranged ``GET`` requests to perform downloads in parallel for large files being downloaded from Cloud Storage. See `sliced object download documentation <https://cloud.google.com/storage/docs/sliced-object-downloads>`_ for a complete discussion. """ _PARALLEL_COMPOSITE_UPLOADS_TEXT = """ <B>PARALLEL COMPOSITE UPLOADS</B> gsutil can automatically use `object composition <https://cloud.google.com/storage/docs/composite-objects>`_ to perform uploads in parallel for large, local files being uploaded to Cloud Storage. See the `parallel composite uploads documentation <https://cloud.google.com/storage/docs/parallel-composite-uploads>`_ for a complete discussion. """ _CHANGING_TEMP_DIRECTORIES_TEXT = """ <B>CHANGING TEMP DIRECTORIES</B> gsutil writes data to a temporary directory in several cases: - when compressing data to be uploaded (see the ``-z`` and ``-Z`` options) - when decompressing data being downloaded (for example, when the data has ``Content-Encoding:gzip`` as a result of being uploaded using gsutil cp -z or gsutil cp -Z) - when running integration tests using the gsutil test command In these cases, it's possible the temporary file location on your system that gsutil selects by default may not have enough space. If gsutil runs out of space during one of these operations (for example, raising "CommandException: Inadequate temp space available to compress <your file>" during a ``gsutil cp -z`` operation), you can change where it writes these temp files by setting the TMPDIR environment variable. On Linux and macOS, you can set the variable as follows: TMPDIR=/some/directory gsutil cp ... You can also add this line to your ~/.bashrc file and restart the shell before running gsutil: export TMPDIR=/some/directory On Windows 7, you can change the TMPDIR environment variable from Start -> Computer -> System -> Advanced System Settings -> Environment Variables. You need to reboot after making this change for it to take effect. Rebooting is not necessary after running the export command on Linux and macOS. """ _COPYING_SPECIAL_FILES_TEXT = """ <B>SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SUCH AS SYMLINKS AND DEVICES)</B> Please see the section about OS-specific file types in "gsutil help rsync". While that section refers to the ``rsync`` command, analogous points apply to the ``cp`` command. """ _OPTIONS_TEXT = """ <B>OPTIONS</B> -a predef_acl Applies the specific predefined ACL to uploaded objects. See "gsutil help acls" for further details. -A Copy all source versions from a source bucket or folder. If not set, only the live version of each source object is copied. NOTE: This option is only useful when the destination bucket has Object Versioning enabled. Additionally, the generation numbers of copied versions do not necessarily match the order of the original generation numbers. -c If an error occurs, continue attempting to copy the remaining files. If any copies are unsuccessful, gsutil's exit status is non-zero, even if this flag is set. This option is implicitly set when running ``gsutil -m cp...``. NOTE: ``-c`` only applies to the actual copying operation. If an error, such as ``invalid Unicode file name``, occurs while iterating over the files in the local directory, gsutil prints an error message and aborts. -D Copy in "daisy chain" mode, which means copying between two buckets by first downloading to the machine where gsutil is run, then uploading to the destination bucket. The default mode is a "copy in the cloud," where data is copied between two buckets without uploading or downloading. During a "copy in the cloud," a source composite object remains composite at its destination. However, you can use "daisy chain" mode to change a composite object into a non-composite object. For example: gsutil cp -D gs://bucket/obj gs://bucket/obj_tmp gsutil mv gs://bucket/obj_tmp gs://bucket/obj NOTE: "Daisy chain" mode is automatically used when copying between providers: for example, when copying data from Cloud Storage to another provider. -e Exclude symlinks. When specified, symbolic links are not copied. -I Use ``stdin`` to specify a list of files or objects to copy. You can use gsutil in a pipeline to upload or download objects as generated by a program. For example: cat filelist | gsutil -m cp -I gs://my-bucket where the output of ``cat filelist`` is a one-per-line list of files, cloud URLs, and wildcards of files and cloud URLs. -j <ext,...> Applies gzip transport encoding to any file upload whose extension matches the ``-j`` extension list. This is useful when uploading files with compressible content such as .js, .css, or .html files. This also saves network bandwidth while leaving the data uncompressed in Cloud Storage. When you specify the ``-j`` option, files being uploaded are compressed in-memory and on-the-wire only. Both the local files and Cloud Storage objects remain uncompressed. The uploaded objects retain the ``Content-Type`` and name of the original files. Note that if you want to use the ``-m`` `top-level option <https://cloud.google.com/storage/docs/gsutil/addlhelp/GlobalCommandLineOptions>`_ to parallelize copies along with the ``-j/-J`` options, your performance may be bottlenecked by the "max_upload_compression_buffer_size" boto config option, which is set to 2 GiB by default. You can change this compression buffer size to a higher limit. For example: gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \\ -m cp -j html,txt -r /local/source/dir gs://bucket/path -J Applies gzip transport encoding to file uploads. This option works like the ``-j`` option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If some of the source files don't compress well, such as binary data, using this option may result in longer uploads. -L <file> Outputs a manifest log file with detailed information about each item that was copied. This manifest contains the following information for each item: - Source path. - Destination path. - Source size. - Bytes transferred. - MD5 hash. - Transfer start time and date in UTC and ISO 8601 format. - Transfer completion time and date in UTC and ISO 8601 format. - Upload id, if a resumable upload was performed. - Final result of the attempted transfer, either success or failure. - Failure details, if any. If the log file already exists, gsutil uses the file as an input to the copy process, and appends log items to the existing file. Objects that are marked in the existing log file as having been successfully copied or skipped are ignored. Objects without entries are copied and ones previously marked as unsuccessful are retried. This option can be used in conjunction with the ``-c`` option to build a script that copies a large number of objects reliably, using a bash script like the following: until gsutil cp -c -L cp.log -r ./dir gs://bucket; do sleep 1 done The -c option enables copying to continue after failures occur, and the -L option allows gsutil to pick up where it left off without duplicating work. The loop continues running as long as gsutil exits with a non-zero status. A non-zero status indicates there was at least one failure during the copy operation. NOTE: If you are synchronizing the contents of a directory and a bucket, or the contents of two buckets, see "gsutil help rsync". -n No-clobber. When specified, existing files or objects at the destination are not replaced. Any items that are skipped by this option are reported as skipped. gsutil performs an additional GET request to check if an item exists before attempting to upload the data. This saves gsutil from retransmitting data, but the additional HTTP requests may make small object transfers slower and more expensive. -p Preserves ACLs when copying in the cloud. Note that this option has performance and cost implications only when using the XML API, as the XML API requires separate HTTP calls for interacting with ACLs. You can mitigate this performance issue using ``gsutil -m cp`` to perform parallel copying. Note that this option only works if you have OWNER access to all objects that are copied. If you want all objects in the destination bucket to end up with the same ACL, you can avoid these performance issues by setting a default object ACL on that bucket instead of using ``cp -p``. See "gsutil help defacl". Note that it's not valid to specify both the ``-a`` and ``-p`` options together. -P Enables POSIX attributes to be preserved when objects are copied. ``gsutil cp`` copies fields provided by ``stat``. These fields are the user ID of the owner, the group ID of the owning group, the mode or permissions of the file, and the access and modification time of the file. For downloads, these attributes are only set if the source objects were uploaded with this flag enabled. On Windows, this flag only sets and restores access time and modification time. This is because Windows doesn't support POSIX uid/gid/mode. -R, -r The ``-R`` and ``-r`` options are synonymous. They enable directories, buckets, and bucket subdirectories to be copied recursively. If you don't use this option for an upload, gsutil copies objects it finds and skips directories. Similarly, if you don't specify this option for a download, gsutil copies objects at the current bucket directory level and skips subdirectories. -s <class> Specifies the storage class of the destination object. If not specified, the default storage class of the destination bucket is used. This option is not valid for copying to non-cloud destinations. -U Skips objects with unsupported object types instead of failing. Unsupported object types include Amazon S3 objects in the GLACIER storage class. -v Prints the version-specific URL for each uploaded object. You can use these URLs to safely make concurrent upload requests, because Cloud Storage refuses to perform an update if the current object version doesn't match the version-specific URL. See `generation numbers <https://cloud.google.com/storage/docs/metadata#generation-number>`_ for more details. -z <ext,...> Applies gzip content-encoding to any file upload whose extension matches the ``-z`` extension list. This is useful when uploading files with compressible content such as .js, .css, or .html files, because it reduces network bandwidth and storage sizes. This can both improve performance and reduce costs. When you specify the ``-z`` option, the data from your files is compressed before it is uploaded, but your actual files are left uncompressed on the local disk. The uploaded objects retain the ``Content-Type`` and name of the original files, but have their ``Content-Encoding`` metadata set to ``gzip`` to indicate that the object data stored are compressed on the Cloud Storage servers and have their ``Cache-Control`` metadata set to ``no-transform``. For example, the following command: gsutil cp -z html \\ cattypes.html tabby.jpeg gs://mycats does the following: - The ``cp`` command uploads the files ``cattypes.html`` and ``tabby.jpeg`` to the bucket ``gs://mycats``. - Based on the file extensions, gsutil sets the ``Content-Type`` of ``cattypes.html`` to ``text/html`` and ``tabby.jpeg`` to ``image/jpeg``. - The ``-z`` option compresses the data in the file ``cattypes.html``. - The ``-z`` option also sets the ``Content-Encoding`` for ``cattypes.html`` to ``gzip`` and the ``Cache-Control`` for ``cattypes.html`` to ``no-transform``. Because the ``-z/-Z`` options compress data prior to upload, they are not subject to the same compression buffer bottleneck that can affect the ``-j/-J`` options. Note that if you download an object with ``Content-Encoding:gzip``, gsutil decompresses the content before writing the local file. -Z Applies gzip content-encoding to file uploads. This option works like the ``-z`` option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If some of the source files don't compress well, such as binary data, using this option may result in files taking up more space in the cloud than they would if left uncompressed. --stet If the STET binary can be found in boto or PATH, cp will use the split-trust encryption tool for end-to-end encryption. """ _DETAILED_HELP_TEXT = '\n\n'.join([ _SYNOPSIS_TEXT, _DESCRIPTION_TEXT, _NAME_CONSTRUCTION_TEXT, _SUBDIRECTORIES_TEXT, _COPY_IN_CLOUD_TEXT, _CHECKSUM_VALIDATION_TEXT, _RETRY_HANDLING_TEXT, _RESUMABLE_TRANSFERS_TEXT, _STREAMING_TRANSFERS_TEXT, _SLICED_OBJECT_DOWNLOADS_TEXT, _PARALLEL_COMPOSITE_UPLOADS_TEXT, _CHANGING_TEMP_DIRECTORIES_TEXT, _COPYING_SPECIAL_FILES_TEXT, _OPTIONS_TEXT, ]) CP_SUB_ARGS = 'a:AcDeIL:MNnpPrRs:tUvz:Zj:J' # May be used by cp or mv. CP_AND_MV_SHIM_FLAG_MAP = { '-A': GcloudStorageFlag('--all-versions'), '-a': GcloudStorageFlag('--predefined-acl'), '-c': GcloudStorageFlag('--continue-on-error'), '-D': GcloudStorageFlag('--daisy-chain'), '-e': GcloudStorageFlag('--ignore-symlinks'), '-I': GcloudStorageFlag('--read-paths-from-stdin'), '-J': GcloudStorageFlag('--gzip-in-flight-all'), '-j': GcloudStorageFlag('--gzip-in-flight'), '-L': GcloudStorageFlag('--manifest-path'), '-n': GcloudStorageFlag('--no-clobber'), '-P': GcloudStorageFlag('--preserve-posix'), '-p': GcloudStorageFlag('--preserve-acl'), '-s': GcloudStorageFlag('--storage-class'), '-v': GcloudStorageFlag('--print-created-message'), '-Z': GcloudStorageFlag('--gzip-local-all'), '-z': GcloudStorageFlag('--gzip-local'), } # Adds recursion flags. CP_SHIM_FLAG_MAP = { k: v for k, v in list(CP_AND_MV_SHIM_FLAG_MAP.items()) + [('-r', GcloudStorageFlag('-r')), ('-R', GcloudStorageFlag('-r'))] } def ShimTranslatePredefinedAclSubOptForCopy(sub_opts): """Gcloud uses camel-case predefined/canned ACLs, and gsutil uses snake-case. The camel-case-snake-case difference is related to gcloud primarily using JSON API rather than the XML API. Predefined ACLs are also called "canned ACLs". Args: sub_opts: List of pairs representing flag keys and values, e.g. [('a', 'public-read')] """ predefined_acl_idx = None for i, (k, _) in enumerate(sub_opts): if k == '-a': predefined_acl_idx = i break if predefined_acl_idx is not None: old_predefined_acl = sub_opts[i][1] sub_opts[i] = (sub_opts[i][0], gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION.get( old_predefined_acl, old_predefined_acl)) def _CopyFuncWrapper(cls, args, thread_state=None): cls.CopyFunc(args, thread_state=thread_state, preserve_posix=cls.preserve_posix_attrs) def _CopyExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) cls.op_failure_count += 1 cls.logger.debug('\n\nEncountered exception while copying:\n%s\n', traceback.format_exc()) def _RmExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) class CpCommand(Command): """Implementation of gsutil cp command. Note that CpCommand is run for both gsutil cp and gsutil mv. The latter happens by MvCommand calling CpCommand and passing the hidden (undocumented) -M option. This allows the copy and remove needed for each mv to run together (rather than first running all the cp's and then all the rm's, as we originally had implemented), which in turn avoids the following problem with removing the wrong objects: starting with a bucket containing only the object gs://bucket/obj, say the user does: gsutil mv gs://bucket/* gs://bucket/d.txt If we ran all the cp's and then all the rm's and we didn't expand the wildcard first, the cp command would first copy gs://bucket/obj to gs://bucket/d.txt, and the rm command would then remove that object. In the implementation prior to gsutil release 3.12 we avoided this by building a list of objects to process and then running the copies and then the removes; but building the list up front limits scalability (compared with the current approach of processing the bucket listing iterator on the fly). """ # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'cp', command_name_aliases=['copy'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, # -t is deprecated but leave intact for now to avoid breakage. supported_sub_args=CP_SUB_ARGS, file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, # Unfortunately, "private" args are the only way to support non-single # character flags. supported_private_args=['stet', 'testcallbackfile='], argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='cp', help_name_aliases=['copy'], help_type='command_help', help_one_line_summary='Copy files and objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): self.logger.warn( "Unlike pure gsutil, this shim won't run composite uploads and sliced" ' downloads in parallel by default. Use the -m flag to enable' ' parallelism (i.e. "gsutil -m cp ...").') ShimTranslatePredefinedAclSubOptForCopy(self.sub_opts) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'cp'], flag_map=CP_SHIM_FLAG_MAP, ) return super().get_gcloud_storage_args(gcloud_storage_map) # pylint: disable=too-many-statements def CopyFunc(self, copy_object_info, thread_state=None, preserve_posix=False): """Worker function for performing the actual copy (and rm, for mv).""" gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) copy_helper_opts = copy_helper.GetCopyHelperOpts() if copy_helper_opts.perform_mv: cmd_name = 'mv' else: cmd_name = self.command_name src_url = copy_object_info.source_storage_url exp_src_url = copy_object_info.expanded_storage_url src_url_names_container = copy_object_info.names_container have_multiple_srcs = copy_object_info.is_multi_source_request if src_url.IsCloudUrl() and src_url.IsProvider(): raise CommandException( 'The %s command does not allow provider-only source URLs (%s)' % (cmd_name, src_url)) if preserve_posix and src_url.IsFileUrl() and src_url.IsStream(): raise CommandException('Cannot preserve POSIX attributes with a stream.') if self.parallel_operations and src_url.IsFileUrl() and src_url.IsStream(): raise CommandException( 'Cannot upload from a stream when using gsutil -m option.') if have_multiple_srcs: copy_helper.InsistDstUrlNamesContainer( copy_object_info.exp_dst_url, copy_object_info.have_existing_dst_container, cmd_name) # Various GUI tools (like the GCS web console) create placeholder objects # ending with '/' when the user creates an empty directory. Normally these # tools should delete those placeholders once objects have been written # "under" the directory, but sometimes the placeholders are left around. We # need to filter them out here, otherwise if the user tries to rsync from # GCS to a local directory it will result in a directory/file conflict # (e.g., trying to download an object called "mydata/" where the local # directory "mydata" exists). if IsCloudSubdirPlaceholder(exp_src_url): # We used to output the message 'Skipping cloud sub-directory placeholder # object...' but we no longer do so because it caused customer confusion. return if copy_helper_opts.use_manifest and self.manifest.WasSuccessful( exp_src_url.url_string): return if copy_helper_opts.perform_mv and copy_object_info.names_container: # Use recursion_requested when performing name expansion for the # directory mv case so we can determine if any of the source URLs are # directories (and then use cp -r and rm -r to perform the move, to # match the behavior of Linux mv (which when moving a directory moves # all the contained files). self.recursion_requested = True if (copy_object_info.exp_dst_url.IsFileUrl() and not os.path.exists(copy_object_info.exp_dst_url.object_name) and have_multiple_srcs): try: os.makedirs(copy_object_info.exp_dst_url.object_name) except OSError as e: if e.errno != errno.EEXIST: raise dst_url = copy_helper.ConstructDstUrl( src_url, exp_src_url, src_url_names_container, have_multiple_srcs, copy_object_info.is_multi_top_level_source_request, copy_object_info.exp_dst_url, copy_object_info.have_existing_dst_container, self.recursion_requested, preserve_posix=preserve_posix) dst_url = copy_helper.FixWindowsNaming(src_url, dst_url) copy_helper.CheckForDirFileConflict(exp_src_url, dst_url) if copy_helper.SrcDstSame(exp_src_url, dst_url): raise CommandException('%s: "%s" and "%s" are the same file - ' 'abort.' % (cmd_name, exp_src_url, dst_url)) if dst_url.IsCloudUrl() and dst_url.HasGeneration(): raise CommandException('%s: a version-specific URL\n(%s)\ncannot be ' 'the destination for gsutil cp - abort.' % (cmd_name, dst_url)) if not dst_url.IsCloudUrl() and copy_helper_opts.dest_storage_class: raise CommandException('Cannot specify storage class for a non-cloud ' 'destination: %s' % dst_url) src_obj_metadata = None if copy_object_info.expanded_result: src_obj_metadata = encoding.JsonToMessage( apitools_messages.Object, copy_object_info.expanded_result) if src_url.IsFileUrl() and preserve_posix: if not src_obj_metadata: src_obj_metadata = apitools_messages.Object() mode, _, _, _, uid, gid, _, atime, mtime, _ = os.stat( exp_src_url.object_name) mode = ConvertModeToBase8(mode) posix_attrs = POSIXAttributes(atime=atime, mtime=mtime, uid=uid, gid=gid, mode=mode) custom_metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) SerializeFileAttributesToObjectMetadata(posix_attrs, custom_metadata, preserve_posix=preserve_posix) src_obj_metadata.metadata = custom_metadata if src_obj_metadata and dst_url.IsFileUrl(): posix_attrs = DeserializeFileAttributesFromObjectMetadata( src_obj_metadata, src_url.url_string) mode = posix_attrs.mode.permissions valid, err = ValidateFilePermissionAccess(src_url.url_string, uid=posix_attrs.uid, gid=posix_attrs.gid, mode=mode) if preserve_posix and not valid: logging.getLogger().critical(err) raise CommandException('This sync will orphan file(s), please fix their' ' permissions before trying again.') bytes_transferred = 0 try: if copy_helper_opts.use_manifest: self.manifest.Initialize(exp_src_url.url_string, dst_url.url_string) if (self.recursion_requested and copy_object_info.exp_dst_url.object_name and dst_url.IsFileUrl()): # exp_dst_url is the wildcard-expanded path passed by the user: # exp_dst_url => ~/dir # container => /usr/name/dir container = os.path.abspath(copy_object_info.exp_dst_url.object_name) # dst_url holds the complete path of the object's destination: # dst_url => /usr/name/dir/../file.txt # abspath => /usr/name/file.txt # # Taking the common path of this and container yields: /usr/name, # which does not start with container when the inclusion of '..' strings # results in a copy outside of the container. if not os.path.commonpath([ container, os.path.abspath(dst_url.object_name) ]).startswith(container): self.logger.warn( 'Skipping copy of source URL %s because it would be copied ' 'outside the expected destination directory: %s.' % (exp_src_url, container)) if copy_helper_opts.use_manifest: self.manifest.SetResult( exp_src_url.url_string, 0, 'skip', 'Would have copied outside the destination directory.') return _, bytes_transferred, result_url, md5 = copy_helper.PerformCopy( self.logger, exp_src_url, dst_url, gsutil_api, self, _CopyExceptionHandler, src_obj_metadata=src_obj_metadata, allow_splitting=True, headers=self.headers, manifest=self.manifest, gzip_encoded=self.gzip_encoded, gzip_exts=self.gzip_exts, preserve_posix=preserve_posix, use_stet=self.use_stet) if copy_helper_opts.use_manifest: if md5: self.manifest.Set(exp_src_url.url_string, 'md5', md5) self.manifest.SetResult(exp_src_url.url_string, bytes_transferred, 'OK') if copy_helper_opts.print_ver: # Some cases don't return a version-specific URL (e.g., if destination # is a file). self.logger.info('Created: %s', result_url) except ItemExistsError: message = 'Skipping existing item: %s' % dst_url self.logger.info(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message) except SkipUnsupportedObjectError as e: message = ('Skipping item %s with unsupported object type %s' % (exp_src_url.url_string, e.unsupported_type)) self.logger.info(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message) except copy_helper.FileConcurrencySkipError as e: self.logger.warn( 'Skipping copy of source URL %s because destination URL ' '%s is already being copied by another gsutil process ' 'or thread (did you specify the same source URL twice?) ' % (src_url, dst_url)) except Exception as e: # pylint: disable=broad-except if (copy_helper_opts.no_clobber and copy_helper.IsNoClobberServerException(e)): message = 'Rejected (noclobber): %s' % dst_url self.logger.info(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message) elif self.continue_on_error: message = 'Error copying %s: %s' % (src_url, str(e)) self.op_failure_count += 1 self.logger.error(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'error', RemoveCRLFFromString(message)) else: if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'error', str(e)) raise else: if copy_helper_opts.perform_mv: self.logger.info('Removing %s...', exp_src_url) if exp_src_url.IsCloudUrl(): gsutil_api.DeleteObject(exp_src_url.bucket_name, exp_src_url.object_name, generation=exp_src_url.generation, provider=exp_src_url.scheme) else: os.unlink(exp_src_url.object_name) with self.stats_lock: # TODO: Remove stats_lock; we should be able to calculate bytes # transferred from StatusMessages posted by operations within PerformCopy. self.total_bytes_transferred += bytes_transferred def _ConstructNameExpansionIteratorDstTupleIterator(self, src_url_strs_iter, dst_url_strs): copy_helper_opts = copy_helper.GetCopyHelperOpts() for src_url_str, dst_url_str in zip(src_url_strs_iter, dst_url_strs): # Getting the destination information for each (sources, destination) # tuple. This assumes that the same destination is never provided in # multiple tuples, and doing so may result in an inconsistent behavior # especially when using the -m multi-threading option. # # Example for the inconsistent behavior, the following commands will # behave differently: # # gsutil cp -r dir1 dir2 gs://bucket/non-existent-dir # gsutil cp -r [ # (dir1, gs://bucket/non-existent-dir), # (dir2, gs://bucket/non-existent-dir) # ] # # When multiple threads execute on a non existing destination directory. # These threads might encounter different states of the destination # directory. The first thread to execute the command finds that the # destination directory does not exist, it will create the destination # directory and copies the files inside the source directories to the # destination directory. The following threads find that the destination # directory already exists and copy the source directories in the # destination directory. In another scenario, all the threads might find # that the destination directory does not exist and copy the source # directories to the destination directory. exp_dst_url, have_existing_dst_container = ( copy_helper.ExpandUrlToSingleBlr(dst_url_str, self.gsutil_api, self.project_id, logger=self.logger)) name_expansion_iterator_dst_tuple = NameExpansionIteratorDestinationTuple( NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, src_url_str, self.recursion_requested or copy_helper_opts.perform_mv, project_id=self.project_id, all_versions=self.all_versions, ignore_symlinks=self.exclude_symlinks, continue_on_error=(self.continue_on_error or self.parallel_operations), bucket_listing_fields=GetSourceFieldsNeededForCopy( exp_dst_url.IsCloudUrl(), copy_helper_opts.skip_unsupported_objects, copy_helper_opts.preserve_acl, preserve_posix=self.preserve_posix_attrs, delete_source=copy_helper_opts.perform_mv, file_size_will_change=self.use_stet)), DestinationInfo(exp_dst_url, have_existing_dst_container)) self.has_file_dst = self.has_file_dst or exp_dst_url.IsFileUrl() self.has_cloud_dst = self.has_cloud_dst or exp_dst_url.IsCloudUrl() self.provider_types.add(exp_dst_url.scheme) self.combined_src_urls = itertools.chain(self.combined_src_urls, src_url_str) yield name_expansion_iterator_dst_tuple # Command entry point. def RunCommand(self): copy_helper_opts = self._ParseOpts() self.total_bytes_transferred = 0 dst_url = StorageUrlFromString(self.args[-1]) if dst_url.IsFileUrl() and (dst_url.object_name == '-' or dst_url.IsFifo()): if self.preserve_posix_attrs: raise CommandException('Cannot preserve POSIX attributes with a ' 'stream or a named pipe.') cat_out_fd = (GetStreamFromFileUrl(dst_url, mode='wb') if dst_url.IsFifo() else None) return cat_helper.CatHelper(self).CatUrlStrings(self.args[:-1], cat_out_fd=cat_out_fd) if copy_helper_opts.read_args_from_stdin: if len(self.args) != 1: raise CommandException('Source URLs cannot be specified with -I option') # Use StdinIteratorCls instead of StdinIterator here to avoid Python 3 # generator pickling errors when multiprocessing a command. src_url_strs = [StdinIteratorCls()] else: if len(self.args) < 2: raise CommandException('Wrong number of arguments for "cp" command.') src_url_strs = [self.args[:-1]] dst_url_strs = [self.args[-1]] self.combined_src_urls = [] self.has_file_dst = False self.has_cloud_dst = False self.provider_types = set() # Because cp may have multiple source URLs and multiple destinations, we # wrap the name expansion iterator in order to collect analytics. name_expansion_iterator = CopyObjectsIterator( self._ConstructNameExpansionIteratorDstTupleIterator( src_url_strs, dst_url_strs), copy_helper_opts.daisy_chain, ) process_count, thread_count = self._GetProcessAndThreadCount( process_count=None, thread_count=None, parallel_operations_override=None, print_macos_warning=False) copy_helper.TriggerReauthForDestinationProviderIfNecessary( dst_url, self.gsutil_api, process_count * thread_count) seek_ahead_iterator = None # Cannot seek ahead with stdin args, since we can only iterate them # once without buffering in memory. if not copy_helper_opts.read_args_from_stdin: seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), self.combined_src_urls, self.recursion_requested or copy_helper_opts.perform_mv, all_versions=self.all_versions, project_id=self.project_id, ignore_symlinks=self.exclude_symlinks, file_size_will_change=self.use_stet) # Use a lock to ensure accurate statistics in the face of # multi-threading/multi-processing. self.stats_lock = parallelism_framework_util.CreateLock() # Tracks if any copies failed. self.op_failure_count = 0 # Start the clock. start_time = time.time() # Tuple of attributes to share/manage across multiple processes in # parallel (-m) mode. shared_attrs = ('op_failure_count', 'total_bytes_transferred') # Perform copy requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(_CopyFuncWrapper, name_expansion_iterator, _CopyExceptionHandler, shared_attrs, fail_on_error=(not self.continue_on_error), seek_ahead_iterator=seek_ahead_iterator) self.logger.debug('total_bytes_transferred: %d', self.total_bytes_transferred) end_time = time.time() self.total_elapsed_time = end_time - start_time self.total_bytes_per_second = CalculateThroughput( self.total_bytes_transferred, self.total_elapsed_time) LogPerformanceSummaryParams( has_file_dst=self.has_file_dst, has_cloud_dst=self.has_cloud_dst, avg_throughput=self.total_bytes_per_second, total_bytes_transferred=self.total_bytes_transferred, total_elapsed_time=self.total_elapsed_time, uses_fan=self.parallel_operations, is_daisy_chain=copy_helper_opts.daisy_chain, provider_types=list(self.provider_types)) if self.debug >= DEBUGLEVEL_DUMP_REQUESTS: # Note that this only counts the actual GET and PUT bytes for the copy # - not any transfers for doing wildcard expansion, the initial # HEAD/GET request performed to get the object metadata, etc. if self.total_bytes_transferred != 0: self.logger.info( 'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)', self.total_bytes_transferred, self.total_elapsed_time, MakeHumanReadable(self.total_bytes_per_second)) if self.op_failure_count: plural_str = 's' if self.op_failure_count > 1 else '' raise CommandException('{count} file{pl}/object{pl} could ' 'not be transferred.'.format( count=self.op_failure_count, pl=plural_str)) return 0 def _ParseOpts(self): # TODO: Arrange variables initialized here in alphabetical order. perform_mv = False # exclude_symlinks is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.exclude_symlinks = False no_clobber = False # continue_on_error is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.continue_on_error = False daisy_chain = False read_args_from_stdin = False print_ver = False use_manifest = False preserve_acl = False self.preserve_posix_attrs = False canned_acl = None # canned_acl is handled by a helper function in parent # Command class, so save in Command state rather than CopyHelperOpts. self.canned = None self.all_versions = False self.skip_unsupported_objects = False # Files matching these extensions should be compressed. # The gzip_encoded flag marks if the files should be compressed during # the upload. The gzip_local flag marks if the files should be compressed # before uploading. Files compressed prior to uploaded are stored # compressed, while files compressed during the upload are stored # uncompressed. These flags cannot be mixed. gzip_encoded = False gzip_local = False gzip_arg_exts = None gzip_arg_all = None test_callback_file = None dest_storage_class = None self.use_stet = False # self.recursion_requested initialized in command.py (so can be checked # in parent class for all commands). self.manifest = None if self.sub_opts: for o, a in self.sub_opts: if o == '-a': canned_acl = a self.canned = True if o == '-A': self.all_versions = True if o == '-c': self.continue_on_error = True elif o == '-D': daisy_chain = True elif o == '-e': self.exclude_symlinks = True elif o == '--testcallbackfile': # File path of a pickled class that implements ProgressCallback.call. # Used for testing transfer interruptions and resumes. test_callback_file = a elif o == '-I': read_args_from_stdin = True elif o == '-j': gzip_encoded = True gzip_arg_exts = [x.strip() for x in a.split(',')] elif o == '-J': gzip_encoded = True gzip_arg_all = GZIP_ALL_FILES elif o == '-L': use_manifest = True self.manifest = Manifest(a) elif o == '-M': # Note that we signal to the cp command to perform a move (copy # followed by remove) and use directory-move naming rules by passing # the undocumented (for internal use) -M option when running the cp # command from mv.py. perform_mv = True elif o == '-n': no_clobber = True elif o == '-p': preserve_acl = True elif o == '-P': self.preserve_posix_attrs = True InitializePreservePosixData() elif o == '-r' or o == '-R': self.recursion_requested = True elif o == '-s': dest_storage_class = NormalizeStorageClass(a) elif o == '-U': self.skip_unsupported_objects = True elif o == '-v': print_ver = True elif o == '-z': gzip_local = True gzip_arg_exts = [x.strip() for x in a.split(',')] elif o == '-Z': gzip_local = True gzip_arg_all = GZIP_ALL_FILES elif o == '--stet': self.use_stet = True if preserve_acl and canned_acl: raise CommandException( 'Specifying both the -p and -a options together is invalid.') if self.all_versions and self.parallel_operations: raise CommandException( 'The gsutil -m option is not supported with the cp -A flag, to ' 'ensure that object version ordering is preserved. Please re-run ' 'the command without the -m option.') if gzip_encoded and gzip_local: raise CommandException( 'Specifying both the -j/-J and -z/-Z options together is invalid.') if gzip_arg_exts and gzip_arg_all: if gzip_encoded: raise CommandException( 'Specifying both the -j and -J options together is invalid.') else: raise CommandException( 'Specifying both the -z and -Z options together is invalid.') self.gzip_exts = gzip_arg_exts or gzip_arg_all self.gzip_encoded = gzip_encoded return CreateCopyHelperOpts( perform_mv=perform_mv, no_clobber=no_clobber, daisy_chain=daisy_chain, read_args_from_stdin=read_args_from_stdin, print_ver=print_ver, use_manifest=use_manifest, preserve_acl=preserve_acl, canned_acl=canned_acl, skip_unsupported_objects=self.skip_unsupported_objects, test_callback_file=test_callback_file, dest_storage_class=dest_storage_class) commands/perfdiag.py 0000644 00000272627 15025013616 0010517 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains the perfdiag gsutil command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import calendar from collections import defaultdict from collections import namedtuple import contextlib import datetime import json import logging import math import multiprocessing import os import random import re import socket import string import subprocess import tempfile import time import boto import boto.gs.connection import six from six.moves import cStringIO from six.moves import http_client from six.moves import range from six.moves import range import gslib from gslib.cloud_api import NotFoundException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import DummyArgChecker from gslib.command_argument import CommandArgument from gslib.commands import config from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.file_part import FilePart from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import text_util from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import ResumableThreshold from gslib.utils.constants import UTF8 from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.cloud_api_helper import GetDownloadSerializationData from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents from gslib.utils.system_util import CheckFreeSpace from gslib.utils.system_util import GetDiskCounters from gslib.utils.system_util import GetFileSize from gslib.utils.system_util import IS_LINUX from gslib.utils.system_util import IsRunningInCiEnvironment from gslib.utils.unit_util import DivideAndCeil from gslib.utils.unit_util import HumanReadableToBytes from gslib.utils.unit_util import MakeBitsHumanReadable from gslib.utils.unit_util import MakeHumanReadable from gslib.utils.unit_util import Percentile _SYNOPSIS = """ gsutil perfdiag [-i in.json] gsutil perfdiag [-o out.json] [-n objects] [-c processes] [-k threads] [-p parallelism type] [-y slices] [-s size] [-d directory] [-t tests] [-j ratio] gs://<bucket_name>... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The ``perfdiag`` command runs a suite of diagnostic tests for a given Cloud Storage bucket. The ``bucket_name`` parameter must name an existing bucket to which the user has write permission. Several test files will be uploaded to and downloaded from this bucket. All test files will be deleted at the completion of the diagnostic if it finishes successfully. For a list of relevant permissions, see `Cloud IAM permissions for gsutil commands <https://cloud.google.com/storage/docs/access-control/iam-gsutil>`_. gsutil performance can be influenced by a number of factors originating at the client, server, or network level. Some examples include the following: + CPU speed + Available memory + The access path to the local disk + Network bandwidth + Contention and error rates along the path between gsutil and Google servers + Operating system buffering configuration + Firewalls and other network elements The `perfdiag` command is provided so that customers can run a known measurement suite when troubleshooting performance problems. <B>PROVIDING DIAGNOSTIC OUTPUT TO THE CLOUD STORAGE TEAM</B> If the Cloud Storage team asks you to run a performance diagnostic please use the following command, and email the output file (output.json) to the @google.com address provided by the Cloud Storage team: gsutil perfdiag -o output.json gs://your-bucket Additional resources for discussing ``perfdiag`` results include the `Stack Overflow tag for Cloud Storage <https://stackoverflow.com/questions/tagged/google-cloud-storage>`_ and the `gsutil GitHub repository <https://github.com/GoogleCloudPlatform/gsutil/issues>`_. <B>OPTIONS</B> -n Sets the number of objects to use when downloading and uploading files during tests. Defaults to 5. -c Sets the number of `processes <https://en.wikipedia.org/wiki/Process_(computing)>`_ to use while running throughput experiments. The default value is 1. -k Sets the number of `threads <https://en.wikipedia.org/wiki/Thread_(computing)>`_ per process to use while running throughput experiments. Each process will receive an equal number of threads. The default value is 1. NOTE: All specified threads and processes will be created, but may not by saturated with work if too few objects (specified with ``-n``) and too few components (specified with ``-y``) are specified. -p Sets the type of parallelism to be used (only applicable when threads or processes are specified and threads * processes > 1). The default is to use ``fan``. Must be one of the following: fan Use one thread per object. This is akin to using gsutil ``-m cp``, with sliced object download / parallel composite upload disabled. slice Use Y (specified with ``-y``) threads for each object, transferring one object at a time. This is akin to using parallel object download / parallel composite upload, without ``-m``. Sliced uploads not supported for s3. both Use Y (specified with ``-y``) threads for each object, transferring multiple objects at a time. This is akin to simultaneously using sliced object download / parallel composite upload and ``gsutil -m cp``. Parallel composite uploads not supported for s3. -y Sets the number of slices to divide each file/object into while transferring data. Only applicable with the slice (or both) parallelism type. The default is 4 slices. -s Sets the size (in bytes) for each of the N (set with ``-n``) objects used in the read and write throughput tests. The default is 1 MiB. This can also be specified using byte suffixes such as 500K or 1M. NOTE: these values are interpreted as multiples of 1024 (K=1024, M=1024*1024, etc.) NOTE: If ``rthru_file`` or ``wthru_file`` are performed, N (set with ``-n``) times as much disk space as specified will be required for the operation. -d Sets the directory to store temporary local files in. If not specified, a default temporary directory will be used. -t Sets the list of diagnostic tests to perform. The default is to run the ``lat``, ``rthru``, and ``wthru`` diagnostic tests. Must be a comma-separated list containing one or more of the following: lat For N (set with ``-n``) objects, write the object, retrieve its metadata, read the object, and finally delete the object. Record the latency of each operation. list Write N (set with ``-n``) objects to the bucket, record how long it takes for the eventually consistent listing call to return the N objects in its result, delete the N objects, then record how long it takes listing to stop returning the N objects. rthru Runs N (set with ``-n``) read operations, with at most C (set with -c) reads outstanding at any given time. rthru_file The same as ``rthru``, but simultaneously writes data to the disk, to gauge the performance impact of the local disk on downloads. wthru Runs N (set with ``-n``) write operations, with at most C (set with ``-c``) writes outstanding at any given time. wthru_file The same as wthru, but simultaneously reads data from the disk, to gauge the performance impact of the local disk on uploads. -m Adds metadata to the result JSON file. Multiple ``-m`` values can be specified. Example: gsutil perfdiag -m "key1:val1" -m "key2:val2" gs://bucketname Each metadata key will be added to the top-level "metadata" dictionary in the output JSON file. -o Writes the results of the diagnostic to an output file. The output is a JSON file containing system information and performance diagnostic results. The file can be read and reported later using the ``-i`` option. -i Reads the JSON output file created using the ``-o`` command and prints a formatted description of the results. -j Applies gzip transport encoding and sets the target compression ratio for the generated test files. This ratio can be an integer between 0 and 100 (inclusive), with 0 generating a file with uniform data, and 100 generating random data. When you specify the ``-j`` option, files being uploaded are compressed in-memory and on-the-wire only. See `cp -j <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_ for specific semantics. <B>MEASURING AVAILABILITY</B> The ``perfdiag`` command ignores the boto num_retries configuration parameter. Instead, it always retries on HTTP errors in the 500 range and keeps track of how many 500 errors were encountered during the test. The availability measurement is reported at the end of the test. Note that HTTP responses are only recorded when the request was made in a single process. When using multiple processes or threads, read and write throughput measurements are performed in an external process, so the availability numbers reported won't include the throughput measurements. <B>NOTE</B> The ``perfdiag`` command runs a series of tests that collects system information, such as the following: + Retrieves requester's IP address. + Executes DNS queries to Google servers and collects the results. + Collects network statistics information from the output of ``netstat -s`` and evaluates the BIOS product name string. + If a proxy server is configured, attempts to connect to it to retrieve the location and storage class of the bucket being used for performance testing. None of this information will be sent to Google unless you proactively choose to send it. """) FileDataTuple = namedtuple('FileDataTuple', 'size md5 data') # Describes one object in a fanned download. If need_to_slice is specified as # True, the object should be downloaded with the slice strategy. Other field # names are the same as documented in PerfDiagCommand.Download. FanDownloadTuple = namedtuple( 'FanDownloadTuple', 'need_to_slice object_name file_name serialization_data') # Describes one slice in a sliced download. # Field names are the same as documented in PerfDiagCommand.Download. SliceDownloadTuple = namedtuple( 'SliceDownloadTuple', 'object_name file_name serialization_data start_byte end_byte') # Describes one file in a fanned upload. If need_to_slice is specified as # True, the file should be uploaded with the slice strategy. Other field # names are the same as documented in PerfDiagCommand.Upload. FanUploadTuple = namedtuple( 'FanUploadTuple', 'need_to_slice file_name object_name use_file gzip_encoded') # Describes one slice in a sliced upload. # Field names are the same as documented in PerfDiagCommand.Upload. SliceUploadTuple = namedtuple( 'SliceUploadTuple', 'file_name object_name use_file file_start file_size gzip_encoded') # Dict storing file_path:FileDataTuple for each temporary file used by # perfdiag. This data should be kept outside of the PerfDiagCommand class # since calls to Apply will make copies of all member data. temp_file_dict = {} class Error(Exception): """Base exception class for this module.""" pass class InvalidArgument(Error): """Raised on invalid arguments to functions.""" pass def _DownloadObject(cls, args, thread_state=None): """Function argument to apply for performing fanned parallel downloads. Args: cls: The calling PerfDiagCommand class instance. args: A FanDownloadTuple object describing this download. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) if args.need_to_slice: cls.PerformSlicedDownload(args.object_name, args.file_name, args.serialization_data) else: cls.Download(args.object_name, gsutil_api, args.file_name, args.serialization_data) def _DownloadSlice(cls, args, thread_state=None): """Function argument to apply for performing sliced downloads. Args: cls: The calling PerfDiagCommand class instance. args: A SliceDownloadTuple object describing this download. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) cls.Download(args.object_name, gsutil_api, args.file_name, args.serialization_data, args.start_byte, args.end_byte) def _UploadObject(cls, args, thread_state=None): """Function argument to apply for performing fanned parallel uploads. Args: cls: The calling PerfDiagCommand class instance. args: A FanUploadTuple object describing this upload. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) if args.need_to_slice: cls.PerformSlicedUpload(args.file_name, args.object_name, args.use_file, gsutil_api, gzip_encoded=args.gzip_encoded) else: cls.Upload(args.file_name, args.object_name, gsutil_api, args.use_file, gzip_encoded=args.gzip_encoded) def _UploadSlice(cls, args, thread_state=None): """Function argument to apply for performing sliced parallel uploads. Args: cls: The calling PerfDiagCommand class instance. args: A SliceUploadTuple object describing this upload. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) cls.Upload(args.file_name, args.object_name, gsutil_api, args.use_file, args.file_start, args.file_size, gzip_encoded=args.gzip_encoded) def _DeleteWrapper(cls, object_name, thread_state=None): """Function argument to apply for performing parallel object deletions. Args: cls: The calling PerfDiagCommand class instance. object_name: The object name to delete from the test bucket. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) cls.Delete(object_name, gsutil_api) def _PerfdiagExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) def _DummyTrackerCallback(_): pass class DummyFile(object): """A dummy, file-like object that throws away everything written to it.""" # Because Python2 is so loose about bytes and text, Python3 code really # works best with a hint about a file object's mode. mode = 'bw' def write(self, *args, **kwargs): # pylint: disable=invalid-name pass def close(self): # pylint: disable=invalid-name pass def _GenerateFileData(fp, file_size=0, random_ratio=100, max_unique_random_bytes=5242883): """Writes data into a file like object. Args: fp: A file like object to write the data to. file_size: The amount of data to write to the file. random_ratio: The percentage of randomly generated data to write. This can be any number between 0 and 100 (inclusive), with 0 producing uniform data, and 100 producing random data. max_unique_random_bytes: The maximum number of bytes to generate pseudo-randomly before beginning to repeat bytes. The default was chosen as the next prime larger than 5 MiB. """ # Normalize the ratio. random_ratio /= 100.0 random_bytes = os.urandom(min(file_size, max_unique_random_bytes)) total_bytes_written = 0 while total_bytes_written < file_size: num_bytes = min(max_unique_random_bytes, file_size - total_bytes_written) # Calculate the amount of random and sequential data to write and # resolve rounding errors by adjusting the random bytes to write. num_bytes_seq = int(num_bytes * (1 - random_ratio)) num_bytes_random = num_bytes - num_bytes_seq fp.write(random_bytes[:num_bytes_random]) fp.write(b'x' * num_bytes_seq) total_bytes_written += num_bytes # Many functions in perfdiag re-define a temporary function based on a # variable from a loop, resulting in a false positive from the linter. # pylint: disable=cell-var-from-loop class PerfDiagCommand(Command): """Implementation of gsutil perfdiag command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'perfdiag', command_name_aliases=['diag', 'diagnostic', 'perf', 'performance'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=1, supported_sub_args='n:c:k:p:y:s:d:t:m:i:o:j:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeNCloudBucketURLsArgument(1)]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='perfdiag', help_name_aliases=[], help_type='command_help', help_one_line_summary='Run performance diagnostic', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) # Byte sizes to use for latency testing files. # TODO: Consider letting the user specify these sizes with a configuration # parameter. test_lat_file_sizes = ( 0, # 0 bytes 1024, # 1 KiB 102400, # 100 KiB 1048576, # 1 MiB ) # Test names. RTHRU = 'rthru' RTHRU_FILE = 'rthru_file' WTHRU = 'wthru' WTHRU_FILE = 'wthru_file' LAT = 'lat' LIST = 'list' # Parallelism strategies. FAN = 'fan' SLICE = 'slice' BOTH = 'both' # List of all diagnostic tests. ALL_DIAG_TESTS = (RTHRU, RTHRU_FILE, WTHRU, WTHRU_FILE, LAT, LIST) # List of diagnostic tests to run by default. DEFAULT_DIAG_TESTS = (RTHRU, WTHRU, LAT) # List of parallelism strategies. PARALLEL_STRATEGIES = (FAN, SLICE, BOTH) # Google Cloud Storage XML API endpoint host. XML_API_HOST = boto.config.get('Credentials', 'gs_host', boto.gs.connection.GSConnection.DefaultHost) # Google Cloud Storage XML API endpoint port. XML_API_PORT = boto.config.getint('Credentials', 'gs_port', 80) # Maximum number of times to retry requests on 5xx errors. MAX_SERVER_ERROR_RETRIES = 5 # Maximum number of times to retry requests on more serious errors like # the socket breaking. MAX_TOTAL_RETRIES = 10 # The default buffer size in boto's Key object is set to 8 KiB. This becomes a # bottleneck at high throughput rates, so we increase it. KEY_BUFFER_SIZE = 16384 # Maximum amount of time, in seconds, we will wait for object listings to # reflect what we expect in the listing tests. MAX_LISTING_WAIT_TIME = 60.0 def _Exec(self, cmd, raise_on_error=True, return_output=False, mute_stderr=False): """Executes a command in a subprocess. Args: cmd: List containing the command to execute. raise_on_error: Whether or not to raise an exception when a process exits with a non-zero return code. return_output: If set to True, the return value of the function is the stdout of the process. mute_stderr: If set to True, the stderr of the process is not printed to the console. Returns: The return code of the process or the stdout if return_output is set. Raises: Exception: If raise_on_error is set to True and any process exits with a non-zero return code. """ self.logger.debug('Running command: %s', cmd) stderr = subprocess.PIPE if mute_stderr else None p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr) (stdoutdata, _) = p.communicate() if six.PY3: if isinstance(stdoutdata, bytes): stdoutdata = stdoutdata.decode(UTF8) if raise_on_error and p.returncode: raise CommandException("Received non-zero return code (%d) from " "subprocess '%s'." % (p.returncode, ' '.join(cmd))) return stdoutdata if return_output else p.returncode def _WarnIfLargeData(self): """Outputs a warning message if a large amount of data is being used.""" if self.num_objects * self.thru_filesize > HumanReadableToBytes('2GiB'): self.logger.info('This is a large operation, and could take a while.') def _MakeTempFile(self, file_size=0, mem_metadata=False, mem_data=False, prefix='gsutil_test_file', random_ratio=100): """Creates a temporary file of the given size and returns its path. Args: file_size: The size of the temporary file to create. mem_metadata: If true, store md5 and file size in memory at temp_file_dict[fpath].md5, tempfile_data[fpath].file_size. mem_data: If true, store the file data in memory at temp_file_dict[fpath].data prefix: The prefix to use for the temporary file. Defaults to gsutil_test_file. random_ratio: The percentage of randomly generated data to write. This can be any number between 0 and 100 (inclusive), with 0 producing uniform data, and 100 producing random data. Returns: The file path of the created temporary file. """ fd, fpath = tempfile.mkstemp(suffix='.bin', prefix=prefix, dir=self.directory, text=False) with os.fdopen(fd, 'wb') as fp: _GenerateFileData(fp, file_size, random_ratio) if mem_metadata or mem_data: with open(fpath, 'rb') as fp: file_size = GetFileSize(fp) if mem_metadata else None md5 = CalculateB64EncodedMd5FromContents(fp) if mem_metadata else None data = fp.read() if mem_data else None temp_file_dict[fpath] = FileDataTuple(file_size, md5, data) self.temporary_files.add(fpath) return fpath def _SetUp(self): """Performs setup operations needed before diagnostics can be run.""" # Stores test result data. self.results = {} # Set of file paths for local temporary files. self.temporary_files = set() # Set of names for test objects that exist in the test bucket. self.temporary_objects = set() # Total number of HTTP requests made. self.total_requests = 0 # Total number of HTTP 5xx errors. self.request_errors = 0 # Number of responses, keyed by response code. self.error_responses_by_code = defaultdict(int) # Total number of socket errors. self.connection_breaks = 0 # Boolean to prevent doing cleanup twice. self.teardown_completed = False # Create files for latency test. if self.LAT in self.diag_tests: self.latency_files = [] for file_size in self.test_lat_file_sizes: fpath = self._MakeTempFile(file_size, mem_metadata=True, mem_data=True) self.latency_files.append(fpath) # Create files for throughput tests. if self.diag_tests.intersection( (self.RTHRU, self.WTHRU, self.RTHRU_FILE, self.WTHRU_FILE)): # Create a file for warming up the TCP connection. self.tcp_warmup_file = self._MakeTempFile(5 * 1024 * 1024, mem_metadata=True, mem_data=True) # For in memory tests, throughput tests transfer the same object N times # instead of creating N objects, in order to avoid excessive memory usage. if self.diag_tests.intersection((self.RTHRU, self.WTHRU)): self.mem_thru_file_name = self._MakeTempFile( self.thru_filesize, mem_metadata=True, mem_data=True, random_ratio=self.gzip_compression_ratio) self.mem_thru_object_name = os.path.basename(self.mem_thru_file_name) # For tests that use disk I/O, it is necessary to create N objects in # in order to properly measure the performance impact of seeks. if self.diag_tests.intersection((self.RTHRU_FILE, self.WTHRU_FILE)): # List of file names and corresponding object names to use for file # throughput tests. self.thru_file_names = [] self.thru_object_names = [] free_disk_space = CheckFreeSpace(self.directory) if free_disk_space >= self.thru_filesize * self.num_objects: self.logger.info( '\nCreating %d local files each of size %s.' % (self.num_objects, MakeHumanReadable(self.thru_filesize))) self._WarnIfLargeData() for _ in range(self.num_objects): file_name = self._MakeTempFile( self.thru_filesize, mem_metadata=True, random_ratio=self.gzip_compression_ratio) self.thru_file_names.append(file_name) self.thru_object_names.append(os.path.basename(file_name)) else: raise CommandException( 'Not enough free disk space for throughput files: ' '%s of disk space required, but only %s available.' % (MakeHumanReadable(self.thru_filesize * self.num_objects), MakeHumanReadable(free_disk_space))) # Dummy file buffer to use for downloading that goes nowhere. self.discard_sink = DummyFile() # Filter out misleading progress callback output and the incorrect # suggestion to use gsutil -m perfdiag. self.logger.addFilter(self._PerfdiagFilter()) def _TearDown(self): """Performs operations to clean things up after performing diagnostics.""" if not self.teardown_completed: temp_file_dict.clear() try: for fpath in self.temporary_files: os.remove(fpath) if self.delete_directory: os.rmdir(self.directory) except OSError: pass # I usually hate leaving commented out code in commits, but this is a # rare exception. This commented out code attempted to clean up the # files created by the perfdiag command. If there is only one process and # thread, it worked by serializing the deletes. If it had more than one # process (multiple threads on a single process seemed to work fine, but # don't quote me on that), it tried to fire up a multi-processing delete # that used the existing process pool created during the previous part of # the perfdiag command execution. This randomly fails in Python 3. So I # have left this code here, commented out, and have replaced it with the # sequential delete part only. That this fails in Python 3 will probably # present as other problems later. At least a multiprocessing rm using # gsutil -m appears to always work, so it is something to do with this # usage of parallel override and existing process pools, I think. # PS - only fails when using prefer XML. # if self.threads > 1 or self.processes > 1: # args = [obj for obj in self.temporary_objects] # self.Apply( # _DeleteWrapper, args, _PerfdiagExceptionHandler, # arg_checker=DummyArgChecker, # parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, # process_count=self.processes, thread_count=self.threads) # else: # for object_name in self.temporary_objects: # self.Delete(object_name, self.gsutil_api) for object_name in self.temporary_objects: self.Delete(object_name, self.gsutil_api) self.teardown_completed = True @contextlib.contextmanager def _Time(self, key, bucket): """A context manager that measures time. A context manager that prints a status message before and after executing the inner command and times how long the inner command takes. Keeps track of the timing, aggregated by the given key. Args: key: The key to insert the timing value into a dictionary bucket. bucket: A dictionary to place the timing value in. Yields: For the context manager. """ self.logger.info('%s starting...', key) t0 = time.time() yield t1 = time.time() bucket[key].append(t1 - t0) self.logger.info('%s done.', key) def _RunOperation(self, func): """Runs an operation with retry logic. Args: func: The function to run. Returns: True if the operation succeeds, False if aborted. """ # We retry on httplib exceptions that can happen if the socket was closed # by the remote party or the connection broke because of network issues. # Only the BotoServerError is counted as a 5xx error towards the retry # limit. success = False server_error_retried = 0 total_retried = 0 i = 0 return_val = None while not success: next_sleep = min(random.random() * (2**i) + 1, GetMaxRetryDelay()) try: return_val = func() self.total_requests += 1 success = True except tuple(self.exceptions) as e: total_retried += 1 if total_retried > self.MAX_TOTAL_RETRIES: self.logger.info('Reached maximum total retries. Not retrying.') break if isinstance(e, ServiceException): if e.status >= 500: self.error_responses_by_code[e.status] += 1 self.total_requests += 1 self.request_errors += 1 server_error_retried += 1 time.sleep(next_sleep) else: raise if server_error_retried > self.MAX_SERVER_ERROR_RETRIES: self.logger.info( 'Reached maximum server error retries. Not retrying.') break else: self.connection_breaks += 1 return return_val def _RunLatencyTests(self): """Runs latency tests.""" # Stores timing information for each category of operation. self.results['latency'] = defaultdict(list) for i in range(self.num_objects): self.logger.info('\nRunning latency iteration %d...', i + 1) for fpath in self.latency_files: file_data = temp_file_dict[fpath] url = self.bucket_url.Clone() url.object_name = os.path.basename(fpath) file_size = file_data.size readable_file_size = MakeHumanReadable(file_size) self.logger.info( "\nFile of size %s located on disk at '%s' being diagnosed in the " "cloud at '%s'.", readable_file_size, fpath, url) upload_target = StorageUrlToUploadObjectMetadata(url) def _Upload(): io_fp = six.BytesIO(file_data.data) with self._Time('UPLOAD_%d' % file_size, self.results['latency']): self.gsutil_api.UploadObject(io_fp, upload_target, size=file_size, provider=self.provider, fields=['name']) self._RunOperation(_Upload) def _Metadata(): with self._Time('METADATA_%d' % file_size, self.results['latency']): return self.gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, provider=self.provider, fields=['name', 'contentType', 'mediaLink', 'size']) # Download will get the metadata first if we don't pass it in. download_metadata = self._RunOperation(_Metadata) serialization_data = GetDownloadSerializationData(download_metadata) def _Download(): with self._Time('DOWNLOAD_%d' % file_size, self.results['latency']): self.gsutil_api.GetObjectMedia( url.bucket_name, url.object_name, self.discard_sink, provider=self.provider, serialization_data=serialization_data) self._RunOperation(_Download) def _Delete(): with self._Time('DELETE_%d' % file_size, self.results['latency']): self.gsutil_api.DeleteObject(url.bucket_name, url.object_name, provider=self.provider) self._RunOperation(_Delete) class _PerfdiagFilter(logging.Filter): def filter(self, record): # Used to prevent unnecessary output when using multiprocessing. msg = record.getMessage() return not (('Copying file:///' in msg) or ('Copying gs://' in msg) or ('Computing CRC' in msg) or ('gsutil -m perfdiag' in msg)) def _PerfdiagExceptionHandler(self, e): """Simple exception handler to allow post-completion status.""" self.logger.error(str(e)) def PerformFannedDownload(self, need_to_slice, object_names, file_names, serialization_data): """Performs a parallel download of multiple objects using the fan strategy. Args: need_to_slice: If True, additionally apply the slice strategy to each object in object_names. object_names: A list of object names to be downloaded. Each object must already exist in the test bucket. file_names: A list, corresponding by index to object_names, of file names for downloaded data. If None, discard downloaded data. serialization_data: A list, corresponding by index to object_names, of serialization data for each object. """ args = [] for i in range(len(object_names)): file_name = file_names[i] if file_names else None args.append( FanDownloadTuple(need_to_slice, object_names[i], file_name, serialization_data[i])) self.Apply( _DownloadObject, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def PerformSlicedDownload(self, object_name, file_name, serialization_data): """Performs a download of an object using the slice strategy. Args: object_name: The name of the object to download. file_name: The name of the file to download data to, or None if data should be discarded. serialization_data: The serialization data for the object. """ if file_name: with open(file_name, 'ab') as fp: fp.truncate(self.thru_filesize) component_size = DivideAndCeil(self.thru_filesize, self.num_slices) args = [] for i in range(self.num_slices): start_byte = i * component_size end_byte = min((i + 1) * (component_size) - 1, self.thru_filesize - 1) args.append( SliceDownloadTuple(object_name, file_name, serialization_data, start_byte, end_byte)) self.Apply( _DownloadSlice, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def PerformFannedUpload(self, need_to_slice, file_names, object_names, use_file, gzip_encoded=False): """Performs a parallel upload of multiple files using the fan strategy. The metadata for file_name should be present in temp_file_dict prior to calling. Also, the data for file_name should be present in temp_file_dict if use_file is specified as False. Args: need_to_slice: If True, additionally apply the slice strategy to each file in file_names. file_names: A list of file names to be uploaded. object_names: A list, corresponding by by index to file_names, of object names to upload data to. use_file: If true, use disk I/O, otherwise read upload data from memory. gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. """ args = [] for i in range(len(file_names)): args.append( FanUploadTuple(need_to_slice, file_names[i], object_names[i], use_file, gzip_encoded)) self.Apply( _UploadObject, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def PerformSlicedUpload(self, file_name, object_name, use_file, gsutil_api, gzip_encoded=False): """Performs a parallel upload of a file using the slice strategy. The metadata for file_name should be present in temp_file_dict prior to calling. Also, the data from for file_name should be present in temp_file_dict if use_file is specified as False. Args: file_name: The name of the file to upload. object_name: The name of the object to upload to. use_file: If true, use disk I/O, otherwise read upload data from memory. gsutil_api: CloudApi instance to use for operations in this thread. gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a semaphore is used to limit resource usage. """ # Divide the file into components. component_size = DivideAndCeil(self.thru_filesize, self.num_slices) component_object_names = ([ object_name + str(i) for i in range(self.num_slices) ]) args = [] for i in range(self.num_slices): component_start = i * component_size component_size = min(component_size, temp_file_dict[file_name].size - component_start) args.append( SliceUploadTuple(file_name, component_object_names[i], use_file, component_start, component_size, gzip_encoded)) # Upload the components in parallel. try: self.Apply( _UploadSlice, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) # Compose the components into an object. request_components = [] for i in range(self.num_slices): src_obj_metadata = ( apitools_messages.ComposeRequest.SourceObjectsValueListEntry( name=component_object_names[i])) request_components.append(src_obj_metadata) def _Compose(): dst_obj_metadata = apitools_messages.Object() dst_obj_metadata.name = object_name dst_obj_metadata.bucket = self.bucket_url.bucket_name gsutil_api.ComposeObject(request_components, dst_obj_metadata, provider=self.provider) self._RunOperation(_Compose) finally: # Delete the temporary components. self.Apply( _DeleteWrapper, component_object_names, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def _RunReadThruTests(self, use_file=False): """Runs read throughput tests.""" test_name = 'read_throughput_file' if use_file else 'read_throughput' file_io_string = 'with file I/O' if use_file else '' self.logger.info( '\nRunning read throughput tests %s (%s objects of size %s)' % (file_io_string, self.num_objects, MakeHumanReadable( self.thru_filesize))) self._WarnIfLargeData() self.results[test_name] = { 'file_size': self.thru_filesize, 'processes': self.processes, 'threads': self.threads, 'parallelism': self.parallel_strategy } # Copy the file(s) to the test bucket, and also get the serialization data # so that we can pass it to download. if use_file: # For test with file I/O use N files on disk to preserve seek performance. file_names = self.thru_file_names object_names = self.thru_object_names serialization_data = [] for i in range(self.num_objects): self.temporary_objects.add(self.thru_object_names[i]) if self.WTHRU_FILE in self.diag_tests: # If we ran the WTHRU_FILE test, then the objects already exist. obj_metadata = self.gsutil_api.GetObjectMetadata( self.bucket_url.bucket_name, self.thru_object_names[i], fields=['size', 'mediaLink'], provider=self.bucket_url.scheme) else: obj_metadata = self.Upload(self.thru_file_names[i], self.thru_object_names[i], self.gsutil_api, use_file) # File overwrite causes performance issues with sliced downloads. # Delete the file and reopen it for download. This matches what a real # download would look like. os.unlink(self.thru_file_names[i]) open(self.thru_file_names[i], 'ab').close() serialization_data.append(GetDownloadSerializationData(obj_metadata)) else: # For in-memory test only use one file but copy it num_objects times, to # allow scalability in num_objects. self.temporary_objects.add(self.mem_thru_object_name) obj_metadata = self.Upload(self.mem_thru_file_name, self.mem_thru_object_name, self.gsutil_api, use_file) file_names = None object_names = [self.mem_thru_object_name] * self.num_objects serialization_data = ([GetDownloadSerializationData(obj_metadata)] * self.num_objects) # Warmup the TCP connection. warmup_obj_name = os.path.basename(self.tcp_warmup_file) self.temporary_objects.add(warmup_obj_name) self.Upload(self.tcp_warmup_file, warmup_obj_name, self.gsutil_api) self.Download(warmup_obj_name, self.gsutil_api) t0 = time.time() if self.processes == 1 and self.threads == 1: for i in range(self.num_objects): file_name = file_names[i] if use_file else None self.Download(object_names[i], self.gsutil_api, file_name, serialization_data[i]) else: if self.parallel_strategy in (self.FAN, self.BOTH): need_to_slice = (self.parallel_strategy == self.BOTH) self.PerformFannedDownload(need_to_slice, object_names, file_names, serialization_data) elif self.parallel_strategy == self.SLICE: for i in range(self.num_objects): file_name = file_names[i] if use_file else None self.PerformSlicedDownload(object_names[i], file_name, serialization_data[i]) t1 = time.time() time_took = t1 - t0 total_bytes_copied = self.thru_filesize * self.num_objects bytes_per_second = total_bytes_copied / time_took self.results[test_name]['time_took'] = time_took self.results[test_name]['total_bytes_copied'] = total_bytes_copied self.results[test_name]['bytes_per_second'] = bytes_per_second def _RunWriteThruTests(self, use_file=False): """Runs write throughput tests.""" test_name = 'write_throughput_file' if use_file else 'write_throughput' file_io_string = 'with file I/O' if use_file else '' self.logger.info( '\nRunning write throughput tests %s (%s objects of size %s)' % (file_io_string, self.num_objects, MakeHumanReadable( self.thru_filesize))) self._WarnIfLargeData() self.results[test_name] = { 'file_size': self.thru_filesize, 'processes': self.processes, 'threads': self.threads, 'parallelism': self.parallel_strategy, 'gzip_encoded_writes': self.gzip_encoded_writes, 'gzip_compression_ratio': self.gzip_compression_ratio } # Warmup the TCP connection. warmup_obj_name = os.path.basename(self.tcp_warmup_file) self.temporary_objects.add(warmup_obj_name) self.Upload(self.tcp_warmup_file, warmup_obj_name, self.gsutil_api) if use_file: # For test with file I/O use N files on disk to preserve seek performance. file_names = self.thru_file_names object_names = self.thru_object_names else: # For in-memory test only use one file but copy it num_objects times, to # allow for scalability in num_objects. file_names = [self.mem_thru_file_name] * self.num_objects object_names = ([ self.mem_thru_object_name + str(i) for i in range(self.num_objects) ]) for object_name in object_names: self.temporary_objects.add(object_name) t0 = time.time() if self.processes == 1 and self.threads == 1: for i in range(self.num_objects): self.Upload(file_names[i], object_names[i], self.gsutil_api, use_file, gzip_encoded=self.gzip_encoded_writes) else: if self.parallel_strategy in (self.FAN, self.BOTH): need_to_slice = (self.parallel_strategy == self.BOTH) self.PerformFannedUpload(need_to_slice, file_names, object_names, use_file, gzip_encoded=self.gzip_encoded_writes) elif self.parallel_strategy == self.SLICE: for i in range(self.num_objects): self.PerformSlicedUpload(file_names[i], object_names[i], use_file, self.gsutil_api, gzip_encoded=self.gzip_encoded_writes) t1 = time.time() time_took = t1 - t0 total_bytes_copied = self.thru_filesize * self.num_objects bytes_per_second = total_bytes_copied / time_took self.results[test_name]['time_took'] = time_took self.results[test_name]['total_bytes_copied'] = total_bytes_copied self.results[test_name]['bytes_per_second'] = bytes_per_second def _RunListTests(self): """Runs eventual consistency listing latency tests.""" self.results['listing'] = {'num_files': self.num_objects} # Generate N random objects to put into the bucket. list_objects = [] args = [] # Differentiate objects created by each perfdiag execution so that leftovers # from a previous run (if perfdiag could not exit gracefully and delete # them) do not affect this run. random_id = ''.join( [random.choice(string.ascii_lowercase) for _ in range(10)]) list_prefix = 'gsutil-perfdiag-list-' + random_id + '-' for _ in range(self.num_objects): fpath = self._MakeTempFile(0, mem_data=True, mem_metadata=True, prefix=list_prefix) object_name = os.path.basename(fpath) list_objects.append(object_name) args.append(FanUploadTuple(False, fpath, object_name, False, False)) self.temporary_objects.add(object_name) # Add the objects to the bucket. self.logger.info('\nWriting %s objects for listing test...', self.num_objects) self.Apply(_UploadObject, args, _PerfdiagExceptionHandler, arg_checker=DummyArgChecker) list_latencies = [] files_seen = [] total_start_time = time.time() expected_objects = set(list_objects) found_objects = set() def _List(): """Lists and returns objects in the bucket. Also records latency.""" t0 = time.time() objects = list( self.gsutil_api.ListObjects(self.bucket_url.bucket_name, prefix=list_prefix, delimiter='/', provider=self.provider, fields=['items/name'])) if len(objects) > self.num_objects: self.logger.warning( 'Listing produced more than the expected %d object(s).', self.num_objects) t1 = time.time() list_latencies.append(t1 - t0) return set([obj.data.name for obj in objects]) def _ListAfterUpload(): names = _List() found_objects.update(names & expected_objects) files_seen.append(len(found_objects)) self.logger.info('Listing bucket %s waiting for %s objects to appear...', self.bucket_url.bucket_name, self.num_objects) while expected_objects - found_objects: self._RunOperation(_ListAfterUpload) if expected_objects - found_objects: if time.time() - total_start_time > self.MAX_LISTING_WAIT_TIME: self.logger.warning('Maximum time reached waiting for listing.') break total_end_time = time.time() self.results['listing']['insert'] = { 'num_listing_calls': len(list_latencies), 'list_latencies': list_latencies, 'files_seen_after_listing': files_seen, 'time_took': total_end_time - total_start_time, } args = [object_name for object_name in list_objects] self.logger.info('Deleting %s objects for listing test...', self.num_objects) self.Apply(_DeleteWrapper, args, _PerfdiagExceptionHandler, arg_checker=DummyArgChecker) self.logger.info('Listing bucket %s waiting for %s objects to disappear...', self.bucket_url.bucket_name, self.num_objects) list_latencies = [] files_seen = [] total_start_time = time.time() found_objects = set(list_objects) while found_objects: def _ListAfterDelete(): names = _List() found_objects.intersection_update(names) files_seen.append(len(found_objects)) self._RunOperation(_ListAfterDelete) if found_objects: if time.time() - total_start_time > self.MAX_LISTING_WAIT_TIME: self.logger.warning('Maximum time reached waiting for listing.') break total_end_time = time.time() self.results['listing']['delete'] = { 'num_listing_calls': len(list_latencies), 'list_latencies': list_latencies, 'files_seen_after_listing': files_seen, 'time_took': total_end_time - total_start_time, } def Upload(self, file_name, object_name, gsutil_api, use_file=False, file_start=0, file_size=None, gzip_encoded=False): """Performs an upload to the test bucket. The file is uploaded to the bucket referred to by self.bucket_url, and has name object_name. Args: file_name: The path to the local file, and the key to its entry in temp_file_dict. object_name: The name of the remote object. gsutil_api: Cloud API instance to use for this upload. use_file: If true, use disk I/O, otherwise read everything from memory. file_start: The first byte in the file to upload to the object. (only should be specified for sliced uploads) file_size: The size of the file to upload. (only should be specified for sliced uploads) gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. Returns: Uploaded Object Metadata. """ fp = None if file_size is None: file_size = temp_file_dict[file_name].size upload_url = self.bucket_url.Clone() upload_url.object_name = object_name upload_target = StorageUrlToUploadObjectMetadata(upload_url) try: if use_file: fp = FilePart(file_name, file_start, file_size) else: data = temp_file_dict[file_name].data[file_start:file_start + file_size] fp = six.BytesIO(data) def _InnerUpload(): if file_size < ResumableThreshold(): return gsutil_api.UploadObject(fp, upload_target, provider=self.provider, size=file_size, fields=['name', 'mediaLink', 'size'], gzip_encoded=gzip_encoded) else: return gsutil_api.UploadObjectResumable( fp, upload_target, provider=self.provider, size=file_size, fields=['name', 'mediaLink', 'size'], tracker_callback=_DummyTrackerCallback, gzip_encoded=gzip_encoded) # Because perfdiag does not go through copy_helper, it misses the # resource lock. Here, if the upload is compressed, the lock is applied # manually. def _InnerUploadLock(): with gslib.command.concurrent_compressed_upload_lock: return _InnerUpload() upload_func = _InnerUploadLock if gzip_encoded else _InnerUpload return self._RunOperation(upload_func) finally: if fp: fp.close() def Download(self, object_name, gsutil_api, file_name=None, serialization_data=None, start_byte=0, end_byte=None): """Downloads an object from the test bucket. Args: object_name: The name of the object (in the test bucket) to download. gsutil_api: CloudApi instance to use for the download. file_name: Optional file name to write downloaded data to. If None, downloaded data is discarded immediately. serialization_data: Optional serialization data, used so that we don't have to get the metadata before downloading. start_byte: The first byte in the object to download. (only should be specified for sliced downloads) end_byte: The last byte in the object to download. (only should be specified for sliced downloads) """ fp = None try: if file_name is not None: fp = open(file_name, 'r+b') fp.seek(start_byte) else: fp = self.discard_sink def _InnerDownload(): gsutil_api.GetObjectMedia(self.bucket_url.bucket_name, object_name, fp, provider=self.provider, start_byte=start_byte, end_byte=end_byte, serialization_data=serialization_data) self._RunOperation(_InnerDownload) finally: if fp: fp.close() def Delete(self, object_name, gsutil_api): """Deletes an object from the test bucket. Args: object_name: The name of the object to delete. gsutil_api: Cloud API instance to use for this delete. """ try: def _InnerDelete(): gsutil_api.DeleteObject(self.bucket_url.bucket_name, object_name, provider=self.provider) self._RunOperation(_InnerDelete) except NotFoundException: pass def _GetDiskCounters(self): """Retrieves disk I/O statistics for all disks. Adapted from the psutil module's psutil._pslinux.disk_io_counters: http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py Originally distributed under under a BSD license. Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola. Returns: A dictionary containing disk names mapped to the disk counters from /disk/diskstats. """ # iostat documentation states that sectors are equivalent with blocks and # have a size of 512 bytes since 2.4 kernels. This value is needed to # calculate the amount of disk I/O in bytes. sector_size = 512 partitions = [] with open('/proc/partitions', 'r') as f: lines = f.readlines()[2:] for line in lines: _, _, _, name = line.split() if name[-1].isdigit(): partitions.append(name) retdict = {} with open('/proc/diskstats', 'r') as f: for line in f: values = line.split()[:11] _, _, name, reads, _, rbytes, rtime, writes, _, wbytes, wtime = values if name in partitions: rbytes = int(rbytes) * sector_size wbytes = int(wbytes) * sector_size reads = int(reads) writes = int(writes) rtime = int(rtime) wtime = int(wtime) retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime) return retdict def _GetTcpStats(self): """Tries to parse out TCP packet information from netstat output. Returns: A dictionary containing TCP information, or None if netstat is not available. """ # netstat return code is non-zero for -s on Linux, so don't raise on error. try: netstat_output = self._Exec(['netstat', '-s'], return_output=True, raise_on_error=False) except OSError: self.logger.warning('netstat not found on your system; some measurement ' 'data will be missing') return None netstat_output = netstat_output.strip().lower() found_tcp = False tcp_retransmit = None tcp_received = None tcp_sent = None for line in netstat_output.split('\n'): # Header for TCP section is "Tcp:" in Linux/Mac and # "TCP Statistics for" in Windows. if 'tcp:' in line or 'tcp statistics' in line: found_tcp = True # Linux == "segments retransmited" (sic), Mac == "retransmit timeouts" # Windows == "segments retransmitted". if (found_tcp and tcp_retransmit is None and ('segments retransmited' in line or 'retransmit timeouts' in line or 'segments retransmitted' in line)): tcp_retransmit = ''.join(c for c in line if c in string.digits) # Linux+Windows == "segments received", Mac == "packets received". if (found_tcp and tcp_received is None and ('segments received' in line or 'packets received' in line)): tcp_received = ''.join(c for c in line if c in string.digits) # Linux == "segments send out" (sic), Mac+Windows == "packets sent". if (found_tcp and tcp_sent is None and ('segments send out' in line or 'packets sent' in line or 'segments sent' in line)): tcp_sent = ''.join(c for c in line if c in string.digits) result = {} try: result['tcp_retransmit'] = int(tcp_retransmit) result['tcp_received'] = int(tcp_received) result['tcp_sent'] = int(tcp_sent) except (ValueError, TypeError): result['tcp_retransmit'] = None result['tcp_received'] = None result['tcp_sent'] = None return result def _CollectSysInfo(self): """Collects system information.""" sysinfo = {} # All exceptions that might be raised from socket module calls. socket_errors = (socket.error, socket.herror, socket.gaierror, socket.timeout) # Find out whether HTTPS is enabled in Boto. sysinfo['boto_https_enabled'] = boto.config.get('Boto', 'is_secure', True) # Look up proxy info. proxy_host = boto.config.get('Boto', 'proxy', None) proxy_port = boto.config.getint('Boto', 'proxy_port', 0) sysinfo['using_proxy'] = bool(proxy_host) if boto.config.get('Boto', 'proxy_rdns', True if proxy_host else False): self.logger.info( 'DNS lookups are disallowed in this environment, so some information ' 'is not included in this perfdiag run. To allow local DNS lookups ' 'while using a proxy, set proxy_rdns to False in your boto file.') # Get the local IP address from socket lib. try: sysinfo['ip_address'] = socket.gethostbyname(socket.gethostname()) except socket_errors: sysinfo['ip_address'] = '' # Record the temporary directory used since it can affect performance, e.g. # when on a networked filesystem. sysinfo['tempdir'] = self.directory # Produces an RFC 2822 compliant GMT timestamp. sysinfo['gmt_timestamp'] = time.strftime('%a, %d %b %Y %H:%M:%S +0000', time.gmtime()) # Execute a CNAME lookup on Google DNS to find what Google server # it's routing to. cmd = ['nslookup', '-type=CNAME', self.XML_API_HOST] try: nslookup_cname_output = self._Exec(cmd, return_output=True, mute_stderr=True) m = re.search(r' = (?P<googserv>[^.]+)\.', nslookup_cname_output) sysinfo['googserv_route'] = m.group('googserv') if m else None except (CommandException, OSError): sysinfo['googserv_route'] = '' # Check the BIOS product name, to determine if we're running on a GCE # instance. # TODO: Add support for checking if we're on GCE from Windows, via WMI. if IS_LINUX: try: with open('/sys/class/dmi/id/product_name', 'r') as fp: sysinfo['on_gce'] = (fp.readline() == 'Google Compute Engine\n') except OSError: pass # If we're running on GCE include details about the instance. if sysinfo.get('on_gce', False): hostname = socket.gethostname() cmd = ['gcloud', 'compute', 'instances', 'list', '--filter=', hostname] try: # "gcloud compute" commands will (hopefully) fail on these envs due to # lack of credentials/permissions to access compute resources. mute_stderr = IsRunningInCiEnvironment() sysinfo['gce_instance_info'] = (self._Exec(cmd, return_output=True, mute_stderr=mute_stderr)) except (CommandException, OSError): sysinfo['gce_instance_info'] = '' # Record info about location and storage class of the bucket being used for # performance testing. bucket_info = self.gsutil_api.GetBucket(self.bucket_url.bucket_name, fields=['location', 'storageClass'], provider=self.bucket_url.scheme) sysinfo['bucket_location'] = bucket_info.location sysinfo['bucket_storageClass'] = bucket_info.storageClass # Try to determine the latency of a DNS lookup for the Google hostname # endpoint. Note: we don't piggyback on gethostbyname_ex below because # the _ex version requires an extra RTT. try: t0 = time.time() socket.gethostbyname(self.XML_API_HOST) t1 = time.time() sysinfo['google_host_dns_latency'] = t1 - t0 except socket_errors: pass # Look up IP addresses for Google Server. try: (hostname, _, ipaddrlist) = socket.gethostbyname_ex(self.XML_API_HOST) sysinfo['googserv_ips'] = ipaddrlist except socket_errors: ipaddrlist = [] sysinfo['googserv_ips'] = [] # Reverse lookup the hostnames for the Google Server IPs. sysinfo['googserv_hostnames'] = [] for googserv_ip in ipaddrlist: try: (hostname, _, ipaddrlist) = socket.gethostbyaddr(googserv_ip) sysinfo['googserv_hostnames'].append(hostname) except socket_errors: pass # Query o-o to find out what the Google DNS thinks is the user's IP. try: cmd = ['nslookup', '-type=TXT', 'o-o.myaddr.google.com.'] nslookup_txt_output = self._Exec(cmd, return_output=True, mute_stderr=True) m = re.search(r'text\s+=\s+"(?P<dnsip>[\.\d]+)"', nslookup_txt_output) sysinfo['dns_o-o_ip'] = m.group('dnsip') if m else None except (CommandException, OSError): sysinfo['dns_o-o_ip'] = '' # Try to determine the latency of connecting to the Google hostname # endpoint. sysinfo['google_host_connect_latencies'] = {} for googserv_ip in ipaddrlist: try: sock = socket.socket() t0 = time.time() sock.connect((googserv_ip, self.XML_API_PORT)) t1 = time.time() sysinfo['google_host_connect_latencies'][googserv_ip] = t1 - t0 except socket_errors: pass # If using a proxy, try to determine the latency of a DNS lookup to resolve # the proxy hostname and the latency of connecting to the proxy. if proxy_host: proxy_ip = None try: t0 = time.time() proxy_ip = socket.gethostbyname(proxy_host) t1 = time.time() sysinfo['proxy_dns_latency'] = t1 - t0 except socket_errors: pass try: sock = socket.socket() t0 = time.time() sock.connect((proxy_ip or proxy_host, proxy_port)) t1 = time.time() sysinfo['proxy_host_connect_latency'] = t1 - t0 except socket_errors: pass # Try and find the number of CPUs in the system if available. try: sysinfo['cpu_count'] = multiprocessing.cpu_count() except NotImplementedError: sysinfo['cpu_count'] = None # For *nix platforms, obtain the CPU load. try: sysinfo['load_avg'] = list(os.getloadavg()) except (AttributeError, OSError): sysinfo['load_avg'] = None # Try and collect memory information from /proc/meminfo if possible. mem_total = None mem_free = None mem_buffers = None mem_cached = None try: with open('/proc/meminfo', 'r') as f: for line in f: if line.startswith('MemTotal'): mem_total = (int(''.join(c for c in line if c in string.digits)) * 1000) elif line.startswith('MemFree'): mem_free = (int(''.join(c for c in line if c in string.digits)) * 1000) elif line.startswith('Buffers'): mem_buffers = (int(''.join(c for c in line if c in string.digits)) * 1000) elif line.startswith('Cached'): mem_cached = (int(''.join(c for c in line if c in string.digits)) * 1000) except (IOError, ValueError): pass sysinfo['meminfo'] = { 'mem_total': mem_total, 'mem_free': mem_free, 'mem_buffers': mem_buffers, 'mem_cached': mem_cached } # Get configuration attributes from config module. sysinfo['gsutil_config'] = {} for attr in dir(config): attr_value = getattr(config, attr) # Filter out multiline strings that are not useful. if attr.isupper() and not (isinstance(attr_value, six.string_types) and '\n' in attr_value): sysinfo['gsutil_config'][attr] = attr_value sysinfo['tcp_proc_values'] = {} stats_to_check = [ '/proc/sys/net/core/rmem_default', '/proc/sys/net/core/rmem_max', '/proc/sys/net/core/wmem_default', '/proc/sys/net/core/wmem_max', '/proc/sys/net/ipv4/tcp_timestamps', '/proc/sys/net/ipv4/tcp_sack', '/proc/sys/net/ipv4/tcp_window_scaling', ] for fname in stats_to_check: try: with open(fname, 'r') as f: value = f.read() sysinfo['tcp_proc_values'][os.path.basename(fname)] = value.strip() except IOError: pass self.results['sysinfo'] = sysinfo def _DisplayStats(self, trials): """Prints out mean, standard deviation, median, and 90th percentile.""" n = len(trials) mean = float(sum(trials)) / n stdev = math.sqrt(sum((x - mean)**2 for x in trials) / n) text_util.print_to_fd(str(n).rjust(6), '', end=' ') text_util.print_to_fd(('%.1f' % (mean * 1000)).rjust(9), '', end=' ') text_util.print_to_fd(('%.1f' % (stdev * 1000)).rjust(12), '', end=' ') text_util.print_to_fd(('%.1f' % (Percentile(trials, 0.5) * 1000)).rjust(11), '', end=' ') text_util.print_to_fd(('%.1f' % (Percentile(trials, 0.9) * 1000)).rjust(11), '') def _DisplayResults(self): """Displays results collected from diagnostic run.""" text_util.print_to_fd() text_util.print_to_fd('=' * 78) text_util.print_to_fd('DIAGNOSTIC RESULTS'.center(78)) text_util.print_to_fd('=' * 78) if 'latency' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Latency'.center(78)) text_util.print_to_fd('-' * 78) text_util.print_to_fd( 'Operation Size Trials Mean (ms) Std Dev (ms) ' 'Median (ms) 90th % (ms)') text_util.print_to_fd( '========= ========= ====== ========= ============ ' '=========== ===========') for key in sorted(self.results['latency']): trials = sorted(self.results['latency'][key]) op, numbytes = key.split('_') numbytes = int(numbytes) if op == 'METADATA': text_util.print_to_fd('Metadata'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if op == 'DOWNLOAD': text_util.print_to_fd('Download'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if op == 'UPLOAD': text_util.print_to_fd('Upload'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if op == 'DELETE': text_util.print_to_fd('Delete'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if 'write_throughput' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Write Throughput'.center(78)) text_util.print_to_fd('-' * 78) write_thru = self.results['write_throughput'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(write_thru['file_size']), MakeHumanReadable(write_thru['total_bytes_copied']))) text_util.print_to_fd( 'Write throughput: %s/s.' % (MakeBitsHumanReadable(write_thru['bytes_per_second'] * 8))) if 'parallelism' in write_thru: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % write_thru['parallelism']) if 'write_throughput_file' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Write Throughput With File I/O'.center(78)) text_util.print_to_fd('-' * 78) write_thru_file = self.results['write_throughput_file'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(write_thru_file['file_size']), MakeHumanReadable(write_thru_file['total_bytes_copied']))) text_util.print_to_fd( 'Write throughput: %s/s.' % (MakeBitsHumanReadable(write_thru_file['bytes_per_second'] * 8))) if 'parallelism' in write_thru_file: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % write_thru_file['parallelism']) if 'read_throughput' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Read Throughput'.center(78)) text_util.print_to_fd('-' * 78) read_thru = self.results['read_throughput'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(read_thru['file_size']), MakeHumanReadable(read_thru['total_bytes_copied']))) text_util.print_to_fd( 'Read throughput: %s/s.' % (MakeBitsHumanReadable(read_thru['bytes_per_second'] * 8))) if 'parallelism' in read_thru: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % read_thru['parallelism']) if 'read_throughput_file' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Read Throughput With File I/O'.center(78)) text_util.print_to_fd('-' * 78) read_thru_file = self.results['read_throughput_file'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(read_thru_file['file_size']), MakeHumanReadable(read_thru_file['total_bytes_copied']))) text_util.print_to_fd( 'Read throughput: %s/s.' % (MakeBitsHumanReadable(read_thru_file['bytes_per_second'] * 8))) if 'parallelism' in read_thru_file: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % read_thru_file['parallelism']) if 'listing' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Listing'.center(78)) text_util.print_to_fd('-' * 78) listing = self.results['listing'] insert = listing['insert'] delete = listing['delete'] text_util.print_to_fd('After inserting %s objects:' % listing['num_files']) text_util.print_to_fd( (' Total time for objects to appear: %.2g seconds' % insert['time_took'])) text_util.print_to_fd(' Number of listing calls made: %s' % insert['num_listing_calls']) text_util.print_to_fd( (' Individual listing call latencies: [%s]' % ', '.join('%.2gs' % lat for lat in insert['list_latencies']))) text_util.print_to_fd( (' Files reflected after each call: [%s]' % ', '.join(map(str, insert['files_seen_after_listing'])))) text_util.print_to_fd('After deleting %s objects:' % listing['num_files']) text_util.print_to_fd( (' Total time for objects to appear: %.2g seconds' % delete['time_took'])) text_util.print_to_fd(' Number of listing calls made: %s' % delete['num_listing_calls']) text_util.print_to_fd( (' Individual listing call latencies: [%s]' % ', '.join('%.2gs' % lat for lat in delete['list_latencies']))) text_util.print_to_fd( (' Files reflected after each call: [%s]' % ', '.join(map(str, delete['files_seen_after_listing'])))) if 'sysinfo' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('System Information'.center(78)) text_util.print_to_fd('-' * 78) info = self.results['sysinfo'] text_util.print_to_fd('IP Address: \n %s' % info['ip_address']) text_util.print_to_fd('Temporary Directory: \n %s' % info['tempdir']) text_util.print_to_fd('Bucket URI: \n %s' % self.results['bucket_uri']) text_util.print_to_fd('gsutil Version: \n %s' % self.results.get('gsutil_version', 'Unknown')) text_util.print_to_fd('boto Version: \n %s' % self.results.get('boto_version', 'Unknown')) if 'gmt_timestamp' in info: ts_string = info['gmt_timestamp'] timetuple = None try: # Convert RFC 2822 string to Linux timestamp. timetuple = time.strptime(ts_string, '%a, %d %b %Y %H:%M:%S +0000') except ValueError: pass if timetuple: # Converts the GMT time tuple to local Linux timestamp. localtime = calendar.timegm(timetuple) localdt = datetime.datetime.fromtimestamp(localtime) text_util.print_to_fd('Measurement time: \n %s' % localdt.strftime('%Y-%m-%d %I:%M:%S %p %Z')) if 'on_gce' in info: text_util.print_to_fd('Running on GCE: \n %s' % info['on_gce']) if info['on_gce']: text_util.print_to_fd('GCE Instance:\n\t%s' % info['gce_instance_info'].replace('\n', '\n\t')) text_util.print_to_fd('Bucket location: \n %s' % info['bucket_location']) text_util.print_to_fd('Bucket storage class: \n %s' % info['bucket_storageClass']) text_util.print_to_fd('Google Server: \n %s' % info['googserv_route']) text_util.print_to_fd('Google Server IP Addresses: \n %s' % ('\n '.join(info['googserv_ips']))) text_util.print_to_fd('Google Server Hostnames: \n %s' % ('\n '.join(info['googserv_hostnames']))) text_util.print_to_fd('Google DNS thinks your IP is: \n %s' % info['dns_o-o_ip']) text_util.print_to_fd('CPU Count: \n %s' % info['cpu_count']) text_util.print_to_fd('CPU Load Average: \n %s' % info['load_avg']) try: text_util.print_to_fd(('Total Memory: \n %s' % MakeHumanReadable(info['meminfo']['mem_total']))) # Free memory is really MemFree + Buffers + Cached. text_util.print_to_fd('Free Memory: \n %s' % MakeHumanReadable(info['meminfo']['mem_free'] + info['meminfo']['mem_buffers'] + info['meminfo']['mem_cached'])) except TypeError: pass if 'netstat_end' in info and 'netstat_start' in info: netstat_after = info['netstat_end'] netstat_before = info['netstat_start'] for tcp_type in ('sent', 'received', 'retransmit'): try: delta = (netstat_after['tcp_%s' % tcp_type] - netstat_before['tcp_%s' % tcp_type]) text_util.print_to_fd('TCP segments %s during test:\n %d' % (tcp_type, delta)) except TypeError: pass else: text_util.print_to_fd( 'TCP segment counts not available because "netstat" was not ' 'found during test runs') if 'disk_counters_end' in info and 'disk_counters_start' in info: text_util.print_to_fd('Disk Counter Deltas:\n', end=' ') disk_after = info['disk_counters_end'] disk_before = info['disk_counters_start'] text_util.print_to_fd('', 'disk'.rjust(6), end=' ') for colname in [ 'reads', 'writes', 'rbytes', 'wbytes', 'rtime', 'wtime' ]: text_util.print_to_fd(colname.rjust(8), end=' ') text_util.print_to_fd() for diskname in sorted(disk_after): before = disk_before[diskname] after = disk_after[diskname] (reads1, writes1, rbytes1, wbytes1, rtime1, wtime1) = before (reads2, writes2, rbytes2, wbytes2, rtime2, wtime2) = after text_util.print_to_fd('', diskname.rjust(6), end=' ') deltas = [ reads2 - reads1, writes2 - writes1, rbytes2 - rbytes1, wbytes2 - wbytes1, rtime2 - rtime1, wtime2 - wtime1 ] for delta in deltas: text_util.print_to_fd(str(delta).rjust(8), end=' ') text_util.print_to_fd() if 'tcp_proc_values' in info: text_util.print_to_fd('TCP /proc values:\n', end=' ') for item in six.iteritems(info['tcp_proc_values']): text_util.print_to_fd(' %s = %s' % item) if 'boto_https_enabled' in info: text_util.print_to_fd('Boto HTTPS Enabled: \n %s' % info['boto_https_enabled']) if 'using_proxy' in info: text_util.print_to_fd('Requests routed through proxy: \n %s' % info['using_proxy']) if 'google_host_dns_latency' in info: text_util.print_to_fd( ('Latency of the DNS lookup for Google Storage server (ms): ' '\n %.1f' % (info['google_host_dns_latency'] * 1000.0))) if 'google_host_connect_latencies' in info: text_util.print_to_fd( 'Latencies connecting to Google Storage server IPs (ms):') for ip, latency in six.iteritems(info['google_host_connect_latencies']): text_util.print_to_fd(' %s = %.1f' % (ip, latency * 1000.0)) if 'proxy_dns_latency' in info: text_util.print_to_fd( ('Latency of the DNS lookup for the configured proxy (ms): ' '\n %.1f' % (info['proxy_dns_latency'] * 1000.0))) if 'proxy_host_connect_latency' in info: text_util.print_to_fd( ('Latency connecting to the configured proxy (ms): \n %.1f' % (info['proxy_host_connect_latency'] * 1000.0))) if 'request_errors' in self.results and 'total_requests' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('In-Process HTTP Statistics'.center(78)) text_util.print_to_fd('-' * 78) total = int(self.results['total_requests']) numerrors = int(self.results['request_errors']) numbreaks = int(self.results['connection_breaks']) availability = (((total - numerrors) / float(total)) * 100 if total > 0 else 100) text_util.print_to_fd('Total HTTP requests made: %d' % total) text_util.print_to_fd('HTTP 5xx errors: %d' % numerrors) text_util.print_to_fd('HTTP connections broken: %d' % numbreaks) text_util.print_to_fd('Availability: %.7g%%' % availability) if 'error_responses_by_code' in self.results: sorted_codes = sorted( six.iteritems(self.results['error_responses_by_code'])) if sorted_codes: text_util.print_to_fd('Error responses by code:') text_util.print_to_fd('\n'.join(' %s: %s' % c for c in sorted_codes)) if self.output_file: with open(self.output_file, 'w') as f: json.dump(self.results, f, indent=2) text_util.print_to_fd() text_util.print_to_fd("Output file written to '%s'." % self.output_file) text_util.print_to_fd() def _ParsePositiveInteger(self, val, msg): """Tries to convert val argument to a positive integer. Args: val: The value (as a string) to convert to a positive integer. msg: The error message to place in the CommandException on an error. Returns: A valid positive integer. Raises: CommandException: If the supplied value is not a valid positive integer. """ try: val = int(val) if val < 1: raise CommandException(msg) return val except ValueError: raise CommandException(msg) def _ParseArgs(self): """Parses arguments for perfdiag command.""" # From -n. self.num_objects = 5 # From -c. self.processes = 1 # From -k. self.threads = 1 # From -p self.parallel_strategy = None # From -y self.num_slices = 4 # From -s. self.thru_filesize = 1048576 # From -d. self.directory = tempfile.gettempdir() # Keep track of whether or not to delete the directory upon completion. self.delete_directory = False # From -t. self.diag_tests = set(self.DEFAULT_DIAG_TESTS) # From -o. self.output_file = None # From -i. self.input_file = None # From -m. self.metadata_keys = {} # From -j. self.gzip_encoded_writes = False self.gzip_compression_ratio = 100 if self.sub_opts: for o, a in self.sub_opts: if o == '-n': self.num_objects = self._ParsePositiveInteger( a, 'The -n parameter must be a positive integer.') if o == '-c': self.processes = self._ParsePositiveInteger( a, 'The -c parameter must be a positive integer.') if o == '-k': self.threads = self._ParsePositiveInteger( a, 'The -k parameter must be a positive integer.') if o == '-p': if a.lower() in self.PARALLEL_STRATEGIES: self.parallel_strategy = a.lower() else: raise CommandException("'%s' is not a valid parallelism strategy." % a) if o == '-y': self.num_slices = self._ParsePositiveInteger( a, 'The -y parameter must be a positive integer.') if o == '-s': try: self.thru_filesize = HumanReadableToBytes(a) except ValueError: raise CommandException('Invalid -s parameter.') if o == '-d': self.directory = a if not os.path.exists(self.directory): self.delete_directory = True os.makedirs(self.directory) if o == '-t': self.diag_tests = set() for test_name in a.strip().split(','): if test_name.lower() not in self.ALL_DIAG_TESTS: raise CommandException("List of test names (-t) contains invalid " "test name '%s'." % test_name) self.diag_tests.add(test_name) if o == '-m': pieces = a.split(':') if len(pieces) != 2: raise CommandException( "Invalid metadata key-value combination '%s'." % a) key, value = pieces self.metadata_keys[key] = value if o == '-o': self.output_file = os.path.abspath(a) if o == '-i': self.input_file = os.path.abspath(a) if not os.path.isfile(self.input_file): raise CommandException("Invalid input file (-i): '%s'." % a) try: with open(self.input_file, 'r') as f: self.results = json.load(f) self.logger.info("Read input file: '%s'.", self.input_file) except ValueError: raise CommandException("Could not decode input file (-i): '%s'." % a) return if o == '-j': self.gzip_encoded_writes = True try: self.gzip_compression_ratio = int(a) except ValueError: self.gzip_compression_ratio = -1 if (self.gzip_compression_ratio < 0 or self.gzip_compression_ratio > 100): raise CommandException( 'The -j parameter must be between 0 and 100 (inclusive).') # If parallelism is specified, default parallelism strategy to fan. if (self.processes > 1 or self.threads > 1) and not self.parallel_strategy: self.parallel_strategy = self.FAN elif self.processes == 1 and self.threads == 1 and self.parallel_strategy: raise CommandException( 'Cannot specify parallelism strategy (-p) without also specifying ' 'multiple threads and/or processes (-c and/or -k).') if not self.args: self.RaiseWrongNumberOfArgumentsException() self.bucket_url = StorageUrlFromString(self.args[0]) self.provider = self.bucket_url.scheme if not self.bucket_url.IsCloudUrl() and self.bucket_url.IsBucket(): raise CommandException('The perfdiag command requires a URL that ' 'specifies a bucket.\n"%s" is not ' 'valid.' % self.args[0]) if (self.thru_filesize > HumanReadableToBytes('2GiB') and (self.RTHRU in self.diag_tests or self.WTHRU in self.diag_tests)): raise CommandException( 'For in-memory tests maximum file size is 2GiB. For larger file ' 'sizes, specify rthru_file and/or wthru_file with the -t option.') perform_slice = self.parallel_strategy in (self.SLICE, self.BOTH) slice_not_available = (self.provider == 's3' and self.diag_tests.intersection(self.WTHRU, self.WTHRU_FILE)) if perform_slice and slice_not_available: raise CommandException('Sliced uploads are not available for s3. ' 'Use -p fan or sequential uploads for s3.') # Ensure the bucket exists. self.gsutil_api.GetBucket(self.bucket_url.bucket_name, provider=self.bucket_url.scheme, fields=['id']) self.exceptions = [ http_client.HTTPException, socket.error, socket.gaierror, socket.timeout, http_client.BadStatusLine, ServiceException ] # Command entry point. def RunCommand(self): """Called by gsutil when the command is being invoked.""" self._ParseArgs() if self.input_file: self._DisplayResults() return 0 # We turn off retries in the underlying boto library because the # _RunOperation function handles errors manually so it can count them. boto.config.set('Boto', 'num_retries', '0') self.logger.info( 'Number of iterations to run: %d\n' 'Base bucket URI: %s\n' 'Number of processes: %d\n' 'Number of threads: %d\n' 'Parallelism strategy: %s\n' 'Throughput file size: %s\n' 'Diagnostics to run: %s\n' 'Gzip compression ratio: %s\n' 'Gzip transport encoding writes: %s', self.num_objects, self.bucket_url, self.processes, self.threads, self.parallel_strategy, MakeHumanReadable(self.thru_filesize), (', '.join(self.diag_tests)), self.gzip_compression_ratio, self.gzip_encoded_writes) try: self._SetUp() # Collect generic system info. self._CollectSysInfo() # Collect netstat info and disk counters before tests (and again later). netstat_output = self._GetTcpStats() if netstat_output: self.results['sysinfo']['netstat_start'] = netstat_output if IS_LINUX: self.results['sysinfo']['disk_counters_start'] = GetDiskCounters() # Record bucket URL. self.results['bucket_uri'] = str(self.bucket_url) self.results['json_format'] = 'perfdiag' self.results['metadata'] = self.metadata_keys if self.LAT in self.diag_tests: self._RunLatencyTests() if self.RTHRU in self.diag_tests: self._RunReadThruTests() # Run WTHRU_FILE before RTHRU_FILE. If data is created in WTHRU_FILE it # will be used in RTHRU_FILE to save time and bandwidth. if self.WTHRU_FILE in self.diag_tests: self._RunWriteThruTests(use_file=True) if self.RTHRU_FILE in self.diag_tests: self._RunReadThruTests(use_file=True) if self.WTHRU in self.diag_tests: self._RunWriteThruTests() if self.LIST in self.diag_tests: self._RunListTests() # Collect netstat info and disk counters after tests. netstat_output = self._GetTcpStats() if netstat_output: self.results['sysinfo']['netstat_end'] = netstat_output if IS_LINUX: self.results['sysinfo']['disk_counters_end'] = GetDiskCounters() self.results['total_requests'] = self.total_requests self.results['request_errors'] = self.request_errors self.results['error_responses_by_code'] = self.error_responses_by_code self.results['connection_breaks'] = self.connection_breaks self.results['gsutil_version'] = gslib.VERSION self.results['boto_version'] = boto.__version__ self._TearDown() self._DisplayResults() finally: # TODO: Install signal handlers so this is performed in response to a # terminating signal; consider multi-threaded object deletes during # cleanup so it happens quickly. self._TearDown() return 0 def StorageUrlToUploadObjectMetadata(storage_url): if storage_url.IsCloudUrl() and storage_url.IsObject(): upload_target = apitools_messages.Object() upload_target.name = storage_url.object_name upload_target.bucket = storage_url.bucket_name return upload_target else: raise CommandException('Non-cloud URL upload target %s was created in ' 'perfdiag implemenation.' % storage_url) commands/notification.py 0000644 00000103705 15025013616 0011412 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the notification command to gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import getopt import re import time import uuid from datetime import datetime from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PublishPermissionDeniedException from gslib.command import Command from gslib.command import NO_MAX from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.project_id import PopulateProjectId from gslib.pubsub_api import PubsubApi from gslib.storage_url import StorageUrlFromString from gslib.third_party.pubsub_apitools.pubsub_v1_messages import Binding from gslib.utils import copy_helper from gslib.utils import shim_util from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap # Cloud Pub/Sub commands _LIST_SYNOPSIS = """ gsutil notification list gs://<bucket_name>... """ _DELETE_SYNOPSIS = """ gsutil notification delete (<notificationConfigName>|gs://<bucket_name>)... """ _CREATE_SYNOPSIS = """ gsutil notification create -f (json|none) [-p <prefix>] [-t <topic>] \\ [-m <key>:<value>]... [-e <eventType>]... gs://<bucket_name> """ # Object Change Notification commands _WATCHBUCKET_SYNOPSIS = """ gsutil notification watchbucket [-i <id>] [-t <token>] <app_url> gs://<bucket_name> """ _STOPCHANNEL_SYNOPSIS = """ gsutil notification stopchannel <channel_id> <resource_id> """ _SYNOPSIS = ( _CREATE_SYNOPSIS + _DELETE_SYNOPSIS.lstrip('\n') + _LIST_SYNOPSIS.lstrip('\n') + _WATCHBUCKET_SYNOPSIS + _STOPCHANNEL_SYNOPSIS.lstrip('\n') + '\n') # yapf: disable _LIST_DESCRIPTION = """ <B>LIST</B> The list sub-command provides a list of notification configs belonging to a given bucket. The listed name of each notification config can be used with the delete sub-command to delete that specific notification config. For listing Object Change Notifications instead of Cloud Pub/Sub notification subscription configs, add a -o flag. <B>LIST EXAMPLES</B> Fetch the list of notification configs for the bucket example-bucket: gsutil notification list gs://example-bucket The same as above, but for Object Change Notifications instead of Cloud Pub/Sub notification subscription configs: gsutil notification list -o gs://example-bucket Fetch the notification configs in all buckets matching a wildcard: gsutil notification list gs://example-* Fetch all of the notification configs for buckets in the default project: gsutil notification list gs://* """ _DELETE_DESCRIPTION = """ <B>DELETE</B> The delete sub-command deletes notification configs from a bucket. If a notification config name is passed as a parameter, that notification config alone is deleted. If a bucket name is passed, all notification configs associated with that bucket are deleted. Cloud Pub/Sub topics associated with this notification config are not deleted by this command. Those must be deleted separately, for example with the gcloud command `gcloud beta pubsub topics delete`. Object Change Notification subscriptions cannot be deleted with this command. For that, see the command `gsutil notification stopchannel`. <B>DELETE EXAMPLES</B> Delete a single notification config (with ID 3) in the bucket example-bucket: gsutil notification delete projects/_/buckets/example-bucket/notificationConfigs/3 Delete all notification configs in the bucket example-bucket: gsutil notification delete gs://example-bucket """ _CREATE_DESCRIPTION = """ <B>CREATE</B> The create sub-command creates a notification config on a bucket, establishing a flow of event notifications from Cloud Storage to a Cloud Pub/Sub topic. As part of creating this flow, the create command also verifies that the destination Cloud Pub/Sub topic exists, creating it if necessary, and verifies that the Cloud Storage bucket has permission to publish events to that topic, granting the permission if necessary. If a destination Cloud Pub/Sub topic is not specified with the -t flag, Cloud Storage chooses a topic name in the default project whose ID is the same as the bucket name. For example, if the default project ID specified is 'default-project' and the bucket being configured is gs://example-bucket, the create command uses the Cloud Pub/Sub topic "projects/default-project/topics/example-bucket". In order to enable notifications, your project's `Cloud Storage service agent <https://cloud.google.com/storage/docs/projects#service-accounts>`_ must have the IAM permission "pubsub.topics.publish". This command checks to see if the destination Cloud Pub/Sub topic grants the service agent this permission. If not, the create command attempts to grant it. A bucket can have up to 100 total notification configurations and up to 10 notification configurations set to trigger for a specific event. <B>CREATE EXAMPLES</B> Begin sending notifications of all changes to the bucket example-bucket to the Cloud Pub/Sub topic projects/default-project/topics/example-bucket: gsutil notification create -f json gs://example-bucket The same as above, but specifies the destination topic ID 'files-to-process' in the default project: gsutil notification create -f json \\ -t files-to-process gs://example-bucket The same as above, but specifies a Cloud Pub/Sub topic belonging to the specific cloud project 'example-project': gsutil notification create -f json \\ -t projects/example-project/topics/files-to-process gs://example-bucket Create a notification config that only sends an event when a new object has been created: gsutil notification create -f json -e OBJECT_FINALIZE gs://example-bucket Create a topic and notification config that only sends an event when an object beginning with "photos/" is affected: gsutil notification create -p photos/ gs://example-bucket List all of the notificationConfigs in bucket example-bucket: gsutil notification list gs://example-bucket Delete all notitificationConfigs for bucket example-bucket: gsutil notification delete gs://example-bucket Delete one specific notificationConfig for bucket example-bucket: gsutil notification delete \\ projects/_/buckets/example-bucket/notificationConfigs/1 <B>OPTIONS</B> The create sub-command has the following options -e Specify an event type filter for this notification config. Cloud Storage only sends notifications of this type. You may specify this parameter multiple times to allow multiple event types. If not specified, Cloud Storage sends notifications for all event types. The valid types are: OBJECT_FINALIZE - An object has been created. OBJECT_METADATA_UPDATE - The metadata of an object has changed. OBJECT_DELETE - An object has been permanently deleted. OBJECT_ARCHIVE - A live version of an object has become a noncurrent version. -f Specifies the payload format of notification messages. Must be either "json" for a payload matches the object metadata for the JSON API, or "none" to specify no payload at all. In either case, notification details are available in the message attributes. -m Specifies a key:value attribute that is appended to the set of attributes sent to Cloud Pub/Sub for all events associated with this notification config. You may specify this parameter multiple times to set multiple attributes. -p Specifies a prefix path filter for this notification config. Cloud Storage only sends notifications for objects in this bucket whose names begin with the specified prefix. -s Skips creation and permission assignment of the Cloud Pub/Sub topic. This is useful if the caller does not have permission to access the topic in question, or if the topic already exists and has the appropriate publish permission assigned. -t The Cloud Pub/Sub topic to which notifications should be sent. If not specified, this command chooses a topic whose project is your default project and whose ID is the same as the Cloud Storage bucket name. <B>NEXT STEPS</B> Once the create command has succeeded, Cloud Storage publishes a message to the specified Cloud Pub/Sub topic when eligible changes occur. In order to receive these messages, you must create a Pub/Sub subscription for your Pub/Sub topic. To learn more about creating Pub/Sub subscriptions, see `the Pub/Sub Subscriber Overview <https://cloud.google.com/pubsub/docs/subscriber>`_. You can create a simple Pub/Sub subscription using the ``gcloud`` command-line tool. For example, to create a new subscription on the topic "myNewTopic" and attempt to pull messages from it, you could run: gcloud beta pubsub subscriptions create --topic myNewTopic testSubscription gcloud beta pubsub subscriptions pull --auto-ack testSubscription """ _WATCHBUCKET_DESCRIPTION = """ <B>WATCHBUCKET</B> The watchbucket sub-command can be used to watch a bucket for object changes. A service account must be used when running this command. The app_url parameter must be an HTTPS URL to an application that will be notified of changes to any object in the bucket. The optional id parameter can be used to assign a unique identifier to the created notification channel. If not provided, a random UUID string is generated. The optional token parameter can be used to validate notifications events. To do this, set this custom token and store it to later verify that notification events contain the client token you expect. <B>WATCHBUCKET EXAMPLES</B> Watch the bucket example-bucket for changes and send notifications to an application server running at example.com: gsutil notification watchbucket https://example.com/notify \\ gs://example-bucket Assign identifier my-channel-id to the created notification channel: gsutil notification watchbucket -i my-channel-id \\ https://example.com/notify gs://example-bucket Set a custom client token that is included with each notification event: gsutil notification watchbucket -t my-client-token \\ https://example.com/notify gs://example-bucket """ _STOPCHANNEL_DESCRIPTION = """ <B>STOPCHANNEL</B> The stopchannel sub-command can be used to stop sending change events to a notification channel. The channel_id and resource_id parameters should match the values from the response of a bucket watch request. <B>STOPCHANNEL EXAMPLES</B> Stop the notification event channel with channel identifier channel1 and resource identifier SoGqan08XDIFWr1Fv_nGpRJBHh8: gsutil notification stopchannel channel1 SoGqan08XDIFWr1Fv_nGpRJBHh8 """ _DESCRIPTION = """ You can use the ``notification`` command to configure `Pub/Sub notifications for Cloud Storage <https://cloud.google.com/storage/docs/pubsub-notifications>`_ and `Object change notification <https://cloud.google.com/storage/docs/object-change-notification>`_ channels. <B>CLOUD PUB/SUB</B> The "create", "list", and "delete" sub-commands deal with configuring Cloud Storage integration with Google Cloud Pub/Sub. """ + _CREATE_DESCRIPTION + _LIST_DESCRIPTION + _DELETE_DESCRIPTION + """ <B>OBJECT CHANGE NOTIFICATIONS</B> Object change notification is a separate, older feature within Cloud Storage for generating notifications. This feature sends HTTPS messages to a client application that you've set up separately. This feature is generally not recommended, because Pub/Sub notifications are cheaper, easier to use, and more flexible. For more information, see `Object change notification <https://cloud.google.com/storage/docs/object-change-notification>`_. The "watchbucket" and "stopchannel" sub-commands enable and disable Object change notifications. """ + _WATCHBUCKET_DESCRIPTION + _STOPCHANNEL_DESCRIPTION + """ <B>NOTIFICATIONS AND PARALLEL COMPOSITE UPLOADS</B> gsutil supports `parallel composite uploads <https://cloud.google.com/storage/docs/uploads-downloads#parallel-composite-uploads>`_. If enabled, an upload can result in multiple temporary component objects being uploaded before the actual intended object is created. Any subscriber to notifications for this bucket then sees a notification for each of these components being created and deleted. If this is a concern for you, note that parallel composite uploads can be disabled by setting "parallel_composite_upload_threshold = 0" in your .boto config file. Alternately, your subscriber code can filter out gsutil's parallel composite uploads by ignoring any notification about objects whose names contain (but do not start with) the following string: "{composite_namespace}". """.format(composite_namespace=copy_helper.PARALLEL_UPLOAD_TEMP_NAMESPACE) NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE = """ Watch bucket attempt failed: {watch_error} You attempted to watch a bucket with an application URL of: {watch_url} which is not authorized for your project. Please ensure that you are using Service Account authentication and that the Service Account's project is authorized for the application URL. Notification endpoint URLs must also be whitelisted in your Cloud Console project. To do that, the domain must also be verified using Google Webmaster Tools. For instructions, please see `Notification Authorization <https://cloud.google.com/storage/docs/object-change-notification#_Authorization>`_. """ _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) # yapf: disable _create_help_text = ( CreateHelpText(_CREATE_SYNOPSIS, _CREATE_DESCRIPTION)) _list_help_text = ( CreateHelpText(_LIST_SYNOPSIS, _LIST_DESCRIPTION)) _delete_help_text = ( CreateHelpText(_DELETE_SYNOPSIS, _DELETE_DESCRIPTION)) _watchbucket_help_text = ( CreateHelpText(_WATCHBUCKET_SYNOPSIS, _WATCHBUCKET_DESCRIPTION)) _stopchannel_help_text = ( CreateHelpText(_STOPCHANNEL_SYNOPSIS, _STOPCHANNEL_DESCRIPTION)) # yapf: enable PAYLOAD_FORMAT_MAP = { 'none': 'NONE', 'json': 'JSON_API_V1', } class NotificationCommand(Command): """Implementation of gsutil notification command.""" # Notification names might look like one of these: # canonical form: projects/_/buckets/bucket/notificationConfigs/3 # JSON API form: b/bucket/notificationConfigs/5 # Either of the above might start with a / if a user is copying & pasting. def _GetNotificationPathRegex(self): if not NotificationCommand._notification_path_regex: NotificationCommand._notification_path_regex = re.compile( ('/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/' 'notificationConfigs/(?P<notification>[0-9]+)')) return NotificationCommand._notification_path_regex _notification_path_regex = None # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'notification', command_name_aliases=[ 'notify', 'notifyconfig', 'notifications', 'notif', ], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='i:t:m:t:of:e:p:s', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'watchbucket': [ CommandArgument.MakeFreeTextArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'stopchannel': [], 'list': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], 'delete': [ # Takes a list of one of the following: # notification: projects/_/buckets/bla/notificationConfigs/5, # bucket: gs://foobar CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], 'create': [ CommandArgument.MakeFreeTextArgument(), # Cloud Pub/Sub topic CommandArgument.MakeNCloudBucketURLsArgument(1), ] }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='notification', help_name_aliases=[ 'watchbucket', 'stopchannel', 'notifyconfig', ], help_type='command_help', help_one_line_summary='Configure object change notification', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'create': _create_help_text, 'list': _list_help_text, 'delete': _delete_help_text, 'watchbucket': _watchbucket_help_text, 'stopchannel': _stopchannel_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'create': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'notifications', 'create' ], flag_map={ '-m': GcloudStorageFlag( '--custom-attributes', repeat_type=shim_util.RepeatFlagType.DICT), '-e': GcloudStorageFlag( '--event-types', repeat_type=shim_util.RepeatFlagType.LIST), '-p': GcloudStorageFlag('--object-prefix'), '-f': GcloudStorageFlag('--payload-format'), '-s': GcloudStorageFlag('--skip-topic-setup'), '-t': GcloudStorageFlag('--topic'), }, ), 'delete': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'notifications', 'delete' ], flag_map={}, ), 'list': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'notifications', 'list', '--human-readable', ], flag_map={}, supports_output_translation=True, ), }, flag_map={}, ) def _WatchBucket(self): """Creates a watch on a bucket given in self.args.""" self.CheckArguments() identifier = None client_token = None if self.sub_opts: for o, a in self.sub_opts: if o == '-i': identifier = a if o == '-t': client_token = a identifier = identifier or str(uuid.uuid4()) watch_url = self.args[0] bucket_arg = self.args[-1] if not watch_url.lower().startswith('https://'): raise CommandException('The application URL must be an https:// URL.') bucket_url = StorageUrlFromString(bucket_arg) if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'): raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) if not bucket_url.IsBucket(): raise CommandException('URL must name a bucket for the %s command.' % self.command_name) self.logger.info('Watching bucket %s with application URL %s ...', bucket_url, watch_url) try: channel = self.gsutil_api.WatchBucket(bucket_url.bucket_name, watch_url, identifier, token=client_token, provider=bucket_url.scheme) except AccessDeniedException as e: self.logger.warn( NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format(watch_error=str(e), watch_url=watch_url)) raise channel_id = channel.id resource_id = channel.resourceId client_token = channel.token self.logger.info('Successfully created watch notification channel.') self.logger.info('Watch channel identifier: %s', channel_id) self.logger.info('Canonicalized resource identifier: %s', resource_id) self.logger.info('Client state token: %s', client_token) return 0 def _StopChannel(self): channel_id = self.args[0] resource_id = self.args[1] self.logger.info('Removing channel %s with resource identifier %s ...', channel_id, resource_id) self.gsutil_api.StopChannel(channel_id, resource_id, provider='gs') self.logger.info('Succesfully removed channel.') return 0 def _ListChannels(self, bucket_arg): """Lists active channel watches on a bucket given in self.args.""" bucket_url = StorageUrlFromString(bucket_arg) if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'): raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) if not bucket_url.IsBucket(): raise CommandException('URL must name a bucket for the %s command.' % self.command_name) channels = self.gsutil_api.ListChannels(bucket_url.bucket_name, provider='gs').items self.logger.info( 'Bucket %s has the following active Object Change Notifications:', bucket_url.bucket_name) for idx, channel in enumerate(channels): self.logger.info('\tNotification channel %d:', idx + 1) self.logger.info('\t\tChannel identifier: %s', channel.channel_id) self.logger.info('\t\tResource identifier: %s', channel.resource_id) self.logger.info('\t\tApplication URL: %s', channel.push_url) self.logger.info('\t\tCreated by: %s', channel.subscriber_email) self.logger.info( '\t\tCreation time: %s', str(datetime.fromtimestamp(channel.creation_time_ms / 1000))) return 0 def _Create(self): self.CheckArguments() # User-specified options pubsub_topic = None payload_format = None custom_attributes = {} event_types = [] object_name_prefix = None should_setup_topic = True if self.sub_opts: for o, a in self.sub_opts: if o == '-e': event_types.append(a) elif o == '-f': payload_format = a elif o == '-m': if ':' not in a: raise CommandException( 'Custom attributes specified with -m should be of the form ' 'key:value') key, value = a.split(':', 1) custom_attributes[key] = value elif o == '-p': object_name_prefix = a elif o == '-s': should_setup_topic = False elif o == '-t': pubsub_topic = a if payload_format not in PAYLOAD_FORMAT_MAP: raise CommandException( "Must provide a payload format with -f of either 'json' or 'none'") payload_format = PAYLOAD_FORMAT_MAP[payload_format] bucket_arg = self.args[-1] bucket_url = StorageUrlFromString(bucket_arg) if not bucket_url.IsCloudUrl() or not bucket_url.IsBucket(): raise CommandException( "%s %s requires a GCS bucket name, but got '%s'" % (self.command_name, self.subcommand_name, bucket_arg)) if bucket_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) bucket_name = bucket_url.bucket_name self.logger.debug('Creating notification for bucket %s', bucket_url) # Find the project this bucket belongs to bucket_metadata = self.gsutil_api.GetBucket(bucket_name, fields=['projectNumber'], provider=bucket_url.scheme) bucket_project_number = bucket_metadata.projectNumber # If not specified, choose a sensible default for the Cloud Pub/Sub topic # name. if not pubsub_topic: pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None), bucket_name) if not pubsub_topic.startswith('projects/'): # If a user picks a topic ID (mytopic) but doesn't pass the whole name ( # projects/my-project/topics/mytopic ), pick a default project. pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None), pubsub_topic) self.logger.debug('Using Cloud Pub/Sub topic %s', pubsub_topic) just_modified_topic_permissions = False if should_setup_topic: # Ask GCS for the email address that represents GCS's permission to # publish to a Cloud Pub/Sub topic from this project. service_account = self.gsutil_api.GetProjectServiceAccount( bucket_project_number, provider=bucket_url.scheme).email_address self.logger.debug('Service account for project %d: %s', bucket_project_number, service_account) just_modified_topic_permissions = self._CreateTopic( pubsub_topic, service_account) for attempt_number in range(0, 2): try: create_response = self.gsutil_api.CreateNotificationConfig( bucket_name, pubsub_topic=pubsub_topic, payload_format=payload_format, custom_attributes=custom_attributes, event_types=event_types if event_types else None, object_name_prefix=object_name_prefix, provider=bucket_url.scheme) break except PublishPermissionDeniedException: if attempt_number == 0 and just_modified_topic_permissions: # If we have just set the IAM policy, it may take up to 10 seconds to # take effect. self.logger.info( 'Retrying create notification in 10 seconds ' '(new permissions may take up to 10 seconds to take effect.)') time.sleep(10) else: raise notification_name = 'projects/_/buckets/%s/notificationConfigs/%s' % ( bucket_name, create_response.id) self.logger.info('Created notification config %s', notification_name) return 0 def _CreateTopic(self, pubsub_topic, service_account): """Assures that a topic exists, creating it if necessary. Also adds GCS as a publisher on that bucket, if necessary. Args: pubsub_topic: name of the Cloud Pub/Sub topic to use/create. service_account: the GCS service account that needs publish permission. Returns: true if we modified IAM permissions, otherwise false. """ pubsub_api = PubsubApi(logger=self.logger) # Verify that the Pub/Sub topic exists. If it does not, create it. try: pubsub_api.GetTopic(topic_name=pubsub_topic) self.logger.debug('Topic %s already exists', pubsub_topic) except NotFoundException: self.logger.debug('Creating topic %s', pubsub_topic) pubsub_api.CreateTopic(topic_name=pubsub_topic) self.logger.info('Created Cloud Pub/Sub topic %s', pubsub_topic) # Verify that the service account is in the IAM policy. policy = pubsub_api.GetTopicIamPolicy(topic_name=pubsub_topic) binding = Binding(role='roles/pubsub.publisher', members=['serviceAccount:%s' % service_account]) # This could be more extensive. We could, for instance, check for roles # that are stronger that pubsub.publisher, like owner. We could also # recurse up the hierarchy looking to see if there are project-level # permissions. This can get very complex very quickly, as the caller # may not necessarily have access to the project-level IAM policy. # There's no danger in double-granting permission just to make sure it's # there, though. if binding not in policy.bindings: policy.bindings.append(binding) # transactional safety via etag field. pubsub_api.SetTopicIamPolicy(topic_name=pubsub_topic, policy=policy) return True else: self.logger.debug('GCS already has publish permission to topic %s.', pubsub_topic) return False def _EnumerateNotificationsFromArgs(self, accept_notification_configs=True): """Yields bucket/notification tuples from command-line args. Given a list of strings that are bucket names (gs://foo) or notification config IDs, yield tuples of bucket names and their associated notifications. Args: accept_notification_configs: whether notification configs are valid args. Yields: Tuples of the form (bucket_name, Notification) """ path_regex = self._GetNotificationPathRegex() for list_entry in self.args: match = path_regex.match(list_entry) if match: if not accept_notification_configs: raise CommandException( '%s %s accepts only bucket names, but you provided %s' % (self.command_name, self.subcommand_name, list_entry)) bucket_name = match.group('bucket') notification_id = match.group('notification') found = False for notification in self.gsutil_api.ListNotificationConfigs( bucket_name, provider='gs'): if notification.id == notification_id: yield (bucket_name, notification) found = True break if not found: raise NotFoundException('Could not find notification %s' % list_entry) else: storage_url = StorageUrlFromString(list_entry) if not storage_url.IsCloudUrl(): raise CommandException( 'The %s command must be used on cloud buckets or notification ' 'config names.' % self.command_name) if storage_url.scheme != 'gs': raise CommandException('The %s command only works on gs:// buckets.') path = None if storage_url.IsProvider(): path = 'gs://*' elif storage_url.IsBucket(): path = list_entry if not path: raise CommandException( 'The %s command cannot be used on cloud objects, only buckets' % self.command_name) for blr in self.WildcardIterator(path).IterBuckets( bucket_fields=['id']): for notification in self.gsutil_api.ListNotificationConfigs( blr.storage_url.bucket_name, provider='gs'): yield (blr.storage_url.bucket_name, notification) def _List(self): self.CheckArguments() if self.sub_opts: if '-o' in dict(self.sub_opts): for bucket_name in self.args: self._ListChannels(bucket_name) else: for bucket_name, notification in self._EnumerateNotificationsFromArgs( accept_notification_configs=False): self._PrintNotificationDetails(bucket_name, notification) return 0 def _PrintNotificationDetails(self, bucket, notification): print('projects/_/buckets/{bucket}/notificationConfigs/{notification}\n' '\tCloud Pub/Sub topic: {topic}'.format( bucket=bucket, notification=notification.id, topic=notification.topic[len('//pubsub.googleapis.com/'):])) if notification.custom_attributes: print('\tCustom attributes:') for attr in notification.custom_attributes.additionalProperties: print('\t\t%s: %s' % (attr.key, attr.value)) filters = [] if notification.event_types: filters.append('\t\tEvent Types: %s' % ', '.join(notification.event_types)) if notification.object_name_prefix: filters.append("\t\tObject name prefix: '%s'" % notification.object_name_prefix) if filters: print('\tFilters:') for line in filters: print(line) self.logger.info('') def _Delete(self): for bucket_name, notification in self._EnumerateNotificationsFromArgs(): self._DeleteNotification(bucket_name, notification.id) return 0 def _DeleteNotification(self, bucket_name, notification_id): self.gsutil_api.DeleteNotificationConfig(bucket_name, notification=notification_id, provider='gs') return 0 def _RunSubCommand(self, func): try: (self.sub_opts, self.args) = getopt.getopt(self.args, self.command_spec.supported_sub_args) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) return func(self) except getopt.GetoptError: self.RaiseInvalidArgumentException() SUBCOMMANDS = { 'create': _Create, 'list': _List, 'delete': _Delete, 'watchbucket': _WatchBucket, 'stopchannel': _StopChannel } def RunCommand(self): """Command entry point for the notification command.""" self.subcommand_name = self.args.pop(0) if self.subcommand_name in NotificationCommand.SUBCOMMANDS: metrics.LogCommandParams(subcommands=[self.subcommand_name]) return self._RunSubCommand( NotificationCommand.SUBCOMMANDS[self.subcommand_name]) else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (self.subcommand_name, self.command_name)) commands/test.py 0000644 00000070440 15025013616 0007702 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of gsutil test command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import namedtuple import logging import os import subprocess import re import sys import tempfile import textwrap import time import traceback import six from six.moves import range import gslib from gslib.cloud_api import ProjectIdException from gslib.command import Command from gslib.command import ResetFailureCount from gslib.exception import CommandException from gslib.project_id import PopulateProjectId import gslib.tests as tests from gslib.tests.util import GetTestNames from gslib.tests.util import InvokedFromParFile from gslib.tests.util import unittest from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 from gslib.utils.system_util import IS_WINDOWS # pylint: disable=g-import-not-at-top try: import coverage except ImportError: coverage = None if six.PY3: long = int _DEFAULT_TEST_PARALLEL_PROCESSES = 5 _DEFAULT_S3_TEST_PARALLEL_PROCESSES = 50 _SEQUENTIAL_ISOLATION_FLAG = 'sequential_only' _SYNOPSIS = """ gsutil test [-l] [-u] [-f] [command command...] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil test command runs the gsutil unit tests and integration tests. The unit tests use an in-memory mock storage service implementation, while the integration tests send requests to the production service using the `preferred API <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ set in the boto configuration file. CAUTION: The ``test`` command creates test buckets and objects in your project. Force quitting the ``test`` command can leave behind stale buckets, objects, and HMAC keys in your project. To run both the unit tests and integration tests, run the command with no arguments: gsutil test To run the unit tests only (which run quickly): gsutil test -u Tests run in parallel regardless of whether the top-level -m flag is present. To limit the number of tests run in parallel to 10 at a time: gsutil test -p 10 To force tests to run sequentially: gsutil test -p 1 To have sequentially-run tests stop running immediately when an error occurs: gsutil test -f To run tests for one or more individual commands add those commands as arguments. For example, the following command will run the cp and mv command tests: gsutil test cp mv To list available tests, run the test command with the -l argument: gsutil test -l The tests are defined in the code under the gslib/tests module. Each test file is of the format test_[name].py where [name] is the test name you can pass to this command. For example, running "gsutil test ls" would run the tests in "gslib/tests/test_ls.py". You can also run an individual test class or function name by passing the test module followed by the class name and optionally a test name. For example, to run the an entire test class by name: gsutil test naming.GsutilNamingTests or an individual test function: gsutil test cp.TestCp.test_streaming You can list the available tests under a module or class by passing arguments with the -l option. For example, to list all available test functions in the cp module: gsutil test -l cp To output test coverage: gsutil test -c -p 500 coverage html This will output an HTML report to a directory named 'htmlcov'. Test coverage is compatible with v4.1 of the coverage module (https://pypi.python.org/pypi/coverage). <B>OPTIONS</B> -b Run tests against multi-regional US buckets. By default, tests run against regional buckets in us-central1. -c Output coverage information. -f Exit on first sequential test failure. -l List available tests. -p N Run at most N tests in parallel. The default value is %d. -s Run tests against S3 instead of GS. -u Only run unit tests. """ % _DEFAULT_TEST_PARALLEL_PROCESSES) TestProcessData = namedtuple('TestProcessData', 'name return_code stdout stderr') def MakeCustomTestResultClass(total_tests): """Creates a closure of CustomTestResult. Args: total_tests: The total number of tests being run. Returns: An instance of CustomTestResult. """ class CustomTestResult(unittest.TextTestResult): """A subclass of unittest.TextTestResult that prints a progress report.""" def startTest(self, test): super(CustomTestResult, self).startTest(test) if self.dots: test_id = '.'.join(test.id().split('.')[-2:]) message = ('\r%d/%d finished - E[%d] F[%d] s[%d] - %s' % (self.testsRun, total_tests, len(self.errors), len(self.failures), len(self.skipped), test_id)) message = message[:73] message = message.ljust(73) self.stream.write('%s - ' % message) return CustomTestResult def GetTestNamesFromSuites(test_suite): """Takes a list of test suites and returns a list of contained test names.""" suites = [test_suite] test_names = [] while suites: suite = suites.pop() for test in suite: if isinstance(test, unittest.TestSuite): suites.append(test) else: test_names.append(test.id()[len('gslib.tests.test_'):]) return test_names # pylint: disable=protected-access # Need to get into the guts of unittest to evaluate test cases for parallelism. def TestCaseToName(test_case): """Converts a python.unittest to its gsutil test-callable name.""" return (str(test_case.__class__).split('\'')[1] + '.' + test_case._testMethodName) # pylint: disable=protected-access # Need to get into the guts of unittest to evaluate test cases for parallelism. def SplitParallelizableTestSuite(test_suite): """Splits a test suite into groups with different running properties. Args: test_suite: A python unittest test suite. Returns: 4-part tuple of lists of test names: (tests that must be run sequentially, tests that must be isolated in a separate process but can be run either sequentially or in parallel, unit tests that can be run in parallel, integration tests that can run in parallel) """ # pylint: disable=import-not-at-top # Need to import this after test globals are set so that skip functions work. from gslib.tests.testcase.unit_testcase import GsUtilUnitTestCase isolated_tests = [] sequential_tests = [] parallelizable_integration_tests = [] parallelizable_unit_tests = [] items_to_evaluate = [test_suite] cases_to_evaluate = [] # Expand the test suites into individual test cases: while items_to_evaluate: suite_or_case = items_to_evaluate.pop() if isinstance(suite_or_case, unittest.suite.TestSuite): for item in suite_or_case._tests: items_to_evaluate.append(item) elif isinstance(suite_or_case, unittest.TestCase): cases_to_evaluate.append(suite_or_case) for test_case in cases_to_evaluate: test_method = getattr(test_case, test_case._testMethodName, None) if getattr(test_method, 'requires_isolation', False): # Test must be isolated to a separate process, even it if is being # run sequentially. isolated_tests.append(TestCaseToName(test_case)) elif not getattr(test_method, 'is_parallelizable', True): sequential_tests.append(TestCaseToName(test_case)) elif not getattr(test_case, 'is_parallelizable', True): sequential_tests.append(TestCaseToName(test_case)) elif isinstance(test_case, GsUtilUnitTestCase): parallelizable_unit_tests.append(TestCaseToName(test_case)) else: parallelizable_integration_tests.append(TestCaseToName(test_case)) return (sorted(sequential_tests), sorted(isolated_tests), sorted(parallelizable_unit_tests), sorted(parallelizable_integration_tests)) def CountFalseInList(input_list): """Counts number of falses in the input list.""" num_false = 0 for item in input_list: if not item: num_false += 1 return num_false def CreateTestProcesses(parallel_tests, test_index, process_list, process_done, max_parallel_tests, root_coverage_file=None): """Creates test processes to run tests in parallel. Args: parallel_tests: List of all parallel tests. test_index: List index of last created test before this function call. process_list: List of running subprocesses. Created processes are appended to this list. process_done: List of booleans indicating process completion. One 'False' will be added per process created. max_parallel_tests: Maximum number of tests to run in parallel. root_coverage_file: The root .coverage filename if coverage is requested. Returns: Index of last created test. """ orig_test_index = test_index # checking to see if test was invoked from a par file (bundled archive) # if not, add python executable path to ensure correct version of python # is used for testing executable_prefix = [sys.executable] if not InvokedFromParFile() else [] s3_argument = ['-s'] if tests.util.RUN_S3_TESTS else [] multiregional_buckets = ['-b'] if tests.util.USE_MULTIREGIONAL_BUCKETS else [] project_id_arg = [] try: project_id_arg = [ '-o', 'GSUtil:default_project_id=%s' % PopulateProjectId() ] except ProjectIdException: # If we don't have a project ID, unit tests should still be able to pass. pass process_create_start_time = time.time() last_log_time = process_create_start_time while (CountFalseInList(process_done) < max_parallel_tests and test_index < len(parallel_tests)): env = os.environ.copy() if root_coverage_file: env['GSUTIL_COVERAGE_OUTPUT_FILE'] = root_coverage_file envstr = dict() # constructing command list and ensuring each part is str cmd = [ six.ensure_str(part) for part in list( executable_prefix + [gslib.GSUTIL_PATH] + project_id_arg + ['test'] + s3_argument + multiregional_buckets + ['--' + _SEQUENTIAL_ISOLATION_FLAG] + [parallel_tests[test_index][len('gslib.tests.test_'):]] ) ] # yapf: disable for k, v in six.iteritems(env): envstr[six.ensure_str(k)] = six.ensure_str(v) process_list.append( subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=envstr)) test_index += 1 process_done.append(False) if time.time() - last_log_time > 5: print(('Created %d new processes (total %d/%d created)' % (test_index - orig_test_index, len(process_list), len(parallel_tests)))) last_log_time = time.time() if test_index == len(parallel_tests): print(('Test process creation finished (%d/%d created)' % (len(process_list), len(parallel_tests)))) return test_index class TestCommand(Command): """Implementation of gsutil test command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'test', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='buflp:sc', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, supported_private_args=[_SEQUENTIAL_ISOLATION_FLAG], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='test', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Run gsutil unit/integration tests (for developers)'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def RunParallelTests(self, parallel_integration_tests, max_parallel_tests, coverage_filename): """Executes the parallel/isolated portion of the test suite. Args: parallel_integration_tests: List of tests to execute. max_parallel_tests: Maximum number of parallel tests to run at once. coverage_filename: If not None, filename for coverage output. Returns: (int number of test failures, float elapsed time) """ process_list = [] process_done = [] process_results = [] # Tuples of (name, return code, stdout, stderr) num_parallel_failures = 0 # Number of logging cycles we ran with no progress. progress_less_logging_cycles = 0 completed_as_of_last_log = 0 num_parallel_tests = len(parallel_integration_tests) parallel_start_time = last_log_time = time.time() test_index = CreateTestProcesses(parallel_integration_tests, 0, process_list, process_done, max_parallel_tests, root_coverage_file=coverage_filename) while len(process_results) < num_parallel_tests: for proc_num in range(len(process_list)): if process_done[proc_num] or process_list[proc_num].poll() is None: continue process_done[proc_num] = True stdout, stderr = process_list[proc_num].communicate() process_list[proc_num].stdout.close() process_list[proc_num].stderr.close() # TODO: Differentiate test failures from errors. if process_list[proc_num].returncode != 0: num_parallel_failures += 1 process_results.append( TestProcessData(name=parallel_integration_tests[proc_num], return_code=process_list[proc_num].returncode, stdout=stdout, stderr=stderr)) if len(process_list) < num_parallel_tests: test_index = CreateTestProcesses(parallel_integration_tests, test_index, process_list, process_done, max_parallel_tests, root_coverage_file=coverage_filename) if len(process_results) < num_parallel_tests: if time.time() - last_log_time > 5: print( '%d/%d finished - %d failures' % (len(process_results), num_parallel_tests, num_parallel_failures)) if len(process_results) == completed_as_of_last_log: progress_less_logging_cycles += 1 else: completed_as_of_last_log = len(process_results) # A process completed, so we made progress. progress_less_logging_cycles = 0 if progress_less_logging_cycles > 4: # Ran 5 or more logging cycles with no progress, let the user # know which tests are running slowly or hanging. still_running = [] for proc_num in range(len(process_list)): if not process_done[proc_num]: still_running.append(parallel_integration_tests[proc_num]) elapsed = time.time() - parallel_start_time print(('{sec} seconds elapsed since beginning parallel tests.\n' 'Still running: {procs}').format( sec=str(int(elapsed)), procs=still_running, )) # TODO: Terminate still-running processes if they # hang for a long time. last_log_time = time.time() time.sleep(1) process_run_finish_time = time.time() if num_parallel_failures: for result in process_results: if result.return_code != 0: new_stderr = result.stderr.split(b'\n') print('Results for failed test %s:' % result.name) for line in new_stderr: print(line.decode(UTF8).strip()) return (num_parallel_failures, (process_run_finish_time - parallel_start_time)) def PrintTestResults(self, num_sequential_tests, sequential_success, sequential_skipped, sequential_time_elapsed, num_parallel_tests, num_parallel_failures, parallel_time_elapsed): """Prints test results for parallel and sequential tests.""" # TODO: Properly track test skips. print('Parallel tests complete. Success: %s Fail: %s' % (num_parallel_tests - num_parallel_failures, num_parallel_failures)) print(( 'Ran %d tests in %.3fs (%d sequential in %.3fs, %d parallel in %.3fs)' % (num_parallel_tests + num_sequential_tests, float(sequential_time_elapsed + parallel_time_elapsed), num_sequential_tests, float(sequential_time_elapsed), num_parallel_tests, float(parallel_time_elapsed)))) self.PrintSkippedTests(sequential_skipped) print() if not num_parallel_failures and sequential_success: print('OK') else: if num_parallel_failures: print('FAILED (parallel tests)') if not sequential_success: print('FAILED (sequential tests)') # TODO: Parallel skipped tests are never gathered anywhere, this needs implementation in RunParallelTests def PrintSkippedTests(self, sequential_skipped=set(), parallel_skipped=set()): """Prints all skipped tests, and the reasons they were skipped. Takes the union of sequentual_skipped and parallel_skipped, and pretty-prints the resulting methods and reasons. Note that these two arguments are lists of tuples from TestResult.skipped as described here: https://docs.python.org/2/library/unittest.html#unittest.TestResult.skipped Args: sequentual_skipped: An instance of TestResult.skipped. parallel_skipped: An instance of TestResult.skipped. """ if len(sequential_skipped) > 0 or len(parallel_skipped) > 0: sequential_skipped = set(sequential_skipped) parallel_skipped = set(parallel_skipped) all_skipped = sequential_skipped.union(parallel_skipped) print('Tests skipped:') for method, reason in all_skipped: print(' ' + method.id()) print(' Reason: ' + reason) def RunCommand(self): """Command entry point for the test command.""" failfast = False list_tests = False max_parallel_tests = _DEFAULT_TEST_PARALLEL_PROCESSES perform_coverage = False sequential_only = False if self.sub_opts: for o, a in self.sub_opts: if o == '-b': tests.util.USE_MULTIREGIONAL_BUCKETS = True elif o == '-c': perform_coverage = True elif o == '-f': failfast = True elif o == '-l': list_tests = True elif o == ('--' + _SEQUENTIAL_ISOLATION_FLAG): # Called to isolate a single test in a separate process. # Don't try to isolate it again (would lead to an infinite loop). sequential_only = True elif o == '-p': max_parallel_tests = long(a) elif o == '-s': if not tests.util.HAS_S3_CREDS: raise CommandException('S3 tests require S3 credentials. Please ' 'add appropriate credentials to your .boto ' 'file and re-run.') tests.util.RUN_S3_TESTS = True elif o == '-u': tests.util.RUN_INTEGRATION_TESTS = False if perform_coverage and not coverage: raise CommandException( 'Coverage has been requested but the coverage module was not found. ' 'You can install it with "pip install coverage".') if (tests.util.RUN_S3_TESTS and max_parallel_tests > _DEFAULT_S3_TEST_PARALLEL_PROCESSES): self.logger.warn( 'Reducing parallel tests to %d due to S3 maximum bucket ' 'limitations.', _DEFAULT_S3_TEST_PARALLEL_PROCESSES) max_parallel_tests = _DEFAULT_S3_TEST_PARALLEL_PROCESSES test_names = sorted(GetTestNames()) if list_tests and not self.args: print('Found %d test names:' % len(test_names)) print(' ', '\n '.join(sorted(test_names))) return 0 # Set list of commands to test if supplied. if self.args: commands_to_test = [] for name in self.args: if name in test_names or name.split('.')[0] in test_names: commands_to_test.append('gslib.tests.test_%s' % name) else: commands_to_test.append(name) else: commands_to_test = ['gslib.tests.test_%s' % name for name in test_names] # Installs a ctrl-c handler that tries to cleanly tear down tests. unittest.installHandler() loader = unittest.TestLoader() if commands_to_test: suite = unittest.TestSuite() for command_name in commands_to_test: try: suite_for_current_command = loader.loadTestsFromName(command_name) suite.addTests(suite_for_current_command) except (ImportError, AttributeError) as e: msg = ('Failed to import test code from file %s. TestLoader provided ' 'this error:\n\n%s' % (command_name, str(e))) # Try to give a better error message; by default, unittest swallows # ImportErrors and only shows that an import failed, not why. E.g.: # "'module' object has no attribute 'test_cp' try: __import__(command_name) except Exception as e: stack_trace = traceback.format_exc() err = re.sub('\\n', '\n ', stack_trace) msg += '\n\nAdditional traceback:\n\n%s' % (err) raise CommandException(msg) if list_tests: test_names = GetTestNamesFromSuites(suite) print('Found %d test names:' % len(test_names)) print(' ', '\n '.join(sorted(test_names))) return 0 if logging.getLogger().getEffectiveLevel() <= logging.INFO: verbosity = 1 else: verbosity = 2 logging.disable(logging.ERROR) if perform_coverage: # We want to run coverage over the gslib module, but filter out the test # modules and any third-party code. We also filter out anything under the # temporary directory. Otherwise, the gsutil update test (which copies # code to the temporary directory) gets included in the output. coverage_controller = coverage.coverage(source=['gslib'], omit=[ 'gslib/third_party/*', 'gslib/tests/*', tempfile.gettempdir() + '*', ]) coverage_controller.erase() coverage_controller.start() num_parallel_failures = 0 sequential_success = False (sequential_tests, isolated_tests, parallel_unit_tests, parallel_integration_tests) = (SplitParallelizableTestSuite(suite)) # Since parallel integration tests are run in a separate process, they # won't get the override to tests.util, so skip them here. if not tests.util.RUN_INTEGRATION_TESTS: parallel_integration_tests = [] logging.debug('Sequential tests to run: %s', sequential_tests) logging.debug('Isolated tests to run: %s', isolated_tests) logging.debug('Parallel unit tests to run: %s', parallel_unit_tests) logging.debug('Parallel integration tests to run: %s', parallel_integration_tests) # If we're running an already-isolated test (spawned in isolation by a # previous test process), or we have no parallel tests to run, # just run sequentially. For now, unit tests are always run sequentially. run_tests_sequentially = (sequential_only or (len(parallel_integration_tests) <= 1 and not isolated_tests)) # Disable analytics for the duration of testing. This is set as an # environment variable so that the subprocesses will also not report. os.environ['GSUTIL_TEST_ANALYTICS'] = '1' if run_tests_sequentially: total_tests = suite.countTestCases() resultclass = MakeCustomTestResultClass(total_tests) runner = unittest.TextTestRunner(verbosity=verbosity, resultclass=resultclass, failfast=failfast) ret = runner.run(suite) sequential_success = ret.wasSuccessful() else: if max_parallel_tests == 1: # We can't take advantage of parallelism, though we may have tests that # need isolation. sequential_tests += parallel_integration_tests parallel_integration_tests = [] sequential_start_time = time.time() # TODO: For now, run unit tests sequentially because they are fast. # We could potentially shave off several seconds of execution time # by executing them in parallel with the integration tests. if len(sequential_tests) + len(parallel_unit_tests): print('Running %d tests sequentially.' % (len(sequential_tests) + len(parallel_unit_tests))) sequential_tests_to_run = sequential_tests + parallel_unit_tests suite = loader.loadTestsFromNames( sorted([test_name for test_name in sequential_tests_to_run])) num_sequential_tests = suite.countTestCases() resultclass = MakeCustomTestResultClass(num_sequential_tests) runner = unittest.TextTestRunner(verbosity=verbosity, resultclass=resultclass, failfast=failfast) ret = runner.run(suite) sequential_success = ret.wasSuccessful() sequential_skipped = ret.skipped else: num_sequential_tests = 0 sequential_success = True sequential_time_elapsed = time.time() - sequential_start_time # At this point, all tests get their own process so just treat the # isolated tests as parallel tests. parallel_integration_tests += isolated_tests num_parallel_tests = len(parallel_integration_tests) if not num_parallel_tests: pass else: sequential_skipped = [] num_processes = min(max_parallel_tests, num_parallel_tests) if num_parallel_tests > 1 and max_parallel_tests > 1: message = 'Running %d tests in parallel mode (%d processes).' if num_processes > _DEFAULT_TEST_PARALLEL_PROCESSES: message += ( ' Please be patient while your CPU is incinerated. ' 'If your machine becomes unresponsive, consider reducing ' 'the amount of parallel test processes by running ' '\'gsutil test -p <num_processes>\'.') print(('\n'.join( textwrap.wrap(message % (num_parallel_tests, num_processes))))) else: print(('Running %d tests sequentially in isolated processes.' % num_parallel_tests)) (num_parallel_failures, parallel_time_elapsed) = self.RunParallelTests( parallel_integration_tests, max_parallel_tests, coverage_controller.data_files.filename if perform_coverage else None) self.PrintTestResults(num_sequential_tests, sequential_success, sequential_skipped, sequential_time_elapsed, num_parallel_tests, num_parallel_failures, parallel_time_elapsed) if perform_coverage: coverage_controller.stop() coverage_controller.combine() coverage_controller.save() print(('Coverage information was saved to: %s' % coverage_controller.data_files.filename)) # Re-enable analytics to report the test command. os.environ['GSUTIL_TEST_ANALYTICS'] = '0' if sequential_success and not num_parallel_failures: ResetFailureCount() return 0 return 1 tracker_file.py 0000644 00000060352 15025013616 0007555 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for tracker file functionality.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import errno import hashlib import json import os import re import sys import six from boto import config from gslib.exception import CommandException from gslib.utils.boto_util import GetGsutilStateDir from gslib.utils.boto_util import ResumableThreshold from gslib.utils.constants import UTF8 from gslib.utils.hashing_helper import GetMd5 from gslib.utils.system_util import CreateDirIfNeeded # The maximum length of a file name can vary wildly between different # operating systems, so we always ensure that tracker files are less # than 100 characters in order to avoid any such issues. MAX_TRACKER_FILE_NAME_LENGTH = 100 TRACKER_FILE_UNWRITABLE_EXCEPTION_TEXT = ( 'Couldn\'t write tracker file (%s): %s. This can happen if gsutil is ' 'configured to save tracker files to an unwritable directory)') # Format for upload tracker files. ENCRYPTION_UPLOAD_TRACKER_ENTRY = 'encryption_key_sha256' SERIALIZATION_UPLOAD_TRACKER_ENTRY = 'serialization_data' class TrackerFileType(object): UPLOAD = 'upload' DOWNLOAD = 'download' DOWNLOAD_COMPONENT = 'download_component' PARALLEL_UPLOAD = 'parallel_upload' SLICED_DOWNLOAD = 'sliced_download' REWRITE = 'rewrite' def _HashFilename(filename): """Apply a hash function (SHA1) to shorten the passed file name. The spec for the hashed file name is as follows: TRACKER_<hash>_<trailing> where hash is a SHA1 hash on the original file name and trailing is the last 16 chars from the original file name. Max file name lengths vary by operating system so the goal of this function is to ensure the hashed version takes fewer than 100 characters. Args: filename: file name to be hashed. May be unicode or bytes. Returns: shorter, hashed version of passed file name """ if isinstance(filename, six.text_type): filename_bytes = filename.encode(UTF8) filename_str = filename else: filename_bytes = filename filename_str = filename.decode(UTF8) m = hashlib.sha1(filename_bytes) return 'TRACKER_' + m.hexdigest() + '.' + filename_str[-16:] def CreateTrackerDirIfNeeded(): """Looks up or creates the gsutil tracker file directory. This is the configured directory where gsutil keeps its resumable transfer tracker files. This function creates it if it doesn't already exist. Returns: The pathname to the tracker directory. """ tracker_dir = config.get('GSUtil', 'resumable_tracker_dir', os.path.join(GetGsutilStateDir(), 'tracker-files')) CreateDirIfNeeded(tracker_dir) return tracker_dir def GetRewriteTrackerFilePath(src_bucket_name, src_obj_name, dst_bucket_name, dst_obj_name, api_selector): """Gets the tracker file name described by the arguments. Args: src_bucket_name: Source bucket (string). src_obj_name: Source object (string). dst_bucket_name: Destination bucket (string). dst_obj_name: Destination object (string) api_selector: API to use for this operation. Returns: File path to tracker file. """ # Encode the src and dest bucket and object names into the tracker file # name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'rewrite__%s__%s__%s__%s__%s.token' % (src_bucket_name, src_obj_name, dst_bucket_name, dst_obj_name, api_selector))) return _HashAndReturnPath(res_tracker_file_name, TrackerFileType.REWRITE) def GetTrackerFilePath(dst_url, tracker_file_type, api_selector, src_url=None, component_num=None): """Gets the tracker file name described by the arguments. Args: dst_url: Destination URL for tracker file. tracker_file_type: TrackerFileType for this operation. api_selector: API to use for this operation. src_url: Source URL for the source file name for parallel uploads. component_num: Component number if this is a download component, else None. Returns: File path to tracker file. """ if tracker_file_type == TrackerFileType.UPLOAD: # Encode the dest bucket and object name into the tracker file name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'resumable_upload__%s__%s__%s.url' % (dst_url.bucket_name, dst_url.object_name, api_selector))) elif tracker_file_type == TrackerFileType.DOWNLOAD: # Encode the fully-qualified dest file name into the tracker file name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'resumable_download__%s__%s.etag' % (os.path.realpath(dst_url.object_name), api_selector))) elif tracker_file_type == TrackerFileType.DOWNLOAD_COMPONENT: # Encode the fully-qualified dest file name and the component number # into the tracker file name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'resumable_download__%s__%s__%d.etag' % (os.path.realpath(dst_url.object_name), api_selector, component_num))) elif tracker_file_type == TrackerFileType.PARALLEL_UPLOAD: # Encode the dest bucket and object names as well as the source file name # into the tracker file name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'parallel_upload__%s__%s__%s__%s.url' % (dst_url.bucket_name, dst_url.object_name, src_url, api_selector))) elif tracker_file_type == TrackerFileType.SLICED_DOWNLOAD: # Encode the fully-qualified dest file name into the tracker file name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'sliced_download__%s__%s.etag' % (os.path.realpath(dst_url.object_name), api_selector))) elif tracker_file_type == TrackerFileType.REWRITE: # Should use GetRewriteTrackerFilePath instead. raise NotImplementedError() return _HashAndReturnPath(res_tracker_file_name, tracker_file_type) def DeleteDownloadTrackerFiles(dst_url, api_selector): """Deletes all tracker files corresponding to an object download. Args: dst_url: StorageUrl describing the destination file. api_selector: The Cloud API implementation used. """ # Delete non-sliced download tracker file. DeleteTrackerFile( GetTrackerFilePath(dst_url, TrackerFileType.DOWNLOAD, api_selector)) # Delete all sliced download tracker files. tracker_files = GetSlicedDownloadTrackerFilePaths(dst_url, api_selector) for tracker_file in tracker_files: DeleteTrackerFile(tracker_file) def GetSlicedDownloadTrackerFilePaths(dst_url, api_selector, num_components=None): """Gets a list of sliced download tracker file paths. The list consists of the parent tracker file path in index 0, and then any existing component tracker files in [1:]. Args: dst_url: Destination URL for tracker file. api_selector: API to use for this operation. num_components: The number of component tracker files, if already known. If not known, the number will be retrieved from the parent tracker file on disk. Returns: File path to tracker file. """ parallel_tracker_file_path = GetTrackerFilePath( dst_url, TrackerFileType.SLICED_DOWNLOAD, api_selector) tracker_file_paths = [parallel_tracker_file_path] # If we don't know the number of components, check the tracker file. if num_components is None: tracker_file = None try: tracker_file = open(parallel_tracker_file_path, 'r') num_components = json.load(tracker_file)['num_components'] except (IOError, ValueError): return tracker_file_paths finally: if tracker_file: tracker_file.close() for i in range(num_components): tracker_file_paths.append( GetTrackerFilePath(dst_url, TrackerFileType.DOWNLOAD_COMPONENT, api_selector, component_num=i)) return tracker_file_paths def _HashAndReturnPath(res_tracker_file_name, tracker_file_type): """Hashes and returns a tracker file path. Args: res_tracker_file_name: The tracker file name prior to it being hashed. tracker_file_type: The TrackerFileType of res_tracker_file_name. Returns: Final (hashed) tracker file path. """ resumable_tracker_dir = CreateTrackerDirIfNeeded() hashed_tracker_file_name = _HashFilename(res_tracker_file_name) tracker_file_name = '%s_%s' % (str(tracker_file_type).lower(), hashed_tracker_file_name) tracker_file_path = '%s%s%s' % (resumable_tracker_dir, os.sep, tracker_file_name) assert len(tracker_file_name) < MAX_TRACKER_FILE_NAME_LENGTH return tracker_file_path def DeleteTrackerFile(tracker_file_name): if tracker_file_name and os.path.exists(tracker_file_name): os.unlink(tracker_file_name) def HashRewriteParameters(src_obj_metadata, dst_obj_metadata, projection, src_generation=None, gen_match=None, meta_gen_match=None, canned_acl=None, max_bytes_per_call=None, src_dec_key_sha256=None, dst_enc_key_sha256=None, fields=None): """Creates an MD5 hex digest of the parameters for a rewrite call. Resuming rewrites requires that the input parameters are identical. Thus, the rewrite tracker file needs to represent the input parameters. For easy comparison, hash the input values. If a user does a performs a same-source/same-destination rewrite via a different command (for example, with a changed ACL), the hashes will not match and we will restart the rewrite from the beginning. Args: src_obj_metadata: apitools Object describing source object. Must include bucket, name, and etag. dst_obj_metadata: apitools Object describing destination object. Must include bucket and object name projection: Projection used for the API call. src_generation: Optional source generation. gen_match: Optional generation precondition. meta_gen_match: Optional metageneration precondition. canned_acl: Optional canned ACL string. max_bytes_per_call: Optional maximum bytes rewritten per call. src_dec_key_sha256: Optional SHA256 hash string of decryption key for source object. dst_enc_key_sha256: Optional SHA256 hash string of encryption key for destination object. fields: Optional fields to include in response to call. Returns: MD5 hex digest Hash of the input parameters, or None if required parameters are missing. """ if (not src_obj_metadata or not src_obj_metadata.bucket or not src_obj_metadata.name or not src_obj_metadata.etag or not dst_obj_metadata or not dst_obj_metadata.bucket or not dst_obj_metadata.name or not projection): return md5_hash = GetMd5() for input_param in ( src_obj_metadata, dst_obj_metadata, projection, src_generation, gen_match, meta_gen_match, canned_acl, fields, max_bytes_per_call, src_dec_key_sha256, dst_enc_key_sha256, ): # Tracker file matching changed between gsutil 4.15 -> 4.16 and will cause # rewrites to start over from the beginning on a gsutil version upgrade. if input_param is not None: md5_hash.update(six.text_type(input_param).encode('UTF8')) return md5_hash.hexdigest() def ReadRewriteTrackerFile(tracker_file_name, rewrite_params_hash): """Attempts to read a rewrite tracker file. Args: tracker_file_name: Tracker file path string. rewrite_params_hash: MD5 hex digest of rewrite call parameters constructed by HashRewriteParameters. Returns: String rewrite_token for resuming rewrite requests if a matching tracker file exists, None otherwise (which will result in starting a new rewrite). """ # Check to see if we already have a matching tracker file. tracker_file = None if not rewrite_params_hash: return try: tracker_file = open(tracker_file_name, 'r') existing_hash = tracker_file.readline().rstrip('\n') if existing_hash == rewrite_params_hash: # Next line is the rewrite token. return tracker_file.readline().rstrip('\n') except IOError as e: # Ignore non-existent file (happens first time a rewrite is attempted. if e.errno != errno.ENOENT: sys.stderr.write( ('Couldn\'t read Copy tracker file (%s): %s. Restarting copy ' 'from scratch.' % (tracker_file_name, e.strerror))) finally: if tracker_file: tracker_file.close() def WriteRewriteTrackerFile(tracker_file_name, rewrite_params_hash, rewrite_token): """Writes a rewrite tracker file. Args: tracker_file_name: Tracker file path string. rewrite_params_hash: MD5 hex digest of rewrite call parameters constructed by HashRewriteParameters. rewrite_token: Rewrite token string returned by the service. """ _WriteTrackerFile(tracker_file_name, '%s\n%s\n' % (rewrite_params_hash, rewrite_token)) def ReadOrCreateDownloadTrackerFile(src_obj_metadata, dst_url, logger, api_selector, start_byte, existing_file_size, component_num=None): """Checks for a download tracker file and creates one if it does not exist. The methodology for determining the download start point differs between normal and sliced downloads. For normal downloads, the existing bytes in the file are presumed to be correct and have been previously downloaded from the server (if a tracker file exists). In this case, the existing file size is used to determine the download start point. For sliced downloads, the number of bytes previously retrieved from the server cannot be determined from the existing file size, and so the number of bytes known to have been previously downloaded is retrieved from the tracker file. Args: src_obj_metadata: Metadata for the source object. Must include etag and generation. dst_url: Destination URL for tracker file. logger: For outputting log messages. api_selector: API to use for this operation. start_byte: The start byte of the byte range for this download. existing_file_size: Size of existing file for this download on disk. component_num: The component number, if this is a component of a parallel download, else None. Returns: tracker_file_name: The name of the tracker file, if one was used. download_start_byte: The first byte that still needs to be downloaded. """ assert src_obj_metadata.etag tracker_file_name = None if src_obj_metadata.size < ResumableThreshold(): # Don't create a tracker file for a small downloads; cross-process resumes # won't work, but restarting a small download is inexpensive. return tracker_file_name, start_byte download_name = dst_url.object_name if component_num is None: tracker_file_type = TrackerFileType.DOWNLOAD else: tracker_file_type = TrackerFileType.DOWNLOAD_COMPONENT download_name += ' component %d' % component_num tracker_file_name = GetTrackerFilePath(dst_url, tracker_file_type, api_selector, component_num=component_num) tracker_file = None # Check to see if we already have a matching tracker file. try: tracker_file = open(tracker_file_name, 'r') if tracker_file_type is TrackerFileType.DOWNLOAD: etag_value = tracker_file.readline().rstrip('\n') if etag_value == src_obj_metadata.etag: return tracker_file_name, existing_file_size elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: component_data = json.loads(tracker_file.read()) if (component_data['etag'] == src_obj_metadata.etag and component_data['generation'] == src_obj_metadata.generation): return tracker_file_name, component_data['download_start_byte'] logger.warn('Tracker file doesn\'t match for download of %s. Restarting ' 'download from scratch.' % download_name) except (IOError, ValueError) as e: # Ignore non-existent file (happens first time a download # is attempted on an object), but warn user for other errors. if isinstance(e, ValueError) or e.errno != errno.ENOENT: logger.warn('Couldn\'t read download tracker file (%s): %s. Restarting ' 'download from scratch.' % (tracker_file_name, str(e))) finally: if tracker_file: tracker_file.close() # There wasn't a matching tracker file, so create one and then start the # download from scratch. if tracker_file_type is TrackerFileType.DOWNLOAD: _WriteTrackerFile(tracker_file_name, '%s\n' % src_obj_metadata.etag) elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: WriteDownloadComponentTrackerFile(tracker_file_name, src_obj_metadata, start_byte) return tracker_file_name, start_byte def GetDownloadStartByte(src_obj_metadata, dst_url, api_selector, start_byte, existing_file_size, component_num=None): """Returns the download starting point. The methodology of this function is the same as in ReadOrCreateDownloadTrackerFile, with the difference that we are not interested here in possibly creating a tracker file. In case there is no tracker file, this means the download starting point is start_byte. Args: src_obj_metadata: Metadata for the source object. Must include etag and generation. dst_url: Destination URL for tracker file. api_selector: API to use for this operation. start_byte: The start byte of the byte range for this download. existing_file_size: Size of existing file for this download on disk. component_num: The component number, if this is a component of a parallel download, else None. Returns: download_start_byte: The first byte that still needs to be downloaded. """ assert src_obj_metadata.etag tracker_file_name = None if src_obj_metadata.size < ResumableThreshold(): # There is no tracker file for small downloads; this means we start from # scratch. return start_byte if component_num is None: tracker_file_type = TrackerFileType.DOWNLOAD else: tracker_file_type = TrackerFileType.DOWNLOAD_COMPONENT tracker_file_name = GetTrackerFilePath(dst_url, tracker_file_type, api_selector, component_num=component_num) tracker_file = None # Check to see if we already have a matching tracker file. try: tracker_file = open(tracker_file_name, 'r') if tracker_file_type is TrackerFileType.DOWNLOAD: etag_value = tracker_file.readline().rstrip('\n') if etag_value == src_obj_metadata.etag: return existing_file_size elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: component_data = json.loads(tracker_file.read()) if (component_data['etag'] == src_obj_metadata.etag and component_data['generation'] == src_obj_metadata.generation): return component_data['download_start_byte'] except (IOError, ValueError): # If the file does not exist, there is not much we can do at this point. pass finally: if tracker_file: tracker_file.close() # There wasn't a matching tracker file, which means our starting point is # start_byte. return start_byte def WriteDownloadComponentTrackerFile(tracker_file_name, src_obj_metadata, current_file_pos): """Updates or creates a download component tracker file on disk. Args: tracker_file_name: The name of the tracker file. src_obj_metadata: Metadata for the source object. Must include etag. current_file_pos: The current position in the file. """ component_data = { 'etag': src_obj_metadata.etag, 'generation': src_obj_metadata.generation, 'download_start_byte': current_file_pos, } _WriteTrackerFile(tracker_file_name, json.dumps(component_data)) def _WriteTrackerFile(tracker_file_name, data): """Creates a tracker file, storing the input data.""" try: fd = os.open(tracker_file_name, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) with os.fdopen(fd, 'w') as tf: tf.write(data) return False except (IOError, OSError) as e: raise RaiseUnwritableTrackerFileException(tracker_file_name, e.strerror) def WriteJsonDataToTrackerFile(tracker_file_name, data): """Create a tracker file and write json data to it. Raises: TypeError: If the data is not JSON serializable """ try: json_str = json.dumps(data) except TypeError as err: raise RaiseUnwritableTrackerFileException(tracker_file_name, err.strerror) _WriteTrackerFile(tracker_file_name, json_str) def GetUploadTrackerData(tracker_file_name, logger, encryption_key_sha256=None): """Reads tracker data from an upload tracker file if it exists. Deletes the tracker file if it uses an old format or the desired encryption key has changed. Args: tracker_file_name: Tracker file name for this upload. logger: logging.Logger for outputting log messages. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. Returns: Serialization data if the tracker file already exists (resume existing upload), None otherwise. """ tracker_file = None remove_tracker_file = False encryption_restart = False # If we already have a matching tracker file, get the serialization data # so that we can resume the upload. try: tracker_file = open(tracker_file_name, 'r') tracker_data = tracker_file.read() tracker_json = json.loads(tracker_data) if tracker_json[ENCRYPTION_UPLOAD_TRACKER_ENTRY] != encryption_key_sha256: encryption_restart = True remove_tracker_file = True else: return tracker_json[SERIALIZATION_UPLOAD_TRACKER_ENTRY] except IOError as e: # Ignore non-existent file (happens first time a upload is attempted on an # object, or when re-starting an upload after a # ResumableUploadStartOverException), but warn user for other errors. if e.errno != errno.ENOENT: logger.warn( 'Couldn\'t read upload tracker file (%s): %s. Restarting ' 'upload from scratch.', tracker_file_name, e.strerror) except (KeyError, ValueError) as e: # Old tracker files used a non-JSON format; rewrite it and assume no # encryption key. remove_tracker_file = True if encryption_key_sha256 is not None: encryption_restart = True else: # If encryption key is still None, we can resume using the old format. return tracker_data finally: if tracker_file: tracker_file.close() if encryption_restart: logger.warn( 'Upload tracker file (%s) does not match current encryption ' 'key. Restarting upload from scratch with a new tracker ' 'file that uses the current encryption key.', tracker_file_name) if remove_tracker_file: DeleteTrackerFile(tracker_file_name) def RaiseUnwritableTrackerFileException(tracker_file_name, error_str): """Raises an exception when unable to write the tracker file.""" raise CommandException(TRACKER_FILE_UNWRITABLE_EXCEPTION_TEXT % (tracker_file_name, error_str)) help_provider.py 0000644 00000010406 15025013616 0007760 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Module defining help types and providers for gsutil commands.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections from gslib.exception import CommandException ALL_HELP_TYPES = ['command_help', 'additional_help'] # Constants enforced by SanityCheck # Mainly here for help output formatting purposes, can be changed if necessary. MAX_HELP_NAME_LEN = 17 MIN_ONE_LINE_SUMMARY_LEN = 10 MAX_ONE_LINE_SUMMARY_LEN = 80 - MAX_HELP_NAME_LEN DESCRIPTION_PREFIX = """ <B>DESCRIPTION</B>""" SYNOPSIS_PREFIX = """ <B>SYNOPSIS</B>""" class HelpProvider(object): """Interface for providing help.""" # Each subclass of HelpProvider define a property named 'help_spec' that is # an instance of the following class. HelpSpec = collections.namedtuple( 'HelpSpec', [ # Name of command or auxiliary help info for which this help applies. 'help_name', # List of help name aliases. 'help_name_aliases', # Type of help. 'help_type', # One line summary of this help. 'help_one_line_summary', # The full help text. 'help_text', # Help text for subcommands of the command's help being specified. 'subcommand_help_text', ]) # Each subclass must override this with an instance of HelpSpec. help_spec = None # This is a static helper instead of a class method because the help loader # (gslib.commands.help._LoadHelpMaps()) operates on classes not instances. def SanityCheck(help_provider, help_name_map): """Helper for checking that a HelpProvider has minimally adequate content.""" # Sanity check the content. help_name_len = len(help_provider.help_spec.help_name) assert (help_name_len > 1 and help_name_len < MAX_HELP_NAME_LEN ), 'The help name "{text}" must be less then {max}'.format( text=help_provider.help_spec.help_name, max=MAX_HELP_NAME_LEN) for hna in help_provider.help_spec.help_name_aliases: assert hna one_line_summary_len = len(help_provider.help_spec.help_one_line_summary) assert (one_line_summary_len >= MIN_ONE_LINE_SUMMARY_LEN), ( 'The one line summary "{text}" with a length of {length} must be ' + 'more then {min} characters').format( text=help_provider.help_spec.help_one_line_summary, length=one_line_summary_len, min=MIN_ONE_LINE_SUMMARY_LEN) assert (one_line_summary_len <= MAX_ONE_LINE_SUMMARY_LEN), ( 'The one line summary "{text}" with a length of {length} must be ' + 'less then {max} characters').format( text=help_provider.help_spec.help_one_line_summary, length=one_line_summary_len, max=MAX_ONE_LINE_SUMMARY_LEN) assert len(help_provider.help_spec.help_text ) > 10, 'The length of "{text}" must be less then 10'.format( text=help_provider.help_spec.help_text) # Ensure there are no dupe help names or aliases across commands. name_check_list = [help_provider.help_spec.help_name] name_check_list.extend(help_provider.help_spec.help_name_aliases) for name_or_alias in name_check_list: if name_or_alias in help_name_map: raise CommandException( 'Duplicate help name/alias "%s" found while loading help from %s. ' 'That name/alias was already taken by %s' % (name_or_alias, help_provider.__module__, help_name_map[name_or_alias].__module__)) def CreateHelpText(synopsis, description): """Helper for adding help text headers given synopsis and description.""" return SYNOPSIS_PREFIX + synopsis + DESCRIPTION_PREFIX + description plurality_checkable_iterator.py 0000644 00000010630 15025013616 0013034 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Iterator wrapper for checking wrapped iterator's emptiness or plurality.""" # TODO: Here and elsewhere (wildcard_iterator, name_expansion), do not reference # __iter__ directly because it causes the first element to be instantiated. # Instead, implement __iter__ as a return self and implement the next() function # which returns (not yields) the values. This necessitates that in the case # of the iterator classes, the iterator is used once per class instantiation # so that next() calls do not collide, but this semantic has been long-assumed # by the iterator classes for the use of __iter__ anyway. from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys import six class PluralityCheckableIterator(six.Iterator): """Iterator wrapper class. Allows you to check whether the wrapped iterator is empty and whether it has more than 1 element. This iterator accepts three types of values from the iterator it wraps: 1. A yielded element (this is the normal case). 2. A raised exception, which will be buffered and re-raised when it is reached in this iterator. 3. A yielded tuple of (exception, stack trace), which will be buffered and raised with it is reached in this iterator. """ def __init__(self, it): # Need to get the iterator function here so that we don't immediately # instantiate the first element (which could raise an exception). self.orig_iterator = it self.base_iterator = None self.head = [] self.underlying_iter_empty = False def _PopulateHead(self, num_elements=1): """Populates self.head from the underlying iterator. Args: num_elements: Populate until self.head contains this many elements (or until the underlying iterator runs out). Returns: Number of elements at self.head after execution complete. """ while not self.underlying_iter_empty and len(self.head) < num_elements: try: if not self.base_iterator: self.base_iterator = iter(self.orig_iterator) e = next(self.base_iterator) self.underlying_iter_empty = False if isinstance(e, tuple) and isinstance(e[0], Exception): self.head.append(('exception', e[0], e[1])) else: self.head.append(('element', e)) except StopIteration: # Indicates we can no longer call next() on underlying iterator, but # there could still be elements left to iterate in head. self.underlying_iter_empty = True except Exception as e: # pylint: disable=broad-except # Buffer the exception and raise it when the element is accessed. # Also, preserve the original stack trace, as the stack trace from # within plurality_checkable_iterator.next is not very useful. self.head.append(('exception', e, sys.exc_info()[2])) return len(self.head) def __iter__(self): return self def __next__(self): if self._PopulateHead(): item_tuple = self.head.pop(0) if item_tuple[0] == 'element': return item_tuple[1] else: # buffered exception raise six.reraise(item_tuple[1].__class__, item_tuple[1], item_tuple[2]) raise StopIteration() def IsEmpty(self): return not self._PopulateHead() def HasPlurality(self): # Populate 2 elements (if possible) into head so we can check whether # iterator has more than 1 item remaining. return self._PopulateHead(num_elements=2) > 1 def PeekException(self): """Raises an exception if the first iterated element raised.""" if self._PopulateHead() and self.head[0][0] == 'exception': exception_tuple = self.head[0] raise six.reraise(exception_tuple[1].__class__, exception_tuple[1], exception_tuple[2]) third_party/__pycache__/__init__.cpython-39.pyc 0000644 00000000204 15025013616 0015470 0 ustar 00 a +(Wg � @ s d S )N� r r r �)/opt/gsutil/gslib/third_party/__init__.py�<module> � third_party/iamcredentials_apitools/__pycache__/iamcredentials_v1_messages.cpython-39.pyc 0000644 00000027364 15025013616 0026130 0 ustar 00 a +(Wg�) � @ s< d Z ddlmZ ddlmZ dZG dd� dej�ZG dd� dej�Z G d d � d ej�Z G d d � d ej�Z G d d� dej�Z G dd� dej�Z G dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd � d ej�ZG d d � d ej�Ze�ed d � e�ejd!d"� e�ejd#d$� d%S )&z�Generated message classes for iamcredentials version v1. Creates short-lived, limited-privilege credentials for IAM service accounts. � )�messages)�encoding�iamcredentialsc @ s6 e Zd ZdZejddd�Ze�d�Zejddd�ZdS )�GenerateAccessTokenRequestak A GenerateAccessTokenRequest object. Fields: delegates: The sequence of service accounts in a delegation chain. Each service account must be granted the `roles/iam.serviceAccountTokenCreator` role on its next service account in the chain. The last service account in the chain must be granted the `roles/iam.serviceAccountTokenCreator` role on the service account that is specified in the `name` field of the request. The delegates must have the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. lifetime: The desired lifetime duration of the access token in seconds. Must be set to a value less than or equal to 3600 (1 hour). If a value is not specified, the token's lifetime will be set to a default value of one hour. scope: Code to identify the scopes to be included in the OAuth 2.0 access token. See https://developers.google.com/identity/protocols/googlescopes for more information. At least one value required. � T��repeated� � N) �__name__� __module__� __qualname__�__doc__� _messages� StringField� delegates�lifetime�scope� r r �S/opt/gsutil/gslib/third_party/iamcredentials_apitools/iamcredentials_v1_messages.pyr s r c @ s$ e Zd ZdZe�d�Ze�d�ZdS )�GenerateAccessTokenResponsez�A GenerateAccessTokenResponse object. Fields: accessToken: The OAuth 2.0 access token. expireTime: Token expiration time. The expiration time is always set. r r N)r r r r r r � accessToken� expireTimer r r r r * s r c @ s2 e Zd ZdZe�d�Zejddd�Ze�d�Z dS )�GenerateIdTokenRequesta� A GenerateIdTokenRequest object. Fields: audience: The audience for the token, such as the API or account that this token grants access to. delegates: The sequence of service accounts in a delegation chain. Each service account must be granted the `roles/iam.serviceAccountTokenCreator` role on its next service account in the chain. The last service account in the chain must be granted the `roles/iam.serviceAccountTokenCreator` role on the service account that is specified in the `name` field of the request. The delegates must have the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. includeEmail: Include the service account email in the token. If set to `true`, the token will contain `email` and `email_verified` claims. r r Tr r N) r r r r r r �audiencer � BooleanFieldZ includeEmailr r r r r 6 s r c @ s e Zd ZdZe�d�ZdS )�GenerateIdTokenResponsezWA GenerateIdTokenResponse object. Fields: token: The OpenId Connect ID token. r N)r r r r r r �tokenr r r r r O s r c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�?IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequesta� A IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequest object. Fields: generateAccessTokenRequest: A GenerateAccessTokenRequest resource to be passed as the request body. name: The resource name of the service account for which the credentials are requested, in the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. r r r T��requiredN) r r r r r � MessageField�generateAccessTokenRequestr �namer r r r r Y s r c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�;IamcredentialsProjectsServiceAccountsGenerateIdTokenRequesta� A IamcredentialsProjectsServiceAccountsGenerateIdTokenRequest object. Fields: generateIdTokenRequest: A GenerateIdTokenRequest resource to be passed as the request body. name: The resource name of the service account for which the credentials are requested, in the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. r r r Tr N) r r r r r r! �generateIdTokenRequestr r# r r r r r$ k s r$ c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�4IamcredentialsProjectsServiceAccountsSignBlobRequesta� A IamcredentialsProjectsServiceAccountsSignBlobRequest object. Fields: name: The resource name of the service account for which the credentials are requested, in the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. signBlobRequest: A SignBlobRequest resource to be passed as the request body. r Tr �SignBlobRequestr N) r r r r r r r# r! �signBlobRequestr r r r r& | s r& c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�3IamcredentialsProjectsServiceAccountsSignJwtRequesta� A IamcredentialsProjectsServiceAccountsSignJwtRequest object. Fields: name: The resource name of the service account for which the credentials are requested, in the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. signJwtRequest: A SignJwtRequest resource to be passed as the request body. r Tr �SignJwtRequestr N) r r r r r r r# r! �signJwtRequestr r r r r) � s r) c @ s( e Zd ZdZejddd�Ze�d�ZdS )r' a� A SignBlobRequest object. Fields: delegates: The sequence of service accounts in a delegation chain. Each service account must be granted the `roles/iam.serviceAccountTokenCreator` role on its next service account in the chain. The last service account in the chain must be granted the `roles/iam.serviceAccountTokenCreator` role on the service account that is specified in the `name` field of the request. The delegates must have the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. payload: The bytes to sign. r Tr r N) r r r r r r r � BytesField�payloadr r r r r' � s r' c @ s$ e Zd ZdZe�d�Ze�d�ZdS )�SignBlobResponsez}A SignBlobResponse object. Fields: keyId: The ID of the key used to sign the blob. signedBlob: The signed blob. r r N) r r r r r r �keyIdr, � signedBlobr r r r r. � s r. c @ s( e Zd ZdZejddd�Ze�d�ZdS )r* a� A SignJwtRequest object. Fields: delegates: The sequence of service accounts in a delegation chain. Each service account must be granted the `roles/iam.serviceAccountTokenCreator` role on its next service account in the chain. The last service account in the chain must be granted the `roles/iam.serviceAccountTokenCreator` role on the service account that is specified in the `name` field of the request. The delegates must have the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. payload: The JWT payload to sign: a JSON object that contains a JWT Claims Set. r Tr r N)r r r r r r r r- r r r r r* � s r* c @ s$ e Zd ZdZe�d�Ze�d�ZdS )�SignJwtResponsezyA SignJwtResponse object. Fields: keyId: The ID of the key used to sign the JWT. signedJwt: The signed JWT. r r N)r r r r r r r/ Z signedJwtr r r r r1 � s r1 c @ s� e Zd ZdZG dd� dej�ZG dd� dej�Ze�dd�Z e� d�Z ejddd d �Z e� d �Z e� d �Ze� d �Ze� d�Zejddd �Ze� d�Ze� d�Ze� d�Ze� d�ZdS )�StandardQueryParametersa4 Query parameters accepted by all methods. Enums: FXgafvValueValuesEnum: V1 error format. AltValueValuesEnum: Data format for response. Fields: f__xgafv: V1 error format. access_token: OAuth access token. alt: Data format for response. callback: JSONP fields: Selector specifying which fields to include in a partial response. key: API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token. oauth_token: OAuth 2.0 token for the current user. prettyPrint: Returns response with indentations and line breaks. quotaUser: Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. trace: A tracing token of the form "token:<tokenid>" to include in api requests. uploadType: Legacy upload protocol for media (e.g. "media", "multipart"). upload_protocol: Upload protocol for media (e.g. "raw", "multipart"). c @ s e Zd ZdZdZdZdZdS )z*StandardQueryParameters.AltValueValuesEnumz�Data format for response. Values: json: Responses with Content-Type of application/json media: Media download with context-dependent Content-Type proto: Responses with Content-Type of application/x-protobuf r r r N)r r r r �json�media�protor r r r �AltValueValuesEnum� s r6 c @ s e Zd ZdZdZdZdS )z-StandardQueryParameters.FXgafvValueValuesEnumzVV1 error format. Values: _1: v1 error format _2: v2 error format r r N)r r r r �_1�_2r r r r �FXgafvValueValuesEnum s r9 r r r r3 )�default� � � � � T� � � � N)r r r r r �Enumr6 r9 � EnumField�f__xgafvr � access_token�alt�callback�fields�key� oauth_tokenr � prettyPrint� quotaUser�trace� uploadTypeZupload_protocolr r r r r2 � s r2 rF z$.xgafvr7 �1r8 �2N)r �apitools.base.protorpcliter r �apitools.base.pyr �package�Messager r r r r r$ r& r) r' r. r* r1 r2 �AddCustomJsonFieldMapping�AddCustomJsonEnumMappingr9 r r r r �<module> s2 ?��� third_party/iamcredentials_apitools/__pycache__/__init__.cpython-39.pyc 0000644 00000000234 15025013616 0022371 0 ustar 00 a +(Wg � @ s d S )N� r r r �A/opt/gsutil/gslib/third_party/iamcredentials_apitools/__init__.py�<module> � third_party/iamcredentials_apitools/__pycache__/iamcredentials_v1_client.cpython-39.pyc 0000644 00000016134 15025013616 0025570 0 ustar 00 a +(Wg� � @ sz d Z ddlmZ ddlZddlZddlmZ ddlm m m Z ddl Z ddl mZ ddlmZ G dd� dej�ZdS ) z7Generated client library for iamcredentials version v1.� )�absolute_importN)�base_api)�MetricsCollector)� system_utilc s� e Zd ZdZeZdZdZdgZdZ dZ dZ de j e�� ejf Ze�� rhed7 Ze�� rhed e�� 7 Ze�� rzed 7 Zned 7 Zd ZdZd Zd� fdd� ZG dd� dej �Z G dd� dej �Z � Z S )�IamcredentialsV1z?Generated client library for service iamcredentials version v1.z&https://iamcredentials.googleapis.com/Ziamcredentialsz.https://www.googleapis.com/auth/cloud-platform�v1Znomatterz!apitools gsutil/%s Python/%s (%s)z google-cloud-sdkz/%sz analytics/disabledz analytics/enabledN� TFc sL |p| j }tt| �j||||||||| | | d� | �| �| _| �| �| _dS )z#Create a new iamcredentials handle.) � credentials�get_credentials�http�model� log_request� log_response�credentials_args�default_global_params�additional_http_headers�response_encodingN)�BASE_URL�superr �__init__� ProjectsServiceAccountsService�projects_serviceAccounts�ProjectsService�projects) �self�urlr r r r r r r r r r �� __class__� �Q/opt/gsutil/gslib/third_party/iamcredentials_apitools/iamcredentials_v1_client.pyr + s � zIamcredentialsV1.__init__c st e Zd ZdZdZ� fdd�Zddd�Zdd � e_dd d �Zd d � e_dd d�Z dd � e _ddd�Z dd � e _� Z S )z/IamcredentialsV1.ProjectsServiceAccountsServicez8Service class for the projects_serviceAccounts resource.r c s t tj| ��|� i | _d S �N)r r r r �_upload_configs�r �clientr r r r B s z8IamcredentialsV1.ProjectsServiceAccountsService.__init__Nc C s | � d�}| j|||d�S )aK Generates an OAuth 2.0 access token for a service account. Args: request: (IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (GenerateAccessTokenResponse) The response message. �GenerateAccessToken�� global_params��GetMethodConfig� _RunMethod�r �requestr& �configr r r r$ G s �zCIamcredentialsV1.ProjectsServiceAccountsService.GenerateAccessTokenc C s$ t jddddgdgg ddddd d � S ) NzPv1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:generateAccessToken�POSTz;iamcredentials.projects.serviceAccounts.generateAccessToken�namez v1/{+name}:generateAccessToken�generateAccessTokenRequest�?IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequestZGenerateAccessTokenResponseF� � flat_path� http_method� method_id�ordered_params� path_params� query_params� relative_path� request_field�request_type_name�response_type_name�supports_download�r � ApiMethodInfor r r r �<lambda>T s �z8IamcredentialsV1.ProjectsServiceAccountsService.<lambda>c C s | � d�}| j|||d�S )aD Generates an OpenID Connect ID token for a service account. Args: request: (IamcredentialsProjectsServiceAccountsGenerateIdTokenRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (GenerateIdTokenResponse) The response message. �GenerateIdTokenr% r' r* r r r r@ b s �z?IamcredentialsV1.ProjectsServiceAccountsService.GenerateIdTokenc C s$ t jddddgdgg ddddd d � S ) NzLv1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:generateIdTokenr- z7iamcredentials.projects.serviceAccounts.generateIdTokenr. zv1/{+name}:generateIdTokenZgenerateIdTokenRequestZ;IamcredentialsProjectsServiceAccountsGenerateIdTokenRequestZGenerateIdTokenResponseFr1 r= r r r r r? o s �c C s | � d�}| j|||d�S )a= Signs a blob using a service account's system-managed private key. Args: request: (IamcredentialsProjectsServiceAccountsSignBlobRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (SignBlobResponse) The response message. �SignBlobr% r' r* r r r rA } s �z8IamcredentialsV1.ProjectsServiceAccountsService.SignBlobc C s$ t jddddgdgg ddddd d � S ) NzEv1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signBlobr- z0iamcredentials.projects.serviceAccounts.signBlobr. zv1/{+name}:signBlob�signBlobRequest�4IamcredentialsProjectsServiceAccountsSignBlobRequestZSignBlobResponseFr1 r= r r r r r? � s �c C s | � d�}| j|||d�S )a: Signs a JWT using a service account's system-managed private key. Args: request: (IamcredentialsProjectsServiceAccountsSignJwtRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (SignJwtResponse) The response message. �SignJwtr% r' r* r r r rD � s �z7IamcredentialsV1.ProjectsServiceAccountsService.SignJwtc C s$ t jddddgdgg ddddd d � S ) NzDv1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signJwtr- z/iamcredentials.projects.serviceAccounts.signJwtr. zv1/{+name}:signJwtZsignJwtRequestZ3IamcredentialsProjectsServiceAccountsSignJwtRequestZSignJwtResponseFr1 r= r r r r r? � s �)N)N)N)N) �__name__� __module__� __qualname__�__doc__�_NAMEr r$ � method_configr@ rA rD � __classcell__r r r r r = s r c s$ e Zd ZdZdZ� fdd�Z� ZS )z IamcredentialsV1.ProjectsServicez(Service class for the projects resource.r c s t tj| ��|� i | _d S r )r r r r r! r" r r r r � s z)IamcredentialsV1.ProjectsService.__init__)rE rF rG rH rI r rK r r r r r � s r ) r NTNNFFNNNN) rE rF rG rH �messages�MESSAGES_MODULEr �_PACKAGE�_SCOPESZ_VERSION� _CLIENT_ID�_CLIENT_SECRET�gslib�VERSION�platform�python_version�sys� _USER_AGENTr �InvokedViaCloudSdk�CloudSdkVersionr � IsDisabledZ_CLIENT_CLASS_NAMEZ _URL_VERSION�_API_KEYr r �BaseApiServicer r rK r r r r r s: � �vr )rH � __future__r rT rV �apitools.base.pyr ZDgslib.third_party.iamcredentials_apitools.iamcredentials_v1_messages� third_party�iamcredentials_apitools�iamcredentials_v1_messagesrL rR Z gslib.metricsr � gslib.utilsr � BaseApiClientr r r r r �<module> s third_party/iamcredentials_apitools/iamcredentials_v1_messages.py 0000644 00000024725 15025013616 0021637 0 ustar 00 """Generated message classes for iamcredentials version v1. Creates short-lived, limited-privilege credentials for IAM service accounts. """ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.protorpclite import messages as _messages from apitools.base.py import encoding package = 'iamcredentials' class GenerateAccessTokenRequest(_messages.Message): r"""A GenerateAccessTokenRequest object. Fields: delegates: The sequence of service accounts in a delegation chain. Each service account must be granted the `roles/iam.serviceAccountTokenCreator` role on its next service account in the chain. The last service account in the chain must be granted the `roles/iam.serviceAccountTokenCreator` role on the service account that is specified in the `name` field of the request. The delegates must have the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. lifetime: The desired lifetime duration of the access token in seconds. Must be set to a value less than or equal to 3600 (1 hour). If a value is not specified, the token's lifetime will be set to a default value of one hour. scope: Code to identify the scopes to be included in the OAuth 2.0 access token. See https://developers.google.com/identity/protocols/googlescopes for more information. At least one value required. """ delegates = _messages.StringField(1, repeated=True) lifetime = _messages.StringField(2) scope = _messages.StringField(3, repeated=True) class GenerateAccessTokenResponse(_messages.Message): r"""A GenerateAccessTokenResponse object. Fields: accessToken: The OAuth 2.0 access token. expireTime: Token expiration time. The expiration time is always set. """ accessToken = _messages.StringField(1) expireTime = _messages.StringField(2) class GenerateIdTokenRequest(_messages.Message): r"""A GenerateIdTokenRequest object. Fields: audience: The audience for the token, such as the API or account that this token grants access to. delegates: The sequence of service accounts in a delegation chain. Each service account must be granted the `roles/iam.serviceAccountTokenCreator` role on its next service account in the chain. The last service account in the chain must be granted the `roles/iam.serviceAccountTokenCreator` role on the service account that is specified in the `name` field of the request. The delegates must have the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. includeEmail: Include the service account email in the token. If set to `true`, the token will contain `email` and `email_verified` claims. """ audience = _messages.StringField(1) delegates = _messages.StringField(2, repeated=True) includeEmail = _messages.BooleanField(3) class GenerateIdTokenResponse(_messages.Message): r"""A GenerateIdTokenResponse object. Fields: token: The OpenId Connect ID token. """ token = _messages.StringField(1) class IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequest(_messages.Message): r"""A IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequest object. Fields: generateAccessTokenRequest: A GenerateAccessTokenRequest resource to be passed as the request body. name: The resource name of the service account for which the credentials are requested, in the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. """ generateAccessTokenRequest = _messages.MessageField('GenerateAccessTokenRequest', 1) name = _messages.StringField(2, required=True) class IamcredentialsProjectsServiceAccountsGenerateIdTokenRequest(_messages.Message): r"""A IamcredentialsProjectsServiceAccountsGenerateIdTokenRequest object. Fields: generateIdTokenRequest: A GenerateIdTokenRequest resource to be passed as the request body. name: The resource name of the service account for which the credentials are requested, in the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. """ generateIdTokenRequest = _messages.MessageField('GenerateIdTokenRequest', 1) name = _messages.StringField(2, required=True) class IamcredentialsProjectsServiceAccountsSignBlobRequest(_messages.Message): r"""A IamcredentialsProjectsServiceAccountsSignBlobRequest object. Fields: name: The resource name of the service account for which the credentials are requested, in the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. signBlobRequest: A SignBlobRequest resource to be passed as the request body. """ name = _messages.StringField(1, required=True) signBlobRequest = _messages.MessageField('SignBlobRequest', 2) class IamcredentialsProjectsServiceAccountsSignJwtRequest(_messages.Message): r"""A IamcredentialsProjectsServiceAccountsSignJwtRequest object. Fields: name: The resource name of the service account for which the credentials are requested, in the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. signJwtRequest: A SignJwtRequest resource to be passed as the request body. """ name = _messages.StringField(1, required=True) signJwtRequest = _messages.MessageField('SignJwtRequest', 2) class SignBlobRequest(_messages.Message): r"""A SignBlobRequest object. Fields: delegates: The sequence of service accounts in a delegation chain. Each service account must be granted the `roles/iam.serviceAccountTokenCreator` role on its next service account in the chain. The last service account in the chain must be granted the `roles/iam.serviceAccountTokenCreator` role on the service account that is specified in the `name` field of the request. The delegates must have the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. payload: The bytes to sign. """ delegates = _messages.StringField(1, repeated=True) payload = _messages.BytesField(2) class SignBlobResponse(_messages.Message): r"""A SignBlobResponse object. Fields: keyId: The ID of the key used to sign the blob. signedBlob: The signed blob. """ keyId = _messages.StringField(1) signedBlob = _messages.BytesField(2) class SignJwtRequest(_messages.Message): r"""A SignJwtRequest object. Fields: delegates: The sequence of service accounts in a delegation chain. Each service account must be granted the `roles/iam.serviceAccountTokenCreator` role on its next service account in the chain. The last service account in the chain must be granted the `roles/iam.serviceAccountTokenCreator` role on the service account that is specified in the `name` field of the request. The delegates must have the following format: `projects/-/serviceAccounts/{ACCOUNT_EMAIL_OR_UNIQUEID}`. The `-` wildcard character is required; replacing it with a project ID is invalid. payload: The JWT payload to sign: a JSON object that contains a JWT Claims Set. """ delegates = _messages.StringField(1, repeated=True) payload = _messages.StringField(2) class SignJwtResponse(_messages.Message): r"""A SignJwtResponse object. Fields: keyId: The ID of the key used to sign the JWT. signedJwt: The signed JWT. """ keyId = _messages.StringField(1) signedJwt = _messages.StringField(2) class StandardQueryParameters(_messages.Message): r"""Query parameters accepted by all methods. Enums: FXgafvValueValuesEnum: V1 error format. AltValueValuesEnum: Data format for response. Fields: f__xgafv: V1 error format. access_token: OAuth access token. alt: Data format for response. callback: JSONP fields: Selector specifying which fields to include in a partial response. key: API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token. oauth_token: OAuth 2.0 token for the current user. prettyPrint: Returns response with indentations and line breaks. quotaUser: Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. trace: A tracing token of the form "token:<tokenid>" to include in api requests. uploadType: Legacy upload protocol for media (e.g. "media", "multipart"). upload_protocol: Upload protocol for media (e.g. "raw", "multipart"). """ class AltValueValuesEnum(_messages.Enum): r"""Data format for response. Values: json: Responses with Content-Type of application/json media: Media download with context-dependent Content-Type proto: Responses with Content-Type of application/x-protobuf """ json = 0 media = 1 proto = 2 class FXgafvValueValuesEnum(_messages.Enum): r"""V1 error format. Values: _1: v1 error format _2: v2 error format """ _1 = 0 _2 = 1 f__xgafv = _messages.EnumField('FXgafvValueValuesEnum', 1) access_token = _messages.StringField(2) alt = _messages.EnumField('AltValueValuesEnum', 3, default=u'json') callback = _messages.StringField(4) fields = _messages.StringField(5) key = _messages.StringField(6) oauth_token = _messages.StringField(7) prettyPrint = _messages.BooleanField(8, default=True) quotaUser = _messages.StringField(9) trace = _messages.StringField(10) uploadType = _messages.StringField(11) upload_protocol = _messages.StringField(12) encoding.AddCustomJsonFieldMapping( StandardQueryParameters, 'f__xgafv', '$.xgafv') encoding.AddCustomJsonEnumMapping( StandardQueryParameters.FXgafvValueValuesEnum, '_1', '1') encoding.AddCustomJsonEnumMapping( StandardQueryParameters.FXgafvValueValuesEnum, '_2', '2') third_party/iamcredentials_apitools/iamcredentials_v1_client.py 0000644 00000016745 15025013616 0021311 0 ustar 00 """Generated client library for iamcredentials version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. # NOTENOTE: This file has been gently hand-edited. Do not simply replace # with an autogenerated version - merge with this version! # For instance, the __future__ import immediately below... from __future__ import absolute_import import platform import sys from apitools.base.py import base_api import gslib.third_party.iamcredentials_apitools.iamcredentials_v1_messages as messages import gslib from gslib.metrics import MetricsCollector from gslib.utils import system_util class IamcredentialsV1(base_api.BaseApiClient): """Generated client library for service iamcredentials version v1.""" MESSAGES_MODULE = messages BASE_URL = u'https://iamcredentials.googleapis.com/' _PACKAGE = u'iamcredentials' _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform'] _VERSION = u'v1' _CLIENT_ID = 'nomatter' _CLIENT_SECRET = 'nomatter' _USER_AGENT = 'apitools gsutil/%s Python/%s (%s)' % ( gslib.VERSION, platform.python_version(), sys.platform) if system_util.InvokedViaCloudSdk(): _USER_AGENT += ' google-cloud-sdk' if system_util.CloudSdkVersion(): _USER_AGENT += '/%s' % system_util.CloudSdkVersion() if MetricsCollector.IsDisabled(): _USER_AGENT += ' analytics/disabled' else: _USER_AGENT += ' analytics/enabled' _CLIENT_CLASS_NAME = u'IamcredentialsV1' _URL_VERSION = u'v1' _API_KEY = None def __init__(self, url='', credentials=None, get_credentials=True, http=None, model=None, log_request=False, log_response=False, credentials_args=None, default_global_params=None, additional_http_headers=None, response_encoding=None): """Create a new iamcredentials handle.""" url = url or self.BASE_URL super(IamcredentialsV1, self).__init__( url, credentials=credentials, get_credentials=get_credentials, http=http, model=model, log_request=log_request, log_response=log_response, credentials_args=credentials_args, default_global_params=default_global_params, additional_http_headers=additional_http_headers, response_encoding=response_encoding) self.projects_serviceAccounts = self.ProjectsServiceAccountsService(self) self.projects = self.ProjectsService(self) class ProjectsServiceAccountsService(base_api.BaseApiService): """Service class for the projects_serviceAccounts resource.""" _NAME = u'projects_serviceAccounts' def __init__(self, client): super(IamcredentialsV1.ProjectsServiceAccountsService, self).__init__(client) self._upload_configs = { } def GenerateAccessToken(self, request, global_params=None): """Generates an OAuth 2.0 access token for a service account. Args: request: (IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (GenerateAccessTokenResponse) The response message. """ config = self.GetMethodConfig('GenerateAccessToken') return self._RunMethod( config, request, global_params=global_params) GenerateAccessToken.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:generateAccessToken', http_method=u'POST', method_id=u'iamcredentials.projects.serviceAccounts.generateAccessToken', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}:generateAccessToken', request_field=u'generateAccessTokenRequest', request_type_name=u'IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequest', response_type_name=u'GenerateAccessTokenResponse', supports_download=False, ) def GenerateIdToken(self, request, global_params=None): r"""Generates an OpenID Connect ID token for a service account. Args: request: (IamcredentialsProjectsServiceAccountsGenerateIdTokenRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (GenerateIdTokenResponse) The response message. """ config = self.GetMethodConfig('GenerateIdToken') return self._RunMethod( config, request, global_params=global_params) GenerateIdToken.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:generateIdToken', http_method=u'POST', method_id=u'iamcredentials.projects.serviceAccounts.generateIdToken', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}:generateIdToken', request_field=u'generateIdTokenRequest', request_type_name=u'IamcredentialsProjectsServiceAccountsGenerateIdTokenRequest', response_type_name=u'GenerateIdTokenResponse', supports_download=False, ) def SignBlob(self, request, global_params=None): r"""Signs a blob using a service account's system-managed private key. Args: request: (IamcredentialsProjectsServiceAccountsSignBlobRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (SignBlobResponse) The response message. """ config = self.GetMethodConfig('SignBlob') return self._RunMethod( config, request, global_params=global_params) SignBlob.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signBlob', http_method=u'POST', method_id=u'iamcredentials.projects.serviceAccounts.signBlob', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}:signBlob', request_field=u'signBlobRequest', request_type_name=u'IamcredentialsProjectsServiceAccountsSignBlobRequest', response_type_name=u'SignBlobResponse', supports_download=False, ) def SignJwt(self, request, global_params=None): r"""Signs a JWT using a service account's system-managed private key. Args: request: (IamcredentialsProjectsServiceAccountsSignJwtRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (SignJwtResponse) The response message. """ config = self.GetMethodConfig('SignJwt') return self._RunMethod( config, request, global_params=global_params) SignJwt.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/serviceAccounts/{serviceAccountsId}:signJwt', http_method=u'POST', method_id=u'iamcredentials.projects.serviceAccounts.signJwt', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}:signJwt', request_field=u'signJwtRequest', request_type_name=u'IamcredentialsProjectsServiceAccountsSignJwtRequest', response_type_name=u'SignJwtResponse', supports_download=False, ) class ProjectsService(base_api.BaseApiService): """Service class for the projects resource.""" _NAME = u'projects' def __init__(self, client): super(IamcredentialsV1.ProjectsService, self).__init__(client) self._upload_configs = { } third_party/iamcredentials_apitools/__init__.py 0000644 00000000000 15025013616 0016071 0 ustar 00 third_party/pubsub_apitools/__pycache__/pubsub_v1_messages.cpython-39.pyc 0000644 00000104675 15025013616 0022761 0 ustar 00 a +(Wg�| � @ sX d Z ddlmZ ddlmZ dZG dd� dej�ZG dd� dej�Z G d d � d ej�Z G d d � d ej�Z G d d� dej�Z G dd� dej�Z G dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd � d ej�ZG d d � d ej�ZG d d � d ej�ZG d!d"� d"ej�ZG d#d$� d$ej�ZG d%d&� d&ej�ZG d'd(� d(ej�ZG d)d*� d*ej�ZG d+d,� d,ej�ZG d-d.� d.ej�Z G d/d0� d0ej�Z G d1d2� d2ej�Z G d3d4� d4ej�Z G d5d6� d6ej�Z G d7d8� d8ej�Z!G d9d:� d:ej�Z"G d;d<� d<ej�Z#G d=d>� d>ej�Z$G d?d@� d@ej�Z%G dAdB� dBej�Z&G dCdD� dDej�Z'G dEdF� dFej�Z(G dGdH� dHej�Z)G dIdJ� dJej�Z*G dKdL� dLej�Z+G dMdN� dNej�Z,G dOdP� dPej�Z-G dQdR� dRej�Z.G dSdT� dTej�Z/G dUdV� dVej�Z0G dWdX� dXej�Z1G dYdZ� dZej�Z2e�3e.d[d\� e�4e.j5d]d^� e�4e.j5d_d`� daS )bz�Generated message classes for pubsub version v1. Provides reliable, many-to-many, asynchronous messaging between applications. � )�messages)�encoding�pubsubc @ s e Zd ZdZejddd�ZdS )�AcknowledgeRequestz�Request for the Acknowledge method. Fields: ackIds: The acknowledgment ID for the messages being acknowledged that was returned by the Pub/Sub system in the `Pull` response. Must not be empty. � T��repeatedN)�__name__� __module__� __qualname__�__doc__� _messages� StringField�ackIds� r r �C/opt/gsutil/gslib/third_party/pubsub_apitools/pubsub_v1_messages.pyr s r c @ s( e Zd ZdZejddd�Ze�d�ZdS )�Bindinga� Associates `members` with a `role`. Fields: members: Specifies the identities requesting access for a Cloud Platform resource. `members` can have the following values: * `allUsers`: A special identifier that represents anyone who is on the internet; with or without a Google account. * `allAuthenticatedUsers`: A special identifier that represents anyone who is authenticated with a Google account or a service account. * `user:{emailid}`: An email address that represents a specific Google account. For example, `alice@gmail.com` or `joe@example.com`. * `serviceAccount:{emailid}`: An email address that represents a service account. For example, `my-other- app@appspot.gserviceaccount.com`. * `group:{emailid}`: An email address that represents a Google group. For example, `admins@example.com`. * `domain:{domain}`: A Google Apps domain name that represents all the users of that domain. For example, `google.com` or `example.com`. role: Role that is assigned to `members`. For example, `roles/viewer`, `roles/editor`, or `roles/owner`. Required r Tr � N)r r r r r r �members�roler r r r r s r c @ s e Zd ZdZdS )�Emptyal A generic empty message that you can re-use to avoid defining duplicated empty messages in your APIs. A typical example is to use it as the request or the response type of an API method. For instance: service Foo { rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); } The JSON representation for `Empty` is empty JSON object `{}`. N)r r r r r r r r r 3 s r c @ s* e Zd ZdZe�d�Zejdddd�ZdS )�ListSubscriptionsResponseaF Response for the `ListSubscriptions` method. Fields: nextPageToken: If not empty, indicates that there may be more subscriptions that match the request; this value should be passed in a new `ListSubscriptionsRequest` to get more subscriptions. subscriptions: The subscriptions that match the request. r � Subscriptionr Tr N) r r r r r r � nextPageToken� MessageField� subscriptionsr r r r r = s r c @ s( e Zd ZdZe�d�Zejddd�ZdS )� ListTopicSubscriptionsResponsea] Response for the `ListTopicSubscriptions` method. Fields: nextPageToken: If not empty, indicates that there may be more subscriptions that match the request; this value should be passed in a new `ListTopicSubscriptionsRequest` to get more subscriptions. subscriptions: The names of the subscriptions that match the request. r r Tr N)r r r r r r r r r r r r r K s r c @ s* e Zd ZdZe�d�Zejdddd�ZdS )�ListTopicsResponsez�Response for the `ListTopics` method. Fields: nextPageToken: If not empty, indicates that there may be more topics that match the request; this value should be passed in a new `ListTopicsRequest`. topics: The resulting topics. r �Topicr Tr N) r r r r r r r r Ztopicsr r r r r Y s r c @ s0 e Zd ZdZejdejjd�Zej ddd�Z dS )�ModifyAckDeadlineRequesta: Request for the ModifyAckDeadline method. Fields: ackDeadlineSeconds: The new ack deadline with respect to the time this request was sent to the Pub/Sub system. For example, if the value is 10, the new ack deadline will expire 10 seconds after the `ModifyAckDeadline` call was made. Specifying zero may immediately make the message available for another pull request. The minimum deadline you can specify is 0 seconds. The maximum deadline you can specify is 600 seconds (10 minutes). ackIds: List of acknowledgment IDs. r ��variantr Tr N) r r r r r � IntegerField�Variant�INT32�ackDeadlineSecondsr r r r r r r g s r c @ s e Zd ZdZe�dd�ZdS )�ModifyPushConfigRequesta� Request for the ModifyPushConfig method. Fields: pushConfig: The push configuration for future deliveries. An empty `pushConfig` indicates that the Pub/Sub system should stop pushing messages from the given subscription and allow messages to be pulled and acknowledged - effectively pausing the subscription if `Pull` or `StreamingPull` is not called. � PushConfigr N)r r r r r r � pushConfigr r r r r& y s r& c @ s< e Zd ZdZejdddd�Ze�d�Zej dej j d�Z d S ) �Policya� Defines an Identity and Access Management (IAM) policy. It is used to specify access control policies for Cloud Platform resources. A `Policy` consists of a list of `bindings`. A `Binding` binds a list of `members` to a `role`, where the members can be user accounts, Google groups, Google domains, and service accounts. A `role` is a named list of permissions defined by IAM. **Example** { "bindings": [ { "role": "roles/owner", "members": [ "user:mike@example.com", "group:admins@example.com", "domain:google.com", "serviceAccount:my-other- app@appspot.gserviceaccount.com", ] }, { "role": "roles/viewer", "members": ["user:sean@example.com"] } ] } For a description of IAM and its features, see the [IAM developer's guide](https://cloud.google.com/iam). Fields: bindings: Associates a list of `members` to a `role`. `bindings` with no members will result in an error. etag: `etag` is used for optimistic concurrency control as a way to help prevent simultaneous updates of a policy from overwriting each other. It is strongly suggested that systems make use of the `etag` in the read- modify-write cycle to perform policy updates in order to avoid race conditions: An `etag` is returned in the response to `getIamPolicy`, and systems are expected to put that etag in the request to `setIamPolicy` to ensure that their change will be applied to the same version of the policy. If no `etag` is provided in the call to `setIamPolicy`, then the existing policy is overwritten blindly. version: Version of the `Policy`. The default version is 0. r r Tr r � r N) r r r r r r �bindings� BytesField�etagr" r# r$ �versionr r r r r) � s r) c @ s e Zd ZdZejdddd�ZdS )�PublishRequestzTRequest for the Publish method. Fields: messages: The messages to publish. � PubsubMessager Tr N)r r r r r r r r r r r r/ � s r/ c @ s e Zd ZdZejddd�ZdS )�PublishResponsez�Response for the `Publish` method. Fields: messageIds: The server-assigned ID of each published message, in the same order as the messages in the request. IDs are guaranteed to be unique within the topic. r Tr N)r r r r r r Z messageIdsr r r r r1 � s r1 c @ sV e Zd ZdZe�d�G dd� dej��Ze� dd�Z e� d�Z e� d�Ze� d�Zd S ) r0 aM A message data and its attributes. The message payload must not be empty; it must contain either a non-empty data field, or at least one attribute. Messages: AttributesValue: Optional attributes for this message. Fields: attributes: Optional attributes for this message. data: The message payload. messageId: ID of this message, assigned by the server when the message is published. Guaranteed to be unique within the topic. This value may be read by a subscriber that receives a `PubsubMessage` via a `Pull` call or a push delivery. It must not be populated by the publisher in a `Publish` call. publishTime: The time at which the message was published, populated by the server when it receives the `Publish` call. It must not be populated by the publisher in a `Publish` call. �additionalPropertiesc @ s2 e Zd ZdZG dd� dej�Zejdddd�ZdS )z PubsubMessage.AttributesValuez�Optional attributes for this message. Messages: AdditionalProperty: An additional property for a AttributesValue object. Fields: additionalProperties: Additional properties of type AttributesValue c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z0PubsubMessage.AttributesValue.AdditionalProperty��An additional property for a AttributesValue object. Fields: key: Name of the additional property. value: A string attribute. r r N�r r r r r r �key�valuer r r r �AdditionalProperty� s r7 r Tr N� r r r r r �Messager7 r r2 r r r r �AttributesValue� s r: r r r* � N)r r r r r �MapUnrecognizedFieldsr r9 r: r � attributesr, �datar Z messageIdZ publishTimer r r r r0 � s r0 c @ s e Zd ZdZejddd�ZdS )�*PubsubProjectsSnapshotsGetIamPolicyRequestz�A PubsubProjectsSnapshotsGetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being requested. See the operation documentation for the appropriate value for this field. r T��requiredN�r r r r r r �resourcer r r r r? � s r? c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�*PubsubProjectsSnapshotsSetIamPolicyRequestaF A PubsubProjectsSnapshotsSetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being specified. See the operation documentation for the appropriate value for this field. setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the request body. r Tr@ �SetIamPolicyRequestr N� r r r r r r rC r �setIamPolicyRequestr r r r rD � s rD c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�0PubsubProjectsSnapshotsTestIamPermissionsRequesta_ A PubsubProjectsSnapshotsTestIamPermissionsRequest object. Fields: resource: REQUIRED: The resource for which the policy detail is being requested. See the operation documentation for the appropriate value for this field. testIamPermissionsRequest: A TestIamPermissionsRequest resource to be passed as the request body. r Tr@ �TestIamPermissionsRequestr N� r r r r r r rC r �testIamPermissionsRequestr r r r rH s rH c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�-PubsubProjectsSubscriptionsAcknowledgeRequesta% A PubsubProjectsSubscriptionsAcknowledgeRequest object. Fields: acknowledgeRequest: A AcknowledgeRequest resource to be passed as the request body. subscription: The subscription whose message is being acknowledged. Format is `projects/{project}/subscriptions/{sub}`. r r r Tr@ N) r r r r r r �acknowledgeRequestr � subscriptionr r r r rL s rL c @ s e Zd ZdZejddd�ZdS )�(PubsubProjectsSubscriptionsDeleteRequestz�A PubsubProjectsSubscriptionsDeleteRequest object. Fields: subscription: The subscription to delete. Format is `projects/{project}/subscriptions/{sub}`. r Tr@ N�r r r r r r rN r r r r rO * s rO c @ s e Zd ZdZejddd�ZdS )�.PubsubProjectsSubscriptionsGetIamPolicyRequestz�A PubsubProjectsSubscriptionsGetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being requested. See the operation documentation for the appropriate value for this field. r Tr@ NrB r r r r rQ 5 s rQ c @ s e Zd ZdZejddd�ZdS )�%PubsubProjectsSubscriptionsGetRequestz�A PubsubProjectsSubscriptionsGetRequest object. Fields: subscription: The name of the subscription to get. Format is `projects/{project}/subscriptions/{sub}`. r Tr@ NrP r r r r rR A s rR c @ s: e Zd ZdZejdejjd�Ze� d�Z ej ddd�Z dS ) �&PubsubProjectsSubscriptionsListRequesta� A PubsubProjectsSubscriptionsListRequest object. Fields: pageSize: Maximum number of subscriptions to return. pageToken: The value returned by the last `ListSubscriptionsResponse`; indicates that this is a continuation of a prior `ListSubscriptions` call, and that the system should return the next page of data. project: The name of the cloud project that subscriptions belong to. Format is `projects/{project}`. r r r r* Tr@ N� r r r r r r" r# r$ �pageSizer � pageToken�projectr r r r rS L s rS c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�3PubsubProjectsSubscriptionsModifyAckDeadlineRequesta A PubsubProjectsSubscriptionsModifyAckDeadlineRequest object. Fields: modifyAckDeadlineRequest: A ModifyAckDeadlineRequest resource to be passed as the request body. subscription: The name of the subscription. Format is `projects/{project}/subscriptions/{sub}`. r r r Tr@ N) r r r r r r �modifyAckDeadlineRequestr rN r r r r rX ] s rX c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�2PubsubProjectsSubscriptionsModifyPushConfigRequesta A PubsubProjectsSubscriptionsModifyPushConfigRequest object. Fields: modifyPushConfigRequest: A ModifyPushConfigRequest resource to be passed as the request body. subscription: The name of the subscription. Format is `projects/{project}/subscriptions/{sub}`. r& r r Tr@ N) r r r r r r �modifyPushConfigRequestr rN r r r r rZ k s rZ c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�&PubsubProjectsSubscriptionsPullRequesta A PubsubProjectsSubscriptionsPullRequest object. Fields: pullRequest: A PullRequest resource to be passed as the request body. subscription: The subscription from which messages should be pulled. Format is `projects/{project}/subscriptions/{sub}`. � PullRequestr r Tr@ N) r r r r r r � pullRequestr rN r r r r r\ y s r\ c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�.PubsubProjectsSubscriptionsSetIamPolicyRequestaJ A PubsubProjectsSubscriptionsSetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being specified. See the operation documentation for the appropriate value for this field. setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the request body. r Tr@ rE r NrF r r r r r_ � s r_ c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�4PubsubProjectsSubscriptionsTestIamPermissionsRequestac A PubsubProjectsSubscriptionsTestIamPermissionsRequest object. Fields: resource: REQUIRED: The resource for which the policy detail is being requested. See the operation documentation for the appropriate value for this field. testIamPermissionsRequest: A TestIamPermissionsRequest resource to be passed as the request body. r Tr@ rI r NrJ r r r r r` � s r` c @ s e Zd ZdZejddd�ZdS )�!PubsubProjectsTopicsDeleteRequestz�A PubsubProjectsTopicsDeleteRequest object. Fields: topic: Name of the topic to delete. Format is `projects/{project}/topics/{topic}`. r Tr@ N�r r r r r r �topicr r r r ra � s ra c @ s e Zd ZdZejddd�ZdS )�'PubsubProjectsTopicsGetIamPolicyRequestz�A PubsubProjectsTopicsGetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being requested. See the operation documentation for the appropriate value for this field. r Tr@ NrB r r r r rd � s rd c @ s e Zd ZdZejddd�ZdS )� PubsubProjectsTopicsGetRequestz�A PubsubProjectsTopicsGetRequest object. Fields: topic: The name of the topic to get. Format is `projects/{project}/topics/{topic}`. r Tr@ Nrb r r r r re � s re c @ s: e Zd ZdZejdejjd�Ze� d�Z ej ddd�Z dS ) � PubsubProjectsTopicsListRequesta� A PubsubProjectsTopicsListRequest object. Fields: pageSize: Maximum number of topics to return. pageToken: The value returned by the last `ListTopicsResponse`; indicates that this is a continuation of a prior `ListTopics` call, and that the system should return the next page of data. project: The name of the cloud project that topics belong to. Format is `projects/{project}`. r r r r* Tr@ NrT r r r r rf � s rf c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�"PubsubProjectsTopicsPublishRequesta A PubsubProjectsTopicsPublishRequest object. Fields: publishRequest: A PublishRequest resource to be passed as the request body. topic: The messages in the request will be published on this topic. Format is `projects/{project}/topics/{topic}`. r/ r r Tr@ N) r r r r r r �publishRequestr rc r r r r rg � s rg c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�'PubsubProjectsTopicsSetIamPolicyRequestaC A PubsubProjectsTopicsSetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being specified. See the operation documentation for the appropriate value for this field. setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the request body. r Tr@ rE r NrF r r r r ri � s ri c @ s: e Zd ZdZejdejjd�Ze� d�Z ej ddd�Z dS ) �,PubsubProjectsTopicsSubscriptionsListRequesta� A PubsubProjectsTopicsSubscriptionsListRequest object. Fields: pageSize: Maximum number of subscription names to return. pageToken: The value returned by the last `ListTopicSubscriptionsResponse`; indicates that this is a continuation of a prior `ListTopicSubscriptions` call, and that the system should return the next page of data. topic: The name of the topic that subscriptions are attached to. Format is `projects/{project}/topics/{topic}`. r r r r* Tr@ N) r r r r r r" r# r$ rU r rV rc r r r r rj � s rj c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�-PubsubProjectsTopicsTestIamPermissionsRequesta\ A PubsubProjectsTopicsTestIamPermissionsRequest object. Fields: resource: REQUIRED: The resource for which the policy detail is being requested. See the operation documentation for the appropriate value for this field. testIamPermissionsRequest: A TestIamPermissionsRequest resource to be passed as the request body. r Tr@ rI r NrJ r r r r rk s rk c @ s, e Zd ZdZejdejjd�Ze� d�Z dS )r] ab Request for the `Pull` method. Fields: maxMessages: The maximum number of messages returned for this request. The Pub/Sub system may return fewer than the number specified. returnImmediately: If this field set to true, the system will respond immediately even if it there are no messages available to return in the `Pull` response. Otherwise, the system may wait (for a bounded amount of time) until at least one message is available, rather than returning no messages. The client may cancel the request if it does not wish to wait any longer for the response. r r r N) r r r r r r" r# r$ Z maxMessages� BooleanFieldZreturnImmediatelyr r r r r] s r] c @ s e Zd ZdZejdddd�ZdS )� PullResponseaK Response for the `Pull` method. Fields: receivedMessages: Received Pub/Sub messages. The Pub/Sub system will return zero messages if there are no more available in the backlog. The Pub/Sub system may return fewer than the `maxMessages` requested even if there are more messages available in the backlog. �ReceivedMessager Tr N)r r r r r r ZreceivedMessagesr r r r rm ' s rm c @ sB e Zd ZdZe�d�G dd� dej��Ze� dd�Z e� d�Z dS )r' a� Configuration for a push delivery endpoint. Messages: AttributesValue: Endpoint configuration attributes. Every endpoint has a set of API supported attributes that can be used to control different aspects of the message delivery. The currently supported attribute is `x-goog-version`, which you can use to change the format of the pushed message. This attribute indicates the version of the data expected by the endpoint. This controls the shape of the pushed message (i.e., its fields and metadata). The endpoint version is based on the version of the Pub/Sub API. If not present during the `CreateSubscription` call, it will default to the version of the API used to make such call. If not present during a `ModifyPushConfig` call, its value will not be changed. `GetSubscription` calls will always return a valid version, even if the subscription was created without this attribute. The possible values for this attribute are: * `v1beta1`: uses the push format defined in the v1beta1 Pub/Sub API. * `v1` or `v1beta2`: uses the push format defined in the v1 Pub/Sub API. Fields: attributes: Endpoint configuration attributes. Every endpoint has a set of API supported attributes that can be used to control different aspects of the message delivery. The currently supported attribute is `x-goog-version`, which you can use to change the format of the pushed message. This attribute indicates the version of the data expected by the endpoint. This controls the shape of the pushed message (i.e., its fields and metadata). The endpoint version is based on the version of the Pub/Sub API. If not present during the `CreateSubscription` call, it will default to the version of the API used to make such call. If not present during a `ModifyPushConfig` call, its value will not be changed. `GetSubscription` calls will always return a valid version, even if the subscription was created without this attribute. The possible values for this attribute are: * `v1beta1`: uses the push format defined in the v1beta1 Pub/Sub API. * `v1` or `v1beta2`: uses the push format defined in the v1 Pub/Sub API. pushEndpoint: A URL locating the endpoint to which messages should be pushed. For example, a Webhook endpoint might use "https://example.com/push". r2 c @ s2 e Zd ZdZG dd� dej�Zejdddd�ZdS )zPushConfig.AttributesValuea� Endpoint configuration attributes. Every endpoint has a set of API supported attributes that can be used to control different aspects of the message delivery. The currently supported attribute is `x-goog-version`, which you can use to change the format of the pushed message. This attribute indicates the version of the data expected by the endpoint. This controls the shape of the pushed message (i.e., its fields and metadata). The endpoint version is based on the version of the Pub/Sub API. If not present during the `CreateSubscription` call, it will default to the version of the API used to make such call. If not present during a `ModifyPushConfig` call, its value will not be changed. `GetSubscription` calls will always return a valid version, even if the subscription was created without this attribute. The possible values for this attribute are: * `v1beta1`: uses the push format defined in the v1beta1 Pub/Sub API. * `v1` or `v1beta2`: uses the push format defined in the v1 Pub/Sub API. Messages: AdditionalProperty: An additional property for a AttributesValue object. Fields: additionalProperties: Additional properties of type AttributesValue c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z-PushConfig.AttributesValue.AdditionalPropertyr3 r r Nr4 r r r r r7 v s r7 r Tr Nr8 r r r r r: ] s r: r r N) r r r r r r< r r9 r: r r= r Z pushEndpointr r r r r' 4 s (% r' c @ s& e Zd ZdZe�d�Ze�dd�ZdS )rn z�A message and its corresponding acknowledgment ID. Fields: ackId: This ID can be used to acknowledge the received message. message: The message. r r0 r N) r r r r r r ZackIdr �messager r r r rn � s rn c @ s e Zd ZdZe�dd�ZdS )rE a7 Request message for `SetIamPolicy` method. Fields: policy: REQUIRED: The complete policy to be applied to the `resource`. The size of the policy is limited to a few 10s of KB. An empty policy is a valid policy but certain Cloud Platform services (such as Projects) might reject them. r) r N)r r r r r r �policyr r r r rE � s rE c @ s� e Zd ZdZG dd� dej�ZG dd� dej�Ze�dd�Z e� d�Z ejddd d �Z e� d �Z e� d �Ze� d �Ze� d�Ze� d�Zejddd �Zejddd �Ze� d�Ze� d�Ze� d�Ze� d�ZdS )�StandardQueryParametersay Query parameters accepted by all methods. Enums: FXgafvValueValuesEnum: V1 error format. AltValueValuesEnum: Data format for response. Fields: f__xgafv: V1 error format. access_token: OAuth access token. alt: Data format for response. bearer_token: OAuth bearer token. callback: JSONP fields: Selector specifying which fields to include in a partial response. key: API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token. oauth_token: OAuth 2.0 token for the current user. pp: Pretty-print response. prettyPrint: Returns response with indentations and line breaks. quotaUser: Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. trace: A tracing token of the form "token:<tokenid>" to include in api requests. uploadType: Legacy upload protocol for media (e.g. "media", "multipart"). upload_protocol: Upload protocol for media (e.g. "raw", "multipart"). c @ s e Zd ZdZdZdZdZdS )z*StandardQueryParameters.AltValueValuesEnumz�Data format for response. Values: json: Responses with Content-Type of application/json media: Media download with context-dependent Content-Type proto: Responses with Content-Type of application/x-protobuf r r r N)r r r r �json�media�protor r r r �AltValueValuesEnum� s ru c @ s e Zd ZdZdZdZdS )z-StandardQueryParameters.FXgafvValueValuesEnumzVV1 error format. Values: _1: v1 error format _2: v2 error format r r N)r r r r �_1�_2r r r r �FXgafvValueValuesEnum� s rx r r r* rr )�defaultr; � � � � � T� � � � � N)r r r r r �Enumru rx � EnumField�f__xgafvr � access_token�alt� bearer_token�callback�fieldsr5 � oauth_tokenrl �pp� prettyPrint� quotaUser�trace� uploadType�upload_protocolr r r r rq � s" rq c @ sB e Zd ZdZejdejjd�Ze� d�Z e� dd�Z e� d�Z dS ) r a A subscription resource. Fields: ackDeadlineSeconds: This value is the maximum time after a subscriber receives a message before the subscriber should acknowledge the message. After message delivery but before the ack deadline expires and before the message is acknowledged, it is an outstanding message and will not be delivered again during that time (on a best-effort basis). For pull subscriptions, this value is used as the initial value for the ack deadline. To override this value for a given message, call `ModifyAckDeadline` with the corresponding `ack_id` if using non- streaming pull or send the `ack_id` in a `StreamingModifyAckDeadlineRequest` if using streaming pull. The minimum custom deadline you can specify is 10 seconds. The maximum custom deadline you can specify is 600 seconds (10 minutes). If this parameter is 0, a default value of 10 seconds is used. For push delivery, this value is also used to set the request timeout for the call to the push endpoint. If the subscriber never acknowledges the message, the Pub/Sub system will eventually redeliver the message. name: The name of the subscription. It must have the format `"projects/{project}/subscriptions/{subscription}"`. `{subscription}` must start with a letter, and contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`), underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent signs (`%`). It must be between 3 and 255 characters in length, and it must not start with `"goog"`. pushConfig: If push delivery is used with this subscription, this field is used to configure it. An empty `pushConfig` signifies that the subscriber will pull and ack messages using API methods. topic: The name of the topic from which this subscription is receiving messages. Format is `projects/{project}/topics/{topic}`. The value of this field will be `_deleted-topic_` if the topic has been deleted. r r r r' r* r; N)r r r r r r" r# r$ r% r �namer r( rc r r r r r � s ! r c @ s e Zd ZdZejddd�ZdS )rI a? Request message for `TestIamPermissions` method. Fields: permissions: The set of permissions to check for the `resource`. Permissions with wildcards (such as '*' or 'storage.*') are not allowed. For more information see [IAM Overview](https://cloud.google.com/iam/docs/overview#permissions). r Tr N�r r r r r r � permissionsr r r r rI s rI c @ s e Zd ZdZejddd�ZdS )�TestIamPermissionsResponsez�Response message for `TestIamPermissions` method. Fields: permissions: A subset of `TestPermissionsRequest.permissions` that the caller is allowed. r Tr Nr� r r r r r� s r� c @ s e Zd ZdZe�d�ZdS )r a� A topic resource. Fields: name: The name of the topic. It must have the format `"projects/{project}/topics/{topic}"`. `{topic}` must start with a letter, and contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`), underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent signs (`%`). It must be between 3 and 255 characters in length, and it must not start with `"goog"`. r N)r r r r r r r� r r r r r # s r r� z$.xgafvrv �1rw �2N)6r �apitools.base.protorpcliter r �apitools.base.pyr �packager9 r r r r r r r r& r) r/ r1 r0 r? rD rH rL rO rQ rR rS rX rZ r\ r_ r` ra rd re rf rg ri rj rk r] rm r' rn rE rq r rI r� r �AddCustomJsonFieldMapping�AddCustomJsonEnumMappingrx r r r r �<module> sn # 2 S C( ��� third_party/pubsub_apitools/__pycache__/__init__.cpython-39.pyc 0000644 00000000224 15025013616 0020704 0 ustar 00 a +(Wg � @ s d S )N� r r r �9/opt/gsutil/gslib/third_party/pubsub_apitools/__init__.py�<module> � third_party/pubsub_apitools/__pycache__/pubsub_v1_client.cpython-39.pyc 0000644 00000064602 15025013616 0022423 0 ustar 00 a +(Wg�| � @ s� d Z ddlmZ ddlZddlZddlZddlmZ ddlm m m Z ddl Z ddlmZ ddlmZ G dd� dej�ZdS ) z/Generated client library for pubsub version v1.� )�absolute_importN)�base_api)�MetricsCollector)� system_utilc s e Zd ZdZeZdZdZddgZdZ dZ dZ de j e�� ejf Ze�� rjed 7 Ze�� rjed e�� 7 Ze�� r|ed 7 Zned 7 Zd ZdZd Zd � fdd� ZG dd� dej �Z G dd� dej �Z G dd� dej �Z G dd� dej �Z G dd � d ej �Z!� Z"S ) �PubsubV1z7Generated client library for service pubsub version v1.z https://pubsub.googleapis.com/Zpubsubz.https://www.googleapis.com/auth/cloud-platformz&https://www.googleapis.com/auth/pubsub�v1�nomatterz!apitools gsutil/%s Python/%s (%s)z google-cloud-sdkz/%sz analytics/disabledz analytics/enabledN� TFc sn |p| j }tt| �j||||||||| | d� | �| �| _| �| �| _| �| �| _ | � | �| _ | � | �| _ dS )zCreate a new pubsub handle.) � credentials�get_credentials�http�model� log_request� log_response�credentials_args�default_global_params�additional_http_headersN)�BASE_URL�superr �__init__�ProjectsSnapshotsService�projects_snapshots� ProjectsSubscriptionsService�projects_subscriptions�"ProjectsTopicsSubscriptionsService� projects_topics_subscriptions�ProjectsTopicsService�projects_topics�ProjectsService�projects) �self�urlr r r r r r r r r �� __class__� �A/opt/gsutil/gslib/third_party/pubsub_apitools/pubsub_v1_client.pyr - s � zPubsubV1.__init__c s` e Zd ZdZdZ� fdd�Zddd�Zdd � e_dd d �Zd d � e_dd d�Z dd � e _� Z S )z!PubsubV1.ProjectsSnapshotsServicez2Service class for the projects_snapshots resource.r c s t tj| ��|� i | _d S �N)r r r r �_upload_configs�r �clientr" r$ r% r F s z*PubsubV1.ProjectsSnapshotsService.__init__Nc C s | � d�}| j|||d�S )ad Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set. Args: request: (PubsubProjectsSnapshotsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. � GetIamPolicy�� global_params��GetMethodConfig� _RunMethod�r �requestr, �configr$ r$ r% r* K s �z.PubsubV1.ProjectsSnapshotsService.GetIamPolicyc C s$ t jddddgdgg ddddd d � S ) Nz=v1/projects/{projectsId}/snapshots/{snapshotsId}:getIamPolicy�GETz&pubsub.projects.snapshots.getIamPolicy�resource�v1/{+resource}:getIamPolicyr Z*PubsubProjectsSnapshotsGetIamPolicyRequest�PolicyF� � flat_path� http_method� method_id�ordered_params� path_params� query_params� relative_path� request_field�request_type_name�response_type_name�supports_download�r � ApiMethodInfor$ r$ r$ r% �<lambda>Z s �z*PubsubV1.ProjectsSnapshotsService.<lambda>c C s | � d�}| j|||d�S )a? Sets the access control policy on the specified resource. Replaces any. existing policy. Args: request: (PubsubProjectsSnapshotsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. � SetIamPolicyr+ r- r0 r$ r$ r% rF h s �z.PubsubV1.ProjectsSnapshotsService.SetIamPolicyc C s$ t jddddgdgg ddddd d � S ) Nz=v1/projects/{projectsId}/snapshots/{snapshotsId}:setIamPolicy�POSTz&pubsub.projects.snapshots.setIamPolicyr4 �v1/{+resource}:setIamPolicy�setIamPolicyRequestZ*PubsubProjectsSnapshotsSetIamPolicyRequestr6 Fr7 rC r$ r$ r$ r% rE v s �c C s | � d�}| j|||d�S )a] Returns permissions that a caller has on the specified resource. If the resource does not exist, this will return an empty set of permissions, not a NOT_FOUND error. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. Args: request: (PubsubProjectsSnapshotsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. �TestIamPermissionsr+ r- r0 r$ r$ r% rJ � s �z4PubsubV1.ProjectsSnapshotsService.TestIamPermissionsc C s$ t jddddgdgg ddddd d � S ) NzCv1/projects/{projectsId}/snapshots/{snapshotsId}:testIamPermissionsrG z,pubsub.projects.snapshots.testIamPermissionsr4 �!v1/{+resource}:testIamPermissions�testIamPermissionsRequestZ0PubsubProjectsSnapshotsTestIamPermissionsRequest�TestIamPermissionsResponseFr7 rC r$ r$ r$ r% rE � s �)N)N)N) �__name__� __module__� __qualname__�__doc__�_NAMEr r* � method_configrF rJ � __classcell__r$ r$ r" r% r A s r c s e Zd ZdZdZ� fdd�Zd(dd�Zdd � e_d)d d �Zd d � e_d*d d�Z dd � e _d+dd�Z dd � e _d,dd�Z dd � e _d-dd�Z dd � e _d.dd�Z dd � e _d/d d �Zd d � e_d0d d �Zd!d � e_d1d"d#�Zd$d � e_d2d%d&�Zd'd � e_� ZS )3z%PubsubV1.ProjectsSubscriptionsServicez6Service class for the projects_subscriptions resource.r c s t tj| ��|� i | _d S r& )r r r r r' r( r" r$ r% r � s z.PubsubV1.ProjectsSubscriptionsService.__init__Nc C s | � d�}| j|||d�S )a? Acknowledges the messages associated with the `ack_ids` in the. `AcknowledgeRequest`. The Pub/Sub system can remove the relevant messages from the subscription. Acknowledging a message whose ack deadline has expired may succeed, but such a message may be redelivered later. Acknowledging a message more than once will not result in an error. Args: request: (PubsubProjectsSubscriptionsAcknowledgeRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Empty) The response message. � Acknowledger+ r- r0 r$ r$ r% rU � s �z1PubsubV1.ProjectsSubscriptionsService.Acknowledgec C s$ t jddddgdgg ddddd d � S ) NzDv1/projects/{projectsId}/subscriptions/{subscriptionsId}:acknowledgerG z)pubsub.projects.subscriptions.acknowledge� subscriptionz v1/{+subscription}:acknowledgeZacknowledgeRequestZ-PubsubProjectsSubscriptionsAcknowledgeRequest�EmptyFr7 rC r$ r$ r$ r% rE � s �z.PubsubV1.ProjectsSubscriptionsService.<lambda>c C s | � d�}| j|||d�S )a� Creates a subscription to a given topic. If the subscription already exists, returns `ALREADY_EXISTS`. If the corresponding topic doesn't exist, returns `NOT_FOUND`. If the name is not provided in the request, the server will assign a random name for this subscription on the same project as the topic, conforming to the [resource name format](https://cloud.google.com/pubsub/docs/overview#names). The generated name is populated in the returned Subscription object. Note that for REST API requests, you must specify a name in the request. Args: request: (Subscription) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Subscription) The response message. �Creater+ r- r0 r$ r$ r% rX � s �z,PubsubV1.ProjectsSubscriptionsService.Createc C s$ t jddddgdgg dddddd � S ) N�8v1/projects/{projectsId}/subscriptions/{subscriptionsId}�PUTz$pubsub.projects.subscriptions.create�name� v1/{+name}� <request>� SubscriptionFr7 rC r$ r$ r$ r% rE � s �c C s | � d�}| j|||d�S )a= Deletes an existing subscription. All messages retained in the subscription. are immediately dropped. Calls to `Pull` after deletion will return `NOT_FOUND`. After a subscription is deleted, a new one may be created with the same name, but the new one has no association with the old subscription or its topic unless the same topic is specified. Args: request: (PubsubProjectsSubscriptionsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Empty) The response message. �Deleter+ r- r0 r$ r$ r% r_ � s �z,PubsubV1.ProjectsSubscriptionsService.Deletec C s$ t jddddgdgg ddddd d � S ) NrY �DELETEz$pubsub.projects.subscriptions.deleterV �v1/{+subscription}r Z(PubsubProjectsSubscriptionsDeleteRequestrW Fr7 rC r$ r$ r$ r% rE s �c C s | � d�}| j|||d�S )a Gets the configuration details of a subscription. Args: request: (PubsubProjectsSubscriptionsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Subscription) The response message. �Getr+ r- r0 r$ r$ r% rb s �z)PubsubV1.ProjectsSubscriptionsService.Getc C s$ t jddddgdgg ddddd d � S ) NrY r3 z!pubsub.projects.subscriptions.getrV ra r Z%PubsubProjectsSubscriptionsGetRequestr^ Fr7 rC r$ r$ r$ r% rE s �c C s | � d�}| j|||d�S )ah Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set. Args: request: (PubsubProjectsSubscriptionsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. r* r+ r- r0 r$ r$ r% r* . s �z2PubsubV1.ProjectsSubscriptionsService.GetIamPolicyc C s$ t jddddgdgg ddddd d � S ) NzEv1/projects/{projectsId}/subscriptions/{subscriptionsId}:getIamPolicyr3 z*pubsub.projects.subscriptions.getIamPolicyr4 r5 r Z.PubsubProjectsSubscriptionsGetIamPolicyRequestr6 Fr7 rC r$ r$ r$ r% rE = s �c C s | � d�}| j|||d�S )a Lists matching subscriptions. Args: request: (PubsubProjectsSubscriptionsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListSubscriptionsResponse) The response message. �Listr+ r- r0 r$ r$ r% rc K s �z*PubsubV1.ProjectsSubscriptionsService.Listc C s( t jddddgdgddgddd d d d � S ) Nz&v1/projects/{projectsId}/subscriptionsr3 z"pubsub.projects.subscriptions.list�project�pageSize� pageTokenzv1/{+project}/subscriptionsr Z&PubsubProjectsSubscriptionsListRequestZListSubscriptionsResponseFr7 rC r$ r$ r$ r% rE X s �c C s | � d�}| j|||d�S )a@ Modifies the ack deadline for a specific message. This method is useful. to indicate that more time is needed to process a message by the subscriber, or to make the message available for redelivery if the processing was interrupted. Note that this does not modify the subscription-level `ackDeadlineSeconds` used for subsequent messages. Args: request: (PubsubProjectsSubscriptionsModifyAckDeadlineRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Empty) The response message. �ModifyAckDeadliner+ r- r0 r$ r$ r% rg f s �z7PubsubV1.ProjectsSubscriptionsService.ModifyAckDeadlinec C s$ t jddddgdgg ddddd d � S ) NzJv1/projects/{projectsId}/subscriptions/{subscriptionsId}:modifyAckDeadlinerG z/pubsub.projects.subscriptions.modifyAckDeadlinerV z$v1/{+subscription}:modifyAckDeadlineZmodifyAckDeadlineRequestZ3PubsubProjectsSubscriptionsModifyAckDeadlineRequestrW Fr7 rC r$ r$ r$ r% rE w s �c C s | � d�}| j|||d�S )aN Modifies the `PushConfig` for a specified subscription. This may be used to change a push subscription to a pull one (signified by an empty `PushConfig`) or vice versa, or change the endpoint URL and other attributes of a push subscription. Messages will accumulate for delivery continuously through the call regardless of changes to the `PushConfig`. Args: request: (PubsubProjectsSubscriptionsModifyPushConfigRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Empty) The response message. �ModifyPushConfigr+ r- r0 r$ r$ r% rh � s �z6PubsubV1.ProjectsSubscriptionsService.ModifyPushConfigc C s$ t jddddgdgg ddddd d � S ) NzIv1/projects/{projectsId}/subscriptions/{subscriptionsId}:modifyPushConfigrG z.pubsub.projects.subscriptions.modifyPushConfigrV z#v1/{+subscription}:modifyPushConfigZmodifyPushConfigRequestZ2PubsubProjectsSubscriptionsModifyPushConfigRequestrW Fr7 rC r$ r$ r$ r% rE � s �c C s | � d�}| j|||d�S )a� Pulls messages from the server. Returns an empty list if there are no. messages available in the backlog. The server may return `UNAVAILABLE` if there are too many concurrent pull requests pending for the given subscription. Args: request: (PubsubProjectsSubscriptionsPullRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (PullResponse) The response message. �Pullr+ r- r0 r$ r$ r% ri � s �z*PubsubV1.ProjectsSubscriptionsService.Pullc C s$ t jddddgdgg ddddd d � S ) Nz=v1/projects/{projectsId}/subscriptions/{subscriptionsId}:pullrG z"pubsub.projects.subscriptions.pullrV zv1/{+subscription}:pullZ pullRequestZ&PubsubProjectsSubscriptionsPullRequestZ PullResponseFr7 rC r$ r$ r$ r% rE � s �c C s | � d�}| j|||d�S )aC Sets the access control policy on the specified resource. Replaces any. existing policy. Args: request: (PubsubProjectsSubscriptionsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. rF r+ r- r0 r$ r$ r% rF � s �z2PubsubV1.ProjectsSubscriptionsService.SetIamPolicyc C s$ t jddddgdgg ddddd d � S ) NzEv1/projects/{projectsId}/subscriptions/{subscriptionsId}:setIamPolicyrG z*pubsub.projects.subscriptions.setIamPolicyr4 rH rI Z.PubsubProjectsSubscriptionsSetIamPolicyRequestr6 Fr7 rC r$ r$ r$ r% rE � s �c C s | � d�}| j|||d�S )aa Returns permissions that a caller has on the specified resource. If the resource does not exist, this will return an empty set of permissions, not a NOT_FOUND error. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. Args: request: (PubsubProjectsSubscriptionsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. rJ r+ r- r0 r$ r$ r% rJ � s �z8PubsubV1.ProjectsSubscriptionsService.TestIamPermissionsc C s$ t jddddgdgg ddddd d � S ) NzKv1/projects/{projectsId}/subscriptions/{subscriptionsId}:testIamPermissionsrG z0pubsub.projects.subscriptions.testIamPermissionsr4 rK rL Z4PubsubProjectsSubscriptionsTestIamPermissionsRequestrM Fr7 rC r$ r$ r$ r% rE � s �)N)N)N)N)N)N)N)N)N)N)N)rN rO rP rQ rR r rU rS rX r_ rb r* rc rg rh ri rF rJ rT r$ r$ r" r% r � s2 r c s8 e Zd ZdZdZ� fdd�Zd dd�Zdd � e_� ZS ) z+PubsubV1.ProjectsTopicsSubscriptionsServicez=Service class for the projects_topics_subscriptions resource.r c s t tj| ��|� i | _d S r& )r r r r r' r( r" r$ r% r s z4PubsubV1.ProjectsTopicsSubscriptionsService.__init__Nc C s | � d�}| j|||d�S )a4 Lists the name of the subscriptions for this topic. Args: request: (PubsubProjectsTopicsSubscriptionsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListTopicSubscriptionsResponse) The response message. rc r+ r- r0 r$ r$ r% rc s �z0PubsubV1.ProjectsTopicsSubscriptionsService.Listc C s( t jddddgdgddgddd d d d � S ) Nz8v1/projects/{projectsId}/topics/{topicsId}/subscriptionsr3 z)pubsub.projects.topics.subscriptions.list�topicre rf zv1/{+topic}/subscriptionsr Z,PubsubProjectsTopicsSubscriptionsListRequestZ ListTopicSubscriptionsResponseFr7 rC r$ r$ r$ r% rE s �z4PubsubV1.ProjectsTopicsSubscriptionsService.<lambda>)N) rN rO rP rQ rR r rc rS rT r$ r$ r" r% r s r c s� e Zd ZdZdZ� fdd�Zd dd�Zdd � e_d d d �Zd d � e_d!d d�Z dd � e _d"dd�Z dd � e _d#dd�Z dd � e _d$dd�Z dd � e _d%dd�Z dd � e _d&d d �Zd d � e_� ZS )'z PubsubV1.ProjectsTopicsServicez/Service class for the projects_topics resource.r c s t tj| ��|� i | _d S r& )r r r r r' r( r" r$ r% r * s z'PubsubV1.ProjectsTopicsService.__init__Nc C s | � d�}| j|||d�S )z�Creates the given topic with the given name. Args: request: (Topic) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Topic) The response message. rX r+ r- r0 r$ r$ r% rX / s �z%PubsubV1.ProjectsTopicsService.Createc C s$ t jddddgdgg dddddd � S ) N�*v1/projects/{projectsId}/topics/{topicsId}rZ z pubsub.projects.topics.creater[ r\ r] �TopicFr7 rC r$ r$ r$ r% rE < s �z'PubsubV1.ProjectsTopicsService.<lambda>c C s | � d�}| j|||d�S )a< Deletes the topic with the given name. Returns `NOT_FOUND` if the topic. does not exist. After a topic is deleted, a new topic may be created with the same name; this is an entirely new topic with none of the old configuration or subscriptions. Existing subscriptions to this topic are not deleted, but their `topic` field is set to `_deleted-topic_`. Args: request: (PubsubProjectsTopicsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Empty) The response message. r_ r+ r- r0 r$ r$ r% r_ J s �z%PubsubV1.ProjectsTopicsService.Deletec C s$ t jddddgdgg ddddd d � S ) Nrk r` z pubsub.projects.topics.deleterj � v1/{+topic}r �!PubsubProjectsTopicsDeleteRequestrW Fr7 rC r$ r$ r$ r% rE [ s �c C s | � d�}| j|||d�S )z�Gets the configuration of a topic. Args: request: (PubsubProjectsTopicsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Topic) The response message. rb r+ r- r0 r$ r$ r% rb i s �z"PubsubV1.ProjectsTopicsService.Getc C s$ t jddddgdgg ddddd d � S ) Nrk r3 zpubsub.projects.topics.getrj rm r � PubsubProjectsTopicsGetRequestrl Fr7 rC r$ r$ r$ r% rE v s �c C s | � d�}| j|||d�S )aa Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set. Args: request: (PubsubProjectsTopicsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. r* r+ r- r0 r$ r$ r% r* � s �z+PubsubV1.ProjectsTopicsService.GetIamPolicyc C s$ t jddddgdgg ddddd d � S ) Nz7v1/projects/{projectsId}/topics/{topicsId}:getIamPolicyr3 z#pubsub.projects.topics.getIamPolicyr4 r5 r �'PubsubProjectsTopicsGetIamPolicyRequestr6 Fr7 rC r$ r$ r$ r% rE � s �c C s | � d�}| j|||d�S )z�Lists matching topics. Args: request: (PubsubProjectsTopicsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListTopicsResponse) The response message. rc r+ r- r0 r$ r$ r% rc � s �z#PubsubV1.ProjectsTopicsService.Listc C s( t jddddgdgddgddd d d d � S ) Nz v1/projects/{projectsId}/topicsr3 zpubsub.projects.topics.listrd re rf zv1/{+project}/topicsr Z PubsubProjectsTopicsListRequestZListTopicsResponseFr7 rC r$ r$ r$ r% rE � s �c C s | � d�}| j|||d�S )a� Adds one or more messages to the topic. Returns `NOT_FOUND` if the topic. does not exist. The message payload must not be empty; it must contain either a non-empty data field, or at least one attribute. Args: request: (PubsubProjectsTopicsPublishRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (PublishResponse) The response message. �Publishr+ r- r0 r$ r$ r% rq � s �z&PubsubV1.ProjectsTopicsService.Publishc C s$ t jddddgdgg ddddd d � S ) Nz2v1/projects/{projectsId}/topics/{topicsId}:publishrG z pubsub.projects.topics.publishrj zv1/{+topic}:publishZpublishRequestZ"PubsubProjectsTopicsPublishRequestZPublishResponseFr7 rC r$ r$ r$ r% rE � s �c C s | � d�}| j|||d�S )a< Sets the access control policy on the specified resource. Replaces any. existing policy. Args: request: (PubsubProjectsTopicsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. rF r+ r- r0 r$ r$ r% rF � s �z+PubsubV1.ProjectsTopicsService.SetIamPolicyc C s$ t jddddgdgg ddddd d � S ) Nz7v1/projects/{projectsId}/topics/{topicsId}:setIamPolicyrG z#pubsub.projects.topics.setIamPolicyr4 rH rI �'PubsubProjectsTopicsSetIamPolicyRequestr6 Fr7 rC r$ r$ r$ r% rE � s �c C s | � d�}| j|||d�S )aZ Returns permissions that a caller has on the specified resource. If the resource does not exist, this will return an empty set of permissions, not a NOT_FOUND error. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. Args: request: (PubsubProjectsTopicsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. rJ r+ r- r0 r$ r$ r% rJ � s �z1PubsubV1.ProjectsTopicsService.TestIamPermissionsc C s$ t jddddgdgg ddddd d � S ) Nz=v1/projects/{projectsId}/topics/{topicsId}:testIamPermissionsrG z)pubsub.projects.topics.testIamPermissionsr4 rK rL Z-PubsubProjectsTopicsTestIamPermissionsRequestrM Fr7 rC r$ r$ r$ r% rE s �)N)N)N)N)N)N)N)N)rN rO rP rQ rR r rX rS r_ rb r* rc rq rF rJ rT r$ r$ r" r% r % s& r c s$ e Zd ZdZdZ� fdd�Z� ZS )zPubsubV1.ProjectsServicez(Service class for the projects resource.r c s t tj| ��|� i | _d S r& )r r r r r' r( r" r$ r% r s z!PubsubV1.ProjectsService.__init__)rN rO rP rQ rR r rT r$ r$ r" r% r s r ) r NTNNFFNNN)#rN rO rP rQ �messages�MESSAGES_MODULEr �_PACKAGE�_SCOPES�_VERSION� _CLIENT_ID�_CLIENT_SECRET�gslib�VERSION�platform�python_version�sys� _USER_AGENTr �InvokedViaCloudSdk�CloudSdkVersionr � IsDisabled�_CLIENT_CLASS_NAME� _URL_VERSION�_API_KEYr r �BaseApiServicer r r r r rT r$ r$ r" r% r sF � �d ]% rr )rQ � __future__r �osr| r~ �apitools.base.pyr �4gslib.third_party.pubsub_apitools.pubsub_v1_messages� third_party�pubsub_apitools�pubsub_v1_messagesrs rz Z gslib.metricsr � gslib.utilsr � BaseApiClientr r$ r$ r$ r% �<module> s third_party/pubsub_apitools/pubsub_v1_client.py 0000644 00000076256 15025013616 0016144 0 ustar 00 """Generated client library for pubsub version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. # NOTENOTE: This file has been gently hand-edited. Do not simply replace # with an autogenerated version - merge with this version! # For instance, the __future__ import immediately below... from __future__ import absolute_import import os import platform import sys from apitools.base.py import base_api import gslib.third_party.pubsub_apitools.pubsub_v1_messages as messages import gslib from gslib.metrics import MetricsCollector from gslib.utils import system_util class PubsubV1(base_api.BaseApiClient): """Generated client library for service pubsub version v1.""" MESSAGES_MODULE = messages BASE_URL = u'https://pubsub.googleapis.com/' _PACKAGE = u'pubsub' _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/pubsub'] _VERSION = u'v1' _CLIENT_ID = 'nomatter' _CLIENT_SECRET = 'nomatter' _USER_AGENT = 'apitools gsutil/%s Python/%s (%s)' % ( gslib.VERSION, platform.python_version(), sys.platform) if system_util.InvokedViaCloudSdk(): _USER_AGENT += ' google-cloud-sdk' if system_util.CloudSdkVersion(): _USER_AGENT += '/%s' % system_util.CloudSdkVersion() if MetricsCollector.IsDisabled(): _USER_AGENT += ' analytics/disabled' else: _USER_AGENT += ' analytics/enabled' _CLIENT_CLASS_NAME = u'PubsubV1' _URL_VERSION = u'v1' _API_KEY = None def __init__(self, url='', credentials=None, get_credentials=True, http=None, model=None, log_request=False, log_response=False, credentials_args=None, default_global_params=None, additional_http_headers=None): """Create a new pubsub handle.""" url = url or self.BASE_URL super(PubsubV1, self).__init__( url, credentials=credentials, get_credentials=get_credentials, http=http, model=model, log_request=log_request, log_response=log_response, credentials_args=credentials_args, default_global_params=default_global_params, additional_http_headers=additional_http_headers) self.projects_snapshots = self.ProjectsSnapshotsService(self) self.projects_subscriptions = self.ProjectsSubscriptionsService(self) self.projects_topics_subscriptions = self.ProjectsTopicsSubscriptionsService(self) self.projects_topics = self.ProjectsTopicsService(self) self.projects = self.ProjectsService(self) class ProjectsSnapshotsService(base_api.BaseApiService): """Service class for the projects_snapshots resource.""" _NAME = u'projects_snapshots' def __init__(self, client): super(PubsubV1.ProjectsSnapshotsService, self).__init__(client) self._upload_configs = { } def GetIamPolicy(self, request, global_params=None): """Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set. Args: request: (PubsubProjectsSnapshotsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('GetIamPolicy') return self._RunMethod( config, request, global_params=global_params) GetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/snapshots/{snapshotsId}:getIamPolicy', http_method=u'GET', method_id=u'pubsub.projects.snapshots.getIamPolicy', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:getIamPolicy', request_field='', request_type_name=u'PubsubProjectsSnapshotsGetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def SetIamPolicy(self, request, global_params=None): """Sets the access control policy on the specified resource. Replaces any. existing policy. Args: request: (PubsubProjectsSnapshotsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('SetIamPolicy') return self._RunMethod( config, request, global_params=global_params) SetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/snapshots/{snapshotsId}:setIamPolicy', http_method=u'POST', method_id=u'pubsub.projects.snapshots.setIamPolicy', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:setIamPolicy', request_field=u'setIamPolicyRequest', request_type_name=u'PubsubProjectsSnapshotsSetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def TestIamPermissions(self, request, global_params=None): """Returns permissions that a caller has on the specified resource. If the resource does not exist, this will return an empty set of permissions, not a NOT_FOUND error. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. Args: request: (PubsubProjectsSnapshotsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. """ config = self.GetMethodConfig('TestIamPermissions') return self._RunMethod( config, request, global_params=global_params) TestIamPermissions.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/snapshots/{snapshotsId}:testIamPermissions', http_method=u'POST', method_id=u'pubsub.projects.snapshots.testIamPermissions', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:testIamPermissions', request_field=u'testIamPermissionsRequest', request_type_name=u'PubsubProjectsSnapshotsTestIamPermissionsRequest', response_type_name=u'TestIamPermissionsResponse', supports_download=False, ) class ProjectsSubscriptionsService(base_api.BaseApiService): """Service class for the projects_subscriptions resource.""" _NAME = u'projects_subscriptions' def __init__(self, client): super(PubsubV1.ProjectsSubscriptionsService, self).__init__(client) self._upload_configs = { } def Acknowledge(self, request, global_params=None): """Acknowledges the messages associated with the `ack_ids` in the. `AcknowledgeRequest`. The Pub/Sub system can remove the relevant messages from the subscription. Acknowledging a message whose ack deadline has expired may succeed, but such a message may be redelivered later. Acknowledging a message more than once will not result in an error. Args: request: (PubsubProjectsSubscriptionsAcknowledgeRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Empty) The response message. """ config = self.GetMethodConfig('Acknowledge') return self._RunMethod( config, request, global_params=global_params) Acknowledge.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions/{subscriptionsId}:acknowledge', http_method=u'POST', method_id=u'pubsub.projects.subscriptions.acknowledge', ordered_params=[u'subscription'], path_params=[u'subscription'], query_params=[], relative_path=u'v1/{+subscription}:acknowledge', request_field=u'acknowledgeRequest', request_type_name=u'PubsubProjectsSubscriptionsAcknowledgeRequest', response_type_name=u'Empty', supports_download=False, ) def Create(self, request, global_params=None): """Creates a subscription to a given topic. If the subscription already exists, returns `ALREADY_EXISTS`. If the corresponding topic doesn't exist, returns `NOT_FOUND`. If the name is not provided in the request, the server will assign a random name for this subscription on the same project as the topic, conforming to the [resource name format](https://cloud.google.com/pubsub/docs/overview#names). The generated name is populated in the returned Subscription object. Note that for REST API requests, you must specify a name in the request. Args: request: (Subscription) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Subscription) The response message. """ config = self.GetMethodConfig('Create') return self._RunMethod( config, request, global_params=global_params) Create.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions/{subscriptionsId}', http_method=u'PUT', method_id=u'pubsub.projects.subscriptions.create', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}', request_field='<request>', request_type_name=u'Subscription', response_type_name=u'Subscription', supports_download=False, ) def Delete(self, request, global_params=None): """Deletes an existing subscription. All messages retained in the subscription. are immediately dropped. Calls to `Pull` after deletion will return `NOT_FOUND`. After a subscription is deleted, a new one may be created with the same name, but the new one has no association with the old subscription or its topic unless the same topic is specified. Args: request: (PubsubProjectsSubscriptionsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Empty) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) Delete.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions/{subscriptionsId}', http_method=u'DELETE', method_id=u'pubsub.projects.subscriptions.delete', ordered_params=[u'subscription'], path_params=[u'subscription'], query_params=[], relative_path=u'v1/{+subscription}', request_field='', request_type_name=u'PubsubProjectsSubscriptionsDeleteRequest', response_type_name=u'Empty', supports_download=False, ) def Get(self, request, global_params=None): """Gets the configuration details of a subscription. Args: request: (PubsubProjectsSubscriptionsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Subscription) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions/{subscriptionsId}', http_method=u'GET', method_id=u'pubsub.projects.subscriptions.get', ordered_params=[u'subscription'], path_params=[u'subscription'], query_params=[], relative_path=u'v1/{+subscription}', request_field='', request_type_name=u'PubsubProjectsSubscriptionsGetRequest', response_type_name=u'Subscription', supports_download=False, ) def GetIamPolicy(self, request, global_params=None): """Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set. Args: request: (PubsubProjectsSubscriptionsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('GetIamPolicy') return self._RunMethod( config, request, global_params=global_params) GetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions/{subscriptionsId}:getIamPolicy', http_method=u'GET', method_id=u'pubsub.projects.subscriptions.getIamPolicy', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:getIamPolicy', request_field='', request_type_name=u'PubsubProjectsSubscriptionsGetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def List(self, request, global_params=None): """Lists matching subscriptions. Args: request: (PubsubProjectsSubscriptionsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListSubscriptionsResponse) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions', http_method=u'GET', method_id=u'pubsub.projects.subscriptions.list', ordered_params=[u'project'], path_params=[u'project'], query_params=[u'pageSize', u'pageToken'], relative_path=u'v1/{+project}/subscriptions', request_field='', request_type_name=u'PubsubProjectsSubscriptionsListRequest', response_type_name=u'ListSubscriptionsResponse', supports_download=False, ) def ModifyAckDeadline(self, request, global_params=None): """Modifies the ack deadline for a specific message. This method is useful. to indicate that more time is needed to process a message by the subscriber, or to make the message available for redelivery if the processing was interrupted. Note that this does not modify the subscription-level `ackDeadlineSeconds` used for subsequent messages. Args: request: (PubsubProjectsSubscriptionsModifyAckDeadlineRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Empty) The response message. """ config = self.GetMethodConfig('ModifyAckDeadline') return self._RunMethod( config, request, global_params=global_params) ModifyAckDeadline.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions/{subscriptionsId}:modifyAckDeadline', http_method=u'POST', method_id=u'pubsub.projects.subscriptions.modifyAckDeadline', ordered_params=[u'subscription'], path_params=[u'subscription'], query_params=[], relative_path=u'v1/{+subscription}:modifyAckDeadline', request_field=u'modifyAckDeadlineRequest', request_type_name=u'PubsubProjectsSubscriptionsModifyAckDeadlineRequest', response_type_name=u'Empty', supports_download=False, ) def ModifyPushConfig(self, request, global_params=None): """Modifies the `PushConfig` for a specified subscription. This may be used to change a push subscription to a pull one (signified by an empty `PushConfig`) or vice versa, or change the endpoint URL and other attributes of a push subscription. Messages will accumulate for delivery continuously through the call regardless of changes to the `PushConfig`. Args: request: (PubsubProjectsSubscriptionsModifyPushConfigRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Empty) The response message. """ config = self.GetMethodConfig('ModifyPushConfig') return self._RunMethod( config, request, global_params=global_params) ModifyPushConfig.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions/{subscriptionsId}:modifyPushConfig', http_method=u'POST', method_id=u'pubsub.projects.subscriptions.modifyPushConfig', ordered_params=[u'subscription'], path_params=[u'subscription'], query_params=[], relative_path=u'v1/{+subscription}:modifyPushConfig', request_field=u'modifyPushConfigRequest', request_type_name=u'PubsubProjectsSubscriptionsModifyPushConfigRequest', response_type_name=u'Empty', supports_download=False, ) def Pull(self, request, global_params=None): """Pulls messages from the server. Returns an empty list if there are no. messages available in the backlog. The server may return `UNAVAILABLE` if there are too many concurrent pull requests pending for the given subscription. Args: request: (PubsubProjectsSubscriptionsPullRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (PullResponse) The response message. """ config = self.GetMethodConfig('Pull') return self._RunMethod( config, request, global_params=global_params) Pull.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions/{subscriptionsId}:pull', http_method=u'POST', method_id=u'pubsub.projects.subscriptions.pull', ordered_params=[u'subscription'], path_params=[u'subscription'], query_params=[], relative_path=u'v1/{+subscription}:pull', request_field=u'pullRequest', request_type_name=u'PubsubProjectsSubscriptionsPullRequest', response_type_name=u'PullResponse', supports_download=False, ) def SetIamPolicy(self, request, global_params=None): """Sets the access control policy on the specified resource. Replaces any. existing policy. Args: request: (PubsubProjectsSubscriptionsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('SetIamPolicy') return self._RunMethod( config, request, global_params=global_params) SetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions/{subscriptionsId}:setIamPolicy', http_method=u'POST', method_id=u'pubsub.projects.subscriptions.setIamPolicy', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:setIamPolicy', request_field=u'setIamPolicyRequest', request_type_name=u'PubsubProjectsSubscriptionsSetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def TestIamPermissions(self, request, global_params=None): """Returns permissions that a caller has on the specified resource. If the resource does not exist, this will return an empty set of permissions, not a NOT_FOUND error. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. Args: request: (PubsubProjectsSubscriptionsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. """ config = self.GetMethodConfig('TestIamPermissions') return self._RunMethod( config, request, global_params=global_params) TestIamPermissions.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/subscriptions/{subscriptionsId}:testIamPermissions', http_method=u'POST', method_id=u'pubsub.projects.subscriptions.testIamPermissions', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:testIamPermissions', request_field=u'testIamPermissionsRequest', request_type_name=u'PubsubProjectsSubscriptionsTestIamPermissionsRequest', response_type_name=u'TestIamPermissionsResponse', supports_download=False, ) class ProjectsTopicsSubscriptionsService(base_api.BaseApiService): """Service class for the projects_topics_subscriptions resource.""" _NAME = u'projects_topics_subscriptions' def __init__(self, client): super(PubsubV1.ProjectsTopicsSubscriptionsService, self).__init__(client) self._upload_configs = { } def List(self, request, global_params=None): """Lists the name of the subscriptions for this topic. Args: request: (PubsubProjectsTopicsSubscriptionsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListTopicSubscriptionsResponse) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/topics/{topicsId}/subscriptions', http_method=u'GET', method_id=u'pubsub.projects.topics.subscriptions.list', ordered_params=[u'topic'], path_params=[u'topic'], query_params=[u'pageSize', u'pageToken'], relative_path=u'v1/{+topic}/subscriptions', request_field='', request_type_name=u'PubsubProjectsTopicsSubscriptionsListRequest', response_type_name=u'ListTopicSubscriptionsResponse', supports_download=False, ) class ProjectsTopicsService(base_api.BaseApiService): """Service class for the projects_topics resource.""" _NAME = u'projects_topics' def __init__(self, client): super(PubsubV1.ProjectsTopicsService, self).__init__(client) self._upload_configs = { } def Create(self, request, global_params=None): """Creates the given topic with the given name. Args: request: (Topic) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Topic) The response message. """ config = self.GetMethodConfig('Create') return self._RunMethod( config, request, global_params=global_params) Create.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/topics/{topicsId}', http_method=u'PUT', method_id=u'pubsub.projects.topics.create', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}', request_field='<request>', request_type_name=u'Topic', response_type_name=u'Topic', supports_download=False, ) def Delete(self, request, global_params=None): """Deletes the topic with the given name. Returns `NOT_FOUND` if the topic. does not exist. After a topic is deleted, a new topic may be created with the same name; this is an entirely new topic with none of the old configuration or subscriptions. Existing subscriptions to this topic are not deleted, but their `topic` field is set to `_deleted-topic_`. Args: request: (PubsubProjectsTopicsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Empty) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) Delete.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/topics/{topicsId}', http_method=u'DELETE', method_id=u'pubsub.projects.topics.delete', ordered_params=[u'topic'], path_params=[u'topic'], query_params=[], relative_path=u'v1/{+topic}', request_field='', request_type_name=u'PubsubProjectsTopicsDeleteRequest', response_type_name=u'Empty', supports_download=False, ) def Get(self, request, global_params=None): """Gets the configuration of a topic. Args: request: (PubsubProjectsTopicsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Topic) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/topics/{topicsId}', http_method=u'GET', method_id=u'pubsub.projects.topics.get', ordered_params=[u'topic'], path_params=[u'topic'], query_params=[], relative_path=u'v1/{+topic}', request_field='', request_type_name=u'PubsubProjectsTopicsGetRequest', response_type_name=u'Topic', supports_download=False, ) def GetIamPolicy(self, request, global_params=None): """Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set. Args: request: (PubsubProjectsTopicsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('GetIamPolicy') return self._RunMethod( config, request, global_params=global_params) GetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/topics/{topicsId}:getIamPolicy', http_method=u'GET', method_id=u'pubsub.projects.topics.getIamPolicy', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:getIamPolicy', request_field='', request_type_name=u'PubsubProjectsTopicsGetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def List(self, request, global_params=None): """Lists matching topics. Args: request: (PubsubProjectsTopicsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListTopicsResponse) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/topics', http_method=u'GET', method_id=u'pubsub.projects.topics.list', ordered_params=[u'project'], path_params=[u'project'], query_params=[u'pageSize', u'pageToken'], relative_path=u'v1/{+project}/topics', request_field='', request_type_name=u'PubsubProjectsTopicsListRequest', response_type_name=u'ListTopicsResponse', supports_download=False, ) def Publish(self, request, global_params=None): """Adds one or more messages to the topic. Returns `NOT_FOUND` if the topic. does not exist. The message payload must not be empty; it must contain either a non-empty data field, or at least one attribute. Args: request: (PubsubProjectsTopicsPublishRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (PublishResponse) The response message. """ config = self.GetMethodConfig('Publish') return self._RunMethod( config, request, global_params=global_params) Publish.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/topics/{topicsId}:publish', http_method=u'POST', method_id=u'pubsub.projects.topics.publish', ordered_params=[u'topic'], path_params=[u'topic'], query_params=[], relative_path=u'v1/{+topic}:publish', request_field=u'publishRequest', request_type_name=u'PubsubProjectsTopicsPublishRequest', response_type_name=u'PublishResponse', supports_download=False, ) def SetIamPolicy(self, request, global_params=None): """Sets the access control policy on the specified resource. Replaces any. existing policy. Args: request: (PubsubProjectsTopicsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('SetIamPolicy') return self._RunMethod( config, request, global_params=global_params) SetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/topics/{topicsId}:setIamPolicy', http_method=u'POST', method_id=u'pubsub.projects.topics.setIamPolicy', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:setIamPolicy', request_field=u'setIamPolicyRequest', request_type_name=u'PubsubProjectsTopicsSetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def TestIamPermissions(self, request, global_params=None): """Returns permissions that a caller has on the specified resource. If the resource does not exist, this will return an empty set of permissions, not a NOT_FOUND error. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. Args: request: (PubsubProjectsTopicsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. """ config = self.GetMethodConfig('TestIamPermissions') return self._RunMethod( config, request, global_params=global_params) TestIamPermissions.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/topics/{topicsId}:testIamPermissions', http_method=u'POST', method_id=u'pubsub.projects.topics.testIamPermissions', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:testIamPermissions', request_field=u'testIamPermissionsRequest', request_type_name=u'PubsubProjectsTopicsTestIamPermissionsRequest', response_type_name=u'TestIamPermissionsResponse', supports_download=False, ) class ProjectsService(base_api.BaseApiService): """Service class for the projects resource.""" _NAME = u'projects' def __init__(self, client): super(PubsubV1.ProjectsService, self).__init__(client) self._upload_configs = { } third_party/pubsub_apitools/__init__.py 0000644 00000000000 15025013616 0014405 0 ustar 00 third_party/pubsub_apitools/pubsub_v1_messages.py 0000644 00000076270 15025013616 0016471 0 ustar 00 """Generated message classes for pubsub version v1. Provides reliable, many-to-many, asynchronous messaging between applications. """ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.protorpclite import messages as _messages from apitools.base.py import encoding package = 'pubsub' class AcknowledgeRequest(_messages.Message): """Request for the Acknowledge method. Fields: ackIds: The acknowledgment ID for the messages being acknowledged that was returned by the Pub/Sub system in the `Pull` response. Must not be empty. """ ackIds = _messages.StringField(1, repeated=True) class Binding(_messages.Message): """Associates `members` with a `role`. Fields: members: Specifies the identities requesting access for a Cloud Platform resource. `members` can have the following values: * `allUsers`: A special identifier that represents anyone who is on the internet; with or without a Google account. * `allAuthenticatedUsers`: A special identifier that represents anyone who is authenticated with a Google account or a service account. * `user:{emailid}`: An email address that represents a specific Google account. For example, `alice@gmail.com` or `joe@example.com`. * `serviceAccount:{emailid}`: An email address that represents a service account. For example, `my-other- app@appspot.gserviceaccount.com`. * `group:{emailid}`: An email address that represents a Google group. For example, `admins@example.com`. * `domain:{domain}`: A Google Apps domain name that represents all the users of that domain. For example, `google.com` or `example.com`. role: Role that is assigned to `members`. For example, `roles/viewer`, `roles/editor`, or `roles/owner`. Required """ members = _messages.StringField(1, repeated=True) role = _messages.StringField(2) class Empty(_messages.Message): """A generic empty message that you can re-use to avoid defining duplicated empty messages in your APIs. A typical example is to use it as the request or the response type of an API method. For instance: service Foo { rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); } The JSON representation for `Empty` is empty JSON object `{}`. """ class ListSubscriptionsResponse(_messages.Message): """Response for the `ListSubscriptions` method. Fields: nextPageToken: If not empty, indicates that there may be more subscriptions that match the request; this value should be passed in a new `ListSubscriptionsRequest` to get more subscriptions. subscriptions: The subscriptions that match the request. """ nextPageToken = _messages.StringField(1) subscriptions = _messages.MessageField('Subscription', 2, repeated=True) class ListTopicSubscriptionsResponse(_messages.Message): """Response for the `ListTopicSubscriptions` method. Fields: nextPageToken: If not empty, indicates that there may be more subscriptions that match the request; this value should be passed in a new `ListTopicSubscriptionsRequest` to get more subscriptions. subscriptions: The names of the subscriptions that match the request. """ nextPageToken = _messages.StringField(1) subscriptions = _messages.StringField(2, repeated=True) class ListTopicsResponse(_messages.Message): """Response for the `ListTopics` method. Fields: nextPageToken: If not empty, indicates that there may be more topics that match the request; this value should be passed in a new `ListTopicsRequest`. topics: The resulting topics. """ nextPageToken = _messages.StringField(1) topics = _messages.MessageField('Topic', 2, repeated=True) class ModifyAckDeadlineRequest(_messages.Message): """Request for the ModifyAckDeadline method. Fields: ackDeadlineSeconds: The new ack deadline with respect to the time this request was sent to the Pub/Sub system. For example, if the value is 10, the new ack deadline will expire 10 seconds after the `ModifyAckDeadline` call was made. Specifying zero may immediately make the message available for another pull request. The minimum deadline you can specify is 0 seconds. The maximum deadline you can specify is 600 seconds (10 minutes). ackIds: List of acknowledgment IDs. """ ackDeadlineSeconds = _messages.IntegerField(1, variant=_messages.Variant.INT32) ackIds = _messages.StringField(2, repeated=True) class ModifyPushConfigRequest(_messages.Message): """Request for the ModifyPushConfig method. Fields: pushConfig: The push configuration for future deliveries. An empty `pushConfig` indicates that the Pub/Sub system should stop pushing messages from the given subscription and allow messages to be pulled and acknowledged - effectively pausing the subscription if `Pull` or `StreamingPull` is not called. """ pushConfig = _messages.MessageField('PushConfig', 1) class Policy(_messages.Message): """Defines an Identity and Access Management (IAM) policy. It is used to specify access control policies for Cloud Platform resources. A `Policy` consists of a list of `bindings`. A `Binding` binds a list of `members` to a `role`, where the members can be user accounts, Google groups, Google domains, and service accounts. A `role` is a named list of permissions defined by IAM. **Example** { "bindings": [ { "role": "roles/owner", "members": [ "user:mike@example.com", "group:admins@example.com", "domain:google.com", "serviceAccount:my-other- app@appspot.gserviceaccount.com", ] }, { "role": "roles/viewer", "members": ["user:sean@example.com"] } ] } For a description of IAM and its features, see the [IAM developer's guide](https://cloud.google.com/iam). Fields: bindings: Associates a list of `members` to a `role`. `bindings` with no members will result in an error. etag: `etag` is used for optimistic concurrency control as a way to help prevent simultaneous updates of a policy from overwriting each other. It is strongly suggested that systems make use of the `etag` in the read- modify-write cycle to perform policy updates in order to avoid race conditions: An `etag` is returned in the response to `getIamPolicy`, and systems are expected to put that etag in the request to `setIamPolicy` to ensure that their change will be applied to the same version of the policy. If no `etag` is provided in the call to `setIamPolicy`, then the existing policy is overwritten blindly. version: Version of the `Policy`. The default version is 0. """ bindings = _messages.MessageField('Binding', 1, repeated=True) etag = _messages.BytesField(2) version = _messages.IntegerField(3, variant=_messages.Variant.INT32) class PublishRequest(_messages.Message): """Request for the Publish method. Fields: messages: The messages to publish. """ messages = _messages.MessageField('PubsubMessage', 1, repeated=True) class PublishResponse(_messages.Message): """Response for the `Publish` method. Fields: messageIds: The server-assigned ID of each published message, in the same order as the messages in the request. IDs are guaranteed to be unique within the topic. """ messageIds = _messages.StringField(1, repeated=True) class PubsubMessage(_messages.Message): """A message data and its attributes. The message payload must not be empty; it must contain either a non-empty data field, or at least one attribute. Messages: AttributesValue: Optional attributes for this message. Fields: attributes: Optional attributes for this message. data: The message payload. messageId: ID of this message, assigned by the server when the message is published. Guaranteed to be unique within the topic. This value may be read by a subscriber that receives a `PubsubMessage` via a `Pull` call or a push delivery. It must not be populated by the publisher in a `Publish` call. publishTime: The time at which the message was published, populated by the server when it receives the `Publish` call. It must not be populated by the publisher in a `Publish` call. """ @encoding.MapUnrecognizedFields('additionalProperties') class AttributesValue(_messages.Message): """Optional attributes for this message. Messages: AdditionalProperty: An additional property for a AttributesValue object. Fields: additionalProperties: Additional properties of type AttributesValue """ class AdditionalProperty(_messages.Message): """An additional property for a AttributesValue object. Fields: key: Name of the additional property. value: A string attribute. """ key = _messages.StringField(1) value = _messages.StringField(2) additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) attributes = _messages.MessageField('AttributesValue', 1) data = _messages.BytesField(2) messageId = _messages.StringField(3) publishTime = _messages.StringField(4) class PubsubProjectsSnapshotsGetIamPolicyRequest(_messages.Message): """A PubsubProjectsSnapshotsGetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being requested. See the operation documentation for the appropriate value for this field. """ resource = _messages.StringField(1, required=True) class PubsubProjectsSnapshotsSetIamPolicyRequest(_messages.Message): """A PubsubProjectsSnapshotsSetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being specified. See the operation documentation for the appropriate value for this field. setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the request body. """ resource = _messages.StringField(1, required=True) setIamPolicyRequest = _messages.MessageField('SetIamPolicyRequest', 2) class PubsubProjectsSnapshotsTestIamPermissionsRequest(_messages.Message): """A PubsubProjectsSnapshotsTestIamPermissionsRequest object. Fields: resource: REQUIRED: The resource for which the policy detail is being requested. See the operation documentation for the appropriate value for this field. testIamPermissionsRequest: A TestIamPermissionsRequest resource to be passed as the request body. """ resource = _messages.StringField(1, required=True) testIamPermissionsRequest = _messages.MessageField('TestIamPermissionsRequest', 2) class PubsubProjectsSubscriptionsAcknowledgeRequest(_messages.Message): """A PubsubProjectsSubscriptionsAcknowledgeRequest object. Fields: acknowledgeRequest: A AcknowledgeRequest resource to be passed as the request body. subscription: The subscription whose message is being acknowledged. Format is `projects/{project}/subscriptions/{sub}`. """ acknowledgeRequest = _messages.MessageField('AcknowledgeRequest', 1) subscription = _messages.StringField(2, required=True) class PubsubProjectsSubscriptionsDeleteRequest(_messages.Message): """A PubsubProjectsSubscriptionsDeleteRequest object. Fields: subscription: The subscription to delete. Format is `projects/{project}/subscriptions/{sub}`. """ subscription = _messages.StringField(1, required=True) class PubsubProjectsSubscriptionsGetIamPolicyRequest(_messages.Message): """A PubsubProjectsSubscriptionsGetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being requested. See the operation documentation for the appropriate value for this field. """ resource = _messages.StringField(1, required=True) class PubsubProjectsSubscriptionsGetRequest(_messages.Message): """A PubsubProjectsSubscriptionsGetRequest object. Fields: subscription: The name of the subscription to get. Format is `projects/{project}/subscriptions/{sub}`. """ subscription = _messages.StringField(1, required=True) class PubsubProjectsSubscriptionsListRequest(_messages.Message): """A PubsubProjectsSubscriptionsListRequest object. Fields: pageSize: Maximum number of subscriptions to return. pageToken: The value returned by the last `ListSubscriptionsResponse`; indicates that this is a continuation of a prior `ListSubscriptions` call, and that the system should return the next page of data. project: The name of the cloud project that subscriptions belong to. Format is `projects/{project}`. """ pageSize = _messages.IntegerField(1, variant=_messages.Variant.INT32) pageToken = _messages.StringField(2) project = _messages.StringField(3, required=True) class PubsubProjectsSubscriptionsModifyAckDeadlineRequest(_messages.Message): """A PubsubProjectsSubscriptionsModifyAckDeadlineRequest object. Fields: modifyAckDeadlineRequest: A ModifyAckDeadlineRequest resource to be passed as the request body. subscription: The name of the subscription. Format is `projects/{project}/subscriptions/{sub}`. """ modifyAckDeadlineRequest = _messages.MessageField('ModifyAckDeadlineRequest', 1) subscription = _messages.StringField(2, required=True) class PubsubProjectsSubscriptionsModifyPushConfigRequest(_messages.Message): """A PubsubProjectsSubscriptionsModifyPushConfigRequest object. Fields: modifyPushConfigRequest: A ModifyPushConfigRequest resource to be passed as the request body. subscription: The name of the subscription. Format is `projects/{project}/subscriptions/{sub}`. """ modifyPushConfigRequest = _messages.MessageField('ModifyPushConfigRequest', 1) subscription = _messages.StringField(2, required=True) class PubsubProjectsSubscriptionsPullRequest(_messages.Message): """A PubsubProjectsSubscriptionsPullRequest object. Fields: pullRequest: A PullRequest resource to be passed as the request body. subscription: The subscription from which messages should be pulled. Format is `projects/{project}/subscriptions/{sub}`. """ pullRequest = _messages.MessageField('PullRequest', 1) subscription = _messages.StringField(2, required=True) class PubsubProjectsSubscriptionsSetIamPolicyRequest(_messages.Message): """A PubsubProjectsSubscriptionsSetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being specified. See the operation documentation for the appropriate value for this field. setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the request body. """ resource = _messages.StringField(1, required=True) setIamPolicyRequest = _messages.MessageField('SetIamPolicyRequest', 2) class PubsubProjectsSubscriptionsTestIamPermissionsRequest(_messages.Message): """A PubsubProjectsSubscriptionsTestIamPermissionsRequest object. Fields: resource: REQUIRED: The resource for which the policy detail is being requested. See the operation documentation for the appropriate value for this field. testIamPermissionsRequest: A TestIamPermissionsRequest resource to be passed as the request body. """ resource = _messages.StringField(1, required=True) testIamPermissionsRequest = _messages.MessageField('TestIamPermissionsRequest', 2) class PubsubProjectsTopicsDeleteRequest(_messages.Message): """A PubsubProjectsTopicsDeleteRequest object. Fields: topic: Name of the topic to delete. Format is `projects/{project}/topics/{topic}`. """ topic = _messages.StringField(1, required=True) class PubsubProjectsTopicsGetIamPolicyRequest(_messages.Message): """A PubsubProjectsTopicsGetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being requested. See the operation documentation for the appropriate value for this field. """ resource = _messages.StringField(1, required=True) class PubsubProjectsTopicsGetRequest(_messages.Message): """A PubsubProjectsTopicsGetRequest object. Fields: topic: The name of the topic to get. Format is `projects/{project}/topics/{topic}`. """ topic = _messages.StringField(1, required=True) class PubsubProjectsTopicsListRequest(_messages.Message): """A PubsubProjectsTopicsListRequest object. Fields: pageSize: Maximum number of topics to return. pageToken: The value returned by the last `ListTopicsResponse`; indicates that this is a continuation of a prior `ListTopics` call, and that the system should return the next page of data. project: The name of the cloud project that topics belong to. Format is `projects/{project}`. """ pageSize = _messages.IntegerField(1, variant=_messages.Variant.INT32) pageToken = _messages.StringField(2) project = _messages.StringField(3, required=True) class PubsubProjectsTopicsPublishRequest(_messages.Message): """A PubsubProjectsTopicsPublishRequest object. Fields: publishRequest: A PublishRequest resource to be passed as the request body. topic: The messages in the request will be published on this topic. Format is `projects/{project}/topics/{topic}`. """ publishRequest = _messages.MessageField('PublishRequest', 1) topic = _messages.StringField(2, required=True) class PubsubProjectsTopicsSetIamPolicyRequest(_messages.Message): """A PubsubProjectsTopicsSetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being specified. See the operation documentation for the appropriate value for this field. setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the request body. """ resource = _messages.StringField(1, required=True) setIamPolicyRequest = _messages.MessageField('SetIamPolicyRequest', 2) class PubsubProjectsTopicsSubscriptionsListRequest(_messages.Message): """A PubsubProjectsTopicsSubscriptionsListRequest object. Fields: pageSize: Maximum number of subscription names to return. pageToken: The value returned by the last `ListTopicSubscriptionsResponse`; indicates that this is a continuation of a prior `ListTopicSubscriptions` call, and that the system should return the next page of data. topic: The name of the topic that subscriptions are attached to. Format is `projects/{project}/topics/{topic}`. """ pageSize = _messages.IntegerField(1, variant=_messages.Variant.INT32) pageToken = _messages.StringField(2) topic = _messages.StringField(3, required=True) class PubsubProjectsTopicsTestIamPermissionsRequest(_messages.Message): """A PubsubProjectsTopicsTestIamPermissionsRequest object. Fields: resource: REQUIRED: The resource for which the policy detail is being requested. See the operation documentation for the appropriate value for this field. testIamPermissionsRequest: A TestIamPermissionsRequest resource to be passed as the request body. """ resource = _messages.StringField(1, required=True) testIamPermissionsRequest = _messages.MessageField('TestIamPermissionsRequest', 2) class PullRequest(_messages.Message): """Request for the `Pull` method. Fields: maxMessages: The maximum number of messages returned for this request. The Pub/Sub system may return fewer than the number specified. returnImmediately: If this field set to true, the system will respond immediately even if it there are no messages available to return in the `Pull` response. Otherwise, the system may wait (for a bounded amount of time) until at least one message is available, rather than returning no messages. The client may cancel the request if it does not wish to wait any longer for the response. """ maxMessages = _messages.IntegerField(1, variant=_messages.Variant.INT32) returnImmediately = _messages.BooleanField(2) class PullResponse(_messages.Message): """Response for the `Pull` method. Fields: receivedMessages: Received Pub/Sub messages. The Pub/Sub system will return zero messages if there are no more available in the backlog. The Pub/Sub system may return fewer than the `maxMessages` requested even if there are more messages available in the backlog. """ receivedMessages = _messages.MessageField('ReceivedMessage', 1, repeated=True) class PushConfig(_messages.Message): """Configuration for a push delivery endpoint. Messages: AttributesValue: Endpoint configuration attributes. Every endpoint has a set of API supported attributes that can be used to control different aspects of the message delivery. The currently supported attribute is `x-goog-version`, which you can use to change the format of the pushed message. This attribute indicates the version of the data expected by the endpoint. This controls the shape of the pushed message (i.e., its fields and metadata). The endpoint version is based on the version of the Pub/Sub API. If not present during the `CreateSubscription` call, it will default to the version of the API used to make such call. If not present during a `ModifyPushConfig` call, its value will not be changed. `GetSubscription` calls will always return a valid version, even if the subscription was created without this attribute. The possible values for this attribute are: * `v1beta1`: uses the push format defined in the v1beta1 Pub/Sub API. * `v1` or `v1beta2`: uses the push format defined in the v1 Pub/Sub API. Fields: attributes: Endpoint configuration attributes. Every endpoint has a set of API supported attributes that can be used to control different aspects of the message delivery. The currently supported attribute is `x-goog-version`, which you can use to change the format of the pushed message. This attribute indicates the version of the data expected by the endpoint. This controls the shape of the pushed message (i.e., its fields and metadata). The endpoint version is based on the version of the Pub/Sub API. If not present during the `CreateSubscription` call, it will default to the version of the API used to make such call. If not present during a `ModifyPushConfig` call, its value will not be changed. `GetSubscription` calls will always return a valid version, even if the subscription was created without this attribute. The possible values for this attribute are: * `v1beta1`: uses the push format defined in the v1beta1 Pub/Sub API. * `v1` or `v1beta2`: uses the push format defined in the v1 Pub/Sub API. pushEndpoint: A URL locating the endpoint to which messages should be pushed. For example, a Webhook endpoint might use "https://example.com/push". """ @encoding.MapUnrecognizedFields('additionalProperties') class AttributesValue(_messages.Message): """Endpoint configuration attributes. Every endpoint has a set of API supported attributes that can be used to control different aspects of the message delivery. The currently supported attribute is `x-goog-version`, which you can use to change the format of the pushed message. This attribute indicates the version of the data expected by the endpoint. This controls the shape of the pushed message (i.e., its fields and metadata). The endpoint version is based on the version of the Pub/Sub API. If not present during the `CreateSubscription` call, it will default to the version of the API used to make such call. If not present during a `ModifyPushConfig` call, its value will not be changed. `GetSubscription` calls will always return a valid version, even if the subscription was created without this attribute. The possible values for this attribute are: * `v1beta1`: uses the push format defined in the v1beta1 Pub/Sub API. * `v1` or `v1beta2`: uses the push format defined in the v1 Pub/Sub API. Messages: AdditionalProperty: An additional property for a AttributesValue object. Fields: additionalProperties: Additional properties of type AttributesValue """ class AdditionalProperty(_messages.Message): """An additional property for a AttributesValue object. Fields: key: Name of the additional property. value: A string attribute. """ key = _messages.StringField(1) value = _messages.StringField(2) additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) attributes = _messages.MessageField('AttributesValue', 1) pushEndpoint = _messages.StringField(2) class ReceivedMessage(_messages.Message): """A message and its corresponding acknowledgment ID. Fields: ackId: This ID can be used to acknowledge the received message. message: The message. """ ackId = _messages.StringField(1) message = _messages.MessageField('PubsubMessage', 2) class SetIamPolicyRequest(_messages.Message): """Request message for `SetIamPolicy` method. Fields: policy: REQUIRED: The complete policy to be applied to the `resource`. The size of the policy is limited to a few 10s of KB. An empty policy is a valid policy but certain Cloud Platform services (such as Projects) might reject them. """ policy = _messages.MessageField('Policy', 1) class StandardQueryParameters(_messages.Message): """Query parameters accepted by all methods. Enums: FXgafvValueValuesEnum: V1 error format. AltValueValuesEnum: Data format for response. Fields: f__xgafv: V1 error format. access_token: OAuth access token. alt: Data format for response. bearer_token: OAuth bearer token. callback: JSONP fields: Selector specifying which fields to include in a partial response. key: API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token. oauth_token: OAuth 2.0 token for the current user. pp: Pretty-print response. prettyPrint: Returns response with indentations and line breaks. quotaUser: Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. trace: A tracing token of the form "token:<tokenid>" to include in api requests. uploadType: Legacy upload protocol for media (e.g. "media", "multipart"). upload_protocol: Upload protocol for media (e.g. "raw", "multipart"). """ class AltValueValuesEnum(_messages.Enum): """Data format for response. Values: json: Responses with Content-Type of application/json media: Media download with context-dependent Content-Type proto: Responses with Content-Type of application/x-protobuf """ json = 0 media = 1 proto = 2 class FXgafvValueValuesEnum(_messages.Enum): """V1 error format. Values: _1: v1 error format _2: v2 error format """ _1 = 0 _2 = 1 f__xgafv = _messages.EnumField('FXgafvValueValuesEnum', 1) access_token = _messages.StringField(2) alt = _messages.EnumField('AltValueValuesEnum', 3, default=u'json') bearer_token = _messages.StringField(4) callback = _messages.StringField(5) fields = _messages.StringField(6) key = _messages.StringField(7) oauth_token = _messages.StringField(8) pp = _messages.BooleanField(9, default=True) prettyPrint = _messages.BooleanField(10, default=True) quotaUser = _messages.StringField(11) trace = _messages.StringField(12) uploadType = _messages.StringField(13) upload_protocol = _messages.StringField(14) class Subscription(_messages.Message): """A subscription resource. Fields: ackDeadlineSeconds: This value is the maximum time after a subscriber receives a message before the subscriber should acknowledge the message. After message delivery but before the ack deadline expires and before the message is acknowledged, it is an outstanding message and will not be delivered again during that time (on a best-effort basis). For pull subscriptions, this value is used as the initial value for the ack deadline. To override this value for a given message, call `ModifyAckDeadline` with the corresponding `ack_id` if using non- streaming pull or send the `ack_id` in a `StreamingModifyAckDeadlineRequest` if using streaming pull. The minimum custom deadline you can specify is 10 seconds. The maximum custom deadline you can specify is 600 seconds (10 minutes). If this parameter is 0, a default value of 10 seconds is used. For push delivery, this value is also used to set the request timeout for the call to the push endpoint. If the subscriber never acknowledges the message, the Pub/Sub system will eventually redeliver the message. name: The name of the subscription. It must have the format `"projects/{project}/subscriptions/{subscription}"`. `{subscription}` must start with a letter, and contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`), underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent signs (`%`). It must be between 3 and 255 characters in length, and it must not start with `"goog"`. pushConfig: If push delivery is used with this subscription, this field is used to configure it. An empty `pushConfig` signifies that the subscriber will pull and ack messages using API methods. topic: The name of the topic from which this subscription is receiving messages. Format is `projects/{project}/topics/{topic}`. The value of this field will be `_deleted-topic_` if the topic has been deleted. """ ackDeadlineSeconds = _messages.IntegerField(1, variant=_messages.Variant.INT32) name = _messages.StringField(2) pushConfig = _messages.MessageField('PushConfig', 3) topic = _messages.StringField(4) class TestIamPermissionsRequest(_messages.Message): """Request message for `TestIamPermissions` method. Fields: permissions: The set of permissions to check for the `resource`. Permissions with wildcards (such as '*' or 'storage.*') are not allowed. For more information see [IAM Overview](https://cloud.google.com/iam/docs/overview#permissions). """ permissions = _messages.StringField(1, repeated=True) class TestIamPermissionsResponse(_messages.Message): """Response message for `TestIamPermissions` method. Fields: permissions: A subset of `TestPermissionsRequest.permissions` that the caller is allowed. """ permissions = _messages.StringField(1, repeated=True) class Topic(_messages.Message): """A topic resource. Fields: name: The name of the topic. It must have the format `"projects/{project}/topics/{topic}"`. `{topic}` must start with a letter, and contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`), underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent signs (`%`). It must be between 3 and 255 characters in length, and it must not start with `"goog"`. """ name = _messages.StringField(1) encoding.AddCustomJsonFieldMapping( StandardQueryParameters, 'f__xgafv', '$.xgafv') encoding.AddCustomJsonEnumMapping( StandardQueryParameters.FXgafvValueValuesEnum, '_1', '1') encoding.AddCustomJsonEnumMapping( StandardQueryParameters.FXgafvValueValuesEnum, '_2', '2') third_party/storage_apitools/__pycache__/storage_v1_messages.cpython-39.pyc 0000644 00000363356 15025013616 0023274 0 ustar 00 a +(Wg*� � @ s d Z ddlmZ ddlmZ ddlmZ ddlmZ dZ G dd� dej �Z G d d � d ej �Z G d d � d ej �Z G d d� dej �ZG dd� dej �ZG dd� dej �ZG dd� dej �ZG dd� dej �ZG dd� dej �ZG dd� dej �ZG dd � d ej �ZG d d � d ej �ZG d d � d ej �ZG d!d"� d"ej �ZG d#d$� d$ej �ZG d%d&� d&ej �ZG d'd(� d(ej �ZG d)d*� d*ej �Z G d+d,� d,ej �Z G d-d.� d.ej �Z G d/d0� d0ej �Z G d1d2� d2ej �Z G d3d4� d4ej �Z!G d5d6� d6ej �Z"G d7d8� d8ej �Z#G d9d:� d:ej �Z$G d;d<� d<ej �Z%G d=d>� d>ej �Z&G d?d@� d@ej �Z'G dAdB� dBej �Z(G dCdD� dDej �Z)G dEdF� dFej �Z*G dGdH� dHej �Z+G dIdJ� dJej �Z,G dKdL� dLej �Z-G dMdN� dNej �Z.G dOdP� dPej �Z/G dQdR� dRej �Z0G dSdT� dTej �Z1G dUdV� dVej �Z2G dWdX� dXej �Z3G dYdZ� dZej �Z4G d[d\� d\ej �Z5G d]d^� d^ej �Z6G d_d`� d`ej �Z7G dadb� dbej �Z8G dcdd� ddej �Z9G dedf� dfej �Z:G dgdh� dhej �Z;G didj� djej �Z<G dkdl� dlej �Z=G dmdn� dnej �Z>G dodp� dpej �Z?G dqdr� drej �Z@G dsdt� dtej �ZAG dudv� dvej �ZBG dwdx� dxej �ZCG dydz� dzej �ZDG d{d|� d|ej �ZEG d}d~� d~ej �ZFG dd�� d�ej �ZGG d�d�� d�ej �ZHG d�d�� d�ej �ZIG d�d�� d�ej �ZJG d�d�� d�ej �ZKG d�d�� d�ej �ZLG d�d�� d�ej �ZMG d�d�� d�ej �ZNG d�d�� d�ej �ZOG d�d�� d�ej �ZPG d�d�� d�ej �ZQG d�d�� d�ej �ZRG d�d�� d�ej �ZSG d�d�� d�ej �ZTG d�d�� d�ej �ZUG d�d�� d�ej �ZVG d�d�� d�ej �ZWG d�d�� d�ej �ZXG d�d�� d�ej �ZYG d�d�� d�ej �ZZG d�d�� d�ej �Z[G d�d�� d�ej �Z\d�S )�zsGenerated message classes for storage version v1. Stores and retrieves potentially large, immutable data objects. � )� message_types)�messages)�encoding)� extra_typesZstoragec @ sf e Zd ZdZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd � d ej�Z G d d � d ej�Z G d d � d ej�Z e � d�G dd� dej��ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd � d ej�Zejd d d d �Ze�dd!�Ze�dd"�Zejdd#d d �Ze�d d$�Ze�d%�Z ejd&d'd d �Z e�d d(�Z e� d)�Z e�d d*�Z!e� d+�Z"ej d,d-d.�Z#e�dd/�Z$e�dd0�Z%e� d1�Z&e� d2�Z'e�dd3�Z(e�)d4�Z*e� d5�Z+e�dd6�Z,ej)d7ej-j.d8�Z/e�dd9�Z0e� d:�Z1e�d;�Z2e� d<�Z3e� d=�Z4e5�6d>�Z7e5�6d?�Z8e�dd@�Z9e�d dA�Z:dBS )C�Bucketa� A bucket. Messages: AutoclassValue: The bucket's Autoclass configuration. BillingValue: The bucket's billing configuration. CorsValueListEntry: A CorsValueListEntry object. CustomPlacementConfigValue: The bucket's custom placement configuration for Custom Dual Regions. EncryptionValue: Encryption configuration for a bucket. IamConfigurationValue: The bucket's IAM configuration. LabelsValue: User-provided labels, in key/value pairs. LifecycleValue: The bucket's lifecycle configuration. See lifecycle management for more information. LoggingValue: The bucket's logging configuration, which defines the destination bucket and optional name prefix for the current bucket's logs. OwnerValue: The owner of the bucket. This is always the project team's owner group. RetentionPolicyValue: The bucket's retention policy. The retention policy enforces a minimum retention time for all objects contained in the bucket, based on their creation time. Any attempt to overwrite or delete objects younger than the retention period will result in a PERMISSION_DENIED error. An unlocked retention policy can be modified or removed from the bucket via a storage.buckets.update operation. A locked retention policy cannot be removed or shortened in duration for the lifetime of the bucket. Attempting to remove or decrease period of a locked retention policy will result in a PERMISSION_DENIED error. VersioningValue: The bucket's versioning configuration. WebsiteValue: The bucket's website configuration, controlling how the service behaves when accessing bucket contents as a web site. See the Static Website Examples for more information. Fields: acl: Access controls on the bucket. autoclass: The bucket's Autoclass configuration. billing: The bucket's billing configuration. cors: The bucket's Cross-Origin Resource Sharing (CORS) configuration. customPlacementConfig: The bucket's custom placement configuration for Custom Dual Regions. defaultEventBasedHold: The default value for event-based hold on newly created objects in this bucket. Event-based hold is a way to retain objects indefinitely until an event occurs, signified by the hold's release. After being released, such objects will be subject to bucket- level retention (if any). One sample use case of this flag is for banks to hold loan documents for at least 3 years after loan is paid in full. Here, bucket-level retention is 3 years and the event is loan being paid in full. In this example, these objects will be held intact for any number of years until the event has occurred (event-based hold on the object is released) and then 3 more years after that. That means retention duration of the objects begins from the moment event-based hold transitioned from true to false. Objects under event-based hold cannot be deleted, overwritten or archived until the hold is removed. defaultObjectAcl: Default access controls to apply to new objects when no ACL is provided. encryption: Encryption configuration for a bucket. etag: HTTP 1.1 Entity tag for the bucket. iamConfiguration: The bucket's IAM configuration. id: The ID of the bucket. For buckets, the id and name properties are the same. kind: The kind of item this is. For buckets, this is always storage#bucket. labels: User-provided labels, in key/value pairs. lifecycle: The bucket's lifecycle configuration. See lifecycle management for more information. location: The location of the bucket. Object data for objects in the bucket resides in physical storage within this region. Defaults to US. See the developer's guide for the authoritative list. locationType: The type of the bucket location. logging: The bucket's logging configuration, which defines the destination bucket and optional name prefix for the current bucket's logs. metageneration: The metadata generation of this bucket. name: The name of the bucket. owner: The owner of the bucket. This is always the project team's owner group. projectNumber: The project number of the project the bucket belongs to. retentionPolicy: The bucket's retention policy. The retention policy enforces a minimum retention time for all objects contained in the bucket, based on their creation time. Any attempt to overwrite or delete objects younger than the retention period will result in a PERMISSION_DENIED error. An unlocked retention policy can be modified or removed from the bucket via a storage.buckets.update operation. A locked retention policy cannot be removed or shortened in duration for the lifetime of the bucket. Attempting to remove or decrease period of a locked retention policy will result in a PERMISSION_DENIED error. rpo: The Recovery Point Objective (RPO) of this bucket. Set to ASYNC_TURBO to turn on turbo replication on a bucket. satisfiesPZS: Reserved for future use. selfLink: The URI of this bucket. storageClass: The bucket's default storage class, used whenever no storageClass is specified for a newly-created object. This defines how objects in the bucket are stored and determines the SLA and the cost of storage. Values include MULTI_REGIONAL, REGIONAL, STANDARD, NEARLINE, COLDLINE, ARCHIVE, and DURABLE_REDUCED_AVAILABILITY. If this value is not specified when the bucket is created, it will default to STANDARD. For more information, see storage classes. timeCreated: The creation time of the bucket in RFC 3339 format. updated: The modification time of the bucket in RFC 3339 format. versioning: The bucket's versioning configuration. website: The bucket's website configuration, controlling how the service behaves when accessing bucket contents as a web site. See the Static Website Examples for more information. c @ s$ e Zd ZdZe�d�Ze�d�Z dS )zBucket.AutoclassValuez�The bucket's Autoclass configuration. Fields: enabled: Whether or not Autoclass is enabled on this bucket toggleTime: A date and time in RFC 3339 format representing the instant at which "enabled" was last toggled. � � N) �__name__� __module__� __qualname__�__doc__� _messages� BooleanField�enabled�_message_types� DateTimeFieldZ toggleTime� r r �E/opt/gsutil/gslib/third_party/storage_apitools/storage_v1_messages.py�AutoclassValue� s r c @ s e Zd ZdZe�d�ZdS )zBucket.BillingValuez�The bucket's billing configuration. Fields: requesterPays: When set to true, Requester Pays is enabled for this bucket. r N)r r r r r r Z requesterPaysr r r r � BillingValue� s r c @ sL e Zd ZdZejdejjd�Zej ddd�Z ej ddd�Z ej ddd�Z d S ) zBucket.CorsValueListEntrya� A CorsValueListEntry object. Fields: maxAgeSeconds: The value, in seconds, to return in the Access-Control- Max-Age header used in preflight responses. method: The list of HTTP methods on which to include CORS response headers, (GET, OPTIONS, POST, etc) Note: "*" is permitted in the list of methods, and means "any method". origin: The list of Origins eligible to receive CORS response headers. Note: "*" is permitted in the list of origins, and means "any Origin". responseHeader: The list of HTTP headers other than the simple response headers to give permission for the user-agent to share across domains. r ��variantr T�Zrepeated� � N) r r r r r � IntegerField�Variant�INT32Z maxAgeSeconds� StringField�method�originZresponseHeaderr r r r �CorsValueListEntry� s r! c @ s e Zd ZdZejddd�ZdS )z!Bucket.CustomPlacementConfigValuez�The bucket's custom placement configuration for Custom Dual Regions. Fields: dataLocations: The list of regional locations in which data is placed. r Tr N)r r r r r r Z dataLocationsr r r r �CustomPlacementConfigValue� s r" c @ s e Zd ZdZe�d�ZdS )zBucket.EncryptionValuez�Encryption configuration for a bucket. Fields: defaultKmsKeyName: A Cloud KMS key that will be used to encrypt objects inserted into this bucket, if no encryption method is specified. r N)r r r r r r ZdefaultKmsKeyNamer r r r �EncryptionValue� s r# c @ s8 e Zd ZdZG dd� dej�Ze�dd�Ze� d�Z dS )z Bucket.IamConfigurationValuea] The bucket's IAM configuration. Messages: BucketPolicyOnlyValue: The bucket's Bucket Policy Only configuration. Fields: bucketPolicyOnly: The bucket's Bucket Policy Only configuration. publicAccessPrevention: The bucket's public access prevention configuration. Valid values are `unspecified` and `enforced`. c @ s$ e Zd ZdZe�d�Ze�d�Z dS )z2Bucket.IamConfigurationValue.BucketPolicyOnlyValuea� The bucket's Bucket Policy Only configuration. Fields: enabled: If set, access checks only use bucket-level IAM policies or above. lockedTime: The deadline time for changing iamConfiguration.bucketPolicyOnly.enabled from true to false in RFC 3339 format. iamConfiguration.bucketPolicyOnly.enabled may be changed from true to false until the locked time, after which the field is immutable. r r N) r r r r r r r r r Z lockedTimer r r r �BucketPolicyOnlyValue� s r$ r r N) r r r r r �Messager$ � MessageFieldZbucketPolicyOnlyr ZpublicAccessPreventionr r r r �IamConfigurationValue� s r' �additionalPropertiesc @ s2 e Zd ZdZG dd� dej�Zejdddd�ZdS )zBucket.LabelsValuez�User-provided labels, in key/value pairs. Messages: AdditionalProperty: An additional property for a LabelsValue object. Fields: additionalProperties: An individual label entry. c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z%Bucket.LabelsValue.AdditionalPropertyz�An additional property for a LabelsValue object. Fields: key: Name of the additional property. value: A string attribute. r r N�r r r r r r �key�valuer r r r �AdditionalProperty� s r, r Tr N� r r r r r r% r, r& r( r r r r � LabelsValue� s �r. c @ s2 e Zd ZdZG dd� dej�Zejdddd�ZdS )zBucket.LifecycleValuea? The bucket's lifecycle configuration. See lifecycle management for more information. Messages: RuleValueListEntry: A RuleValueListEntry object. Fields: rule: A lifecycle management rule, which is made of an action to take and the condition(s) under which the action will be taken. c @ sL e Zd ZdZG dd� dej�ZG dd� dej�Ze�dd�Z e�dd�Z dS ) z(Bucket.LifecycleValue.RuleValueListEntrya) A RuleValueListEntry object. Messages: ActionValue: The action to take. ConditionValue: The condition(s) under which the action will be taken. Fields: action: The action to take. condition: The condition(s) under which the action will be taken. c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z4Bucket.LifecycleValue.RuleValueListEntry.ActionValuea The action to take. Fields: storageClass: Target storage class. Required iff the type of the action is SetStorageClass. type: Type of the action. Currently, only Delete and SetStorageClass are supported. r r N)r r r r r r � storageClass�typer r r r � ActionValue s r1 c @ s� e Zd ZdZejdejjd�Ze � d�Z e � d�Z ejdejjd�Z ejdejjd�Ze�d�Zejd d d �Ze � d �Zejd ejjd�ZdS )z7Bucket.LifecycleValue.RuleValueListEntry.ConditionValueaj The condition(s) under which the action will be taken. Fields: age: Age of an object (in days). This condition is satisfied when an object reaches the specified age. createdBefore: A date in RFC 3339 format with only the date part (for instance, "2013-01-15"). This condition is satisfied when an object is created before midnight of the specified date in UTC. customTimeBefore: A date in RFC 3339 format with only the date part (for instance, "2013-01-15"). This condition is satisfied when the custom time on an object is before this date in UTC. daysSinceCustomTime: Number of days elapsed since the user-specified timestamp set on an object. The condition is satisfied if the days elapsed is at least this number. If no custom timestamp is specified on an object, the condition does not apply. daysSinceNoncurrentTime: Number of days elapsed since the noncurrent timestamp of an object. The condition is satisfied if the days elapsed is at least this number. This condition is relevant only for versioned objects. The value of the field must be a nonnegative integer. If it's zero, the object version will become eligible for Lifecycle action as soon as it becomes noncurrent. isLive: Relevant only for versioned objects. If the value is true, this condition matches live objects; if the value is false, it matches archived objects. matchesStorageClass: Objects having any of the storage classes specified by this condition will be matched. Values include MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, and DURABLE_REDUCED_AVAILABILITY. noncurrentTimeBefore: A date in RFC 3339 format with only the date part (for instance, "2013-01-15"). This condition is satisfied when the noncurrent time on an object is before this date in UTC. This condition is relevant only for versioned objects. numNewerVersions: Relevant only for versioned objects. If the value is N, this condition is satisfied when there are at least N versions (including the live version) newer than this version of the object. r r r � � � r r Tr � � N)r r r r r r r r Zager Z DateFieldZ createdBeforeZcustomTimeBeforeZdaysSinceCustomTimeZdaysSinceNoncurrentTimer ZisLiver ZmatchesStorageClassZnoncurrentTimeBeforeZnumNewerVersionsr r r r �ConditionValue s & �� �r7 r r N) r r r r r r% r1 r7 r& �action� conditionr r r r �RuleValueListEntry s 4 r: r Tr N) r r r r r r% r: r& �ruler r r r �LifecycleValue� s Pr< c @ s$ e Zd ZdZe�d�Ze�d�ZdS )zBucket.LoggingValuea. The bucket's logging configuration, which defines the destination bucket and optional name prefix for the current bucket's logs. Fields: logBucket: The destination bucket where the current bucket's logs should be placed. logObjectPrefix: A prefix for log object names. r r N)r r r r r r Z logBucketZlogObjectPrefixr r r r � LoggingValueX s r= c @ s$ e Zd ZdZe�d�Ze�d�ZdS )zBucket.OwnerValuez�The owner of the bucket. This is always the project team's owner group. Fields: entity: The entity, in the form project-owner-projectId. entityId: The ID for the entity. r r N�r r r r r r �entity�entityIdr r r r � OwnerValuee s rA c @ s. e Zd ZdZe�d�Ze�d�Z e� d�Z dS )zBucket.RetentionPolicyValuea� The bucket's retention policy. The retention policy enforces a minimum retention time for all objects contained in the bucket, based on their creation time. Any attempt to overwrite or delete objects younger than the retention period will result in a PERMISSION_DENIED error. An unlocked retention policy can be modified or removed from the bucket via a storage.buckets.update operation. A locked retention policy cannot be removed or shortened in duration for the lifetime of the bucket. Attempting to remove or decrease period of a locked retention policy will result in a PERMISSION_DENIED error. Fields: effectiveTime: Server-determined value that indicates the time from which policy was enforced and effective. This value is in RFC 3339 format. isLocked: Once locked, an object retention policy cannot be modified. retentionPeriod: The duration in seconds that objects need to be retained. Retention duration must be greater than zero and less than 100 years. Note that enforcement of retention periods less than a day is not guaranteed. Such periods should only be used for testing purposes. r r r N) r r r r r r Z effectiveTimer r ZisLockedr ZretentionPeriodr r r r �RetentionPolicyValueq s rB c @ s e Zd ZdZe�d�ZdS )zBucket.VersioningValuez�The bucket's versioning configuration. Fields: enabled: While set to true, versioning is fully enabled for this bucket. r N)r r r r r r r r r r r �VersioningValue� s rC c @ s$ e Zd ZdZe�d�Ze�d�ZdS )zBucket.WebsiteValuea� The bucket's website configuration, controlling how the service behaves when accessing bucket contents as a web site. See the Static Website Examples for more information. Fields: mainPageSuffix: If the requested object path is missing, the service will ensure the path has a trailing '/', append this suffix, and attempt to retrieve the resulting object. This allows the creation of index.html objects to represent directory pages. notFoundPage: If the requested object path is missing, and any mainPageSuffix object is missing, if applicable, the service will return the named object from this bucket as the content for a 404 Not Found result. r r N)r r r r r r ZmainPageSuffixZ notFoundPager r r r � WebsiteValue� s rD �BucketAccessControlr Tr r r r r6 r2 �ObjectAccessControlr3 r4 r5 � � � zstorage#bucket��default� � � � � � � � � r � � � � � � � � � N);r r r r r r% r r r! r" r# r' r �MapUnrecognizedFieldsr. r< r= rA rB rC rD r& �aclZ autoclass�billing�corsZcustomPlacementConfigr ZdefaultEventBasedHold�defaultObjectAclZ encryptionr �etagZiamConfiguration�id�kind�labels� lifecycle�locationZ locationType�loggingr �metageneration�name�ownerr �UINT64� projectNumberZretentionPolicyZrpoZ satisfiesPZS�selfLinkr/ r r � timeCreated�updated� versioning�websiter r r r r s^ g ^ � r c @ s� e Zd ZdZG dd� dej�Ze�d�Ze�d�Z e�d�Z e�d�Z e�d�Z e�d �Z e�d �Zejd d d �Ze�dd�Ze�d�Ze�d�ZdS )rE a� An access-control entry. Messages: ProjectTeamValue: The project team associated with the entity, if any. Fields: bucket: The name of the bucket. domain: The domain associated with the entity, if any. email: The email address associated with the entity, if any. entity: The entity holding the permission, in one of the following forms: - user-userId - user-email - group-groupId - group-email - domain- domain - project-team-projectId - allUsers - allAuthenticatedUsers Examples: - The user liz@example.com would be user-liz@example.com. - The group example@googlegroups.com would be group- example@googlegroups.com. - To refer to all members of the Google Apps for Business domain example.com, the entity would be domain-example.com. entityId: The ID for the entity, if any. etag: HTTP 1.1 Entity tag for the access-control entry. id: The ID of the access-control entry. kind: The kind of item this is. For bucket access control entries, this is always storage#bucketAccessControl. projectTeam: The project team associated with the entity, if any. role: The access permission for the entity. selfLink: The link to this access-control entry. c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z$BucketAccessControl.ProjectTeamValue��The project team associated with the entity, if any. Fields: projectNumber: The project number. team: The team. r r N�r r r r r r rn Zteamr r r r �ProjectTeamValue� s rv r r r r r6 r2 r3 r4 zstorage#bucketAccessControlrJ r5 rG rH N)r r r r r r% rv r �bucket�domain�emailr? r@ rc rd re r& � projectTeam�rolero r r r r rE � s rE c @ s. e Zd ZdZejdddd�Zejddd�Zd S ) �BucketAccessControlsz�An access-control list. Fields: items: The list of items. kind: The kind of item this is. For lists of bucket access control entries, this is always storage#bucketAccessControls. rE r Tr r z storage#bucketAccessControlsrJ N� r r r r r r& �itemsr re r r r r r| � s r| c @ s8 e Zd ZdZejdddd�Zejddd�Ze�d �Z d S ) �BucketsaL A list of buckets. Fields: items: The list of items. kind: The kind of item this is. For lists of buckets, this is always storage#buckets. nextPageToken: The continuation token, used to page through large result sets. Provide this value in a subsequent request to return the next page of results. r r Tr r zstorage#bucketsrJ r N� r r r r r r& r~ r re � nextPageTokenr r r r r s r c @ s� e Zd ZdZe�d�G dd� dej��Ze� d�Z e� d�Z e� d�Z ej dd d �Ze�dd �Ze�d �Ze� d �Ze� d�Ze� d�Ze� d�ZdS )�Channela� An notification channel used to watch for resource changes. Messages: ParamsValue: Additional parameters controlling delivery channel behavior. Optional. Fields: address: The address where notifications are delivered for this channel. expiration: Date and time of notification channel expiration, expressed as a Unix timestamp, in milliseconds. Optional. id: A UUID or similar unique string that identifies this channel. kind: Identifies this as a notification channel used to watch for changes to a resource. Value: the fixed string "api#channel". params: Additional parameters controlling delivery channel behavior. Optional. payload: A Boolean value to indicate whether payload is wanted. Optional. resourceId: An opaque ID that identifies the resource being watched on this channel. Stable across different API versions. resourceUri: A version-specific identifier for the watched resource. token: An arbitrary string delivered to the target address with each notification delivered over this channel. Optional. type: The type of delivery mechanism used for this channel. r( c @ s2 e Zd ZdZG dd� dej�Zejdddd�ZdS )zChannel.ParamsValuez�Additional parameters controlling delivery channel behavior. Optional. Messages: AdditionalProperty: An additional property for a ParamsValue object. Fields: additionalProperties: Declares a new parameter by name. c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z&Channel.ParamsValue.AdditionalPropertyz�An additional property for a ParamsValue object. Fields: key: Name of the additional property. value: A string attribute. r r Nr) r r r r r, > s r, r Tr Nr- r r r r � ParamsValue3 s �r� r r r r z api#channelrJ r6 r2 r3 r4 r5 rG N)r r r r r r^ r r% r� r �addressr � expirationrd re r& �paramsr �payload� resourceIdZ resourceUri�tokenr0 r r r r r� s r� c @ s@ e Zd ZdZG dd� dej�Zejdddd�Zej ddd �Z d S ) �Channelsz�A channels list response. Messages: ItemsValueListEntry: A ItemsValueListEntry object. Fields: items: The list of notification channels for a bucket. kind: The kind of item this is. c @ sB e Zd ZdZe�d�Ze�d�Ze�d�Z e�d�Z e�d�Z dS )z Channels.ItemsValueListEntrya� A ItemsValueListEntry object. Fields: channel_id: User-specified name for a channel. Needed to unsubscribe. creation_time_ms: 64-bit Unix timestamp in ms of when the channel was created. push_url: Url used to identify where notifications are sent to. resource_id: Opaque value generated by GCS representing a bucket. Needed to unsubscribe. subscriber_email: Email address of the subscriber. r r r r r6 N) r r r r r r � channel_idr Zcreation_time_msZpush_url� resource_idZsubscriber_emailr r r r �ItemsValueListEntryd s r� r Tr r zstorage#channelsrJ N) r r r r r r% r� r& r~ r re r r r r r� Y s r� c @ sV e Zd ZdZG dd� dej�Ze�dd�Zej ddd�Z ejdd d d �Z e� d �Z d S )�ComposeRequesta� A Compose request. Messages: SourceObjectsValueListEntry: A SourceObjectsValueListEntry object. Fields: destination: Properties of the resulting object. kind: The kind of item this is. sourceObjects: The list of source objects that will be concatenated into a single object. userProject: The project to be billed for this request, for Requester Pays buckets. c @ sB e Zd ZdZG dd� dej�Ze�d�Ze� d�Z e� dd�Z dS )z*ComposeRequest.SourceObjectsValueListEntrya� A SourceObjectsValueListEntry object. Messages: ObjectPreconditionsValue: Conditions that must be met for this operation to execute. Fields: generation: The generation of this object to use as the source. name: The source object's name. All source objects must reside in the same bucket. objectPreconditions: Conditions that must be met for this operation to execute. c @ s e Zd ZdZe�d�ZdS )zCComposeRequest.SourceObjectsValueListEntry.ObjectPreconditionsValueaZ Conditions that must be met for this operation to execute. Fields: ifGenerationMatch: Only perform the composition if the generation of the source object that would be used matches this value. If this value and a generation are both specified, they must be the same value or the call will fail. r N)r r r r r r �ifGenerationMatchr r r r �ObjectPreconditionsValue� s r� r r r N) r r r r r r% r� r � generationr rk r& ZobjectPreconditionsr r r r �SourceObjectsValueListEntry� s r� �Objectr r zstorage#composeRequestrJ r Tr r N) r r r r r r% r� r& Z destinationr re Z sourceObjects� userProjectr r r r r� { s �r� c @ sF e Zd ZdZe�d�Ze�d�Zejddd�Ze�d�Z e�d�Z d S ) �Expra� Represents an expression text. Example: title: "User account presence" description: "Determines whether the request has a user account" expression: "size(request.user) > 0" Fields: description: An optional description of the expression. This is a longer text which describes the expression, e.g. when hovered over it in a UI. expression: Textual representation of an expression in Common Expression Language syntax. The application context of the containing message determines which well-known feature set of CEL is supported. kind: The kind of item this is. For storage, this is always storage#expr. This field is ignored on input. location: An optional string indicating the location of the expression for error reporting, e.g. a file name and a position in the file. title: An optional title for the expression, i.e. a short string describing its purpose. This can be used e.g. in UIs which allow to enter the expression. r r r z storage#exprrJ r r6 N) r r r r r r � descriptionZ expressionre rh �titler r r r r� � s r� c @ s� e Zd ZdZe�d�G dd� dej��Ze� dd�Z e� d�Z ej ddd �Z e� d �Zej d d d �Ze� d�Zej ddd �Ze� d�Ze� d�ZdS )� Notificationa A subscription to receive Google PubSub notifications. Messages: CustomAttributesValue: An optional list of additional attributes to attach to each Cloud PubSub message published for this notification subscription. Fields: custom_attributes: An optional list of additional attributes to attach to each Cloud PubSub message published for this notification subscription. etag: HTTP 1.1 Entity tag for this subscription notification. event_types: If present, only send notifications about listed event types. If empty, sent notifications for all event types. id: The ID of the notification. kind: The kind of item this is. For notifications, this is always storage#notification. object_name_prefix: If present, only apply this notification configuration to object names that begin with this prefix. payload_format: The desired content of the Payload. selfLink: The canonical URL of this notification. topic: The Cloud PubSub topic to which this subscription publishes. Formatted as: '//pubsub.googleapis.com/projects/{project- identifier}/topics/{my-topic}' r( c @ s2 e Zd ZdZG dd� dej�Zejdddd�ZdS )z"Notification.CustomAttributesValueaX An optional list of additional attributes to attach to each Cloud PubSub message published for this notification subscription. Messages: AdditionalProperty: An additional property for a CustomAttributesValue object. Fields: additionalProperties: Additional properties of type CustomAttributesValue c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z5Notification.CustomAttributesValue.AdditionalPropertyz�An additional property for a CustomAttributesValue object. Fields: key: Name of the additional property. value: A string attribute. r r Nr) r r r r r, � s r, r Tr Nr- r r r r �CustomAttributesValue� s �r� r r r Tr r r6 zstorage#notificationrJ r2 r3 Z JSON_API_V1r4 r5 N)r r r r r r^ r r% r� r& �custom_attributesr rc � event_typesrd re �object_name_prefix�payload_formatro Ztopicr r r r r� � s r� c @ s. e Zd ZdZejdddd�Zejddd�Zd S ) � Notificationsz�A list of notification subscriptions. Fields: items: The list of items. kind: The kind of item this is. For lists of notifications, this is always storage#notifications. r� r Tr r zstorage#notificationsrJ Nr} r r r r r� s r� c @ s� e Zd ZdZG dd� dej�Ze�d�G dd� dej��Z G dd� dej�Z ej d d d d �Z e� d �Ze� d�Zejdejjd�Ze� d�Ze� d�Ze� d�Ze� d�Ze� d�Ze�d�Ze� dd�Z e� d�Z e� d�Z e�d�Z e� d�Z!ej d d d �Z"e� d �Z#e� d �Z$e� d!�Z%e� dd"�Z&e�d#�Z'e� d$�Z(e� dd%�Z)e�d&�Z*e� d'�Z+ejd(ejj,d�Z-e� d)�Z.e� d*�Z/e�d+�Z0e�d,�Z1e�d-�Z2e�d.�Z3d/S )0r� a� An object. Messages: CustomerEncryptionValue: Metadata of customer-supplied encryption key, if the object is encrypted by such a key. MetadataValue: User-provided metadata, in key/value pairs. OwnerValue: The owner of the object. This will always be the uploader of the object. Fields: acl: Access controls on the object. bucket: The name of the bucket containing this object. cacheControl: Cache-Control directive for the object data. If omitted, and the object is accessible to all anonymous users, the default will be public, max-age=3600. componentCount: Number of underlying components that make up this object. Components are accumulated by compose operations. contentDisposition: Content-Disposition of the object data. contentEncoding: Content-Encoding of the object data. contentLanguage: Content-Language of the object data. contentType: Content-Type of the object data. If an object is stored without a Content-Type, it is served as application/octet-stream. crc32c: CRC32c checksum, as described in RFC 4960, Appendix B; encoded using base64 in big-endian byte order. For more information about using the CRC32c checksum, see Hashes and ETags: Best Practices. customTime: A timestamp in RFC 3339 format specified by the user for an object. customerEncryption: Metadata of customer-supplied encryption key, if the object is encrypted by such a key. etag: HTTP 1.1 Entity tag for the object. eventBasedHold: Whether an object is under event-based hold. Event-based hold is a way to retain objects until an event occurs, which is signified by the hold's release (i.e. this value is set to false). After being released (set to false), such objects will be subject to bucket- level retention (if any). One sample use case of this flag is for banks to hold loan documents for at least 3 years after loan is paid in full. Here, bucket-level retention is 3 years and the event is the loan being paid in full. In this example, these objects will be held intact for any number of years until the event has occurred (event-based hold on the object is released) and then 3 more years after that. That means retention duration of the objects begins from the moment event-based hold transitioned from true to false. generation: The content generation of this object. Used for object versioning. id: The ID of the object, including the bucket name, object name, and generation number. kind: The kind of item this is. For objects, this is always storage#object. kmsKeyName: Cloud KMS Key used to encrypt this object, if the object is encrypted by such a key. md5Hash: MD5 hash of the data; encoded using base64. For more information about using the MD5 hash, see Hashes and ETags: Best Practices. mediaLink: Media download link. metadata: User-provided metadata, in key/value pairs. metageneration: The version of the metadata for this object at this generation. Used for preconditions and for detecting changes in metadata. A metageneration number is only meaningful in the context of a particular generation of a particular object. name: The name of the object. Required if not specified by URL parameter. owner: The owner of the object. This will always be the uploader of the object. retentionExpirationTime: A server-determined value that specifies the earliest time that the object's retention period expires. This value is in RFC 3339 format. Note 1: This field is not provided for objects with an active event-based hold, since retention expiration is unknown until the hold is removed. Note 2: This value can be provided even when temporary hold is set (so that the user can reason about policy without having to first unset the temporary hold). selfLink: The link to this object. size: Content-Length of the data in bytes. storageClass: Storage class of the object. temporaryHold: Whether an object is under temporary hold. While this flag is set to true, the object is protected against deletion and overwrites. A common use case of this flag is regulatory investigations where objects need to be retained while the investigation is ongoing. Note that unlike event-based hold, temporary hold does not impact retention expiration time of an object. timeCreated: The creation time of the object in RFC 3339 format. timeDeleted: The deletion time of the object in RFC 3339 format. Will be returned if and only if this version of the object has been deleted. timeStorageClassUpdated: The time at which the object's storage class was last changed. When the object is initially created, it will be set to timeCreated. updated: The modification time of the object metadata in RFC 3339 format. c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z Object.CustomerEncryptionValuez�Metadata of customer-supplied encryption key, if the object is encrypted by such a key. Fields: encryptionAlgorithm: The encryption algorithm. keySha256: SHA256 hash value of the encryption key. r r N)r r r r r r �encryptionAlgorithmZ keySha256r r r r �CustomerEncryptionValuer s r� r( c @ s2 e Zd ZdZG dd� dej�Zejdddd�ZdS )zObject.MetadataValuez�User-provided metadata, in key/value pairs. Messages: AdditionalProperty: An additional property for a MetadataValue object. Fields: additionalProperties: An individual metadata entry. c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z'Object.MetadataValue.AdditionalPropertyz�An additional property for a MetadataValue object. Fields: key: Name of the additional property. value: A string attribute. r r Nr) r r r r r, � s r, r Tr Nr- r r r r � MetadataValue~ s �r� c @ s$ e Zd ZdZe�d�Ze�d�ZdS )zObject.OwnerValuez�The owner of the object. This will always be the uploader of the object. Fields: entity: The entity, in the form user-userId. entityId: The ID for the entity. r r Nr> r r r r rA � s rA rF r Tr r r r r r6 r2 r3 r4 r5 � rG rH rI rL rM rN zstorage#objectrJ rO rP rQ rR rS rT rU rV rW rX rY rZ r[ r\ r] � N)4r r r r r r% r� r r^ r� rA r& r_ r rw Z cacheControlr r r ZcomponentCountZcontentDisposition�contentEncodingZcontentLanguageZ contentTypeZcrc32cr r Z customTimeZcustomerEncryptionrc r ZeventBasedHoldr� rd re � kmsKeyNameZmd5HashZ mediaLink�metadatarj rk rl ZretentionExpirationTimero rm �sizer/ Z temporaryHoldrp Z timeDeletedZtimeStorageClassUpdatedrq r r r r r� sJ V r� c @ s� e Zd ZdZG dd� dej�Ze�d�Ze�d�Z e�d�Z e�d�Z e�d�Z e�d �Z e�d �Ze�d �Zejd d d�Ze�d�Ze�dd�Ze�d�Ze�d�ZdS )rF an An access-control entry. Messages: ProjectTeamValue: The project team associated with the entity, if any. Fields: bucket: The name of the bucket. domain: The domain associated with the entity, if any. email: The email address associated with the entity, if any. entity: The entity holding the permission, in one of the following forms: - user-userId - user-email - group-groupId - group-email - domain- domain - project-team-projectId - allUsers - allAuthenticatedUsers Examples: - The user liz@example.com would be user-liz@example.com. - The group example@googlegroups.com would be group- example@googlegroups.com. - To refer to all members of the Google Apps for Business domain example.com, the entity would be domain-example.com. entityId: The ID for the entity, if any. etag: HTTP 1.1 Entity tag for the access-control entry. generation: The content generation of the object, if applied to an object. id: The ID of the access-control entry. kind: The kind of item this is. For object access control entries, this is always storage#objectAccessControl. object: The name of the object, if applied to an object. projectTeam: The project team associated with the entity, if any. role: The access permission for the entity. selfLink: The link to this access-control entry. c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z$ObjectAccessControl.ProjectTeamValuert r r Nru r r r r rv � s rv r r r r r6 r2 r3 r4 r5 zstorage#objectAccessControlrJ rG rH rI rL N)r r r r r r% rv r rw rx ry r? r@ rc r r� rd re �objectr& rz r{ ro r r r r rF � s rF c @ s. e Zd ZdZejdddd�Zejddd�Zd S ) �ObjectAccessControlsz�An access-control list. Fields: items: The list of items. kind: The kind of item this is. For lists of object access control entries, this is always storage#objectAccessControls. rF r Tr r z storage#objectAccessControlsrJ Nr} r r r r r� � s r� c @ sF e Zd ZdZejdddd�Zejddd�Ze�d �Z ejd dd�Z d S ) �Objectsa� A list of objects. Fields: items: The list of items. kind: The kind of item this is. For lists of objects, this is always storage#objects. nextPageToken: The continuation token, used to page through large result sets. Provide this value in a subsequent request to return the next page of results. prefixes: The list of prefixes of objects matching-but-not-listed up to and including the requested delimiter. r� r Tr r zstorage#objectsrJ r r N) r r r r r r& r~ r re r� �prefixesr r r r r� s r� c @ sf e Zd ZdZG dd� dej�Zejdddd�Ze� d�Z ej dd d �Z e� d �Z ejd ejjd �ZdS )�Policya� A bucket/object IAM policy. Messages: BindingsValueListEntry: A BindingsValueListEntry object. Fields: bindings: An association between a role, which comes with a set of permissions, and members who may assume that role. etag: HTTP 1.1 Entity tag for the policy. kind: The kind of item this is. For policies, this is always storage#policy. This field is ignored on input. resourceId: The ID of the resource to which this policy belongs. Will be of the form projects/_/buckets/bucket for buckets, and projects/_/buckets/bucket/objects/object for objects. A specific generation may be specified by appending #generationNumber to the end of the object name, e.g. projects/_/buckets/my-bucket/objects/data.txt#17. The current generation can be denoted with #0. This field is ignored on input. version: The policy format version to be returned in the response. c @ s4 e Zd ZdZe�dd�Zejddd�Ze�d�Z dS ) z Policy.BindingsValueListEntrya� A BindingsValueListEntry object. Fields: condition: The condition that is associated with this binding. NOTE: an unsatisfied condition will not allow user access via current binding. Different bindings, including their conditions, are examined independently. members: A collection of identifiers for members who may assume the provided role. Recognized identifiers are as follows: - allUsers - A special identifier that represents anyone on the internet; with or without a Google account. - allAuthenticatedUsers - A special identifier that represents anyone who is authenticated with a Google account or a service account. - user:emailid - An email address that represents a specific account. For example, user:alice@gmail.com or user:joe@example.com. - serviceAccount:emailid - An email address that represents a service account. For example, serviceAccount:my- other-app@appspot.gserviceaccount.com . - group:emailid - An email address that represents a Google group. For example, group:admins@example.com. - domain:domain - A Google Apps domain name that represents all the users of that domain. For example, domain:google.com or domain:example.com. - projectOwner:projectid - Owners of the given project. For example, projectOwner:my-example- project - projectEditor:projectid - Editors of the given project. For example, projectEditor:my-example-project - projectViewer:projectid - Viewers of the given project. For example, projectViewer:my-example-project role: The role to which members belong. Two types of roles are supported: new IAM roles, which grant permissions that do not map directly to those provided by ACLs, and legacy IAM roles, which do map directly to ACL permissions. All roles are of the format roles/storage.specificRole. The new IAM roles are: - roles/storage.admin - Full control of Google Cloud Storage resources. - roles/storage.objectViewer - Read-Only access to Google Cloud Storage objects. - roles/storage.objectCreator - Access to create objects in Google Cloud Storage. - roles/storage.objectAdmin - Full control of Google Cloud Storage objects. The legacy IAM roles are: - roles/storage.legacyObjectReader - Read-only access to objects without listing. Equivalent to an ACL entry on an object with the READER role. - roles/storage.legacyObjectOwner - Read/write access to existing objects without listing. Equivalent to an ACL entry on an object with the OWNER role. - roles/storage.legacyBucketReader - Read access to buckets with object listing. Equivalent to an ACL entry on a bucket with the READER role. - roles/storage.legacyBucketWriter - Read access to buckets with object listing/creation/deletion. Equivalent to an ACL entry on a bucket with the WRITER role. - roles/storage.legacyBucketOwner - Read and write access to existing buckets with object listing/creation/deletion. Equivalent to an ACL entry on a bucket with the OWNER role. r� r r Tr r N) r r r r r r& r9 r �membersr{ r r r r �BindingsValueListEntry4 s 2 r� r Tr r r zstorage#policyrJ r r6 r N)r r r r r r% r� r& �bindingsZ BytesFieldrc r re r� r r r �versionr r r r r� s 7 r� c @ sR e Zd ZdZe�d�Zejddd�Ze� d�Z e� dd�Z e�d �Z e� d �Zd S ) �RewriteResponseaJ A rewrite response. Fields: done: true if the copy is finished; otherwise, false if the copy is in progress. This property is always present in the response. kind: The kind of item this is. objectSize: The total size of the object being copied in bytes. This property is always present in the response. resource: A resource containing the metadata for the copied-to object. This property is present in the response only when copying completes. rewriteToken: A token to use in subsequent requests to continue copying data. This token is present in the response only when there is more data to copy. totalBytesRewritten: The total bytes written so far, which can be used to provide a waiting user with a progress indicator. This property is always present in the response. r r zstorage#rewriteResponserJ r r� r r6 r2 N)r r r r r r Zdoner re r Z objectSizer& �resource� rewriteTokenZtotalBytesRewrittenr r r r r� r s r� c @ s( e Zd ZdZe�d�Zejddd�ZdS )�ServiceAccountz�A subscription to receive Google PubSub notifications. Fields: email_address: The ID of the notification. kind: The kind of item this is. For notifications, this is always storage#notification. r r zstorage#serviceAccountrJ N)r r r r r r � email_addressre r r r r r� � s r� c @ s| e Zd ZdZG dd� dej�Zejdddd�Ze� d�Z e� d�Z e� d �Z ej d d d�Ze� d �Ze� d �Ze� d�ZdS )�StandardQueryParametersa Query parameters accepted by all methods. Enums: AltValueValuesEnum: Data format for the response. Fields: alt: Data format for the response. fields: Selector specifying which fields to include in a partial response. key: API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token. oauth_token: OAuth 2.0 token for the current user. prettyPrint: Returns response with indentations and line breaks. quotaUser: An opaque string that represents a user for quota purposes. Must not exceed 40 characters. trace: A tracing token of the form "token:<tokenid>" to include in api requests. userIp: Deprecated. Please use quotaUser instead. c @ s e Zd ZdZdZdS )z*StandardQueryParameters.AltValueValuesEnumzkData format for the response. Values: json: Responses with Content-Type of application/json r N)r r r r �jsonr r r r �AltValueValuesEnum� s r� r r� rJ r r r r6 Tr2 r3 r4 N)r r r r r �Enumr� � EnumFieldZaltr �fieldsr* Z oauth_tokenr � prettyPrintZ quotaUser�traceZuserIpr r r r r� � s r� c @ s6 e Zd ZdZejddd�Zejddd�Ze�d�ZdS )�(StorageBucketAccessControlsDeleteRequestao A StorageBucketAccessControlsDeleteRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. r T��requiredr r N� r r r r r r rw r? r� r r r r r� � s r� c @ s e Zd ZdZdS )�)StorageBucketAccessControlsDeleteResponsez4An empty StorageBucketAccessControlsDelete response.N�r r r r r r r r r� � s r� c @ s6 e Zd ZdZejddd�Zejddd�Ze�d�ZdS )�%StorageBucketAccessControlsGetRequestal A StorageBucketAccessControlsGetRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r Nr� r r r r r� � s r� c @ s4 e Zd ZdZejddd�Ze�dd�Ze�d�Z dS ) �(StorageBucketAccessControlsInsertRequesta% A StorageBucketAccessControlsInsertRequest object. Fields: bucket: Name of a bucket. bucketAccessControl: A BucketAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� rE r r N) r r r r r r rw r& �bucketAccessControlr� r r r r r� � s r� c @ s( e Zd ZdZejddd�Ze�d�ZdS )�&StorageBucketAccessControlsListRequestz�A StorageBucketAccessControlsListRequest object. Fields: bucket: Name of a bucket. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r N�r r r r r r rw r� r r r r r� � s r� c @ sB e Zd ZdZejddd�Ze�dd�Zejddd�Z e�d�Z d S ) �'StorageBucketAccessControlsPatchRequesta� A StorageBucketAccessControlsPatchRequest object. Fields: bucket: Name of a bucket. bucketAccessControl: A BucketAccessControl resource to be passed as the request body. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� rE r r r N� r r r r r r rw r& r� r? r� r r r r r� s r� c @ sB e Zd ZdZejddd�Ze�dd�Zejddd�Z e�d�Z d S ) �(StorageBucketAccessControlsUpdateRequesta� A StorageBucketAccessControlsUpdateRequest object. Fields: bucket: Name of a bucket. bucketAccessControl: A BucketAccessControl resource to be passed as the request body. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� rE r r r Nr� r r r r r� s r� c @ s< e Zd ZdZejddd�Ze�d�Ze�d�Z e�d�Z dS ) �StorageBucketsDeleteRequesta� A StorageBucketsDeleteRequest object. Fields: bucket: Name of a bucket. ifMetagenerationMatch: If set, only deletes the bucket if its metageneration matches this value. ifMetagenerationNotMatch: If set, only deletes the bucket if its metageneration does not match this value. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r N� r r r r r r rw r �ifMetagenerationMatch�ifMetagenerationNotMatchr� r r r r r� , s r� c @ s e Zd ZdZdS )� StorageBucketsDeleteResponsez'An empty StorageBucketsDelete response.Nr� r r r r r� ? s r� c @ sD e Zd ZdZejddd�Ze�d�Zejdej j d�Z e�d�Z d S ) �!StorageBucketsGetIamPolicyRequesta� A StorageBucketsGetIamPolicyRequest object. Fields: bucket: Name of a bucket. provisionalUserProject: The project to be billed for this request if the target bucket is requester-pays bucket. optionsRequestedPolicyVersion: The policy format version to be returned in the response. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r r N) r r r r r r rw ZprovisionalUserProjectr r r Z optionsRequestedPolicyVersionr� r r r r r� C s �r� c @ sZ e Zd ZdZG dd� dej�Zejddd�Ze� d�Z e� d�Z e� dd �Z e�d �Zd S ) �StorageBucketsGetRequesta� A StorageBucketsGetRequest object. Enums: ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. Fields: bucket: Name of a bucket. ifMetagenerationMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration does not match the given value. projection: Set of properties to return. Defaults to noAcl. userProject: The project to be billed for this request. Required for Requester Pays buckets. c @ s e Zd ZdZdZdZdS )z2StorageBucketsGetRequest.ProjectionValueValuesEnum��Set of properties to return. Defaults to noAcl. Values: full: Include all properties. noAcl: Omit owner, acl and defaultObjectAcl properties. r r N�r r r r �fullZnoAclr r r r �ProjectionValueValuesEnumi s r� r Tr� r r r r6 N)r r r r r r� r� r rw r r� r� r� � projectionr� r r r r r� W s r� c @ s� e Zd ZdZG dd� dej�ZG dd� dej�ZG dd� dej�Ze� dd �Z e� dd �Z e� dd �Z ejd d d�Ze� dd�Ze�d�ZdS )�StorageBucketsInsertRequesta� A StorageBucketsInsertRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this bucket. PredefinedDefaultObjectAclValueValuesEnum: Apply a predefined set of default object access controls to this bucket. ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it defaults to full. Fields: bucket: A Bucket resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this bucket. predefinedDefaultObjectAcl: Apply a predefined set of default object access controls to this bucket. project: A valid API project identifier. projection: Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it defaults to full. userProject: The project to be billed for this request. c @ s$ e Zd ZdZdZdZdZdZdZdS )z8StorageBucketsInsertRequest.PredefinedAclValueValuesEnum� Apply a predefined set of access controls to this bucket. Values: authenticatedRead: Project team owners get OWNER access, and allAuthenticatedUsers get READER access. private: Project team owners get OWNER access. projectPrivate: Project team members get access according to their roles. publicRead: Project team owners get OWNER access, and allUsers get READER access. publicReadWrite: Project team owners get OWNER access, and allUsers get WRITER access. r r r r r N� r r r r �authenticatedRead�private�projectPrivate� publicReadZpublicReadWriter r r r � PredefinedAclValueValuesEnum� s r� c @ s( e Zd ZdZdZdZdZdZdZdZ dS ) zEStorageBucketsInsertRequest.PredefinedDefaultObjectAclValueValuesEnum� Apply a predefined set of default object access controls to this bucket. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. r r r r r r6 N� r r r r r� ZbucketOwnerFullControlZbucketOwnerReadr� r� r� r r r r �)PredefinedDefaultObjectAclValueValuesEnum� s r� c @ s e Zd ZdZdZdZdS )z5StorageBucketsInsertRequest.ProjectionValueValuesEnuma Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it defaults to full. Values: full: Include all properties. noAcl: Omit owner, acl and defaultObjectAcl properties. r r Nr� r r r r r� � s r� r r r r r Tr� r6 r2 N)r r r r r r� r� r� r� r& rw r� � predefinedAcl�predefinedDefaultObjectAclr �projectr� r� r r r r r� z s � r� c @ sn e Zd ZdZG dd� dej�Zejdejj dd�Z e� d�Z e� d�Z ej d d d �Ze�dd �Ze� d �ZdS )�StorageBucketsListRequesta� A StorageBucketsListRequest object. Enums: ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. Fields: maxResults: Maximum number of buckets to return in a single response. The service will use this parameter or 1,000 items, whichever is smaller. pageToken: A previously-returned page token representing part of the larger set of results to view. prefix: Filter results to buckets whose names begin with this prefix. project: A valid API project identifier. projection: Set of properties to return. Defaults to noAcl. userProject: The project to be billed for this request. c @ s e Zd ZdZdZdZdS )z3StorageBucketsListRequest.ProjectionValueValuesEnumr� r r Nr� r r r r r� � s r� r �� �r rK r r r Tr� r6 r2 N)r r r r r r� r� r r �UINT32� maxResultsr � pageToken�prefixr� r� r� r� r r r r r� � s � r� c @ s( e Zd ZdZejddd�Ze�d�ZdS )�!StorageBucketsListChannelsRequestz�A StorageBucketsListChannelsRequest object. Fields: bucket: Name of a bucket. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r Nr� r r r r r� � s r� c @ s6 e Zd ZdZejddd�Zejddd�Ze�d�Z dS )�(StorageBucketsLockRetentionPolicyRequestaJ A StorageBucketsLockRetentionPolicyRequest object. Fields: bucket: Name of a bucket. ifMetagenerationMatch: Makes the operation conditional on whether bucket's current metageneration matches the given value. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r N) r r r r r r rw r r� r� r r r r r� s r� c @ s� e Zd ZdZG dd� dej�ZG dd� dej�ZG dd� dej�Zej dd d �Z e� d d �Z e� d �Ze� d�Ze�dd�Ze�dd�Ze�dd�Ze� d�ZdS )�StorageBucketsPatchRequesta| A StorageBucketsPatchRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this bucket. PredefinedDefaultObjectAclValueValuesEnum: Apply a predefined set of default object access controls to this bucket. ProjectionValueValuesEnum: Set of properties to return. Defaults to full. Fields: bucket: Name of a bucket. bucketResource: A Bucket resource to be passed as the request body. ifMetagenerationMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration does not match the given value. predefinedAcl: Apply a predefined set of access controls to this bucket. predefinedDefaultObjectAcl: Apply a predefined set of default object access controls to this bucket. projection: Set of properties to return. Defaults to full. userProject: The project to be billed for this request. Required for Requester Pays buckets. c @ s$ e Zd ZdZdZdZdZdZdZdS )z7StorageBucketsPatchRequest.PredefinedAclValueValuesEnumr� r r r r r Nr� r r r r r� / s r� c @ s( e Zd ZdZdZdZdZdZdZdZ dS ) zDStorageBucketsPatchRequest.PredefinedDefaultObjectAclValueValuesEnumr� r r r r r r6 Nr� r r r r r� C s r� c @ s e Zd ZdZdZdZdS )z4StorageBucketsPatchRequest.ProjectionValueValuesEnum��Set of properties to return. Defaults to full. Values: full: Include all properties. noAcl: Omit owner, acl and defaultObjectAcl properties. r r Nr� r r r r r� [ s r� r Tr� r r r r r6 r2 r3 r4 N�r r r r r r� r� r� r� r rw r& ZbucketResourcer r� r� r� r� r� r� r� r r r r r� s � r� c @ s4 e Zd ZdZejddd�Ze�dd�Ze�d�Z dS ) �!StorageBucketsSetIamPolicyRequestz�A StorageBucketsSetIamPolicyRequest object. Fields: bucket: Name of a bucket. policy: A Policy resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r� r r N) r r r r r r rw r& �policyr� r r r r r� p s r� c @ s6 e Zd ZdZejddd�Zejddd�Ze�d�ZdS )�'StorageBucketsTestIamPermissionsRequestz�A StorageBucketsTestIamPermissionsRequest object. Fields: bucket: Name of a bucket. permissions: Permissions to test. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r N) r r r r r r rw � permissionsr� r r r r r� s r� c @ s� e Zd ZdZG dd� dej�ZG dd� dej�ZG dd� dej�Zej dd d �Z e� d d �Z e� d �Ze� d�Ze�dd�Ze�dd�Ze�dd�Ze� d�ZdS )�StorageBucketsUpdateRequesta} A StorageBucketsUpdateRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this bucket. PredefinedDefaultObjectAclValueValuesEnum: Apply a predefined set of default object access controls to this bucket. ProjectionValueValuesEnum: Set of properties to return. Defaults to full. Fields: bucket: Name of a bucket. bucketResource: A Bucket resource to be passed as the request body. ifMetagenerationMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration does not match the given value. predefinedAcl: Apply a predefined set of access controls to this bucket. predefinedDefaultObjectAcl: Apply a predefined set of default object access controls to this bucket. projection: Set of properties to return. Defaults to full. userProject: The project to be billed for this request. Required for Requester Pays buckets. c @ s$ e Zd ZdZdZdZdZdZdZdS )z8StorageBucketsUpdateRequest.PredefinedAclValueValuesEnumr� r r r r r Nr� r r r r r� � s r� c @ s( e Zd ZdZdZdZdZdZdZdZ dS ) zEStorageBucketsUpdateRequest.PredefinedDefaultObjectAclValueValuesEnumr� r r r r r r6 Nr� r r r r r� � s r� c @ s e Zd ZdZdZdZdS )z5StorageBucketsUpdateRequest.ProjectionValueValuesEnumr� r r Nr� r r r r r� � s r� r Tr� r r r r r6 r2 r3 r4 Nr� r r r r r� � s � r� c @ s e Zd ZdZdS )�StorageChannelsStopResponsez&An empty StorageChannelsStop response.Nr� r r r r r� � s r� c @ s6 e Zd ZdZejddd�Zejddd�Ze�d�ZdS )�/StorageDefaultObjectAccessControlsDeleteRequestav A StorageDefaultObjectAccessControlsDeleteRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r Nr� r r r r r� � s r� c @ s e Zd ZdZdS )�0StorageDefaultObjectAccessControlsDeleteResponsez;An empty StorageDefaultObjectAccessControlsDelete response.Nr� r r r r r� � s r� c @ s6 e Zd ZdZejddd�Zejddd�Ze�d�ZdS )�,StorageDefaultObjectAccessControlsGetRequestas A StorageDefaultObjectAccessControlsGetRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r Nr� r r r r r� s r� c @ s4 e Zd ZdZejddd�Ze�dd�Ze�d�Z dS ) �/StorageDefaultObjectAccessControlsInsertRequesta, A StorageDefaultObjectAccessControlsInsertRequest object. Fields: bucket: Name of a bucket. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� rF r r N) r r r r r r rw r& �objectAccessControlr� r r r r r� s r� c @ s< e Zd ZdZejddd�Ze�d�Ze�d�Z e�d�Z dS ) �-StorageDefaultObjectAccessControlsListRequesta� A StorageDefaultObjectAccessControlsListRequest object. Fields: bucket: Name of a bucket. ifMetagenerationMatch: If present, only return default ACL listing if the bucket's current metageneration matches this value. ifMetagenerationNotMatch: If present, only return default ACL listing if the bucket's current metageneration does not match the given value. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r Nr� r r r r r� # s r� c @ sB e Zd ZdZejddd�Zejddd�Ze�dd�Z e�d�Z d S ) �.StorageDefaultObjectAccessControlsPatchRequesta� A StorageDefaultObjectAccessControlsPatchRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r rF r r N� r r r r r r rw r? r& r� r� r r r r r� 6 s r� c @ sB e Zd ZdZejddd�Zejddd�Ze�dd�Z e�d�Z d S ) �/StorageDefaultObjectAccessControlsUpdateRequesta� A StorageDefaultObjectAccessControlsUpdateRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r rF r r Nr� r r r r r J s r c @ s6 e Zd ZdZejddd�Zejddd�Ze�d�ZdS )�!StorageNotificationsDeleteRequesta A StorageNotificationsDeleteRequest object. Fields: bucket: The parent bucket of the notification. notification: ID of the notification to delete. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r N� r r r r r r rw � notificationr� r r r r r ^ s r c @ s e Zd ZdZdS )�"StorageNotificationsDeleteResponsez-An empty StorageNotificationsDelete response.Nr� r r r r r m s r c @ s6 e Zd ZdZejddd�Zejddd�Ze�d�ZdS )� StorageNotificationsGetRequestz�A StorageNotificationsGetRequest object. Fields: bucket: The parent bucket of the notification. notification: Notification ID userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r Nr r r r r r q s r c @ s4 e Zd ZdZejddd�Ze�dd�Ze�d�Z dS ) �!StorageNotificationsInsertRequesta A StorageNotificationsInsertRequest object. Fields: bucket: The parent bucket of the notification. notification: A Notification resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r� r r N) r r r r r r rw r& r r� r r r r r � s r c @ s( e Zd ZdZejddd�Ze�d�ZdS )� StorageNotificationsListRequestz�A StorageNotificationsListRequest object. Fields: bucket: Name of a Google Cloud Storage bucket. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r Nr� r r r r r � s r c @ sN e Zd ZdZejddd�Zejddd�Ze�d�Z ejddd�Z e�d�Z d S ) �(StorageObjectAccessControlsDeleteRequestav A StorageObjectAccessControlsDeleteRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r r6 N� r r r r r r rw r? r r� r� r� r r r r r � s r c @ s e Zd ZdZdS )�)StorageObjectAccessControlsDeleteResponsez4An empty StorageObjectAccessControlsDelete response.Nr� r r r r r � s r c @ sN e Zd ZdZejddd�Zejddd�Ze�d�Z ejddd�Z e�d�Z d S ) �%StorageObjectAccessControlsGetRequestas A StorageObjectAccessControlsGetRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r r6 Nr r r r r r � s r c @ sL e Zd ZdZejddd�Ze�d�Zejddd�Z e� dd�Z e�d �Z d S ) �(StorageObjectAccessControlsInsertRequesta, A StorageObjectAccessControlsInsertRequest object. Fields: bucket: Name of a bucket. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r rF r r6 N) r r r r r r rw r r� r� r& r� r� r r r r r � s r c @ s@ e Zd ZdZejddd�Ze�d�Zejddd�Z e�d�Z dS ) �&StorageObjectAccessControlsListRequesta� A StorageObjectAccessControlsListRequest object. Fields: bucket: Name of a bucket. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r N� r r r r r r rw r r� r� r� r r r r r � s r c @ sZ e Zd ZdZejddd�Zejddd�Ze�d�Z ejddd�Z e� dd �Z e�d �Z d S ) �'StorageObjectAccessControlsPatchRequesta� A StorageObjectAccessControlsPatchRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r rF r6 r2 N�r r r r r r rw r? r r� r� r& r� r� r r r r r � s r c @ sZ e Zd ZdZejddd�Zejddd�Ze�d�Z ejddd�Z e� dd �Z e�d �Z d S ) �(StorageObjectAccessControlsUpdateRequesta� A StorageObjectAccessControlsUpdateRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r rF r6 r2 Nr r r r r r s r c @ s~ e Zd ZdZG dd� dej�Ze�dd�Zej ddd�Z ej d dd�Z e� dd �Z e�d �Ze�d �Ze� d �Ze� d�ZdS )� StorageObjectsComposeRequesta^ A StorageObjectsComposeRequest object. Enums: DestinationPredefinedAclValueValuesEnum: Apply a predefined set of access controls to the destination object. Fields: composeRequest: A ComposeRequest resource to be passed as the request body. destinationBucket: Name of the bucket containing the source objects. The destination object is stored in this bucket. destinationObject: Name of the new object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. destinationPredefinedAcl: Apply a predefined set of access controls to the destination object. ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. kmsKeyName: Resource name of the Cloud KMS key, of the form projects/my- project/locations/global/keyRings/my-kr/cryptoKeys/my-key, that will be used to encrypt the object. Overrides the object metadata's kms_key_name value, if any. userProject: The project to be billed for this request. Required for Requester Pays buckets. c @ s( e Zd ZdZdZdZdZdZdZdZ dS ) zDStorageObjectsComposeRequest.DestinationPredefinedAclValueValuesEnum� Apply a predefined set of access controls to the destination object. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. r r r r r r6 Nr� r r r r �'DestinationPredefinedAclValueValuesEnumH s r r� r r Tr� r r r6 r2 r3 r4 N)r r r r r r� r r& ZcomposeRequestr �destinationBucket�destinationObjectr� �destinationPredefinedAclr r� r� r� r� r r r r r + s � r c @ s� e Zd ZdZG dd� dej�ZG dd� dej�Zejddd�Z ejd dd�Z e� dd �Z e� d �Ze� d �Ze� d �Ze� d�Ze� d�Ze� d�Ze� d�Ze� d�Ze�dd�Ze� dd�Zejddd�Ze� d�Zejddd�Ze�d�Z dS )�StorageObjectsCopyRequesta� A StorageObjectsCopyRequest object. Enums: DestinationPredefinedAclValueValuesEnum: Apply a predefined set of access controls to the destination object. ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. Fields: destinationBucket: Name of the bucket in which to store the new object. Overrides the provided object metadata's bucket value, if any.For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. destinationObject: Name of the new object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. destinationPredefinedAcl: Apply a predefined set of access controls to the destination object. ifGenerationMatch: Makes the operation conditional on whether the destination object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the destination object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the destination object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the destination object's current metageneration does not match the given value. ifSourceGenerationMatch: Makes the operation conditional on whether the source object's current generation matches the given value. ifSourceGenerationNotMatch: Makes the operation conditional on whether the source object's current generation does not match the given value. ifSourceMetagenerationMatch: Makes the operation conditional on whether the source object's current metageneration matches the given value. ifSourceMetagenerationNotMatch: Makes the operation conditional on whether the source object's current metageneration does not match the given value. object: A Object resource to be passed as the request body. projection: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. sourceBucket: Name of the bucket in which to find the source object. sourceGeneration: If present, selects a specific revision of the source object (as opposed to the latest version, the default). sourceObject: Name of the source object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. c @ s( e Zd ZdZdZdZdZdZdZdZ dS ) zAStorageObjectsCopyRequest.DestinationPredefinedAclValueValuesEnumr r r r r r r6 Nr� r r r r r � s r c @ s e Zd ZdZdZdZdS )z3StorageObjectsCopyRequest.ProjectionValueValuesEnum��Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. Values: full: Include all properties. noAcl: Omit the owner, acl property. r r Nr� r r r r r� � s r� r Tr� r r r r6 r2 r3 r4 r5 rG rH r� rI rL rM rN rO rP N) r r r r r r� r r� r r r r� r r r� �ifGenerationNotMatchr� r� �ifSourceGenerationMatch�ifSourceGenerationNotMatch�ifSourceMetagenerationMatch� ifSourceMetagenerationNotMatchr& r� r� � sourceBucket�sourceGeneration� sourceObjectr� r r r r r j s, 5 � r c @ sh e Zd ZdZejddd�Ze�d�Ze�d�Z e�d�Z e�d�Z e�d �Z ejd dd�Z e�d �Zd S ) �StorageObjectsDeleteRequesta A StorageObjectsDeleteRequest object. Fields: bucket: Name of the bucket in which the object resides. generation: If present, permanently deletes a specific revision of this object (as opposed to the latest version, the default). ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r r6 r2 r3 r4 N)r r r r r r rw r r� r� r r� r� r� r� r r r r r" � s r" c @ s e Zd ZdZdS )� StorageObjectsDeleteResponsez'An empty StorageObjectsDelete response.Nr� r r r r r# � s r# c @ s@ e Zd ZdZejddd�Ze�d�Zejddd�Z e�d�Z dS ) �!StorageObjectsGetIamPolicyRequesta� A StorageObjectsGetIamPolicyRequest object. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r Nr r r r r r$ � s r$ c @ s� e Zd ZdZG dd� dej�Zejddd�Ze� d�Z e� d�Z e� d �Z e� d �Z e� d �Zejd dd�Ze�dd �Ze�d�ZdS )�StorageObjectsGetRequesta� A StorageObjectsGetRequest object. Enums: ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. projection: Set of properties to return. Defaults to noAcl. userProject: The project to be billed for this request. Required for Requester Pays buckets. c @ s e Zd ZdZdZdZdS )z2StorageObjectsGetRequest.ProjectionValueValuesEnum��Set of properties to return. Defaults to noAcl. Values: full: Include all properties. noAcl: Omit the owner, acl property. r r Nr� r r r r r� + s r� r Tr� r r r r6 r2 r3 r4 r5 N)r r r r r r� r� r rw r r� r� r r� r� r� r� r� r� r r r r r% s r% c @ s� e Zd ZdZG dd� dej�ZG dd� dej�Zejddd�Z e�d �Z e� d �Z e� d �Z e� d �Ze� d �Ze�d�Ze�d�Ze�dd�Ze�dd�Ze�dd�Ze�d�ZdS )�StorageObjectsInsertRequesta� A StorageObjectsInsertRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this object. ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. Fields: bucket: Name of the bucket in which to store the new object. Overrides the provided object metadata's bucket value, if any. contentEncoding: If set, sets the contentEncoding property of the final object to this value. Setting this parameter is equivalent to setting the contentEncoding metadata property. This can be useful when uploading an object with uploadType=media to indicate the encoding of the content being uploaded. ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. kmsKeyName: Resource name of the Cloud KMS key, of the form projects/my- project/locations/global/keyRings/my-kr/cryptoKeys/my-key, that will be used to encrypt the object. Overrides the object metadata's kms_key_name value, if any. name: Name of the object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. object: A Object resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. userProject: The project to be billed for this request. Required for Requester Pays buckets. c @ s( e Zd ZdZdZdZdZdZdZdZ dS ) z8StorageObjectsInsertRequest.PredefinedAclValueValuesEnum� Apply a predefined set of access controls to this object. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. r r r r r r6 Nr� r r r r r� m s r� c @ s e Zd ZdZdZdZdS )z5StorageObjectsInsertRequest.ProjectionValueValuesEnumr r r Nr� r r r r r� � s r� r Tr� r r r r6 r2 r3 r4 r� r5 rG rH rI N)r r r r r r� r� r� r rw r� r r� r r� r� r� rk r& r� r� r� r� r� r r r r r' @ s , r' c @ s� e Zd ZdZG dd� dej�Zejddd�Ze�d�Z e� d�Z ej d ej jd d �Ze�d �Ze�d �Ze�dd�Ze�d�Ze� d�ZdS )�StorageObjectsListRequesta� A StorageObjectsListRequest object. Enums: ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. Fields: bucket: Name of the bucket in which to look for objects. delimiter: Returns results in a directory-like mode. items will contain only objects whose names, aside from the prefix, do not contain delimiter. Objects whose names, aside from the prefix, contain delimiter will have their name, truncated after the delimiter, returned in prefixes. Duplicate prefixes are omitted. includeTrailingDelimiter: If true, objects that end in exactly one instance of delimiter will have their metadata included in items in addition to prefixes. maxResults: Maximum number of items plus prefixes to return in a single page of responses. As duplicate prefixes are omitted, fewer total results may be returned than requested. The service will use this parameter or 1,000 items, whichever is smaller. pageToken: A previously-returned page token representing part of the larger set of results to view. prefix: Filter results to objects whose names begin with this prefix. projection: Set of properties to return. Defaults to noAcl. userProject: The project to be billed for this request. Required for Requester Pays buckets. versions: If true, lists all versions of an object as distinct results. The default is false. For more information, see Object Versioning. c @ s e Zd ZdZdZdZdS )z3StorageObjectsListRequest.ProjectionValueValuesEnumr& r r Nr� r r r r r� � s r� r Tr� r r r r� r� r6 r2 r3 r4 r5 N)r r r r r r� r� r rw � delimiterr �includeTrailingDelimiterr r r� r� r� r� r� r� r� �versionsr r r r r) � s � r) c @ s� e Zd ZdZG dd� dej�ZG dd� dej�Zejddd�Z e� d �Z e� d �Z e� d �Z e� d �Ze� d �Zejddd�Ze�dd�Ze�dd�Ze�dd�Ze�d�ZdS )�StorageObjectsPatchRequesta� A StorageObjectsPatchRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this object. ProjectionValueValuesEnum: Set of properties to return. Defaults to full. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. objectResource: A Object resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to full. userProject: The project to be billed for this request, for Requester Pays buckets. c @ s( e Zd ZdZdZdZdZdZdZdZ dS ) z7StorageObjectsPatchRequest.PredefinedAclValueValuesEnumr( r r r r r r6 Nr� r r r r r� � s r� c @ s e Zd ZdZdZdZdS )z4StorageObjectsPatchRequest.ProjectionValueValuesEnum��Set of properties to return. Defaults to full. Values: full: Include all properties. noAcl: Omit the owner, acl property. r r Nr� r r r r r� s r� r Tr� r r r r6 r2 r3 r� r4 r5 rG rH N�r r r r r r� r� r� r rw r r� r� r r� r� r� r& ZobjectResourcer� r� r� r� r r r r r- � s r- c @ s e Zd ZdZG dd� dej�ZG dd� dej�Zejddd�Z e�d �Z ejd dd�Z e� dd �Z e�d �Ze�d �Ze�d�Ze�d�Ze�d�Ze�d�Ze�d�Ze�d�Ze�d�Ze�dd�Ze� dd�Ze�d�Zejddd�Z e�d�Z ejddd�Z e�d �Z d S ) � StorageObjectsRewriteRequesta; A StorageObjectsRewriteRequest object. Enums: DestinationPredefinedAclValueValuesEnum: Apply a predefined set of access controls to the destination object. ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. Fields: destinationBucket: Name of the bucket in which to store the new object. Overrides the provided object metadata's bucket value, if any. destinationKmsKeyName: Resource name of the Cloud KMS key, of the form projects/my-project/locations/global/keyRings/my-kr/cryptoKeys/my-key, that will be used to encrypt the object. Overrides the object metadata's kms_key_name value, if any. destinationObject: Name of the new object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. destinationPredefinedAcl: Apply a predefined set of access controls to the destination object. ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the destination object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the destination object's current metageneration does not match the given value. ifSourceGenerationMatch: Makes the operation conditional on whether the source object's current generation matches the given value. ifSourceGenerationNotMatch: Makes the operation conditional on whether the source object's current generation does not match the given value. ifSourceMetagenerationMatch: Makes the operation conditional on whether the source object's current metageneration matches the given value. ifSourceMetagenerationNotMatch: Makes the operation conditional on whether the source object's current metageneration does not match the given value. maxBytesRewrittenPerCall: The maximum number of bytes that will be rewritten per rewrite request. Most callers shouldn't need to specify this parameter - it is primarily in place to support testing. If specified the value must be an integral multiple of 1 MiB (1048576). Also, this only applies to requests where the source and destination span locations and/or storage classes. Finally, this value must not change across rewrite calls else you'll get an error that the rewriteToken is invalid. object: A Object resource to be passed as the request body. projection: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. rewriteToken: Include this field (from the previous rewrite response) on each rewrite request after the first one, until the rewrite response 'done' flag is true. Calls that provide a rewriteToken can omit all other request fields, but if included those fields must match the values provided in the first rewrite request. sourceBucket: Name of the bucket in which to find the source object. sourceGeneration: If present, selects a specific revision of the source object (as opposed to the latest version, the default). sourceObject: Name of the source object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. c @ s( e Zd ZdZdZdZdZdZdZdZ dS ) zDStorageObjectsRewriteRequest.DestinationPredefinedAclValueValuesEnumr r r r r r r6 Nr� r r r r r e s r c @ s e Zd ZdZdZdZdS )z6StorageObjectsRewriteRequest.ProjectionValueValuesEnumr r r Nr� r r r r r� | s r� r Tr� r r r r6 r2 r3 r4 r5 rG rH rI rL r� rM rN rO rP rQ rR rS N) r r r r r r� r r� r r ZdestinationKmsKeyNamer r� r r r� r r� r� r r r r ZmaxBytesRewrittenPerCallr& r� r� r� r r r! r� r r r r r0 s2 D � r0 c @ sL e Zd ZdZejddd�Ze�d�Zejddd�Z e� dd�Z e�d �Z d S ) �!StorageObjectsSetIamPolicyRequesta# A StorageObjectsSetIamPolicyRequest object. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. policy: A Policy resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r� r r6 N) r r r r r r rw r r� r� r& r� r� r r r r r1 � s r1 c @ sN e Zd ZdZejddd�Ze�d�Zejddd�Z ejddd�Z e�d�Z d S ) �'StorageObjectsTestIamPermissionsRequesta A StorageObjectsTestIamPermissionsRequest object. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. permissions: Permissions to test. userProject: The project to be billed for this request. Required for Requester Pays buckets. r Tr� r r r r6 N) r r r r r r rw r r� r� r� r� r r r r r2 � s r2 c @ s� e Zd ZdZG dd� dej�ZG dd� dej�Zejddd�Z e� d �Z e� d �Z e� d �Z e� d �Ze� d �Zejddd�Ze�dd�Ze�dd�Ze�dd�Ze�d�ZdS )�StorageObjectsUpdateRequesta� A StorageObjectsUpdateRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this object. ProjectionValueValuesEnum: Set of properties to return. Defaults to full. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. objectResource: A Object resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to full. userProject: The project to be billed for this request. Required for Requester Pays buckets. c @ s( e Zd ZdZdZdZdZdZdZdZ dS ) z8StorageObjectsUpdateRequest.PredefinedAclValueValuesEnumr( r r r r r r6 Nr� r r r r r� � s r� c @ s e Zd ZdZdZdZdS )z5StorageObjectsUpdateRequest.ProjectionValueValuesEnumr. r r Nr� r r r r r� � s r� r Tr� r r r r6 r2 r3 r� r4 r5 rG rH Nr/ r r r r r3 � s r3 c @ s� e Zd ZdZG dd� dej�Zejddd�Ze� dd�Z e�d �Z e� d �Z ejd ejjd d �Ze�d�Ze�d�Ze�dd�Ze�d�Ze� d�ZdS )� StorageObjectsWatchAllRequesta A StorageObjectsWatchAllRequest object. Enums: ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. Fields: bucket: Name of the bucket in which to look for objects. channel: A Channel resource to be passed as the request body. delimiter: Returns results in a directory-like mode. items will contain only objects whose names, aside from the prefix, do not contain delimiter. Objects whose names, aside from the prefix, contain delimiter will have their name, truncated after the delimiter, returned in prefixes. Duplicate prefixes are omitted. includeTrailingDelimiter: If true, objects that end in exactly one instance of delimiter will have their metadata included in items in addition to prefixes. maxResults: Maximum number of items plus prefixes to return in a single page of responses. As duplicate prefixes are omitted, fewer total results may be returned than requested. The service will use this parameter or 1,000 items, whichever is smaller. pageToken: A previously-returned page token representing part of the larger set of results to view. prefix: Filter results to objects whose names begin with this prefix. projection: Set of properties to return. Defaults to noAcl. userProject: The project to be billed for this request. Required for Requester Pays buckets. versions: If true, lists all versions of an object as distinct results. The default is false. For more information, see Object Versioning. c @ s e Zd ZdZdZdZdS )z7StorageObjectsWatchAllRequest.ProjectionValueValuesEnumr& r r Nr� r r r r r� 5 s r� r Tr� r� r r r r6 r� r� r2 r3 r4 r5 rG N)r r r r r r� r� r rw r& Zchannelr* r r+ r r r� r� r� r� r� r� r� r, r r r r r4 s � r4 c @ s( e Zd ZdZejddd�Ze�d�ZdS )�'StorageProjectsServiceAccountGetRequestz�A StorageProjectsServiceAccountGetRequest object. Fields: projectId: Project ID userProject: The project to be billed for this request. r Tr� r N)r r r r r r � projectIdr� r r r r r5 M s r5 c @ s, e Zd ZdZejddd�Zejddd�ZdS ) �TestIamPermissionsResponseaI A storage.(buckets|objects).testIamPermissions response. Fields: kind: The kind of item this is. permissions: The permissions held by the caller. Permissions are always of the format storage.resource.capability, where resource is one of buckets or objects. The supported permissions are as follows: - storage.buckets.delete - Delete bucket. - storage.buckets.get - Read bucket metadata. - storage.buckets.getIamPolicy - Read bucket IAM policy. - storage.buckets.create - Create bucket. - storage.buckets.list - List buckets. - storage.buckets.setIamPolicy - Update bucket IAM policy. - storage.buckets.update - Update bucket metadata. - storage.objects.delete - Delete object. - storage.objects.get - Read object data and metadata. - storage.objects.getIamPolicy - Read object IAM policy. - storage.objects.create - Create object. - storage.objects.list - List objects. - storage.objects.setIamPolicy - Update object IAM policy. - storage.objects.update - Update object metadata. r z"storage#testIamPermissionsResponserJ r Tr N)r r r r r r re r� r r r r r7 Y s r7 c @ s, e Zd ZdZejddd�Zejddd�ZdS )�$StorageProjectsHmacKeysCreateRequestz�A StorageProjectsHmacKeysCreateRequest object. Fields: projectId: Project ID serviceAccountEmail: Email address of the service account for which to create a key. r Tr� r N)r r r r r r r6 �serviceAccountEmailr r r r r8 r s r8 c @ s, e Zd ZdZejddd�Zejddd�ZdS )�$StorageProjectsHmacKeysDeleteRequestz�A StorageProjectsHmacKeysDeleteRequest object. Fields: projectId: Project ID accessId: Name of the HMAC key to be deleted. r Tr� r N�r r r r r r r6 �accessIdr r r r r: s r: c @ s, e Zd ZdZejddd�Zejddd�ZdS )�!StorageProjectsHmacKeysGetRequestz�A StorageProjectsHmacKeysGetRequest object. Fields: projectId: Project ID accessId: Name of the HMAC key for which the metadata is being requested. r Tr� r Nr; r r r r r= � s r= c @ sP e Zd ZdZejddd�Ze�d�Ze�d�Z ej dej j dd �Z e�d �Zd S ) �"StorageProjectsHmacKeysListRequesta A StorageProjectsHmacKeysListRequest object. Fields: projectId: Project ID serviceAccountEmail: If present only, keys for the given service account will be returned. showDeletedKeys: Whether or not to show keys in the DELETED state. maxResults: Maximum number of items to return in a single page of responses. The service will use this parameter or 1,000 items, whichever is smaller. pageToken: A previously-returned page token representing part of the larger set of results to view. r Tr� r r r r� r� r6 N)r r r r r r r6 r9 r ZshowDeletedKeysr r r� r� r� r r r r r> � s �r> c @ s8 e Zd ZdZejddd�Zejddd�Ze�dd�Z dS ) �$StorageProjectsHmacKeysUpdateRequestz�A StorageProjectsHmacKeysUpdateRequest object. Fields: projectId: Project ID accessId: Name of the HMAC key being updated. r Tr� r �HmacKeyMetadatar N) r r r r r r r6 r< r& r� r r r r r? � s r? c @ sx e Zd ZdZe�d�Ze�d�Ze�d�Ze�d�Z e�d�Z e�d�Z e � d�Ze � d �Ze�d �Zejd d d �ZdS )r@ a� An HMAC Key Metadata resource. Fields: id: The ID of the HMAC key, including the Project ID and the Access ID. selfLink: The link to this resource. accessId: The ID of the HMAC Key. projectId: Project ID owning the service account to which the key authenticates. serviceAccountEmail: The email address of the key's associated service account. state: The state of the key. Can be one of ACTIVE, INACTIVE, or DELETED. timeCreated: The creation time of the key in RFC 3339 format. updated: The modification time of the key in RFC 3339 format. etag: HTTP 1.1 Entity tag for the key. kind: The kind of item this is. For HMAC Key metadata, this is always storage#hmacKeyMetadata r r r r r6 r2 r3 r4 r5 rG zstorage#hmacKeyMetadatarJ N)r r r r r r rd ro r< r6 r9 �stater r rp rq rc re r r r r r@ � s r@ c @ s4 e Zd ZdZe�dd�Ze�d�Zejddd�Z dS ) �HmacKeyz�An HMAC Key resource. Fields: metadata: Key metadata. secret: HMAC secret key material. kind: The kind of item this is. For HMAC keys, this is always storage#hmacKey. r@ r r r zstorage#hmacKeyrJ N) r r r r r r& r� r �secretre r r r r rB � s rB c @ s8 e Zd ZdZejdddd�Zejddd�Ze�d �Z d S ) �HmacKeyMetadataListam A list of HMAC key metadata. Fields: items: The list of HMAC key metadata. kind: The kind of item this is. For a list of HMAC key, this is always storage#hmacKeysMetadata. nextPageToken: The continuation token, used to page through large result sets. Provide this value in a subsequent request to return the next page of results. r@ r Tr r zstorage#hmacKeysMetadatarJ r Nr� r r r r rD � s rD c @ s e Zd ZdZdS )�HmacKeysDeleteResponsez!An empty HmacKeysDelete response.Nr� r r r r rE � s rE N)]r Zapitools.base.protorpcliter r r r �apitools.base.pyr r �packager% r rE r| r r� r� r� r� r� r� r� rF r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r r r r r r r r r r r r" r# r$ r% r' r) r- r0 r1 r2 r3 r4 r5 r7 r8 r: r= r> r? r@ rB rD rE r r r r �<module> s� 03 ?"6B ,7 T ' #Y% [[ ?l"1]5N~N7 ! third_party/storage_apitools/__pycache__/__init__.cpython-39.pyc 0000644 00000000225 15025013616 0021051 0 ustar 00 a +(Wg � @ s d S )N� r r r �:/opt/gsutil/gslib/third_party/storage_apitools/__init__.py�<module> � third_party/storage_apitools/__pycache__/storage_v1_client.cpython-39.pyc 0000644 00000136547 15025013616 0022743 0 ustar 00 a +(Wg�� � @ sV d Z ddlZddlZddlmZ ddlZddlmZ ddlm Z G dd� dej �Z dS )z0Generated client library for storage version v1.� N)�base_api)�MetricsCollector)�storage_v1_messagesc s0 e Zd ZdZeZdZdZg d�ZdZ dZ dZ de � � Zeej7 Zd ZdZdZd dd ddd d dde ddf � fd d � ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�Z G dd� dej�Z G d d � d ej�Z G d d � d ej�Z G d d!� d!ej�Z � Z!S )"� StorageV1z8Generated client library for service storage version v1.z&https://www.googleapis.com/storage/v1/�storage)z.https://www.googleapis.com/auth/cloud-platformz8https://www.googleapis.com/auth/cloud-platform.read-onlyz7https://www.googleapis.com/auth/devstorage.full_controlz4https://www.googleapis.com/auth/devstorage.read_onlyz5https://www.googleapis.com/auth/devstorage.read_write�v1�nomatterzapitools Python/%sN� TFc s� |p| j }tt| �j||||||||| | | d� | | _| �| �| _| �| �| _| � | �| _ | � | �| _ | � | �| _| �| �| _| �| �| _| �| �| _| �| �| _| �| �| _dS )z Create a new storage handle.) � credentials�get_credentials�http�model� log_request� log_response�credentials_args�default_global_params�additional_http_headers�response_encodingN)�BASE_URL�superr �__init__�_version�BucketAccessControlsService�bucketAccessControls�BucketsService�buckets�ChannelsService�channels�"DefaultObjectAccessControlsService�defaultObjectAccessControls�NotificationsService� notifications�ObjectAccessControlsService�objectAccessControls�ObjectsService�objects� ProjectsServiceAccountService�projects_serviceAccount�ProjectsService�projects�HmacKeysService�hmacKeys) �self�urlr r r r r r r r �versionr r �� __class__� �C/opt/gsutil/gslib/third_party/storage_apitools/storage_v1_client.pyr , s* � zStorageV1.__init__c s� e Zd ZdZdZ� fdd�Zddd�Zdd � e_dd d �Zd d � e_dd d�Z dd � e _d dd�Z dd � e _d dd�Z dd � e _d dd�Z dd � e _� Z S ) z%StorageV1.BucketAccessControlsServicez4Service class for the bucketAccessControls resource.r c s t tj| ��|� i | _d S �N)r r r r �_upload_configs�r, �clientr/ r1 r2 r M s z.StorageV1.BucketAccessControlsService.__init__Nc C s | � d�}| j|||d�S )a[ Permanently deletes the ACL entry for the specified entity on the specified bucket. Args: request: (StorageBucketAccessControlsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageBucketAccessControlsDeleteResponse) The response message. �Delete�� global_params��GetMethodConfig� _RunMethod�r, �requestr9 �configr1 r1 r2 r7 R s �z,StorageV1.BucketAccessControlsService.Deletec C s( t jddddgddgdgdddd d d � S ) N�DELETEz#storage.bucketAccessControls.delete�bucket�entity� userProject�b/{bucket}/acl/{entity}r �(StorageBucketAccessControlsDeleteRequest�)StorageBucketAccessControlsDeleteResponseF� � http_method� method_id�ordered_params� path_params� query_params� relative_path� request_field�request_type_name�response_type_name�supports_download�r � ApiMethodInfor1 r1 r1 r2 �<lambda>_ s �z.StorageV1.BucketAccessControlsService.<lambda>c C s | � d�}| j|||d�S )a6 Returns the ACL entry for the specified entity on the specified bucket. Args: request: (StorageBucketAccessControlsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (BucketAccessControl) The response message. �Getr8 r: r= r1 r1 r2 rU l s �z)StorageV1.BucketAccessControlsService.Getc C s( t jddddgddgdgdddd d d � S ) N�GETz storage.bucketAccessControls.getrA rB rC rD r �%StorageBucketAccessControlsGetRequest�BucketAccessControlFrG rR r1 r1 r1 r2 rT y s �c C s | � d�}| j|||d�S )a" Creates a new ACL entry on the specified bucket. Args: request: (StorageBucketAccessControlsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (BucketAccessControl) The response message. �Insertr8 r: r= r1 r1 r2 rY � s �z,StorageV1.BucketAccessControlsService.Insertc C s$ t jdddgdgdgddddd d � S ) N�POSTz#storage.bucketAccessControls.insertrA rC �b/{bucket}/acl�bucketAccessControl�(StorageBucketAccessControlsInsertRequestrX FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Retrieves ACL entries on the specified bucket. Args: request: (StorageBucketAccessControlsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (BucketAccessControls) The response message. �Listr8 r: r= r1 r1 r2 r^ � s �z*StorageV1.BucketAccessControlsService.Listc C s$ t jdddgdgdgddddd d � S ) NrV z!storage.bucketAccessControls.listrA rC r[ r �&StorageBucketAccessControlsListRequest�BucketAccessControlsFrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Patches an ACL entry on the specified bucket. Args: request: (StorageBucketAccessControlsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (BucketAccessControl) The response message. �Patchr8 r: r= r1 r1 r2 ra � s �z+StorageV1.BucketAccessControlsService.Patchc C s( t jddddgddgdgdddd d d � S ) N�PATCHz"storage.bucketAccessControls.patchrA rB rC rD r\ �'StorageBucketAccessControlsPatchRequestrX FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Updates an ACL entry on the specified bucket. Args: request: (StorageBucketAccessControlsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (BucketAccessControl) The response message. �Updater8 r: r= r1 r1 r2 rd � s �z,StorageV1.BucketAccessControlsService.Updatec C s( t jddddgddgdgdddd d d � S ) N�PUTz#storage.bucketAccessControls.updaterA rB rC rD r\ �(StorageBucketAccessControlsUpdateRequestrX FrG rR r1 r1 r1 r2 rT � s �)N)N)N)N)N)N��__name__� __module__� __qualname__�__doc__�_NAMEr r7 � method_configrU rY r^ ra rd � __classcell__r1 r1 r/ r2 r H s r c s e Zd ZdZdZ� fdd�Zd(dd�Zdd � e_d)d d �Zd d � e_d*d d�Z dd � e _d+dd�Z dd � e _d,dd�Z dd � e _d-dd�Z dd � e _d.dd�Z dd � e _d/d d �Zd d � e_d0d d �Zd!d � e_d1d"d#�Zd$d � e_d2d%d&�Zd'd � e_� ZS )3zStorageV1.BucketsServicez'Service class for the buckets resource.r c s t tj| ��|� i | _d S r3 )r r r r r4 r5 r/ r1 r2 r � s z!StorageV1.BucketsService.__init__Nc C s | � d�}| j|||d�S )a Permanently deletes an empty bucket. Args: request: (StorageBucketsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageBucketsDeleteResponse) The response message. r7 r8 r: r= r1 r1 r2 r7 � s �z StorageV1.BucketsService.Deletec C s& t jdddgdgg d�ddddd d � S ) Nr@ zstorage.buckets.deleterA ��ifMetagenerationMatch�ifMetagenerationNotMatchrC � b/{bucket}r �StorageBucketsDeleteRequest� StorageBucketsDeleteResponseFrG rR r1 r1 r1 r2 rT s �z!StorageV1.BucketsService.<lambda>c C s | � d�}| j|||d�S )z�Returns metadata for the specified bucket. Args: request: (StorageBucketsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Bucket) The response message. rU r8 r: r= r1 r1 r2 rU s �z StorageV1.BucketsService.Getc C s& t jdddgdgg d�ddddd d � S ) NrV zstorage.buckets.getrA )rp rq � projectionrC rr r �StorageBucketsGetRequest�BucketFrG rR r1 r1 r1 r2 rT s �c C s | � d�}| j|||d�S )a Returns an IAM policy for the specified bucket. Args: request: (StorageBucketsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. � GetIamPolicyr8 r: r= r1 r1 r2 rx , s �z%StorageV1.BucketsService.GetIamPolicyc C s& t jdddgdgg d�ddddd d � S ) NrV z storage.buckets.getIamPolicyrA )�provisionalUserProject� optionsRequestedPolicyVersionrC �b/{bucket}/iamr �!StorageBucketsGetIamPolicyRequest�PolicyFrG rR r1 r1 r1 r2 rT 9 s �c C s | � d�}| j|||d�S )z�Creates a new bucket. Args: request: (StorageBucketsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Bucket) The response message. rY r8 r: r= r1 r1 r2 rY F s �z StorageV1.BucketsService.Insertc C s$ t jdddgg g d�ddddd d � S ) NrZ zstorage.buckets.insert�project)� predefinedAcl�predefinedDefaultObjectAclr~ ru rC �brA �StorageBucketsInsertRequestrw FrG rR r1 r1 r1 r2 rT S s �c C s | � d�}| j|||d�S )a Retrieves a list of buckets for a given project. Args: request: (StorageBucketsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Buckets) The response message. r^ r8 r: r= r1 r1 r2 r^ ` s �z StorageV1.BucketsService.Listc C s$ t jdddgg g d�ddddd d � S ) NrV zstorage.buckets.listr~ )� maxResults� pageToken�prefixr~ ru rC r� r �StorageBucketsListRequest�BucketsFrG rR r1 r1 r1 r2 rT m s �c C s | � d�}| j|||d�S )a List active object change notification channels for this bucket. Args: request: (StorageBucketsListChannelsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Channels) The response message. � ListChannelsr8 r: r= r1 r1 r2 r� z s �z%StorageV1.BucketsService.ListChannelsc C s$ t jdddgdgdgddddd d � S ) NrV z storage.buckets.listChannelsrA rC zb/{bucket}/channelsr �!StorageBucketsListChannelsRequest�ChannelsFrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Locks retention policy on a bucket. Args: request: (StorageBucketsLockRetentionPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Bucket) The response message. �LockRetentionPolicyr8 r: r= r1 r1 r2 r� � s �z,StorageV1.BucketsService.LockRetentionPolicyc C s( t jddddgdgddgdddd d d � S ) NrZ z#storage.buckets.lockRetentionPolicyrA rp rC z b/{bucket}/lockRetentionPolicyr �(StorageBucketsLockRetentionPolicyRequestrw FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a` Patches a bucket. Changes to the bucket will be readable immediately after writing, but configuration changes may take time to propagate. Args: request: (StorageBucketsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Bucket) The response message. ra r8 r: r= r1 r1 r2 ra � s �z StorageV1.BucketsService.Patchc C s& t jdddgdgg d�ddddd d � S ) Nrb zstorage.buckets.patchrA �rp rq r r� ru rC rr �bucketResource�StorageBucketsPatchRequestrw FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Updates an IAM policy for the specified bucket. Args: request: (StorageBucketsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. � SetIamPolicyr8 r: r= r1 r1 r2 r� � s �z%StorageV1.BucketsService.SetIamPolicyc C s$ t jdddgdgdgddddd d � S ) Nre z storage.buckets.setIamPolicyrA rC r{ �policy�!StorageBucketsSetIamPolicyRequestr} FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )aT Tests a set of permissions on the given bucket to see which, if any, are held by the caller. Args: request: (StorageBucketsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. �TestIamPermissionsr8 r: r= r1 r1 r2 r� � s �z+StorageV1.BucketsService.TestIamPermissionsc C s( t jddddgdgddgdddd d d � S ) NrV z"storage.buckets.testIamPermissionsrA � permissionsrC z b/{bucket}/iam/testPermissionsr �'StorageBucketsTestIamPermissionsRequest�TestIamPermissionsResponseFrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )aa Updates a bucket. Changes to the bucket will be readable immediately after writing, but configuration changes may take time to propagate. Args: request: (StorageBucketsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Bucket) The response message. rd r8 r: r= r1 r1 r2 rd � s �z StorageV1.BucketsService.Updatec C s& t jdddgdgg d�ddddd d � S ) Nre zstorage.buckets.updaterA r� rr r� �StorageBucketsUpdateRequestrw FrG rR r1 r1 r1 r2 rT s �)N)N)N)N)N)N)N)N)N)N)N)rh ri rj rk rl r r7 rm rU rx rY r^ r� r� ra r� r� rd rn r1 r1 r/ r2 r � s2 r c s8 e Zd ZdZdZ� fdd�Zd dd�Zdd � e_� ZS ) zStorageV1.ChannelsServicez(Service class for the channels resource.r c s t tj| ��|� i | _d S r3 )r r r r r4 r5 r/ r1 r2 r s z"StorageV1.ChannelsService.__init__Nc C s | � d�}| j|||d�S )a Stop watching resources through this channel. Args: request: (Channel) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageChannelsStopResponse) The response message. �Stopr8 r: r= r1 r1 r2 r� s �z StorageV1.ChannelsService.Stopc C s t jddg g g dddddd� S ) NrZ zstorage.channels.stopz channels/stopz <request>�Channel�StorageChannelsStopResponseFrG rR r1 r1 r1 r2 rT - s �z"StorageV1.ChannelsService.<lambda>)N) rh ri rj rk rl r r� rm rn r1 r1 r/ r2 r s r c s� e Zd ZdZdZ� fdd�Zddd�Zdd � e_dd d �Zd d � e_dd d�Z dd � e _d dd�Z dd � e _d dd�Z dd � e _d dd�Z dd � e _� Z S ) z,StorageV1.DefaultObjectAccessControlsServicez;Service class for the defaultObjectAccessControls resource.r c s t tj| ��|� i | _d S r3 )r r r r r4 r5 r/ r1 r2 r ? s z5StorageV1.DefaultObjectAccessControlsService.__init__Nc C s | � d�}| j|||d�S )ax Permanently deletes the default object ACL entry for the specified entity on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageDefaultObjectAccessControlsDeleteResponse) The response message. r7 r8 r: r= r1 r1 r2 r7 D s �z3StorageV1.DefaultObjectAccessControlsService.Deletec C s( t jddddgddgdgdddd d d � S ) Nr@ z*storage.defaultObjectAccessControls.deleterA rB rC �$b/{bucket}/defaultObjectAcl/{entity}r �/StorageDefaultObjectAccessControlsDeleteRequest�0StorageDefaultObjectAccessControlsDeleteResponseFrG rR r1 r1 r1 r2 rT Q s �z5StorageV1.DefaultObjectAccessControlsService.<lambda>c C s | � d�}| j|||d�S )aL Returns the default object ACL entry for the specified entity on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. rU r8 r: r= r1 r1 r2 rU ^ s �z0StorageV1.DefaultObjectAccessControlsService.Getc C s( t jddddgddgdgdddd d d � S ) NrV z'storage.defaultObjectAccessControls.getrA rB rC r� r �,StorageDefaultObjectAccessControlsGetRequest�ObjectAccessControlFrG rR r1 r1 r1 r2 rT k s �c C s | � d�}| j|||d�S )a8 Creates a new default object ACL entry on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. rY r8 r: r= r1 r1 r2 rY x s �z3StorageV1.DefaultObjectAccessControlsService.Insertc C s$ t jdddgdgdgddddd d � S ) NrZ z*storage.defaultObjectAccessControls.insertrA rC �b/{bucket}/defaultObjectAcl�objectAccessControl�/StorageDefaultObjectAccessControlsInsertRequestr� FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a5 Retrieves default object ACL entries on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControls) The response message. r^ r8 r: r= r1 r1 r2 r^ � s �z1StorageV1.DefaultObjectAccessControlsService.Listc C s& t jdddgdgg d�ddddd d � S ) NrV z(storage.defaultObjectAccessControls.listrA ro r� r �-StorageDefaultObjectAccessControlsListRequest�ObjectAccessControlsFrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a3 Patches a default object ACL entry on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. ra r8 r: r= r1 r1 r2 ra � s �z2StorageV1.DefaultObjectAccessControlsService.Patchc C s( t jddddgddgdgdddd d d � S ) Nrb z)storage.defaultObjectAccessControls.patchrA rB rC r� r� �.StorageDefaultObjectAccessControlsPatchRequestr� FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a4 Updates a default object ACL entry on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. rd r8 r: r= r1 r1 r2 rd � s �z3StorageV1.DefaultObjectAccessControlsService.Updatec C s( t jddddgddgdgdddd d d � S ) Nre z*storage.defaultObjectAccessControls.updaterA rB rC r� r� �/StorageDefaultObjectAccessControlsUpdateRequestr� FrG rR r1 r1 r1 r2 rT � s �)N)N)N)N)N)Nrg r1 r1 r/ r2 r : s r c st e Zd ZdZdZ� fdd�Zddd�Zdd � e_dd d �Zd d � e_dd d�Z dd � e _ddd�Z dd � e _� Z S )z StorageV1.NotificationsServicez-Service class for the notifications resource.r! c s t tj| ��|� i | _d S r3 )r r r r r4 r5 r/ r1 r2 r � s z'StorageV1.NotificationsService.__init__Nc C s | � d�}| j|||d�S )a* Permanently deletes a notification subscription. Args: request: (StorageNotificationsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageNotificationsDeleteResponse) The response message. r7 r8 r: r= r1 r1 r2 r7 � s �z%StorageV1.NotificationsService.Deletec C s( t jddddgddgdgdddd d d � S ) Nr@ z storage.notifications.deleterA � notificationrC �-b/{bucket}/notificationConfigs/{notification}r �!StorageNotificationsDeleteRequest�"StorageNotificationsDeleteResponseFrG rR r1 r1 r1 r2 rT � s �z'StorageV1.NotificationsService.<lambda>c C s | � d�}| j|||d�S )a View a notification configuration. Args: request: (StorageNotificationsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Notification) The response message. rU r8 r: r= r1 r1 r2 rU s �z"StorageV1.NotificationsService.Getc C s( t jddddgddgdgdddd d d � S ) NrV zstorage.notifications.getrA r� rC r� r � StorageNotificationsGetRequest� NotificationFrG rR r1 r1 r1 r2 rT s �c C s | � d�}| j|||d�S )a Creates a notification subscription for a given bucket. Args: request: (StorageNotificationsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Notification) The response message. rY r8 r: r= r1 r1 r2 rY s �z%StorageV1.NotificationsService.Insertc C s$ t jdddgdgdgddddd d � S ) NrZ z storage.notifications.insertrA rC � b/{bucket}/notificationConfigsr� �!StorageNotificationsInsertRequestr� FrG rR r1 r1 r1 r2 rT + s �c C s | � d�}| j|||d�S )a% Retrieves a list of notification subscriptions for a given bucket. Args: request: (StorageNotificationsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Notifications) The response message. r^ r8 r: r= r1 r1 r2 r^ 8 s �z#StorageV1.NotificationsService.Listc C s$ t jdddgdgdgddddd d � S ) NrV zstorage.notifications.listrA rC r� r � StorageNotificationsListRequest� NotificationsFrG rR r1 r1 r1 r2 rT E s �)N)N)N)N) rh ri rj rk rl r r7 rm rU rY r^ rn r1 r1 r/ r2 r � s r c s� e Zd ZdZdZ� fdd�Zddd�Zdd � e_dd d �Zd d � e_dd d�Z dd � e _d dd�Z dd � e _d dd�Z dd � e _d dd�Z dd � e _� Z S ) z%StorageV1.ObjectAccessControlsServicez4Service class for the objectAccessControls resource.r# c s t tj| ��|� i | _d S r3 )r r r" r r4 r5 r/ r1 r2 r W s z.StorageV1.ObjectAccessControlsService.__init__Nc C s | � d�}| j|||d�S )a[ Permanently deletes the ACL entry for the specified entity on the specified object. Args: request: (StorageObjectAccessControlsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageObjectAccessControlsDeleteResponse) The response message. r7 r8 r: r= r1 r1 r2 r7 \ s �z,StorageV1.ObjectAccessControlsService.Deletec C s* t jddg d�g d�ddgddd d d d � S ) Nr@ z#storage.objectAccessControls.delete�rA �objectrB �rA rB r� � generationrC �"b/{bucket}/o/{object}/acl/{entity}r �(StorageObjectAccessControlsDeleteRequest�)StorageObjectAccessControlsDeleteResponseFrG rR r1 r1 r1 r2 rT i s �z.StorageV1.ObjectAccessControlsService.<lambda>c C s | � d�}| j|||d�S )a6 Returns the ACL entry for the specified entity on the specified object. Args: request: (StorageObjectAccessControlsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. rU r8 r: r= r1 r1 r2 rU v s �z)StorageV1.ObjectAccessControlsService.Getc C s* t jddg d�g d�ddgddd d d d � S ) NrV z storage.objectAccessControls.getr� r� r� rC r� r �%StorageObjectAccessControlsGetRequestr� FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a" Creates a new ACL entry on the specified object. Args: request: (StorageObjectAccessControlsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. rY r8 r: r= r1 r1 r2 rY � s �z,StorageV1.ObjectAccessControlsService.Insertc C s* t jddddgddgddgddd d d d � S ) NrZ z#storage.objectAccessControls.insertrA r� r� rC �b/{bucket}/o/{object}/aclr� �(StorageObjectAccessControlsInsertRequestr� FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Retrieves ACL entries on the specified object. Args: request: (StorageObjectAccessControlsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControls) The response message. r^ r8 r: r= r1 r1 r2 r^ � s �z*StorageV1.ObjectAccessControlsService.Listc C s* t jddddgddgddgddd d d d � S ) NrV z!storage.objectAccessControls.listrA r� r� rC r� r �&StorageObjectAccessControlsListRequestr� FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Patches an ACL entry on the specified object. Args: request: (StorageObjectAccessControlsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. ra r8 r: r= r1 r1 r2 ra � s �z+StorageV1.ObjectAccessControlsService.Patchc C s* t jddg d�g d�ddgddd d d d � S ) Nrb z"storage.objectAccessControls.patchr� r� r� rC r� r� �'StorageObjectAccessControlsPatchRequestr� FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Updates an ACL entry on the specified object. Args: request: (StorageObjectAccessControlsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. rd r8 r: r= r1 r1 r2 rd � s �z,StorageV1.ObjectAccessControlsService.Updatec C s* t jddg d�g d�ddgddd d d d � S ) Nre z#storage.objectAccessControls.updater� r� r� rC r� r� �(StorageObjectAccessControlsUpdateRequestr� FrG rR r1 r1 r1 r2 rT � s �)N)N)N)N)N)Nrg r1 r1 r/ r2 r" R s r" c s( e Zd ZdZdZ� fdd�Zd.dd�Zdd � e_d/d d �Zd d � e_d0d d�Z dd � e _d1dd�Z dd � e _d2dd�Z dd � e _d3dd�Z dd � e _d4dd�Z dd � e _d5d d �Zd d � e_d6d d �Zd!d � e_d7d"d#�Zd$d � e_d8d%d&�Zd'd � e_d9d(d)�Zd*d � e_d:d+d,�Zd-d � e_� ZS );zStorageV1.ObjectsServicez'Service class for the objects resource.r% c sL t tj| ��|� dtjdgd dd| jj d dd| jj d d�i| _d S )NrY z*/*Tz/resumable/upload/storage/z /b/{bucket}/oz/upload/storage/)�accept�max_size�resumable_multipart�resumable_path�simple_multipart� simple_path) r r r$ r r � ApiUploadInfo�_clientr r4 r5 r/ r1 r2 r � s ��z!StorageV1.ObjectsService.__init__Nc C s | � d�}| j|||d�S )a& Concatenates a list of existing objects into a new object in the same bucket. Args: request: (StorageObjectsComposeRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Object) The response message. �Composer8 r: r= r1 r1 r2 r� s �z StorageV1.ObjectsService.Composec C s* t jddddgddgg d�dddd d d � S ) NrZ zstorage.objects.compose�destinationBucket�destinationObject)�destinationPredefinedAcl�ifGenerationMatchrp � kmsKeyNamerC z3b/{destinationBucket}/o/{destinationObject}/compose�composeRequest� StorageObjectsComposeRequest�ObjectFrG rR r1 r1 r1 r2 rT s �z!StorageV1.ObjectsService.<lambda>c C s | � d�}| j|||d�S )a$ Copies a source object to a destination object. Optionally overrides metadata. Args: request: (StorageObjectsCopyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Object) The response message. �Copyr8 r: r= r1 r1 r2 r� $ s �z StorageV1.ObjectsService.Copyc C s* t jddg d�g d�g d�dddd d d � S ) NrZ zstorage.objects.copy�� sourceBucket� sourceObjectr� r� �r� r� r� r� ) r� r� �ifGenerationNotMatchrp rq �ifSourceGenerationMatch�ifSourceGenerationNotMatch�ifSourceMetagenerationMatch� ifSourceMetagenerationNotMatchru �sourceGenerationrC zTb/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}r� �StorageObjectsCopyRequestr� FrG rR r1 r1 r1 r2 rT 1 s �c C s | � d�}| j|||d�S )a~ Deletes an object and its metadata. Deletions are permanent if versioning is not enabled for the bucket, or if the generation parameter is used. Args: request: (StorageObjectsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageObjectsDeleteResponse) The response message. r7 r8 r: r= r1 r1 r2 r7 > s �z StorageV1.ObjectsService.Deletec C s* t jddddgddgg d�dddd d d � S ) Nr@ zstorage.objects.deleterA r� )r� r� r� rp rq rC �b/{bucket}/o/{object}r �StorageObjectsDeleteRequest� StorageObjectsDeleteResponseFrG rR r1 r1 r1 r2 rT K s �c C s | � d�}| j||||d�S )am Retrieves an object or its metadata. Args: request: (StorageObjectsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments download: (Download, default: None) If present, download data from the request via this stream. Returns: (Object) The response message. rU )r9 �downloadr: )r, r> r9 r� r? r1 r1 r2 rU X s �z StorageV1.ObjectsService.Getc C s* t jddddgddgg d�dddd d d � S ) NrV zstorage.objects.getrA r� )r� r� r� rp rq ru rC r� r �StorageObjectsGetRequestr� TrG rR r1 r1 r1 r2 rT h s �c C s | � d�}| j|||d�S )a Returns an IAM policy for the specified object. Args: request: (StorageObjectsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. rx r8 r: r= r1 r1 r2 rx u s �z%StorageV1.ObjectsService.GetIamPolicyc C s* t jddddgddgddgddd d d d � S ) NrV z storage.objects.getIamPolicyrA r� r� rC �b/{bucket}/o/{object}/iamr �!StorageObjectsGetIamPolicyRequestr} FrG rR r1 r1 r1 r2 rT � s �c C s( | � d�}| �d�}| j|||||d�S )a^ Stores a new object and metadata. Args: request: (StorageObjectsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments upload: (Upload, default: None) If present, upload this stream with the request. Returns: (Object) The response message. rY )r9 �upload� upload_config)r; �GetUploadConfigr< )r, r> r9 r� r? r� r1 r1 r2 rY � s �z StorageV1.ObjectsService.Insertc C s& t jdddgdgg d�ddddd d � S ) NrZ zstorage.objects.insertrA ) �contentEncodingr� r� rp rq r� �namer ru rC � b/{bucket}/or� �StorageObjectsInsertRequestr� FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Retrieves a list of objects matching the criteria. Args: request: (StorageObjectsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Objects) The response message. r^ r8 r: r= r1 r1 r2 r^ � s �z StorageV1.ObjectsService.Listc C s& t jdddgdgg d�ddddd d � S ) NrV zstorage.objects.listrA �� delimiter�includeTrailingDelimiterr� r� r� ru rC �versionsr� r �StorageObjectsListRequest�ObjectsFrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )z�Patches an object's metadata. Args: request: (StorageObjectsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Object) The response message. ra r8 r: r= r1 r1 r2 ra � s �z StorageV1.ObjectsService.Patchc C s* t jddddgddgg d�dddd d d � S ) Nrb zstorage.objects.patchrA r� �r� r� r� rp rq r ru rC r� �objectResource�StorageObjectsPatchRequestr� FrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a2 Rewrites a source object to a destination object. Optionally overrides metadata. Args: request: (StorageObjectsRewriteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (RewriteResponse) The response message. �Rewriter8 r: r= r1 r1 r2 r� � s �z StorageV1.ObjectsService.Rewritec C s* t jddg d�g d�g d�dddd d d � S ) NrZ zstorage.objects.rewriter� r� )�destinationKmsKeyNamer� r� r� rp rq r� r� r� r� �maxBytesRewrittenPerCallru � rewriteTokenr� rC zWb/{sourceBucket}/o/{sourceObject}/rewriteTo/b/{destinationBucket}/o/{destinationObject}r� � StorageObjectsRewriteRequest�RewriteResponseFrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Updates an IAM policy for the specified object. Args: request: (StorageObjectsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. r� r8 r: r= r1 r1 r2 r� � s �z%StorageV1.ObjectsService.SetIamPolicyc C s* t jddddgddgddgddd d d d � S ) Nre z storage.objects.setIamPolicyrA r� r� rC r� r� �!StorageObjectsSetIamPolicyRequestr} FrG rR r1 r1 r1 r2 rT s �c C s | � d�}| j|||d�S )aT Tests a set of permissions on the given object to see which, if any, are held by the caller. Args: request: (StorageObjectsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. r� r8 r: r= r1 r1 r2 r� s �z+StorageV1.ObjectsService.TestIamPermissionsc C s* t jddg d�ddgg d�ddd d d d � S ) NrV z"storage.objects.testIamPermissions)rA r� r� rA r� )r� r� rC z)b/{bucket}/o/{object}/iam/testPermissionsr �'StorageObjectsTestIamPermissionsRequestr� FrG rR r1 r1 r1 r2 rT " s �c C s | � d�}| j|||d�S )z�Updates an object's metadata. Args: request: (StorageObjectsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Object) The response message. rd r8 r: r= r1 r1 r2 rd / s �z StorageV1.ObjectsService.Updatec C s* t jddddgddgg d�dddd d d � S ) Nre zstorage.objects.updaterA r� r� r� r� �StorageObjectsUpdateRequestr� FrG rR r1 r1 r1 r2 rT < s �c C s | � d�}| j|||d�S )a Watch for changes on all objects in a bucket. Args: request: (StorageObjectsWatchAllRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Channel) The response message. �WatchAllr8 r: r= r1 r1 r2 r� I s �z!StorageV1.ObjectsService.WatchAllc C s& t jdddgdgg d�ddddd d � S ) NrZ zstorage.objects.watchAllrA r� zb/{bucket}/o/watch�channel� StorageObjectsWatchAllRequestr� FrG rR r1 r1 r1 r2 rT V s �)N)N)N)NN)N)NN)N)N)N)N)N)N)N)rh ri rj rk rl r r� rm r� r7 rU rx rY r^ ra r� r� r� rd r� rn r1 r1 r/ r2 r$ � s: r$ c s8 e Zd ZdZdZ� fdd�Zd dd�Zdd � e_� ZS ) z'StorageV1.ProjectsServiceAccountServicez7Service class for the projects_serviceAccount resource.r' c s t tj| ��|� i | _d S r3 )r r r& r r4 r5 r/ r1 r2 r h s z0StorageV1.ProjectsServiceAccountService.__init__Nc C s | � d�}| j|||d�S )a9 Get the email address of this project's Google Cloud Storage service account. Args: request: (StorageProjectsServiceAccountGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ServiceAccount) The response message. rU r8 r: r= r1 r1 r2 rU m s �z+StorageV1.ProjectsServiceAccountService.Getc C s$ t jdddgdgdgddddd d � S ) NrV z#storage.projects.serviceAccount.get� projectIdrC z#projects/{projectId}/serviceAccountr �'StorageProjectsServiceAccountGetRequest�ServiceAccountFrG rR r1 r1 r1 r2 rT z s �z0StorageV1.ProjectsServiceAccountService.<lambda>)N) rh ri rj rk rl r rU rm rn r1 r1 r/ r2 r& c s r& c s$ e Zd ZdZdZ� fdd�Z� ZS )zStorageV1.ProjectsServicez(Service class for the projects resource.r) c s t tj| ��|� i | _d S r3 )r r r( r r4 r5 r/ r1 r2 r � s z"StorageV1.ProjectsService.__init__)rh ri rj rk rl r rn r1 r1 r/ r2 r( � s r( c @ sx e Zd ZdZdZddd�Zdd� e_ddd �Zd d� e_dd d �Zd d� e_ddd�Z dd� e _ddd�Z dd� e _dS )zStorageV1.HmacKeysServicez0Service class for the project_hmacKeys resource.Zprojects_hmacKeysNc C s | � d�}| j|||d�S )a Creates HMAC key for a service account. Args: request: (StorageProjectsHmacKeysCreateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (HmacKey) The response message. �Creater8 r: r= r1 r1 r2 r � s z StorageV1.HmacKeysService.Createc C s& t jddddgdgdgddddd d � S ) NrZ z storage.projects.hmacKeys.create�serviceAccountEmailr� � projects/{projectId}/hmacKeysr �$StorageProjectsHmacKeysCreateRequest�HmacKeyFrG rR r1 r1 r1 r2 rT � s �z"StorageV1.HmacKeysService.<lambda>c C s | � d�}| j|||d�S )a Deletes an HMAC key. Args: request: (StorageProjectsHmacKeysDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (HmacKeyDeleteResponse) The empty response message. r7 r8 r: r= r1 r1 r2 r7 � s z StorageV1.HmacKeysService.Deletec C s& t jddddgddgg ddddd d � S ) Nr@ z storage.projects.hmacKeys.delete�accessIdr� �(projects/{projectId}/hmacKeys/{accessId}r �$StorageProjectsHmacKeysDeleteRequest�HmacKeysDeleteResponseFrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Retrieves an HMAC key's metadata Args: request: (StorageProjectsHmacKeysGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (HmacKeyMetadata) The response message. rU r8 r: r= r1 r1 r2 rU � s z StorageV1.HmacKeysService.Getc C s$ t jddddgddgddddd d � S ) NrV � storage.projects.hmacKeys.getr r� r r �!StorageProjectsHmacKeysGetRequest�HmacKeyMetadataF) rH rI rJ rK rM rN rO rP rQ rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a! Retrieves a list of HMAC keys matching the criteria. Args: request: (StorageProjectsHmacKeysListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (HmacKeyMetadataList) The response message. r^ r8 r: r= r1 r1 r2 r^ � s z StorageV1.HmacKeysService.Listc C s( t jddg d�dgg d�dddd d d � S ) NrV r )r� r �showDeletedKeysr� r� r� )r r r� r� r r �"StorageProjectsHmacKeysListRequest�HmacKeyMetadataListFrG rR r1 r1 r1 r2 rT � s �c C s | � d�}| j|||d�S )a Updates the state of an HMAC key. Args: request: (StorageProjectsHmacKeysUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (HmacKeyMetadata) The response message. rd r8 r: r= r1 r1 r2 rd s z StorageV1.HmacKeysService.Updatec C s& t jddddgddgg ddddd d � S ) Nre z storage.projects.hmacKeys.updater r� r �resource�$StorageProjectsHmacKeysUpdateRequestr FrG rR r1 r1 r1 r2 rT s �)N)N)N)N)N) rh ri rj rk rl r rm r7 rU r^ rd r1 r1 r1 r2 r* � s r* )"rh ri rj rk �messages�MESSAGES_MODULEr �_PACKAGE�_SCOPES�_VERSION� _CLIENT_ID�_CLIENT_SECRET�platform�python_version� _USER_AGENT�gslib� USER_AGENT�_CLIENT_CLASS_NAME� _URL_VERSION�_API_KEYr r �BaseApiServicer r r r r r" r$ r& r( r* rn r1 r1 r/ r2 r sJ � ' *$ 'r ' m$ r ) rk r �sys�apitools.base.pyr r Z gslib.metricsr �"gslib.third_party.storage_apitoolsr r � BaseApiClientr r1 r1 r1 r2 �<module> s third_party/storage_apitools/storage_v1_client.py 0000644 00000171366 15025013616 0016452 0 ustar 00 # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Generated client library for storage version v1.""" import platform import sys from apitools.base.py import base_api import gslib from gslib.metrics import MetricsCollector from gslib.third_party.storage_apitools import storage_v1_messages as messages class StorageV1(base_api.BaseApiClient): """Generated client library for service storage version v1.""" MESSAGES_MODULE = messages BASE_URL = u'https://www.googleapis.com/storage/v1/' _PACKAGE = u'storage' _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/cloud-platform.read-only', u'https://www.googleapis.com/auth/devstorage.full_control', u'https://www.googleapis.com/auth/devstorage.read_only', u'https://www.googleapis.com/auth/devstorage.read_write'] _VERSION = u'v1' _CLIENT_ID = 'nomatter' _CLIENT_SECRET = 'nomatter' _USER_AGENT = 'apitools Python/%s' % platform.python_version() _USER_AGENT += gslib.USER_AGENT _CLIENT_CLASS_NAME = u'StorageV1' _URL_VERSION = u'v1' _API_KEY = None def __init__(self, url='', credentials=None, get_credentials=True, http=None, model=None, log_request=False, log_response=False, credentials_args=None, default_global_params=None, version=_VERSION, additional_http_headers=None, response_encoding=None): """Create a new storage handle.""" url = url or self.BASE_URL super(StorageV1, self).__init__( url, credentials=credentials, get_credentials=get_credentials, http=http, model=model, log_request=log_request, log_response=log_response, credentials_args=credentials_args, default_global_params=default_global_params, additional_http_headers=additional_http_headers, response_encoding=response_encoding) self._version = version self.bucketAccessControls = self.BucketAccessControlsService(self) self.buckets = self.BucketsService(self) self.channels = self.ChannelsService(self) self.defaultObjectAccessControls = self.DefaultObjectAccessControlsService(self) self.notifications = self.NotificationsService(self) self.objectAccessControls = self.ObjectAccessControlsService(self) self.objects = self.ObjectsService(self) self.projects_serviceAccount = self.ProjectsServiceAccountService(self) self.projects = self.ProjectsService(self) self.hmacKeys = self.HmacKeysService(self) class BucketAccessControlsService(base_api.BaseApiService): """Service class for the bucketAccessControls resource.""" _NAME = u'bucketAccessControls' def __init__(self, client): super(StorageV1.BucketAccessControlsService, self).__init__(client) self._upload_configs = { } def Delete(self, request, global_params=None): r"""Permanently deletes the ACL entry for the specified entity on the specified bucket. Args: request: (StorageBucketAccessControlsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageBucketAccessControlsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) Delete.method_config = lambda: base_api.ApiMethodInfo( http_method=u'DELETE', method_id=u'storage.bucketAccessControls.delete', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[u'userProject'], relative_path=u'b/{bucket}/acl/{entity}', request_field='', request_type_name=u'StorageBucketAccessControlsDeleteRequest', response_type_name=u'StorageBucketAccessControlsDeleteResponse', supports_download=False, ) def Get(self, request, global_params=None): r"""Returns the ACL entry for the specified entity on the specified bucket. Args: request: (StorageBucketAccessControlsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.bucketAccessControls.get', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[u'userProject'], relative_path=u'b/{bucket}/acl/{entity}', request_field='', request_type_name=u'StorageBucketAccessControlsGetRequest', response_type_name=u'BucketAccessControl', supports_download=False, ) def Insert(self, request, global_params=None): r"""Creates a new ACL entry on the specified bucket. Args: request: (StorageBucketAccessControlsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Insert') return self._RunMethod( config, request, global_params=global_params) Insert.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.bucketAccessControls.insert', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'userProject'], relative_path=u'b/{bucket}/acl', request_field=u'bucketAccessControl', request_type_name=u'StorageBucketAccessControlsInsertRequest', response_type_name=u'BucketAccessControl', supports_download=False, ) def List(self, request, global_params=None): r"""Retrieves ACL entries on the specified bucket. Args: request: (StorageBucketAccessControlsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (BucketAccessControls) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.bucketAccessControls.list', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'userProject'], relative_path=u'b/{bucket}/acl', request_field='', request_type_name=u'StorageBucketAccessControlsListRequest', response_type_name=u'BucketAccessControls', supports_download=False, ) def Patch(self, request, global_params=None): r"""Patches an ACL entry on the specified bucket. Args: request: (StorageBucketAccessControlsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) Patch.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PATCH', method_id=u'storage.bucketAccessControls.patch', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[u'userProject'], relative_path=u'b/{bucket}/acl/{entity}', request_field=u'bucketAccessControl', request_type_name=u'StorageBucketAccessControlsPatchRequest', response_type_name=u'BucketAccessControl', supports_download=False, ) def Update(self, request, global_params=None): r"""Updates an ACL entry on the specified bucket. Args: request: (StorageBucketAccessControlsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (BucketAccessControl) The response message. """ config = self.GetMethodConfig('Update') return self._RunMethod( config, request, global_params=global_params) Update.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PUT', method_id=u'storage.bucketAccessControls.update', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[u'userProject'], relative_path=u'b/{bucket}/acl/{entity}', request_field=u'bucketAccessControl', request_type_name=u'StorageBucketAccessControlsUpdateRequest', response_type_name=u'BucketAccessControl', supports_download=False, ) class BucketsService(base_api.BaseApiService): """Service class for the buckets resource.""" _NAME = u'buckets' def __init__(self, client): super(StorageV1.BucketsService, self).__init__(client) self._upload_configs = { } def Delete(self, request, global_params=None): r"""Permanently deletes an empty bucket. Args: request: (StorageBucketsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageBucketsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) Delete.method_config = lambda: base_api.ApiMethodInfo( http_method=u'DELETE', method_id=u'storage.buckets.delete', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'userProject'], relative_path=u'b/{bucket}', request_field='', request_type_name=u'StorageBucketsDeleteRequest', response_type_name=u'StorageBucketsDeleteResponse', supports_download=False, ) def Get(self, request, global_params=None): r"""Returns metadata for the specified bucket. Args: request: (StorageBucketsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Bucket) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.buckets.get', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection', u'userProject'], relative_path=u'b/{bucket}', request_field='', request_type_name=u'StorageBucketsGetRequest', response_type_name=u'Bucket', supports_download=False, ) def GetIamPolicy(self, request, global_params=None): r"""Returns an IAM policy for the specified bucket. Args: request: (StorageBucketsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('GetIamPolicy') return self._RunMethod( config, request, global_params=global_params) GetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.buckets.getIamPolicy', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'provisionalUserProject', u'optionsRequestedPolicyVersion', u'userProject'], relative_path=u'b/{bucket}/iam', request_field='', request_type_name=u'StorageBucketsGetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def Insert(self, request, global_params=None): r"""Creates a new bucket. Args: request: (StorageBucketsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Bucket) The response message. """ config = self.GetMethodConfig('Insert') return self._RunMethod( config, request, global_params=global_params) Insert.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.buckets.insert', ordered_params=[u'project'], path_params=[], query_params=[u'predefinedAcl', u'predefinedDefaultObjectAcl', u'project', u'projection', u'userProject'], relative_path=u'b', request_field=u'bucket', request_type_name=u'StorageBucketsInsertRequest', response_type_name=u'Bucket', supports_download=False, ) def List(self, request, global_params=None): r"""Retrieves a list of buckets for a given project. Args: request: (StorageBucketsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Buckets) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.buckets.list', ordered_params=[u'project'], path_params=[], query_params=[u'maxResults', u'pageToken', u'prefix', u'project', u'projection', u'userProject'], relative_path=u'b', request_field='', request_type_name=u'StorageBucketsListRequest', response_type_name=u'Buckets', supports_download=False, ) def ListChannels(self, request, global_params=None): r"""List active object change notification channels for this bucket. Args: request: (StorageBucketsListChannelsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Channels) The response message. """ config = self.GetMethodConfig('ListChannels') return self._RunMethod( config, request, global_params=global_params) ListChannels.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.buckets.listChannels', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'userProject'], relative_path=u'b/{bucket}/channels', request_field='', request_type_name=u'StorageBucketsListChannelsRequest', response_type_name=u'Channels', supports_download=False, ) def LockRetentionPolicy(self, request, global_params=None): r"""Locks retention policy on a bucket. Args: request: (StorageBucketsLockRetentionPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Bucket) The response message. """ config = self.GetMethodConfig('LockRetentionPolicy') return self._RunMethod( config, request, global_params=global_params) LockRetentionPolicy.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.buckets.lockRetentionPolicy', ordered_params=[u'bucket', u'ifMetagenerationMatch'], path_params=[u'bucket'], query_params=[u'ifMetagenerationMatch', u'userProject'], relative_path=u'b/{bucket}/lockRetentionPolicy', request_field='', request_type_name=u'StorageBucketsLockRetentionPolicyRequest', response_type_name=u'Bucket', supports_download=False, ) def Patch(self, request, global_params=None): r"""Patches a bucket. Changes to the bucket will be readable immediately after writing, but configuration changes may take time to propagate. Args: request: (StorageBucketsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Bucket) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) Patch.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PATCH', method_id=u'storage.buckets.patch', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection', u'userProject'], relative_path=u'b/{bucket}', request_field=u'bucketResource', request_type_name=u'StorageBucketsPatchRequest', response_type_name=u'Bucket', supports_download=False, ) def SetIamPolicy(self, request, global_params=None): r"""Updates an IAM policy for the specified bucket. Args: request: (StorageBucketsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('SetIamPolicy') return self._RunMethod( config, request, global_params=global_params) SetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PUT', method_id=u'storage.buckets.setIamPolicy', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'userProject'], relative_path=u'b/{bucket}/iam', request_field=u'policy', request_type_name=u'StorageBucketsSetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def TestIamPermissions(self, request, global_params=None): r"""Tests a set of permissions on the given bucket to see which, if any, are held by the caller. Args: request: (StorageBucketsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. """ config = self.GetMethodConfig('TestIamPermissions') return self._RunMethod( config, request, global_params=global_params) TestIamPermissions.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.buckets.testIamPermissions', ordered_params=[u'bucket', u'permissions'], path_params=[u'bucket'], query_params=[u'permissions', u'userProject'], relative_path=u'b/{bucket}/iam/testPermissions', request_field='', request_type_name=u'StorageBucketsTestIamPermissionsRequest', response_type_name=u'TestIamPermissionsResponse', supports_download=False, ) def Update(self, request, global_params=None): r"""Updates a bucket. Changes to the bucket will be readable immediately after writing, but configuration changes may take time to propagate. Args: request: (StorageBucketsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Bucket) The response message. """ config = self.GetMethodConfig('Update') return self._RunMethod( config, request, global_params=global_params) Update.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PUT', method_id=u'storage.buckets.update', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'predefinedDefaultObjectAcl', u'projection', u'userProject'], relative_path=u'b/{bucket}', request_field=u'bucketResource', request_type_name=u'StorageBucketsUpdateRequest', response_type_name=u'Bucket', supports_download=False, ) class ChannelsService(base_api.BaseApiService): """Service class for the channels resource.""" _NAME = u'channels' def __init__(self, client): super(StorageV1.ChannelsService, self).__init__(client) self._upload_configs = { } def Stop(self, request, global_params=None): r"""Stop watching resources through this channel. Args: request: (Channel) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageChannelsStopResponse) The response message. """ config = self.GetMethodConfig('Stop') return self._RunMethod( config, request, global_params=global_params) Stop.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.channels.stop', ordered_params=[], path_params=[], query_params=[], relative_path=u'channels/stop', request_field='<request>', request_type_name=u'Channel', response_type_name=u'StorageChannelsStopResponse', supports_download=False, ) class DefaultObjectAccessControlsService(base_api.BaseApiService): """Service class for the defaultObjectAccessControls resource.""" _NAME = u'defaultObjectAccessControls' def __init__(self, client): super(StorageV1.DefaultObjectAccessControlsService, self).__init__(client) self._upload_configs = { } def Delete(self, request, global_params=None): r"""Permanently deletes the default object ACL entry for the specified entity on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageDefaultObjectAccessControlsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) Delete.method_config = lambda: base_api.ApiMethodInfo( http_method=u'DELETE', method_id=u'storage.defaultObjectAccessControls.delete', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[u'userProject'], relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', request_field='', request_type_name=u'StorageDefaultObjectAccessControlsDeleteRequest', response_type_name=u'StorageDefaultObjectAccessControlsDeleteResponse', supports_download=False, ) def Get(self, request, global_params=None): r"""Returns the default object ACL entry for the specified entity on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.defaultObjectAccessControls.get', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[u'userProject'], relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', request_field='', request_type_name=u'StorageDefaultObjectAccessControlsGetRequest', response_type_name=u'ObjectAccessControl', supports_download=False, ) def Insert(self, request, global_params=None): r"""Creates a new default object ACL entry on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Insert') return self._RunMethod( config, request, global_params=global_params) Insert.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.defaultObjectAccessControls.insert', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'userProject'], relative_path=u'b/{bucket}/defaultObjectAcl', request_field=u'objectAccessControl', request_type_name=u'StorageDefaultObjectAccessControlsInsertRequest', response_type_name=u'ObjectAccessControl', supports_download=False, ) def List(self, request, global_params=None): r"""Retrieves default object ACL entries on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControls) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.defaultObjectAccessControls.list', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'userProject'], relative_path=u'b/{bucket}/defaultObjectAcl', request_field='', request_type_name=u'StorageDefaultObjectAccessControlsListRequest', response_type_name=u'ObjectAccessControls', supports_download=False, ) def Patch(self, request, global_params=None): r"""Patches a default object ACL entry on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) Patch.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PATCH', method_id=u'storage.defaultObjectAccessControls.patch', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[u'userProject'], relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', request_field=u'objectAccessControl', request_type_name=u'StorageDefaultObjectAccessControlsPatchRequest', response_type_name=u'ObjectAccessControl', supports_download=False, ) def Update(self, request, global_params=None): r"""Updates a default object ACL entry on the specified bucket. Args: request: (StorageDefaultObjectAccessControlsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Update') return self._RunMethod( config, request, global_params=global_params) Update.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PUT', method_id=u'storage.defaultObjectAccessControls.update', ordered_params=[u'bucket', u'entity'], path_params=[u'bucket', u'entity'], query_params=[u'userProject'], relative_path=u'b/{bucket}/defaultObjectAcl/{entity}', request_field=u'objectAccessControl', request_type_name=u'StorageDefaultObjectAccessControlsUpdateRequest', response_type_name=u'ObjectAccessControl', supports_download=False, ) class NotificationsService(base_api.BaseApiService): """Service class for the notifications resource.""" _NAME = u'notifications' def __init__(self, client): super(StorageV1.NotificationsService, self).__init__(client) self._upload_configs = { } def Delete(self, request, global_params=None): r"""Permanently deletes a notification subscription. Args: request: (StorageNotificationsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageNotificationsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) Delete.method_config = lambda: base_api.ApiMethodInfo( http_method=u'DELETE', method_id=u'storage.notifications.delete', ordered_params=[u'bucket', u'notification'], path_params=[u'bucket', u'notification'], query_params=[u'userProject'], relative_path=u'b/{bucket}/notificationConfigs/{notification}', request_field='', request_type_name=u'StorageNotificationsDeleteRequest', response_type_name=u'StorageNotificationsDeleteResponse', supports_download=False, ) def Get(self, request, global_params=None): r"""View a notification configuration. Args: request: (StorageNotificationsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Notification) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.notifications.get', ordered_params=[u'bucket', u'notification'], path_params=[u'bucket', u'notification'], query_params=[u'userProject'], relative_path=u'b/{bucket}/notificationConfigs/{notification}', request_field='', request_type_name=u'StorageNotificationsGetRequest', response_type_name=u'Notification', supports_download=False, ) def Insert(self, request, global_params=None): r"""Creates a notification subscription for a given bucket. Args: request: (StorageNotificationsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Notification) The response message. """ config = self.GetMethodConfig('Insert') return self._RunMethod( config, request, global_params=global_params) Insert.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.notifications.insert', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'userProject'], relative_path=u'b/{bucket}/notificationConfigs', request_field=u'notification', request_type_name=u'StorageNotificationsInsertRequest', response_type_name=u'Notification', supports_download=False, ) def List(self, request, global_params=None): r"""Retrieves a list of notification subscriptions for a given bucket. Args: request: (StorageNotificationsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Notifications) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.notifications.list', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'userProject'], relative_path=u'b/{bucket}/notificationConfigs', request_field='', request_type_name=u'StorageNotificationsListRequest', response_type_name=u'Notifications', supports_download=False, ) class ObjectAccessControlsService(base_api.BaseApiService): """Service class for the objectAccessControls resource.""" _NAME = u'objectAccessControls' def __init__(self, client): super(StorageV1.ObjectAccessControlsService, self).__init__(client) self._upload_configs = { } def Delete(self, request, global_params=None): r"""Permanently deletes the ACL entry for the specified entity on the specified object. Args: request: (StorageObjectAccessControlsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageObjectAccessControlsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) Delete.method_config = lambda: base_api.ApiMethodInfo( http_method=u'DELETE', method_id=u'storage.objectAccessControls.delete', ordered_params=[u'bucket', u'object', u'entity'], path_params=[u'bucket', u'entity', u'object'], query_params=[u'generation', u'userProject'], relative_path=u'b/{bucket}/o/{object}/acl/{entity}', request_field='', request_type_name=u'StorageObjectAccessControlsDeleteRequest', response_type_name=u'StorageObjectAccessControlsDeleteResponse', supports_download=False, ) def Get(self, request, global_params=None): r"""Returns the ACL entry for the specified entity on the specified object. Args: request: (StorageObjectAccessControlsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.objectAccessControls.get', ordered_params=[u'bucket', u'object', u'entity'], path_params=[u'bucket', u'entity', u'object'], query_params=[u'generation', u'userProject'], relative_path=u'b/{bucket}/o/{object}/acl/{entity}', request_field='', request_type_name=u'StorageObjectAccessControlsGetRequest', response_type_name=u'ObjectAccessControl', supports_download=False, ) def Insert(self, request, global_params=None): r"""Creates a new ACL entry on the specified object. Args: request: (StorageObjectAccessControlsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Insert') return self._RunMethod( config, request, global_params=global_params) Insert.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.objectAccessControls.insert', ordered_params=[u'bucket', u'object'], path_params=[u'bucket', u'object'], query_params=[u'generation', u'userProject'], relative_path=u'b/{bucket}/o/{object}/acl', request_field=u'objectAccessControl', request_type_name=u'StorageObjectAccessControlsInsertRequest', response_type_name=u'ObjectAccessControl', supports_download=False, ) def List(self, request, global_params=None): r"""Retrieves ACL entries on the specified object. Args: request: (StorageObjectAccessControlsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControls) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.objectAccessControls.list', ordered_params=[u'bucket', u'object'], path_params=[u'bucket', u'object'], query_params=[u'generation', u'userProject'], relative_path=u'b/{bucket}/o/{object}/acl', request_field='', request_type_name=u'StorageObjectAccessControlsListRequest', response_type_name=u'ObjectAccessControls', supports_download=False, ) def Patch(self, request, global_params=None): r"""Patches an ACL entry on the specified object. Args: request: (StorageObjectAccessControlsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) Patch.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PATCH', method_id=u'storage.objectAccessControls.patch', ordered_params=[u'bucket', u'object', u'entity'], path_params=[u'bucket', u'entity', u'object'], query_params=[u'generation', u'userProject'], relative_path=u'b/{bucket}/o/{object}/acl/{entity}', request_field=u'objectAccessControl', request_type_name=u'StorageObjectAccessControlsPatchRequest', response_type_name=u'ObjectAccessControl', supports_download=False, ) def Update(self, request, global_params=None): r"""Updates an ACL entry on the specified object. Args: request: (StorageObjectAccessControlsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ObjectAccessControl) The response message. """ config = self.GetMethodConfig('Update') return self._RunMethod( config, request, global_params=global_params) Update.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PUT', method_id=u'storage.objectAccessControls.update', ordered_params=[u'bucket', u'object', u'entity'], path_params=[u'bucket', u'entity', u'object'], query_params=[u'generation', u'userProject'], relative_path=u'b/{bucket}/o/{object}/acl/{entity}', request_field=u'objectAccessControl', request_type_name=u'StorageObjectAccessControlsUpdateRequest', response_type_name=u'ObjectAccessControl', supports_download=False, ) class ObjectsService(base_api.BaseApiService): """Service class for the objects resource.""" _NAME = u'objects' def __init__(self, client): super(StorageV1.ObjectsService, self).__init__(client) self._upload_configs = { 'Insert': base_api.ApiUploadInfo( accept=['*/*'], max_size=None, resumable_multipart=True, resumable_path=u'/resumable/upload/storage/' + self._client._version + '/b/{bucket}/o', simple_multipart=True, simple_path=u'/upload/storage/' + self._client._version + '/b/{bucket}/o', ), } def Compose(self, request, global_params=None): r"""Concatenates a list of existing objects into a new object in the same bucket. Args: request: (StorageObjectsComposeRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Object) The response message. """ config = self.GetMethodConfig('Compose') return self._RunMethod( config, request, global_params=global_params) Compose.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.objects.compose', ordered_params=[u'destinationBucket', u'destinationObject'], path_params=[u'destinationBucket', u'destinationObject'], query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifMetagenerationMatch', u'kmsKeyName', u'userProject'], relative_path=u'b/{destinationBucket}/o/{destinationObject}/compose', request_field=u'composeRequest', request_type_name=u'StorageObjectsComposeRequest', response_type_name=u'Object', supports_download=False, ) def Copy(self, request, global_params=None): r"""Copies a source object to a destination object. Optionally overrides metadata. Args: request: (StorageObjectsCopyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Object) The response message. """ config = self.GetMethodConfig('Copy') return self._RunMethod( config, request, global_params=global_params) Copy.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.objects.copy', ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], query_params=[u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'projection', u'sourceGeneration', u'userProject'], relative_path=u'b/{sourceBucket}/o/{sourceObject}/copyTo/b/{destinationBucket}/o/{destinationObject}', request_field=u'object', request_type_name=u'StorageObjectsCopyRequest', response_type_name=u'Object', supports_download=False, ) def Delete(self, request, global_params=None): r"""Deletes an object and its metadata. Deletions are permanent if versioning is not enabled for the bucket, or if the generation parameter is used. Args: request: (StorageObjectsDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (StorageObjectsDeleteResponse) The response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod( config, request, global_params=global_params) Delete.method_config = lambda: base_api.ApiMethodInfo( http_method=u'DELETE', method_id=u'storage.objects.delete', ordered_params=[u'bucket', u'object'], path_params=[u'bucket', u'object'], query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'userProject'], relative_path=u'b/{bucket}/o/{object}', request_field='', request_type_name=u'StorageObjectsDeleteRequest', response_type_name=u'StorageObjectsDeleteResponse', supports_download=False, ) def Get(self, request, global_params=None, download=None): r"""Retrieves an object or its metadata. Args: request: (StorageObjectsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments download: (Download, default: None) If present, download data from the request via this stream. Returns: (Object) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params, download=download) Get.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.objects.get', ordered_params=[u'bucket', u'object'], path_params=[u'bucket', u'object'], query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'projection', u'userProject'], relative_path=u'b/{bucket}/o/{object}', request_field='', request_type_name=u'StorageObjectsGetRequest', response_type_name=u'Object', supports_download=True, ) def GetIamPolicy(self, request, global_params=None): r"""Returns an IAM policy for the specified object. Args: request: (StorageObjectsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('GetIamPolicy') return self._RunMethod( config, request, global_params=global_params) GetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.objects.getIamPolicy', ordered_params=[u'bucket', u'object'], path_params=[u'bucket', u'object'], query_params=[u'generation', u'userProject'], relative_path=u'b/{bucket}/o/{object}/iam', request_field='', request_type_name=u'StorageObjectsGetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def Insert(self, request, global_params=None, upload=None): r"""Stores a new object and metadata. Args: request: (StorageObjectsInsertRequest) input message global_params: (StandardQueryParameters, default: None) global arguments upload: (Upload, default: None) If present, upload this stream with the request. Returns: (Object) The response message. """ config = self.GetMethodConfig('Insert') upload_config = self.GetUploadConfig('Insert') return self._RunMethod( config, request, global_params=global_params, upload=upload, upload_config=upload_config) Insert.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.objects.insert', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'contentEncoding', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'kmsKeyName', u'name', u'predefinedAcl', u'projection', u'userProject'], relative_path=u'b/{bucket}/o', request_field=u'object', request_type_name=u'StorageObjectsInsertRequest', response_type_name=u'Object', supports_download=False, ) def List(self, request, global_params=None): r"""Retrieves a list of objects matching the criteria. Args: request: (StorageObjectsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Objects) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.objects.list', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'delimiter', u'includeTrailingDelimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'userProject', u'versions'], relative_path=u'b/{bucket}/o', request_field='', request_type_name=u'StorageObjectsListRequest', response_type_name=u'Objects', supports_download=False, ) def Patch(self, request, global_params=None): r"""Patches an object's metadata. Args: request: (StorageObjectsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Object) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) Patch.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PATCH', method_id=u'storage.objects.patch', ordered_params=[u'bucket', u'object'], path_params=[u'bucket', u'object'], query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection', u'userProject'], relative_path=u'b/{bucket}/o/{object}', request_field=u'objectResource', request_type_name=u'StorageObjectsPatchRequest', response_type_name=u'Object', supports_download=False, ) def Rewrite(self, request, global_params=None): r"""Rewrites a source object to a destination object. Optionally overrides metadata. Args: request: (StorageObjectsRewriteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (RewriteResponse) The response message. """ config = self.GetMethodConfig('Rewrite') return self._RunMethod( config, request, global_params=global_params) Rewrite.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.objects.rewrite', ordered_params=[u'sourceBucket', u'sourceObject', u'destinationBucket', u'destinationObject'], path_params=[u'destinationBucket', u'destinationObject', u'sourceBucket', u'sourceObject'], query_params=[u'destinationKmsKeyName', u'destinationPredefinedAcl', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'ifSourceGenerationMatch', u'ifSourceGenerationNotMatch', u'ifSourceMetagenerationMatch', u'ifSourceMetagenerationNotMatch', u'maxBytesRewrittenPerCall', u'projection', u'rewriteToken', u'sourceGeneration', u'userProject'], relative_path=u'b/{sourceBucket}/o/{sourceObject}/rewriteTo/b/{destinationBucket}/o/{destinationObject}', request_field=u'object', request_type_name=u'StorageObjectsRewriteRequest', response_type_name=u'RewriteResponse', supports_download=False, ) def SetIamPolicy(self, request, global_params=None): r"""Updates an IAM policy for the specified object. Args: request: (StorageObjectsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('SetIamPolicy') return self._RunMethod( config, request, global_params=global_params) SetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PUT', method_id=u'storage.objects.setIamPolicy', ordered_params=[u'bucket', u'object'], path_params=[u'bucket', u'object'], query_params=[u'generation', u'userProject'], relative_path=u'b/{bucket}/o/{object}/iam', request_field=u'policy', request_type_name=u'StorageObjectsSetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def TestIamPermissions(self, request, global_params=None): r"""Tests a set of permissions on the given object to see which, if any, are held by the caller. Args: request: (StorageObjectsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. """ config = self.GetMethodConfig('TestIamPermissions') return self._RunMethod( config, request, global_params=global_params) TestIamPermissions.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.objects.testIamPermissions', ordered_params=[u'bucket', u'object', u'permissions'], path_params=[u'bucket', u'object'], query_params=[u'generation', u'permissions', u'userProject'], relative_path=u'b/{bucket}/o/{object}/iam/testPermissions', request_field='', request_type_name=u'StorageObjectsTestIamPermissionsRequest', response_type_name=u'TestIamPermissionsResponse', supports_download=False, ) def Update(self, request, global_params=None): r"""Updates an object's metadata. Args: request: (StorageObjectsUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Object) The response message. """ config = self.GetMethodConfig('Update') return self._RunMethod( config, request, global_params=global_params) Update.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PUT', method_id=u'storage.objects.update', ordered_params=[u'bucket', u'object'], path_params=[u'bucket', u'object'], query_params=[u'generation', u'ifGenerationMatch', u'ifGenerationNotMatch', u'ifMetagenerationMatch', u'ifMetagenerationNotMatch', u'predefinedAcl', u'projection', u'userProject'], relative_path=u'b/{bucket}/o/{object}', request_field=u'objectResource', request_type_name=u'StorageObjectsUpdateRequest', response_type_name=u'Object', supports_download=False, ) def WatchAll(self, request, global_params=None): r"""Watch for changes on all objects in a bucket. Args: request: (StorageObjectsWatchAllRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Channel) The response message. """ config = self.GetMethodConfig('WatchAll') return self._RunMethod( config, request, global_params=global_params) WatchAll.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.objects.watchAll', ordered_params=[u'bucket'], path_params=[u'bucket'], query_params=[u'delimiter', u'includeTrailingDelimiter', u'maxResults', u'pageToken', u'prefix', u'projection', u'userProject', u'versions'], relative_path=u'b/{bucket}/o/watch', request_field=u'channel', request_type_name=u'StorageObjectsWatchAllRequest', response_type_name=u'Channel', supports_download=False, ) class ProjectsServiceAccountService(base_api.BaseApiService): """Service class for the projects_serviceAccount resource.""" _NAME = u'projects_serviceAccount' def __init__(self, client): super(StorageV1.ProjectsServiceAccountService, self).__init__(client) self._upload_configs = { } def Get(self, request, global_params=None): r"""Get the email address of this project's Google Cloud Storage service account. Args: request: (StorageProjectsServiceAccountGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ServiceAccount) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.projects.serviceAccount.get', ordered_params=[u'projectId'], path_params=[u'projectId'], query_params=[u'userProject'], relative_path=u'projects/{projectId}/serviceAccount', request_field='', request_type_name=u'StorageProjectsServiceAccountGetRequest', response_type_name=u'ServiceAccount', supports_download=False, ) class ProjectsService(base_api.BaseApiService): """Service class for the projects resource.""" _NAME = u'projects' def __init__(self, client): super(StorageV1.ProjectsService, self).__init__(client) self._upload_configs = { } class HmacKeysService(base_api.BaseApiService): """Service class for the project_hmacKeys resource.""" _NAME = u'projects_hmacKeys' def Create(self, request, global_params=None): """Creates HMAC key for a service account. Args: request: (StorageProjectsHmacKeysCreateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (HmacKey) The response message. """ config = self.GetMethodConfig('Create') return self._RunMethod(config, request, global_params=global_params) Create.method_config = lambda: base_api.ApiMethodInfo( http_method=u'POST', method_id=u'storage.projects.hmacKeys.create', ordered_params=[u'serviceAccountEmail', u'projectId'], path_params=[u'projectId'], query_params=[u'serviceAccountEmail'], relative_path=u'projects/{projectId}/hmacKeys', request_field='', request_type_name=u'StorageProjectsHmacKeysCreateRequest', response_type_name=u'HmacKey', supports_download=False, ) def Delete(self, request, global_params=None): """Deletes an HMAC key. Args: request: (StorageProjectsHmacKeysDeleteRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (HmacKeyDeleteResponse) The empty response message. """ config = self.GetMethodConfig('Delete') return self._RunMethod(config, request, global_params=global_params) Delete.method_config = lambda: base_api.ApiMethodInfo( http_method=u'DELETE', method_id=u'storage.projects.hmacKeys.delete', ordered_params=[u'accessId', u'projectId'], path_params=[u'accessId', u'projectId'], query_params=[], relative_path=u'projects/{projectId}/hmacKeys/{accessId}', request_field='', request_type_name=u'StorageProjectsHmacKeysDeleteRequest', response_type_name=u'HmacKeysDeleteResponse', supports_download=False, ) def Get(self, request, global_params=None): """Retrieves an HMAC key's metadata Args: request: (StorageProjectsHmacKeysGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (HmacKeyMetadata) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod(config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.projects.hmacKeys.get', ordered_params=[u'accessId', u'projectId'], path_params=[u'accessId', u'projectId'], relative_path=u'projects/{projectId}/hmacKeys/{accessId}', request_field='', request_type_name=u'StorageProjectsHmacKeysGetRequest', response_type_name=u'HmacKeyMetadata', supports_download=False, ) def List(self, request, global_params=None): """Retrieves a list of HMAC keys matching the criteria. Args: request: (StorageProjectsHmacKeysListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (HmacKeyMetadataList) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod(config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( http_method=u'GET', method_id=u'storage.projects.hmacKeys.get', ordered_params=[ u'projectId', u'serviceAccountEmail', u'showDeletedKeys', u'maxResults', u'pageToken', ], path_params=[u'projectId'], query_params=[ u'serviceAccountEmail', u'showDeletedKeys', u'maxResults', u'pageToken' ], relative_path=u'projects/{projectId}/hmacKeys', request_field='', request_type_name=u'StorageProjectsHmacKeysListRequest', response_type_name=u'HmacKeyMetadataList', supports_download=False, ) def Update(self, request, global_params=None): """Updates the state of an HMAC key. Args: request: (StorageProjectsHmacKeysUpdateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (HmacKeyMetadata) The response message. """ config = self.GetMethodConfig('Update') return self._RunMethod(config, request, global_params=global_params) Update.method_config = lambda: base_api.ApiMethodInfo( http_method=u'PUT', method_id=u'storage.projects.hmacKeys.update', ordered_params=[u'accessId', u'projectId'], path_params=[u'accessId', u'projectId'], query_params=[], relative_path=u'projects/{projectId}/hmacKeys/{accessId}', request_field='resource', request_type_name=u'StorageProjectsHmacKeysUpdateRequest', response_type_name=u'HmacKeyMetadata', supports_download=False, ) third_party/storage_apitools/credentials_lib.py 0000644 00000002272 15025013616 0016152 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shim for backwards-compatibility for moving GCE credentials. oauth2client loads credentials classes based on the module name where they were created; this means that moving GceAssertionCredentials from here to third_party requires a shim mapping the old name to the new one. Once loaded, the credential will be re-serialized with the new path, meaning that we can (at some point) consider removing this file. """ # TODO: Remove this module once this change has been around long # enough that old credentials are likely to be rare. from apitools.base.py import GceAssertionCredentials third_party/storage_apitools/storage_v1_messages.py 0000644 00000373052 15025013616 0016777 0 ustar 00 # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Generated message classes for storage version v1. Stores and retrieves potentially large, immutable data objects. """ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.protorpclite import message_types as _message_types from apitools.base.protorpclite import messages as _messages from apitools.base.py import encoding from apitools.base.py import extra_types package = 'storage' class Bucket(_messages.Message): r"""A bucket. Messages: AutoclassValue: The bucket's Autoclass configuration. BillingValue: The bucket's billing configuration. CorsValueListEntry: A CorsValueListEntry object. CustomPlacementConfigValue: The bucket's custom placement configuration for Custom Dual Regions. EncryptionValue: Encryption configuration for a bucket. IamConfigurationValue: The bucket's IAM configuration. LabelsValue: User-provided labels, in key/value pairs. LifecycleValue: The bucket's lifecycle configuration. See lifecycle management for more information. LoggingValue: The bucket's logging configuration, which defines the destination bucket and optional name prefix for the current bucket's logs. OwnerValue: The owner of the bucket. This is always the project team's owner group. RetentionPolicyValue: The bucket's retention policy. The retention policy enforces a minimum retention time for all objects contained in the bucket, based on their creation time. Any attempt to overwrite or delete objects younger than the retention period will result in a PERMISSION_DENIED error. An unlocked retention policy can be modified or removed from the bucket via a storage.buckets.update operation. A locked retention policy cannot be removed or shortened in duration for the lifetime of the bucket. Attempting to remove or decrease period of a locked retention policy will result in a PERMISSION_DENIED error. VersioningValue: The bucket's versioning configuration. WebsiteValue: The bucket's website configuration, controlling how the service behaves when accessing bucket contents as a web site. See the Static Website Examples for more information. Fields: acl: Access controls on the bucket. autoclass: The bucket's Autoclass configuration. billing: The bucket's billing configuration. cors: The bucket's Cross-Origin Resource Sharing (CORS) configuration. customPlacementConfig: The bucket's custom placement configuration for Custom Dual Regions. defaultEventBasedHold: The default value for event-based hold on newly created objects in this bucket. Event-based hold is a way to retain objects indefinitely until an event occurs, signified by the hold's release. After being released, such objects will be subject to bucket- level retention (if any). One sample use case of this flag is for banks to hold loan documents for at least 3 years after loan is paid in full. Here, bucket-level retention is 3 years and the event is loan being paid in full. In this example, these objects will be held intact for any number of years until the event has occurred (event-based hold on the object is released) and then 3 more years after that. That means retention duration of the objects begins from the moment event-based hold transitioned from true to false. Objects under event-based hold cannot be deleted, overwritten or archived until the hold is removed. defaultObjectAcl: Default access controls to apply to new objects when no ACL is provided. encryption: Encryption configuration for a bucket. etag: HTTP 1.1 Entity tag for the bucket. iamConfiguration: The bucket's IAM configuration. id: The ID of the bucket. For buckets, the id and name properties are the same. kind: The kind of item this is. For buckets, this is always storage#bucket. labels: User-provided labels, in key/value pairs. lifecycle: The bucket's lifecycle configuration. See lifecycle management for more information. location: The location of the bucket. Object data for objects in the bucket resides in physical storage within this region. Defaults to US. See the developer's guide for the authoritative list. locationType: The type of the bucket location. logging: The bucket's logging configuration, which defines the destination bucket and optional name prefix for the current bucket's logs. metageneration: The metadata generation of this bucket. name: The name of the bucket. owner: The owner of the bucket. This is always the project team's owner group. projectNumber: The project number of the project the bucket belongs to. retentionPolicy: The bucket's retention policy. The retention policy enforces a minimum retention time for all objects contained in the bucket, based on their creation time. Any attempt to overwrite or delete objects younger than the retention period will result in a PERMISSION_DENIED error. An unlocked retention policy can be modified or removed from the bucket via a storage.buckets.update operation. A locked retention policy cannot be removed or shortened in duration for the lifetime of the bucket. Attempting to remove or decrease period of a locked retention policy will result in a PERMISSION_DENIED error. rpo: The Recovery Point Objective (RPO) of this bucket. Set to ASYNC_TURBO to turn on turbo replication on a bucket. satisfiesPZS: Reserved for future use. selfLink: The URI of this bucket. storageClass: The bucket's default storage class, used whenever no storageClass is specified for a newly-created object. This defines how objects in the bucket are stored and determines the SLA and the cost of storage. Values include MULTI_REGIONAL, REGIONAL, STANDARD, NEARLINE, COLDLINE, ARCHIVE, and DURABLE_REDUCED_AVAILABILITY. If this value is not specified when the bucket is created, it will default to STANDARD. For more information, see storage classes. timeCreated: The creation time of the bucket in RFC 3339 format. updated: The modification time of the bucket in RFC 3339 format. versioning: The bucket's versioning configuration. website: The bucket's website configuration, controlling how the service behaves when accessing bucket contents as a web site. See the Static Website Examples for more information. """ class AutoclassValue(_messages.Message): r"""The bucket's Autoclass configuration. Fields: enabled: Whether or not Autoclass is enabled on this bucket toggleTime: A date and time in RFC 3339 format representing the instant at which "enabled" was last toggled. """ enabled = _messages.BooleanField(1) toggleTime = _message_types.DateTimeField(2) class BillingValue(_messages.Message): r"""The bucket's billing configuration. Fields: requesterPays: When set to true, Requester Pays is enabled for this bucket. """ requesterPays = _messages.BooleanField(1) class CorsValueListEntry(_messages.Message): r"""A CorsValueListEntry object. Fields: maxAgeSeconds: The value, in seconds, to return in the Access-Control- Max-Age header used in preflight responses. method: The list of HTTP methods on which to include CORS response headers, (GET, OPTIONS, POST, etc) Note: "*" is permitted in the list of methods, and means "any method". origin: The list of Origins eligible to receive CORS response headers. Note: "*" is permitted in the list of origins, and means "any Origin". responseHeader: The list of HTTP headers other than the simple response headers to give permission for the user-agent to share across domains. """ maxAgeSeconds = _messages.IntegerField(1, variant=_messages.Variant.INT32) method = _messages.StringField(2, repeated=True) origin = _messages.StringField(3, repeated=True) responseHeader = _messages.StringField(4, repeated=True) class CustomPlacementConfigValue(_messages.Message): r"""The bucket's custom placement configuration for Custom Dual Regions. Fields: dataLocations: The list of regional locations in which data is placed. """ dataLocations = _messages.StringField(1, repeated=True) class EncryptionValue(_messages.Message): r"""Encryption configuration for a bucket. Fields: defaultKmsKeyName: A Cloud KMS key that will be used to encrypt objects inserted into this bucket, if no encryption method is specified. """ defaultKmsKeyName = _messages.StringField(1) class IamConfigurationValue(_messages.Message): r"""The bucket's IAM configuration. Messages: BucketPolicyOnlyValue: The bucket's Bucket Policy Only configuration. Fields: bucketPolicyOnly: The bucket's Bucket Policy Only configuration. publicAccessPrevention: The bucket's public access prevention configuration. Valid values are `unspecified` and `enforced`. """ class BucketPolicyOnlyValue(_messages.Message): r"""The bucket's Bucket Policy Only configuration. Fields: enabled: If set, access checks only use bucket-level IAM policies or above. lockedTime: The deadline time for changing iamConfiguration.bucketPolicyOnly.enabled from true to false in RFC 3339 format. iamConfiguration.bucketPolicyOnly.enabled may be changed from true to false until the locked time, after which the field is immutable. """ enabled = _messages.BooleanField(1) lockedTime = _message_types.DateTimeField(2) bucketPolicyOnly = _messages.MessageField('BucketPolicyOnlyValue', 1) publicAccessPrevention = _messages.StringField(2) @encoding.MapUnrecognizedFields('additionalProperties') class LabelsValue(_messages.Message): r"""User-provided labels, in key/value pairs. Messages: AdditionalProperty: An additional property for a LabelsValue object. Fields: additionalProperties: An individual label entry. """ class AdditionalProperty(_messages.Message): r"""An additional property for a LabelsValue object. Fields: key: Name of the additional property. value: A string attribute. """ key = _messages.StringField(1) value = _messages.StringField(2) additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) class LifecycleValue(_messages.Message): r"""The bucket's lifecycle configuration. See lifecycle management for more information. Messages: RuleValueListEntry: A RuleValueListEntry object. Fields: rule: A lifecycle management rule, which is made of an action to take and the condition(s) under which the action will be taken. """ class RuleValueListEntry(_messages.Message): r"""A RuleValueListEntry object. Messages: ActionValue: The action to take. ConditionValue: The condition(s) under which the action will be taken. Fields: action: The action to take. condition: The condition(s) under which the action will be taken. """ class ActionValue(_messages.Message): r"""The action to take. Fields: storageClass: Target storage class. Required iff the type of the action is SetStorageClass. type: Type of the action. Currently, only Delete and SetStorageClass are supported. """ storageClass = _messages.StringField(1) type = _messages.StringField(2) class ConditionValue(_messages.Message): r"""The condition(s) under which the action will be taken. Fields: age: Age of an object (in days). This condition is satisfied when an object reaches the specified age. createdBefore: A date in RFC 3339 format with only the date part (for instance, "2013-01-15"). This condition is satisfied when an object is created before midnight of the specified date in UTC. customTimeBefore: A date in RFC 3339 format with only the date part (for instance, "2013-01-15"). This condition is satisfied when the custom time on an object is before this date in UTC. daysSinceCustomTime: Number of days elapsed since the user-specified timestamp set on an object. The condition is satisfied if the days elapsed is at least this number. If no custom timestamp is specified on an object, the condition does not apply. daysSinceNoncurrentTime: Number of days elapsed since the noncurrent timestamp of an object. The condition is satisfied if the days elapsed is at least this number. This condition is relevant only for versioned objects. The value of the field must be a nonnegative integer. If it's zero, the object version will become eligible for Lifecycle action as soon as it becomes noncurrent. isLive: Relevant only for versioned objects. If the value is true, this condition matches live objects; if the value is false, it matches archived objects. matchesStorageClass: Objects having any of the storage classes specified by this condition will be matched. Values include MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, ARCHIVE, STANDARD, and DURABLE_REDUCED_AVAILABILITY. noncurrentTimeBefore: A date in RFC 3339 format with only the date part (for instance, "2013-01-15"). This condition is satisfied when the noncurrent time on an object is before this date in UTC. This condition is relevant only for versioned objects. numNewerVersions: Relevant only for versioned objects. If the value is N, this condition is satisfied when there are at least N versions (including the live version) newer than this version of the object. """ age = _messages.IntegerField(1, variant=_messages.Variant.INT32) createdBefore = extra_types.DateField(2) customTimeBefore = extra_types.DateField(6) daysSinceCustomTime = _messages.IntegerField( 7, variant=_messages.Variant.INT32) daysSinceNoncurrentTime = _messages.IntegerField( 8, variant=_messages.Variant.INT32) isLive = _messages.BooleanField(3) matchesStorageClass = _messages.StringField(4, repeated=True) noncurrentTimeBefore = extra_types.DateField(9) numNewerVersions = _messages.IntegerField( 5, variant=_messages.Variant.INT32) action = _messages.MessageField('ActionValue', 1) condition = _messages.MessageField('ConditionValue', 2) rule = _messages.MessageField('RuleValueListEntry', 1, repeated=True) class LoggingValue(_messages.Message): r"""The bucket's logging configuration, which defines the destination bucket and optional name prefix for the current bucket's logs. Fields: logBucket: The destination bucket where the current bucket's logs should be placed. logObjectPrefix: A prefix for log object names. """ logBucket = _messages.StringField(1) logObjectPrefix = _messages.StringField(2) class OwnerValue(_messages.Message): r"""The owner of the bucket. This is always the project team's owner group. Fields: entity: The entity, in the form project-owner-projectId. entityId: The ID for the entity. """ entity = _messages.StringField(1) entityId = _messages.StringField(2) class RetentionPolicyValue(_messages.Message): r"""The bucket's retention policy. The retention policy enforces a minimum retention time for all objects contained in the bucket, based on their creation time. Any attempt to overwrite or delete objects younger than the retention period will result in a PERMISSION_DENIED error. An unlocked retention policy can be modified or removed from the bucket via a storage.buckets.update operation. A locked retention policy cannot be removed or shortened in duration for the lifetime of the bucket. Attempting to remove or decrease period of a locked retention policy will result in a PERMISSION_DENIED error. Fields: effectiveTime: Server-determined value that indicates the time from which policy was enforced and effective. This value is in RFC 3339 format. isLocked: Once locked, an object retention policy cannot be modified. retentionPeriod: The duration in seconds that objects need to be retained. Retention duration must be greater than zero and less than 100 years. Note that enforcement of retention periods less than a day is not guaranteed. Such periods should only be used for testing purposes. """ effectiveTime = _message_types.DateTimeField(1) isLocked = _messages.BooleanField(2) retentionPeriod = _messages.IntegerField(3) class VersioningValue(_messages.Message): r"""The bucket's versioning configuration. Fields: enabled: While set to true, versioning is fully enabled for this bucket. """ enabled = _messages.BooleanField(1) class WebsiteValue(_messages.Message): r"""The bucket's website configuration, controlling how the service behaves when accessing bucket contents as a web site. See the Static Website Examples for more information. Fields: mainPageSuffix: If the requested object path is missing, the service will ensure the path has a trailing '/', append this suffix, and attempt to retrieve the resulting object. This allows the creation of index.html objects to represent directory pages. notFoundPage: If the requested object path is missing, and any mainPageSuffix object is missing, if applicable, the service will return the named object from this bucket as the content for a 404 Not Found result. """ mainPageSuffix = _messages.StringField(1) notFoundPage = _messages.StringField(2) acl = _messages.MessageField('BucketAccessControl', 1, repeated=True) autoclass = _messages.MessageField('AutoclassValue', 2) billing = _messages.MessageField('BillingValue', 3) cors = _messages.MessageField('CorsValueListEntry', 4, repeated=True) customPlacementConfig = _messages.MessageField('CustomPlacementConfigValue', 5) defaultEventBasedHold = _messages.BooleanField(6) defaultObjectAcl = _messages.MessageField('ObjectAccessControl', 7, repeated=True) encryption = _messages.MessageField('EncryptionValue', 8) etag = _messages.StringField(9) iamConfiguration = _messages.MessageField('IamConfigurationValue', 10) id = _messages.StringField(11) kind = _messages.StringField(12, default=u'storage#bucket') labels = _messages.MessageField('LabelsValue', 13) lifecycle = _messages.MessageField('LifecycleValue', 14) location = _messages.StringField(15) locationType = _messages.StringField(16) logging = _messages.MessageField('LoggingValue', 17) metageneration = _messages.IntegerField(18) name = _messages.StringField(19) owner = _messages.MessageField('OwnerValue', 20) projectNumber = _messages.IntegerField(21, variant=_messages.Variant.UINT64) retentionPolicy = _messages.MessageField('RetentionPolicyValue', 22) rpo = _messages.StringField(23) satisfiesPZS = _messages.BooleanField(24) selfLink = _messages.StringField(25) storageClass = _messages.StringField(26) timeCreated = _message_types.DateTimeField(27) updated = _message_types.DateTimeField(28) versioning = _messages.MessageField('VersioningValue', 29) website = _messages.MessageField('WebsiteValue', 30) class BucketAccessControl(_messages.Message): r"""An access-control entry. Messages: ProjectTeamValue: The project team associated with the entity, if any. Fields: bucket: The name of the bucket. domain: The domain associated with the entity, if any. email: The email address associated with the entity, if any. entity: The entity holding the permission, in one of the following forms: - user-userId - user-email - group-groupId - group-email - domain- domain - project-team-projectId - allUsers - allAuthenticatedUsers Examples: - The user liz@example.com would be user-liz@example.com. - The group example@googlegroups.com would be group- example@googlegroups.com. - To refer to all members of the Google Apps for Business domain example.com, the entity would be domain-example.com. entityId: The ID for the entity, if any. etag: HTTP 1.1 Entity tag for the access-control entry. id: The ID of the access-control entry. kind: The kind of item this is. For bucket access control entries, this is always storage#bucketAccessControl. projectTeam: The project team associated with the entity, if any. role: The access permission for the entity. selfLink: The link to this access-control entry. """ class ProjectTeamValue(_messages.Message): r"""The project team associated with the entity, if any. Fields: projectNumber: The project number. team: The team. """ projectNumber = _messages.StringField(1) team = _messages.StringField(2) bucket = _messages.StringField(1) domain = _messages.StringField(2) email = _messages.StringField(3) entity = _messages.StringField(4) entityId = _messages.StringField(5) etag = _messages.StringField(6) id = _messages.StringField(7) kind = _messages.StringField(8, default=u'storage#bucketAccessControl') projectTeam = _messages.MessageField('ProjectTeamValue', 9) role = _messages.StringField(10) selfLink = _messages.StringField(11) class BucketAccessControls(_messages.Message): r"""An access-control list. Fields: items: The list of items. kind: The kind of item this is. For lists of bucket access control entries, this is always storage#bucketAccessControls. """ items = _messages.MessageField('BucketAccessControl', 1, repeated=True) kind = _messages.StringField(2, default=u'storage#bucketAccessControls') class Buckets(_messages.Message): r"""A list of buckets. Fields: items: The list of items. kind: The kind of item this is. For lists of buckets, this is always storage#buckets. nextPageToken: The continuation token, used to page through large result sets. Provide this value in a subsequent request to return the next page of results. """ items = _messages.MessageField('Bucket', 1, repeated=True) kind = _messages.StringField(2, default=u'storage#buckets') nextPageToken = _messages.StringField(3) class Channel(_messages.Message): r"""An notification channel used to watch for resource changes. Messages: ParamsValue: Additional parameters controlling delivery channel behavior. Optional. Fields: address: The address where notifications are delivered for this channel. expiration: Date and time of notification channel expiration, expressed as a Unix timestamp, in milliseconds. Optional. id: A UUID or similar unique string that identifies this channel. kind: Identifies this as a notification channel used to watch for changes to a resource. Value: the fixed string "api#channel". params: Additional parameters controlling delivery channel behavior. Optional. payload: A Boolean value to indicate whether payload is wanted. Optional. resourceId: An opaque ID that identifies the resource being watched on this channel. Stable across different API versions. resourceUri: A version-specific identifier for the watched resource. token: An arbitrary string delivered to the target address with each notification delivered over this channel. Optional. type: The type of delivery mechanism used for this channel. """ @encoding.MapUnrecognizedFields('additionalProperties') class ParamsValue(_messages.Message): r"""Additional parameters controlling delivery channel behavior. Optional. Messages: AdditionalProperty: An additional property for a ParamsValue object. Fields: additionalProperties: Declares a new parameter by name. """ class AdditionalProperty(_messages.Message): r"""An additional property for a ParamsValue object. Fields: key: Name of the additional property. value: A string attribute. """ key = _messages.StringField(1) value = _messages.StringField(2) additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) address = _messages.StringField(1) expiration = _messages.IntegerField(2) id = _messages.StringField(3) kind = _messages.StringField(4, default=u'api#channel') params = _messages.MessageField('ParamsValue', 5) payload = _messages.BooleanField(6) resourceId = _messages.StringField(7) resourceUri = _messages.StringField(8) token = _messages.StringField(9) type = _messages.StringField(10) class Channels(_messages.Message): r"""A channels list response. Messages: ItemsValueListEntry: A ItemsValueListEntry object. Fields: items: The list of notification channels for a bucket. kind: The kind of item this is. """ class ItemsValueListEntry(_messages.Message): r"""A ItemsValueListEntry object. Fields: channel_id: User-specified name for a channel. Needed to unsubscribe. creation_time_ms: 64-bit Unix timestamp in ms of when the channel was created. push_url: Url used to identify where notifications are sent to. resource_id: Opaque value generated by GCS representing a bucket. Needed to unsubscribe. subscriber_email: Email address of the subscriber. """ channel_id = _messages.StringField(1) creation_time_ms = _messages.IntegerField(2) push_url = _messages.StringField(3) resource_id = _messages.StringField(4) subscriber_email = _messages.StringField(5) items = _messages.MessageField('ItemsValueListEntry', 1, repeated=True) kind = _messages.StringField(2, default=u'storage#channels') class ComposeRequest(_messages.Message): r"""A Compose request. Messages: SourceObjectsValueListEntry: A SourceObjectsValueListEntry object. Fields: destination: Properties of the resulting object. kind: The kind of item this is. sourceObjects: The list of source objects that will be concatenated into a single object. userProject: The project to be billed for this request, for Requester Pays buckets. """ class SourceObjectsValueListEntry(_messages.Message): r"""A SourceObjectsValueListEntry object. Messages: ObjectPreconditionsValue: Conditions that must be met for this operation to execute. Fields: generation: The generation of this object to use as the source. name: The source object's name. All source objects must reside in the same bucket. objectPreconditions: Conditions that must be met for this operation to execute. """ class ObjectPreconditionsValue(_messages.Message): r"""Conditions that must be met for this operation to execute. Fields: ifGenerationMatch: Only perform the composition if the generation of the source object that would be used matches this value. If this value and a generation are both specified, they must be the same value or the call will fail. """ ifGenerationMatch = _messages.IntegerField(1) generation = _messages.IntegerField(1) name = _messages.StringField(2) objectPreconditions = _messages.MessageField('ObjectPreconditionsValue', 3) destination = _messages.MessageField('Object', 1) kind = _messages.StringField(2, default=u'storage#composeRequest') sourceObjects = _messages.MessageField('SourceObjectsValueListEntry', 3, repeated=True) userProject = _messages.StringField(4) class Expr(_messages.Message): r"""Represents an expression text. Example: title: "User account presence" description: "Determines whether the request has a user account" expression: "size(request.user) > 0" Fields: description: An optional description of the expression. This is a longer text which describes the expression, e.g. when hovered over it in a UI. expression: Textual representation of an expression in Common Expression Language syntax. The application context of the containing message determines which well-known feature set of CEL is supported. kind: The kind of item this is. For storage, this is always storage#expr. This field is ignored on input. location: An optional string indicating the location of the expression for error reporting, e.g. a file name and a position in the file. title: An optional title for the expression, i.e. a short string describing its purpose. This can be used e.g. in UIs which allow to enter the expression. """ description = _messages.StringField(1) expression = _messages.StringField(2) kind = _messages.StringField(3, default=u'storage#expr') location = _messages.StringField(4) title = _messages.StringField(5) class Notification(_messages.Message): r"""A subscription to receive Google PubSub notifications. Messages: CustomAttributesValue: An optional list of additional attributes to attach to each Cloud PubSub message published for this notification subscription. Fields: custom_attributes: An optional list of additional attributes to attach to each Cloud PubSub message published for this notification subscription. etag: HTTP 1.1 Entity tag for this subscription notification. event_types: If present, only send notifications about listed event types. If empty, sent notifications for all event types. id: The ID of the notification. kind: The kind of item this is. For notifications, this is always storage#notification. object_name_prefix: If present, only apply this notification configuration to object names that begin with this prefix. payload_format: The desired content of the Payload. selfLink: The canonical URL of this notification. topic: The Cloud PubSub topic to which this subscription publishes. Formatted as: '//pubsub.googleapis.com/projects/{project- identifier}/topics/{my-topic}' """ @encoding.MapUnrecognizedFields('additionalProperties') class CustomAttributesValue(_messages.Message): r"""An optional list of additional attributes to attach to each Cloud PubSub message published for this notification subscription. Messages: AdditionalProperty: An additional property for a CustomAttributesValue object. Fields: additionalProperties: Additional properties of type CustomAttributesValue """ class AdditionalProperty(_messages.Message): r"""An additional property for a CustomAttributesValue object. Fields: key: Name of the additional property. value: A string attribute. """ key = _messages.StringField(1) value = _messages.StringField(2) additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) custom_attributes = _messages.MessageField('CustomAttributesValue', 1) etag = _messages.StringField(2) event_types = _messages.StringField(3, repeated=True) id = _messages.StringField(4) kind = _messages.StringField(5, default=u'storage#notification') object_name_prefix = _messages.StringField(6) payload_format = _messages.StringField(7, default=u'JSON_API_V1') selfLink = _messages.StringField(8) topic = _messages.StringField(9) class Notifications(_messages.Message): r"""A list of notification subscriptions. Fields: items: The list of items. kind: The kind of item this is. For lists of notifications, this is always storage#notifications. """ items = _messages.MessageField('Notification', 1, repeated=True) kind = _messages.StringField(2, default=u'storage#notifications') class Object(_messages.Message): r"""An object. Messages: CustomerEncryptionValue: Metadata of customer-supplied encryption key, if the object is encrypted by such a key. MetadataValue: User-provided metadata, in key/value pairs. OwnerValue: The owner of the object. This will always be the uploader of the object. Fields: acl: Access controls on the object. bucket: The name of the bucket containing this object. cacheControl: Cache-Control directive for the object data. If omitted, and the object is accessible to all anonymous users, the default will be public, max-age=3600. componentCount: Number of underlying components that make up this object. Components are accumulated by compose operations. contentDisposition: Content-Disposition of the object data. contentEncoding: Content-Encoding of the object data. contentLanguage: Content-Language of the object data. contentType: Content-Type of the object data. If an object is stored without a Content-Type, it is served as application/octet-stream. crc32c: CRC32c checksum, as described in RFC 4960, Appendix B; encoded using base64 in big-endian byte order. For more information about using the CRC32c checksum, see Hashes and ETags: Best Practices. customTime: A timestamp in RFC 3339 format specified by the user for an object. customerEncryption: Metadata of customer-supplied encryption key, if the object is encrypted by such a key. etag: HTTP 1.1 Entity tag for the object. eventBasedHold: Whether an object is under event-based hold. Event-based hold is a way to retain objects until an event occurs, which is signified by the hold's release (i.e. this value is set to false). After being released (set to false), such objects will be subject to bucket- level retention (if any). One sample use case of this flag is for banks to hold loan documents for at least 3 years after loan is paid in full. Here, bucket-level retention is 3 years and the event is the loan being paid in full. In this example, these objects will be held intact for any number of years until the event has occurred (event-based hold on the object is released) and then 3 more years after that. That means retention duration of the objects begins from the moment event-based hold transitioned from true to false. generation: The content generation of this object. Used for object versioning. id: The ID of the object, including the bucket name, object name, and generation number. kind: The kind of item this is. For objects, this is always storage#object. kmsKeyName: Cloud KMS Key used to encrypt this object, if the object is encrypted by such a key. md5Hash: MD5 hash of the data; encoded using base64. For more information about using the MD5 hash, see Hashes and ETags: Best Practices. mediaLink: Media download link. metadata: User-provided metadata, in key/value pairs. metageneration: The version of the metadata for this object at this generation. Used for preconditions and for detecting changes in metadata. A metageneration number is only meaningful in the context of a particular generation of a particular object. name: The name of the object. Required if not specified by URL parameter. owner: The owner of the object. This will always be the uploader of the object. retentionExpirationTime: A server-determined value that specifies the earliest time that the object's retention period expires. This value is in RFC 3339 format. Note 1: This field is not provided for objects with an active event-based hold, since retention expiration is unknown until the hold is removed. Note 2: This value can be provided even when temporary hold is set (so that the user can reason about policy without having to first unset the temporary hold). selfLink: The link to this object. size: Content-Length of the data in bytes. storageClass: Storage class of the object. temporaryHold: Whether an object is under temporary hold. While this flag is set to true, the object is protected against deletion and overwrites. A common use case of this flag is regulatory investigations where objects need to be retained while the investigation is ongoing. Note that unlike event-based hold, temporary hold does not impact retention expiration time of an object. timeCreated: The creation time of the object in RFC 3339 format. timeDeleted: The deletion time of the object in RFC 3339 format. Will be returned if and only if this version of the object has been deleted. timeStorageClassUpdated: The time at which the object's storage class was last changed. When the object is initially created, it will be set to timeCreated. updated: The modification time of the object metadata in RFC 3339 format. """ class CustomerEncryptionValue(_messages.Message): r"""Metadata of customer-supplied encryption key, if the object is encrypted by such a key. Fields: encryptionAlgorithm: The encryption algorithm. keySha256: SHA256 hash value of the encryption key. """ encryptionAlgorithm = _messages.StringField(1) keySha256 = _messages.StringField(2) @encoding.MapUnrecognizedFields('additionalProperties') class MetadataValue(_messages.Message): r"""User-provided metadata, in key/value pairs. Messages: AdditionalProperty: An additional property for a MetadataValue object. Fields: additionalProperties: An individual metadata entry. """ class AdditionalProperty(_messages.Message): r"""An additional property for a MetadataValue object. Fields: key: Name of the additional property. value: A string attribute. """ key = _messages.StringField(1) value = _messages.StringField(2) additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) class OwnerValue(_messages.Message): r"""The owner of the object. This will always be the uploader of the object. Fields: entity: The entity, in the form user-userId. entityId: The ID for the entity. """ entity = _messages.StringField(1) entityId = _messages.StringField(2) acl = _messages.MessageField('ObjectAccessControl', 1, repeated=True) bucket = _messages.StringField(2) cacheControl = _messages.StringField(3) componentCount = _messages.IntegerField(4, variant=_messages.Variant.INT32) contentDisposition = _messages.StringField(5) contentEncoding = _messages.StringField(6) contentLanguage = _messages.StringField(7) contentType = _messages.StringField(8) crc32c = _messages.StringField(9) customTime = _message_types.DateTimeField(32) customerEncryption = _messages.MessageField('CustomerEncryptionValue', 10) etag = _messages.StringField(11) eventBasedHold = _messages.BooleanField(12) generation = _messages.IntegerField(13) id = _messages.StringField(14) kind = _messages.StringField(15, default=u'storage#object') kmsKeyName = _messages.StringField(16) md5Hash = _messages.StringField(17) mediaLink = _messages.StringField(18) metadata = _messages.MessageField('MetadataValue', 19) metageneration = _messages.IntegerField(20) name = _messages.StringField(21) owner = _messages.MessageField('OwnerValue', 22) retentionExpirationTime = _message_types.DateTimeField(23) selfLink = _messages.StringField(24) size = _messages.IntegerField(25, variant=_messages.Variant.UINT64) storageClass = _messages.StringField(26) temporaryHold = _messages.BooleanField(27) timeCreated = _message_types.DateTimeField(28) timeDeleted = _message_types.DateTimeField(29) timeStorageClassUpdated = _message_types.DateTimeField(30) updated = _message_types.DateTimeField(31) class ObjectAccessControl(_messages.Message): r"""An access-control entry. Messages: ProjectTeamValue: The project team associated with the entity, if any. Fields: bucket: The name of the bucket. domain: The domain associated with the entity, if any. email: The email address associated with the entity, if any. entity: The entity holding the permission, in one of the following forms: - user-userId - user-email - group-groupId - group-email - domain- domain - project-team-projectId - allUsers - allAuthenticatedUsers Examples: - The user liz@example.com would be user-liz@example.com. - The group example@googlegroups.com would be group- example@googlegroups.com. - To refer to all members of the Google Apps for Business domain example.com, the entity would be domain-example.com. entityId: The ID for the entity, if any. etag: HTTP 1.1 Entity tag for the access-control entry. generation: The content generation of the object, if applied to an object. id: The ID of the access-control entry. kind: The kind of item this is. For object access control entries, this is always storage#objectAccessControl. object: The name of the object, if applied to an object. projectTeam: The project team associated with the entity, if any. role: The access permission for the entity. selfLink: The link to this access-control entry. """ class ProjectTeamValue(_messages.Message): r"""The project team associated with the entity, if any. Fields: projectNumber: The project number. team: The team. """ projectNumber = _messages.StringField(1) team = _messages.StringField(2) bucket = _messages.StringField(1) domain = _messages.StringField(2) email = _messages.StringField(3) entity = _messages.StringField(4) entityId = _messages.StringField(5) etag = _messages.StringField(6) generation = _messages.IntegerField(7) id = _messages.StringField(8) kind = _messages.StringField(9, default=u'storage#objectAccessControl') object = _messages.StringField(10) projectTeam = _messages.MessageField('ProjectTeamValue', 11) role = _messages.StringField(12) selfLink = _messages.StringField(13) class ObjectAccessControls(_messages.Message): r"""An access-control list. Fields: items: The list of items. kind: The kind of item this is. For lists of object access control entries, this is always storage#objectAccessControls. """ items = _messages.MessageField('ObjectAccessControl', 1, repeated=True) kind = _messages.StringField(2, default=u'storage#objectAccessControls') class Objects(_messages.Message): r"""A list of objects. Fields: items: The list of items. kind: The kind of item this is. For lists of objects, this is always storage#objects. nextPageToken: The continuation token, used to page through large result sets. Provide this value in a subsequent request to return the next page of results. prefixes: The list of prefixes of objects matching-but-not-listed up to and including the requested delimiter. """ items = _messages.MessageField('Object', 1, repeated=True) kind = _messages.StringField(2, default=u'storage#objects') nextPageToken = _messages.StringField(3) prefixes = _messages.StringField(4, repeated=True) class Policy(_messages.Message): r"""A bucket/object IAM policy. Messages: BindingsValueListEntry: A BindingsValueListEntry object. Fields: bindings: An association between a role, which comes with a set of permissions, and members who may assume that role. etag: HTTP 1.1 Entity tag for the policy. kind: The kind of item this is. For policies, this is always storage#policy. This field is ignored on input. resourceId: The ID of the resource to which this policy belongs. Will be of the form projects/_/buckets/bucket for buckets, and projects/_/buckets/bucket/objects/object for objects. A specific generation may be specified by appending #generationNumber to the end of the object name, e.g. projects/_/buckets/my-bucket/objects/data.txt#17. The current generation can be denoted with #0. This field is ignored on input. version: The policy format version to be returned in the response. """ class BindingsValueListEntry(_messages.Message): r"""A BindingsValueListEntry object. Fields: condition: The condition that is associated with this binding. NOTE: an unsatisfied condition will not allow user access via current binding. Different bindings, including their conditions, are examined independently. members: A collection of identifiers for members who may assume the provided role. Recognized identifiers are as follows: - allUsers - A special identifier that represents anyone on the internet; with or without a Google account. - allAuthenticatedUsers - A special identifier that represents anyone who is authenticated with a Google account or a service account. - user:emailid - An email address that represents a specific account. For example, user:alice@gmail.com or user:joe@example.com. - serviceAccount:emailid - An email address that represents a service account. For example, serviceAccount:my- other-app@appspot.gserviceaccount.com . - group:emailid - An email address that represents a Google group. For example, group:admins@example.com. - domain:domain - A Google Apps domain name that represents all the users of that domain. For example, domain:google.com or domain:example.com. - projectOwner:projectid - Owners of the given project. For example, projectOwner:my-example- project - projectEditor:projectid - Editors of the given project. For example, projectEditor:my-example-project - projectViewer:projectid - Viewers of the given project. For example, projectViewer:my-example-project role: The role to which members belong. Two types of roles are supported: new IAM roles, which grant permissions that do not map directly to those provided by ACLs, and legacy IAM roles, which do map directly to ACL permissions. All roles are of the format roles/storage.specificRole. The new IAM roles are: - roles/storage.admin - Full control of Google Cloud Storage resources. - roles/storage.objectViewer - Read-Only access to Google Cloud Storage objects. - roles/storage.objectCreator - Access to create objects in Google Cloud Storage. - roles/storage.objectAdmin - Full control of Google Cloud Storage objects. The legacy IAM roles are: - roles/storage.legacyObjectReader - Read-only access to objects without listing. Equivalent to an ACL entry on an object with the READER role. - roles/storage.legacyObjectOwner - Read/write access to existing objects without listing. Equivalent to an ACL entry on an object with the OWNER role. - roles/storage.legacyBucketReader - Read access to buckets with object listing. Equivalent to an ACL entry on a bucket with the READER role. - roles/storage.legacyBucketWriter - Read access to buckets with object listing/creation/deletion. Equivalent to an ACL entry on a bucket with the WRITER role. - roles/storage.legacyBucketOwner - Read and write access to existing buckets with object listing/creation/deletion. Equivalent to an ACL entry on a bucket with the OWNER role. """ condition = _messages.MessageField('Expr', 1) members = _messages.StringField(2, repeated=True) role = _messages.StringField(3) bindings = _messages.MessageField('BindingsValueListEntry', 1, repeated=True) etag = _messages.BytesField(2) kind = _messages.StringField(3, default=u'storage#policy') resourceId = _messages.StringField(4) version = _messages.IntegerField(5, variant=_messages.Variant.INT32) class RewriteResponse(_messages.Message): r"""A rewrite response. Fields: done: true if the copy is finished; otherwise, false if the copy is in progress. This property is always present in the response. kind: The kind of item this is. objectSize: The total size of the object being copied in bytes. This property is always present in the response. resource: A resource containing the metadata for the copied-to object. This property is present in the response only when copying completes. rewriteToken: A token to use in subsequent requests to continue copying data. This token is present in the response only when there is more data to copy. totalBytesRewritten: The total bytes written so far, which can be used to provide a waiting user with a progress indicator. This property is always present in the response. """ done = _messages.BooleanField(1) kind = _messages.StringField(2, default=u'storage#rewriteResponse') objectSize = _messages.IntegerField(3) resource = _messages.MessageField('Object', 4) rewriteToken = _messages.StringField(5) totalBytesRewritten = _messages.IntegerField(6) class ServiceAccount(_messages.Message): r"""A subscription to receive Google PubSub notifications. Fields: email_address: The ID of the notification. kind: The kind of item this is. For notifications, this is always storage#notification. """ email_address = _messages.StringField(1) kind = _messages.StringField(2, default=u'storage#serviceAccount') class StandardQueryParameters(_messages.Message): r"""Query parameters accepted by all methods. Enums: AltValueValuesEnum: Data format for the response. Fields: alt: Data format for the response. fields: Selector specifying which fields to include in a partial response. key: API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token. oauth_token: OAuth 2.0 token for the current user. prettyPrint: Returns response with indentations and line breaks. quotaUser: An opaque string that represents a user for quota purposes. Must not exceed 40 characters. trace: A tracing token of the form "token:<tokenid>" to include in api requests. userIp: Deprecated. Please use quotaUser instead. """ class AltValueValuesEnum(_messages.Enum): r"""Data format for the response. Values: json: Responses with Content-Type of application/json """ json = 0 alt = _messages.EnumField('AltValueValuesEnum', 1, default=u'json') fields = _messages.StringField(2) key = _messages.StringField(3) oauth_token = _messages.StringField(4) prettyPrint = _messages.BooleanField(5, default=True) quotaUser = _messages.StringField(6) trace = _messages.StringField(7) userIp = _messages.StringField(8) class StorageBucketAccessControlsDeleteRequest(_messages.Message): r"""A StorageBucketAccessControlsDeleteRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) entity = _messages.StringField(2, required=True) userProject = _messages.StringField(3) class StorageBucketAccessControlsDeleteResponse(_messages.Message): r"""An empty StorageBucketAccessControlsDelete response.""" class StorageBucketAccessControlsGetRequest(_messages.Message): r"""A StorageBucketAccessControlsGetRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) entity = _messages.StringField(2, required=True) userProject = _messages.StringField(3) class StorageBucketAccessControlsInsertRequest(_messages.Message): r"""A StorageBucketAccessControlsInsertRequest object. Fields: bucket: Name of a bucket. bucketAccessControl: A BucketAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) bucketAccessControl = _messages.MessageField('BucketAccessControl', 2) userProject = _messages.StringField(3) class StorageBucketAccessControlsListRequest(_messages.Message): r"""A StorageBucketAccessControlsListRequest object. Fields: bucket: Name of a bucket. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) userProject = _messages.StringField(2) class StorageBucketAccessControlsPatchRequest(_messages.Message): r"""A StorageBucketAccessControlsPatchRequest object. Fields: bucket: Name of a bucket. bucketAccessControl: A BucketAccessControl resource to be passed as the request body. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) bucketAccessControl = _messages.MessageField('BucketAccessControl', 2) entity = _messages.StringField(3, required=True) userProject = _messages.StringField(4) class StorageBucketAccessControlsUpdateRequest(_messages.Message): r"""A StorageBucketAccessControlsUpdateRequest object. Fields: bucket: Name of a bucket. bucketAccessControl: A BucketAccessControl resource to be passed as the request body. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) bucketAccessControl = _messages.MessageField('BucketAccessControl', 2) entity = _messages.StringField(3, required=True) userProject = _messages.StringField(4) class StorageBucketsDeleteRequest(_messages.Message): r"""A StorageBucketsDeleteRequest object. Fields: bucket: Name of a bucket. ifMetagenerationMatch: If set, only deletes the bucket if its metageneration matches this value. ifMetagenerationNotMatch: If set, only deletes the bucket if its metageneration does not match this value. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) ifMetagenerationMatch = _messages.IntegerField(2) ifMetagenerationNotMatch = _messages.IntegerField(3) userProject = _messages.StringField(4) class StorageBucketsDeleteResponse(_messages.Message): r"""An empty StorageBucketsDelete response.""" class StorageBucketsGetIamPolicyRequest(_messages.Message): r"""A StorageBucketsGetIamPolicyRequest object. Fields: bucket: Name of a bucket. provisionalUserProject: The project to be billed for this request if the target bucket is requester-pays bucket. optionsRequestedPolicyVersion: The policy format version to be returned in the response. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) provisionalUserProject = _messages.StringField(2) optionsRequestedPolicyVersion = _messages.IntegerField( 3, variant=_messages.Variant.INT32) userProject = _messages.StringField(4) class StorageBucketsGetRequest(_messages.Message): r"""A StorageBucketsGetRequest object. Enums: ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. Fields: bucket: Name of a bucket. ifMetagenerationMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration does not match the given value. projection: Set of properties to return. Defaults to noAcl. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to noAcl. Values: full: Include all properties. noAcl: Omit owner, acl and defaultObjectAcl properties. """ full = 0 noAcl = 1 bucket = _messages.StringField(1, required=True) ifMetagenerationMatch = _messages.IntegerField(2) ifMetagenerationNotMatch = _messages.IntegerField(3) projection = _messages.EnumField('ProjectionValueValuesEnum', 4) userProject = _messages.StringField(5) class StorageBucketsInsertRequest(_messages.Message): r"""A StorageBucketsInsertRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this bucket. PredefinedDefaultObjectAclValueValuesEnum: Apply a predefined set of default object access controls to this bucket. ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it defaults to full. Fields: bucket: A Bucket resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this bucket. predefinedDefaultObjectAcl: Apply a predefined set of default object access controls to this bucket. project: A valid API project identifier. projection: Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it defaults to full. userProject: The project to be billed for this request. """ class PredefinedAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of access controls to this bucket. Values: authenticatedRead: Project team owners get OWNER access, and allAuthenticatedUsers get READER access. private: Project team owners get OWNER access. projectPrivate: Project team members get access according to their roles. publicRead: Project team owners get OWNER access, and allUsers get READER access. publicReadWrite: Project team owners get OWNER access, and allUsers get WRITER access. """ authenticatedRead = 0 private = 1 projectPrivate = 2 publicRead = 3 publicReadWrite = 4 class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of default object access controls to this bucket. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. """ authenticatedRead = 0 bucketOwnerFullControl = 1 bucketOwnerRead = 2 private = 3 projectPrivate = 4 publicRead = 5 class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to noAcl, unless the bucket resource specifies acl or defaultObjectAcl properties, when it defaults to full. Values: full: Include all properties. noAcl: Omit owner, acl and defaultObjectAcl properties. """ full = 0 noAcl = 1 bucket = _messages.MessageField('Bucket', 1) predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 2) predefinedDefaultObjectAcl = _messages.EnumField( 'PredefinedDefaultObjectAclValueValuesEnum', 3) project = _messages.StringField(4, required=True) projection = _messages.EnumField('ProjectionValueValuesEnum', 5) userProject = _messages.StringField(6) class StorageBucketsListRequest(_messages.Message): r"""A StorageBucketsListRequest object. Enums: ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. Fields: maxResults: Maximum number of buckets to return in a single response. The service will use this parameter or 1,000 items, whichever is smaller. pageToken: A previously-returned page token representing part of the larger set of results to view. prefix: Filter results to buckets whose names begin with this prefix. project: A valid API project identifier. projection: Set of properties to return. Defaults to noAcl. userProject: The project to be billed for this request. """ class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to noAcl. Values: full: Include all properties. noAcl: Omit owner, acl and defaultObjectAcl properties. """ full = 0 noAcl = 1 maxResults = _messages.IntegerField(1, variant=_messages.Variant.UINT32, default=1000) pageToken = _messages.StringField(2) prefix = _messages.StringField(3) project = _messages.StringField(4, required=True) projection = _messages.EnumField('ProjectionValueValuesEnum', 5) userProject = _messages.StringField(6) class StorageBucketsListChannelsRequest(_messages.Message): r"""A StorageBucketsListChannelsRequest object. Fields: bucket: Name of a bucket. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) userProject = _messages.StringField(2) class StorageBucketsLockRetentionPolicyRequest(_messages.Message): r"""A StorageBucketsLockRetentionPolicyRequest object. Fields: bucket: Name of a bucket. ifMetagenerationMatch: Makes the operation conditional on whether bucket's current metageneration matches the given value. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) ifMetagenerationMatch = _messages.IntegerField(2, required=True) userProject = _messages.StringField(3) class StorageBucketsPatchRequest(_messages.Message): r"""A StorageBucketsPatchRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this bucket. PredefinedDefaultObjectAclValueValuesEnum: Apply a predefined set of default object access controls to this bucket. ProjectionValueValuesEnum: Set of properties to return. Defaults to full. Fields: bucket: Name of a bucket. bucketResource: A Bucket resource to be passed as the request body. ifMetagenerationMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration does not match the given value. predefinedAcl: Apply a predefined set of access controls to this bucket. predefinedDefaultObjectAcl: Apply a predefined set of default object access controls to this bucket. projection: Set of properties to return. Defaults to full. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ class PredefinedAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of access controls to this bucket. Values: authenticatedRead: Project team owners get OWNER access, and allAuthenticatedUsers get READER access. private: Project team owners get OWNER access. projectPrivate: Project team members get access according to their roles. publicRead: Project team owners get OWNER access, and allUsers get READER access. publicReadWrite: Project team owners get OWNER access, and allUsers get WRITER access. """ authenticatedRead = 0 private = 1 projectPrivate = 2 publicRead = 3 publicReadWrite = 4 class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of default object access controls to this bucket. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. """ authenticatedRead = 0 bucketOwnerFullControl = 1 bucketOwnerRead = 2 private = 3 projectPrivate = 4 publicRead = 5 class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to full. Values: full: Include all properties. noAcl: Omit owner, acl and defaultObjectAcl properties. """ full = 0 noAcl = 1 bucket = _messages.StringField(1, required=True) bucketResource = _messages.MessageField('Bucket', 2) ifMetagenerationMatch = _messages.IntegerField(3) ifMetagenerationNotMatch = _messages.IntegerField(4) predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 5) predefinedDefaultObjectAcl = _messages.EnumField( 'PredefinedDefaultObjectAclValueValuesEnum', 6) projection = _messages.EnumField('ProjectionValueValuesEnum', 7) userProject = _messages.StringField(8) class StorageBucketsSetIamPolicyRequest(_messages.Message): r"""A StorageBucketsSetIamPolicyRequest object. Fields: bucket: Name of a bucket. policy: A Policy resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) policy = _messages.MessageField('Policy', 2) userProject = _messages.StringField(3) class StorageBucketsTestIamPermissionsRequest(_messages.Message): r"""A StorageBucketsTestIamPermissionsRequest object. Fields: bucket: Name of a bucket. permissions: Permissions to test. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) permissions = _messages.StringField(2, required=True) userProject = _messages.StringField(3) class StorageBucketsUpdateRequest(_messages.Message): r"""A StorageBucketsUpdateRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this bucket. PredefinedDefaultObjectAclValueValuesEnum: Apply a predefined set of default object access controls to this bucket. ProjectionValueValuesEnum: Set of properties to return. Defaults to full. Fields: bucket: Name of a bucket. bucketResource: A Bucket resource to be passed as the request body. ifMetagenerationMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the return of the bucket metadata conditional on whether the bucket's current metageneration does not match the given value. predefinedAcl: Apply a predefined set of access controls to this bucket. predefinedDefaultObjectAcl: Apply a predefined set of default object access controls to this bucket. projection: Set of properties to return. Defaults to full. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ class PredefinedAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of access controls to this bucket. Values: authenticatedRead: Project team owners get OWNER access, and allAuthenticatedUsers get READER access. private: Project team owners get OWNER access. projectPrivate: Project team members get access according to their roles. publicRead: Project team owners get OWNER access, and allUsers get READER access. publicReadWrite: Project team owners get OWNER access, and allUsers get WRITER access. """ authenticatedRead = 0 private = 1 projectPrivate = 2 publicRead = 3 publicReadWrite = 4 class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of default object access controls to this bucket. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. """ authenticatedRead = 0 bucketOwnerFullControl = 1 bucketOwnerRead = 2 private = 3 projectPrivate = 4 publicRead = 5 class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to full. Values: full: Include all properties. noAcl: Omit owner, acl and defaultObjectAcl properties. """ full = 0 noAcl = 1 bucket = _messages.StringField(1, required=True) bucketResource = _messages.MessageField('Bucket', 2) ifMetagenerationMatch = _messages.IntegerField(3) ifMetagenerationNotMatch = _messages.IntegerField(4) predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 5) predefinedDefaultObjectAcl = _messages.EnumField( 'PredefinedDefaultObjectAclValueValuesEnum', 6) projection = _messages.EnumField('ProjectionValueValuesEnum', 7) userProject = _messages.StringField(8) class StorageChannelsStopResponse(_messages.Message): r"""An empty StorageChannelsStop response.""" class StorageDefaultObjectAccessControlsDeleteRequest(_messages.Message): r"""A StorageDefaultObjectAccessControlsDeleteRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) entity = _messages.StringField(2, required=True) userProject = _messages.StringField(3) class StorageDefaultObjectAccessControlsDeleteResponse(_messages.Message): r"""An empty StorageDefaultObjectAccessControlsDelete response.""" class StorageDefaultObjectAccessControlsGetRequest(_messages.Message): r"""A StorageDefaultObjectAccessControlsGetRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) entity = _messages.StringField(2, required=True) userProject = _messages.StringField(3) class StorageDefaultObjectAccessControlsInsertRequest(_messages.Message): r"""A StorageDefaultObjectAccessControlsInsertRequest object. Fields: bucket: Name of a bucket. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) objectAccessControl = _messages.MessageField('ObjectAccessControl', 2) userProject = _messages.StringField(3) class StorageDefaultObjectAccessControlsListRequest(_messages.Message): r"""A StorageDefaultObjectAccessControlsListRequest object. Fields: bucket: Name of a bucket. ifMetagenerationMatch: If present, only return default ACL listing if the bucket's current metageneration matches this value. ifMetagenerationNotMatch: If present, only return default ACL listing if the bucket's current metageneration does not match the given value. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) ifMetagenerationMatch = _messages.IntegerField(2) ifMetagenerationNotMatch = _messages.IntegerField(3) userProject = _messages.StringField(4) class StorageDefaultObjectAccessControlsPatchRequest(_messages.Message): r"""A StorageDefaultObjectAccessControlsPatchRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) entity = _messages.StringField(2, required=True) objectAccessControl = _messages.MessageField('ObjectAccessControl', 3) userProject = _messages.StringField(4) class StorageDefaultObjectAccessControlsUpdateRequest(_messages.Message): r"""A StorageDefaultObjectAccessControlsUpdateRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) entity = _messages.StringField(2, required=True) objectAccessControl = _messages.MessageField('ObjectAccessControl', 3) userProject = _messages.StringField(4) class StorageNotificationsDeleteRequest(_messages.Message): r"""A StorageNotificationsDeleteRequest object. Fields: bucket: The parent bucket of the notification. notification: ID of the notification to delete. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) notification = _messages.StringField(2, required=True) userProject = _messages.StringField(3) class StorageNotificationsDeleteResponse(_messages.Message): r"""An empty StorageNotificationsDelete response.""" class StorageNotificationsGetRequest(_messages.Message): r"""A StorageNotificationsGetRequest object. Fields: bucket: The parent bucket of the notification. notification: Notification ID userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) notification = _messages.StringField(2, required=True) userProject = _messages.StringField(3) class StorageNotificationsInsertRequest(_messages.Message): r"""A StorageNotificationsInsertRequest object. Fields: bucket: The parent bucket of the notification. notification: A Notification resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) notification = _messages.MessageField('Notification', 2) userProject = _messages.StringField(3) class StorageNotificationsListRequest(_messages.Message): r"""A StorageNotificationsListRequest object. Fields: bucket: Name of a Google Cloud Storage bucket. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) userProject = _messages.StringField(2) class StorageObjectAccessControlsDeleteRequest(_messages.Message): r"""A StorageObjectAccessControlsDeleteRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) entity = _messages.StringField(2, required=True) generation = _messages.IntegerField(3) object = _messages.StringField(4, required=True) userProject = _messages.StringField(5) class StorageObjectAccessControlsDeleteResponse(_messages.Message): r"""An empty StorageObjectAccessControlsDelete response.""" class StorageObjectAccessControlsGetRequest(_messages.Message): r"""A StorageObjectAccessControlsGetRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) entity = _messages.StringField(2, required=True) generation = _messages.IntegerField(3) object = _messages.StringField(4, required=True) userProject = _messages.StringField(5) class StorageObjectAccessControlsInsertRequest(_messages.Message): r"""A StorageObjectAccessControlsInsertRequest object. Fields: bucket: Name of a bucket. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) generation = _messages.IntegerField(2) object = _messages.StringField(3, required=True) objectAccessControl = _messages.MessageField('ObjectAccessControl', 4) userProject = _messages.StringField(5) class StorageObjectAccessControlsListRequest(_messages.Message): r"""A StorageObjectAccessControlsListRequest object. Fields: bucket: Name of a bucket. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) generation = _messages.IntegerField(2) object = _messages.StringField(3, required=True) userProject = _messages.StringField(4) class StorageObjectAccessControlsPatchRequest(_messages.Message): r"""A StorageObjectAccessControlsPatchRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) entity = _messages.StringField(2, required=True) generation = _messages.IntegerField(3) object = _messages.StringField(4, required=True) objectAccessControl = _messages.MessageField('ObjectAccessControl', 5) userProject = _messages.StringField(6) class StorageObjectAccessControlsUpdateRequest(_messages.Message): r"""A StorageObjectAccessControlsUpdateRequest object. Fields: bucket: Name of a bucket. entity: The entity holding the permission. Can be user-userId, user- emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. objectAccessControl: A ObjectAccessControl resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) entity = _messages.StringField(2, required=True) generation = _messages.IntegerField(3) object = _messages.StringField(4, required=True) objectAccessControl = _messages.MessageField('ObjectAccessControl', 5) userProject = _messages.StringField(6) class StorageObjectsComposeRequest(_messages.Message): r"""A StorageObjectsComposeRequest object. Enums: DestinationPredefinedAclValueValuesEnum: Apply a predefined set of access controls to the destination object. Fields: composeRequest: A ComposeRequest resource to be passed as the request body. destinationBucket: Name of the bucket containing the source objects. The destination object is stored in this bucket. destinationObject: Name of the new object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. destinationPredefinedAcl: Apply a predefined set of access controls to the destination object. ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. kmsKeyName: Resource name of the Cloud KMS key, of the form projects/my- project/locations/global/keyRings/my-kr/cryptoKeys/my-key, that will be used to encrypt the object. Overrides the object metadata's kms_key_name value, if any. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ class DestinationPredefinedAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of access controls to the destination object. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. """ authenticatedRead = 0 bucketOwnerFullControl = 1 bucketOwnerRead = 2 private = 3 projectPrivate = 4 publicRead = 5 composeRequest = _messages.MessageField('ComposeRequest', 1) destinationBucket = _messages.StringField(2, required=True) destinationObject = _messages.StringField(3, required=True) destinationPredefinedAcl = _messages.EnumField( 'DestinationPredefinedAclValueValuesEnum', 4) ifGenerationMatch = _messages.IntegerField(5) ifMetagenerationMatch = _messages.IntegerField(6) kmsKeyName = _messages.StringField(7) userProject = _messages.StringField(8) class StorageObjectsCopyRequest(_messages.Message): r"""A StorageObjectsCopyRequest object. Enums: DestinationPredefinedAclValueValuesEnum: Apply a predefined set of access controls to the destination object. ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. Fields: destinationBucket: Name of the bucket in which to store the new object. Overrides the provided object metadata's bucket value, if any.For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. destinationObject: Name of the new object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. destinationPredefinedAcl: Apply a predefined set of access controls to the destination object. ifGenerationMatch: Makes the operation conditional on whether the destination object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the destination object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the destination object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the destination object's current metageneration does not match the given value. ifSourceGenerationMatch: Makes the operation conditional on whether the source object's current generation matches the given value. ifSourceGenerationNotMatch: Makes the operation conditional on whether the source object's current generation does not match the given value. ifSourceMetagenerationMatch: Makes the operation conditional on whether the source object's current metageneration matches the given value. ifSourceMetagenerationNotMatch: Makes the operation conditional on whether the source object's current metageneration does not match the given value. object: A Object resource to be passed as the request body. projection: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. sourceBucket: Name of the bucket in which to find the source object. sourceGeneration: If present, selects a specific revision of the source object (as opposed to the latest version, the default). sourceObject: Name of the source object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ class DestinationPredefinedAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of access controls to the destination object. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. """ authenticatedRead = 0 bucketOwnerFullControl = 1 bucketOwnerRead = 2 private = 3 projectPrivate = 4 publicRead = 5 class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. Values: full: Include all properties. noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 destinationBucket = _messages.StringField(1, required=True) destinationObject = _messages.StringField(2, required=True) destinationPredefinedAcl = _messages.EnumField( 'DestinationPredefinedAclValueValuesEnum', 3) ifGenerationMatch = _messages.IntegerField(4) ifGenerationNotMatch = _messages.IntegerField(5) ifMetagenerationMatch = _messages.IntegerField(6) ifMetagenerationNotMatch = _messages.IntegerField(7) ifSourceGenerationMatch = _messages.IntegerField(8) ifSourceGenerationNotMatch = _messages.IntegerField(9) ifSourceMetagenerationMatch = _messages.IntegerField(10) ifSourceMetagenerationNotMatch = _messages.IntegerField(11) object = _messages.MessageField('Object', 12) projection = _messages.EnumField('ProjectionValueValuesEnum', 13) sourceBucket = _messages.StringField(14, required=True) sourceGeneration = _messages.IntegerField(15) sourceObject = _messages.StringField(16, required=True) userProject = _messages.StringField(17) class StorageObjectsDeleteRequest(_messages.Message): r"""A StorageObjectsDeleteRequest object. Fields: bucket: Name of the bucket in which the object resides. generation: If present, permanently deletes a specific revision of this object (as opposed to the latest version, the default). ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) generation = _messages.IntegerField(2) ifGenerationMatch = _messages.IntegerField(3) ifGenerationNotMatch = _messages.IntegerField(4) ifMetagenerationMatch = _messages.IntegerField(5) ifMetagenerationNotMatch = _messages.IntegerField(6) object = _messages.StringField(7, required=True) userProject = _messages.StringField(8) class StorageObjectsDeleteResponse(_messages.Message): r"""An empty StorageObjectsDelete response.""" class StorageObjectsGetIamPolicyRequest(_messages.Message): r"""A StorageObjectsGetIamPolicyRequest object. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) generation = _messages.IntegerField(2) object = _messages.StringField(3, required=True) userProject = _messages.StringField(4) class StorageObjectsGetRequest(_messages.Message): r"""A StorageObjectsGetRequest object. Enums: ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. projection: Set of properties to return. Defaults to noAcl. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to noAcl. Values: full: Include all properties. noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 bucket = _messages.StringField(1, required=True) generation = _messages.IntegerField(2) ifGenerationMatch = _messages.IntegerField(3) ifGenerationNotMatch = _messages.IntegerField(4) ifMetagenerationMatch = _messages.IntegerField(5) ifMetagenerationNotMatch = _messages.IntegerField(6) object = _messages.StringField(7, required=True) projection = _messages.EnumField('ProjectionValueValuesEnum', 8) userProject = _messages.StringField(9) class StorageObjectsInsertRequest(_messages.Message): r"""A StorageObjectsInsertRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this object. ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. Fields: bucket: Name of the bucket in which to store the new object. Overrides the provided object metadata's bucket value, if any. contentEncoding: If set, sets the contentEncoding property of the final object to this value. Setting this parameter is equivalent to setting the contentEncoding metadata property. This can be useful when uploading an object with uploadType=media to indicate the encoding of the content being uploaded. ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. kmsKeyName: Resource name of the Cloud KMS key, of the form projects/my- project/locations/global/keyRings/my-kr/cryptoKeys/my-key, that will be used to encrypt the object. Overrides the object metadata's kms_key_name value, if any. name: Name of the object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. object: A Object resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ class PredefinedAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of access controls to this object. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. """ authenticatedRead = 0 bucketOwnerFullControl = 1 bucketOwnerRead = 2 private = 3 projectPrivate = 4 publicRead = 5 class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. Values: full: Include all properties. noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 bucket = _messages.StringField(1, required=True) contentEncoding = _messages.StringField(2) ifGenerationMatch = _messages.IntegerField(3) ifGenerationNotMatch = _messages.IntegerField(4) ifMetagenerationMatch = _messages.IntegerField(5) ifMetagenerationNotMatch = _messages.IntegerField(6) kmsKeyName = _messages.StringField(7) name = _messages.StringField(8) object = _messages.MessageField('Object', 9) predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 10) projection = _messages.EnumField('ProjectionValueValuesEnum', 11) userProject = _messages.StringField(12) class StorageObjectsListRequest(_messages.Message): r"""A StorageObjectsListRequest object. Enums: ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. Fields: bucket: Name of the bucket in which to look for objects. delimiter: Returns results in a directory-like mode. items will contain only objects whose names, aside from the prefix, do not contain delimiter. Objects whose names, aside from the prefix, contain delimiter will have their name, truncated after the delimiter, returned in prefixes. Duplicate prefixes are omitted. includeTrailingDelimiter: If true, objects that end in exactly one instance of delimiter will have their metadata included in items in addition to prefixes. maxResults: Maximum number of items plus prefixes to return in a single page of responses. As duplicate prefixes are omitted, fewer total results may be returned than requested. The service will use this parameter or 1,000 items, whichever is smaller. pageToken: A previously-returned page token representing part of the larger set of results to view. prefix: Filter results to objects whose names begin with this prefix. projection: Set of properties to return. Defaults to noAcl. userProject: The project to be billed for this request. Required for Requester Pays buckets. versions: If true, lists all versions of an object as distinct results. The default is false. For more information, see Object Versioning. """ class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to noAcl. Values: full: Include all properties. noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 bucket = _messages.StringField(1, required=True) delimiter = _messages.StringField(2) includeTrailingDelimiter = _messages.BooleanField(3) maxResults = _messages.IntegerField(4, variant=_messages.Variant.UINT32, default=1000) pageToken = _messages.StringField(5) prefix = _messages.StringField(6) projection = _messages.EnumField('ProjectionValueValuesEnum', 7) userProject = _messages.StringField(8) versions = _messages.BooleanField(9) class StorageObjectsPatchRequest(_messages.Message): r"""A StorageObjectsPatchRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this object. ProjectionValueValuesEnum: Set of properties to return. Defaults to full. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. objectResource: A Object resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to full. userProject: The project to be billed for this request, for Requester Pays buckets. """ class PredefinedAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of access controls to this object. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. """ authenticatedRead = 0 bucketOwnerFullControl = 1 bucketOwnerRead = 2 private = 3 projectPrivate = 4 publicRead = 5 class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to full. Values: full: Include all properties. noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 bucket = _messages.StringField(1, required=True) generation = _messages.IntegerField(2) ifGenerationMatch = _messages.IntegerField(3) ifGenerationNotMatch = _messages.IntegerField(4) ifMetagenerationMatch = _messages.IntegerField(5) ifMetagenerationNotMatch = _messages.IntegerField(6) object = _messages.StringField(7, required=True) objectResource = _messages.MessageField('Object', 8) predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 9) projection = _messages.EnumField('ProjectionValueValuesEnum', 10) userProject = _messages.StringField(11) class StorageObjectsRewriteRequest(_messages.Message): r"""A StorageObjectsRewriteRequest object. Enums: DestinationPredefinedAclValueValuesEnum: Apply a predefined set of access controls to the destination object. ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. Fields: destinationBucket: Name of the bucket in which to store the new object. Overrides the provided object metadata's bucket value, if any. destinationKmsKeyName: Resource name of the Cloud KMS key, of the form projects/my-project/locations/global/keyRings/my-kr/cryptoKeys/my-key, that will be used to encrypt the object. Overrides the object metadata's kms_key_name value, if any. destinationObject: Name of the new object. Required when the object metadata is not otherwise provided. Overrides the object metadata's name value, if any. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. destinationPredefinedAcl: Apply a predefined set of access controls to the destination object. ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the destination object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the destination object's current metageneration does not match the given value. ifSourceGenerationMatch: Makes the operation conditional on whether the source object's current generation matches the given value. ifSourceGenerationNotMatch: Makes the operation conditional on whether the source object's current generation does not match the given value. ifSourceMetagenerationMatch: Makes the operation conditional on whether the source object's current metageneration matches the given value. ifSourceMetagenerationNotMatch: Makes the operation conditional on whether the source object's current metageneration does not match the given value. maxBytesRewrittenPerCall: The maximum number of bytes that will be rewritten per rewrite request. Most callers shouldn't need to specify this parameter - it is primarily in place to support testing. If specified the value must be an integral multiple of 1 MiB (1048576). Also, this only applies to requests where the source and destination span locations and/or storage classes. Finally, this value must not change across rewrite calls else you'll get an error that the rewriteToken is invalid. object: A Object resource to be passed as the request body. projection: Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. rewriteToken: Include this field (from the previous rewrite response) on each rewrite request after the first one, until the rewrite response 'done' flag is true. Calls that provide a rewriteToken can omit all other request fields, but if included those fields must match the values provided in the first rewrite request. sourceBucket: Name of the bucket in which to find the source object. sourceGeneration: If present, selects a specific revision of the source object (as opposed to the latest version, the default). sourceObject: Name of the source object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ class DestinationPredefinedAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of access controls to the destination object. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. """ authenticatedRead = 0 bucketOwnerFullControl = 1 bucketOwnerRead = 2 private = 3 projectPrivate = 4 publicRead = 5 class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full. Values: full: Include all properties. noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 destinationBucket = _messages.StringField(1, required=True) destinationKmsKeyName = _messages.StringField(2) destinationObject = _messages.StringField(3, required=True) destinationPredefinedAcl = _messages.EnumField( 'DestinationPredefinedAclValueValuesEnum', 4) ifGenerationMatch = _messages.IntegerField(5) ifGenerationNotMatch = _messages.IntegerField(6) ifMetagenerationMatch = _messages.IntegerField(7) ifMetagenerationNotMatch = _messages.IntegerField(8) ifSourceGenerationMatch = _messages.IntegerField(9) ifSourceGenerationNotMatch = _messages.IntegerField(10) ifSourceMetagenerationMatch = _messages.IntegerField(11) ifSourceMetagenerationNotMatch = _messages.IntegerField(12) maxBytesRewrittenPerCall = _messages.IntegerField(13) object = _messages.MessageField('Object', 14) projection = _messages.EnumField('ProjectionValueValuesEnum', 15) rewriteToken = _messages.StringField(16) sourceBucket = _messages.StringField(17, required=True) sourceGeneration = _messages.IntegerField(18) sourceObject = _messages.StringField(19, required=True) userProject = _messages.StringField(20) class StorageObjectsSetIamPolicyRequest(_messages.Message): r"""A StorageObjectsSetIamPolicyRequest object. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. policy: A Policy resource to be passed as the request body. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) generation = _messages.IntegerField(2) object = _messages.StringField(3, required=True) policy = _messages.MessageField('Policy', 4) userProject = _messages.StringField(5) class StorageObjectsTestIamPermissionsRequest(_messages.Message): r"""A StorageObjectsTestIamPermissionsRequest object. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. permissions: Permissions to test. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ bucket = _messages.StringField(1, required=True) generation = _messages.IntegerField(2) object = _messages.StringField(3, required=True) permissions = _messages.StringField(4, required=True) userProject = _messages.StringField(5) class StorageObjectsUpdateRequest(_messages.Message): r"""A StorageObjectsUpdateRequest object. Enums: PredefinedAclValueValuesEnum: Apply a predefined set of access controls to this object. ProjectionValueValuesEnum: Set of properties to return. Defaults to full. Fields: bucket: Name of the bucket in which the object resides. generation: If present, selects a specific revision of this object (as opposed to the latest version, the default). ifGenerationMatch: Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object. ifGenerationNotMatch: Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object. ifMetagenerationMatch: Makes the operation conditional on whether the object's current metageneration matches the given value. ifMetagenerationNotMatch: Makes the operation conditional on whether the object's current metageneration does not match the given value. object: Name of the object. For information about how to URL encode object names to be path safe, see Encoding URI Path Parts. objectResource: A Object resource to be passed as the request body. predefinedAcl: Apply a predefined set of access controls to this object. projection: Set of properties to return. Defaults to full. userProject: The project to be billed for this request. Required for Requester Pays buckets. """ class PredefinedAclValueValuesEnum(_messages.Enum): r"""Apply a predefined set of access controls to this object. Values: authenticatedRead: Object owner gets OWNER access, and allAuthenticatedUsers get READER access. bucketOwnerFullControl: Object owner gets OWNER access, and project team owners get OWNER access. bucketOwnerRead: Object owner gets OWNER access, and project team owners get READER access. private: Object owner gets OWNER access. projectPrivate: Object owner gets OWNER access, and project team members get access according to their roles. publicRead: Object owner gets OWNER access, and allUsers get READER access. """ authenticatedRead = 0 bucketOwnerFullControl = 1 bucketOwnerRead = 2 private = 3 projectPrivate = 4 publicRead = 5 class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to full. Values: full: Include all properties. noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 bucket = _messages.StringField(1, required=True) generation = _messages.IntegerField(2) ifGenerationMatch = _messages.IntegerField(3) ifGenerationNotMatch = _messages.IntegerField(4) ifMetagenerationMatch = _messages.IntegerField(5) ifMetagenerationNotMatch = _messages.IntegerField(6) object = _messages.StringField(7, required=True) objectResource = _messages.MessageField('Object', 8) predefinedAcl = _messages.EnumField('PredefinedAclValueValuesEnum', 9) projection = _messages.EnumField('ProjectionValueValuesEnum', 10) userProject = _messages.StringField(11) class StorageObjectsWatchAllRequest(_messages.Message): r"""A StorageObjectsWatchAllRequest object. Enums: ProjectionValueValuesEnum: Set of properties to return. Defaults to noAcl. Fields: bucket: Name of the bucket in which to look for objects. channel: A Channel resource to be passed as the request body. delimiter: Returns results in a directory-like mode. items will contain only objects whose names, aside from the prefix, do not contain delimiter. Objects whose names, aside from the prefix, contain delimiter will have their name, truncated after the delimiter, returned in prefixes. Duplicate prefixes are omitted. includeTrailingDelimiter: If true, objects that end in exactly one instance of delimiter will have their metadata included in items in addition to prefixes. maxResults: Maximum number of items plus prefixes to return in a single page of responses. As duplicate prefixes are omitted, fewer total results may be returned than requested. The service will use this parameter or 1,000 items, whichever is smaller. pageToken: A previously-returned page token representing part of the larger set of results to view. prefix: Filter results to objects whose names begin with this prefix. projection: Set of properties to return. Defaults to noAcl. userProject: The project to be billed for this request. Required for Requester Pays buckets. versions: If true, lists all versions of an object as distinct results. The default is false. For more information, see Object Versioning. """ class ProjectionValueValuesEnum(_messages.Enum): r"""Set of properties to return. Defaults to noAcl. Values: full: Include all properties. noAcl: Omit the owner, acl property. """ full = 0 noAcl = 1 bucket = _messages.StringField(1, required=True) channel = _messages.MessageField('Channel', 2) delimiter = _messages.StringField(3) includeTrailingDelimiter = _messages.BooleanField(4) maxResults = _messages.IntegerField(5, variant=_messages.Variant.UINT32, default=1000) pageToken = _messages.StringField(6) prefix = _messages.StringField(7) projection = _messages.EnumField('ProjectionValueValuesEnum', 8) userProject = _messages.StringField(9) versions = _messages.BooleanField(10) class StorageProjectsServiceAccountGetRequest(_messages.Message): r"""A StorageProjectsServiceAccountGetRequest object. Fields: projectId: Project ID userProject: The project to be billed for this request. """ projectId = _messages.StringField(1, required=True) userProject = _messages.StringField(2) class TestIamPermissionsResponse(_messages.Message): r"""A storage.(buckets|objects).testIamPermissions response. Fields: kind: The kind of item this is. permissions: The permissions held by the caller. Permissions are always of the format storage.resource.capability, where resource is one of buckets or objects. The supported permissions are as follows: - storage.buckets.delete - Delete bucket. - storage.buckets.get - Read bucket metadata. - storage.buckets.getIamPolicy - Read bucket IAM policy. - storage.buckets.create - Create bucket. - storage.buckets.list - List buckets. - storage.buckets.setIamPolicy - Update bucket IAM policy. - storage.buckets.update - Update bucket metadata. - storage.objects.delete - Delete object. - storage.objects.get - Read object data and metadata. - storage.objects.getIamPolicy - Read object IAM policy. - storage.objects.create - Create object. - storage.objects.list - List objects. - storage.objects.setIamPolicy - Update object IAM policy. - storage.objects.update - Update object metadata. """ kind = _messages.StringField(1, default=u'storage#testIamPermissionsResponse') permissions = _messages.StringField(2, repeated=True) class StorageProjectsHmacKeysCreateRequest(_messages.Message): """A StorageProjectsHmacKeysCreateRequest object. Fields: projectId: Project ID serviceAccountEmail: Email address of the service account for which to create a key. """ projectId = _messages.StringField(1, required=True) serviceAccountEmail = _messages.StringField(2, required=True) class StorageProjectsHmacKeysDeleteRequest(_messages.Message): """A StorageProjectsHmacKeysDeleteRequest object. Fields: projectId: Project ID accessId: Name of the HMAC key to be deleted. """ projectId = _messages.StringField(1, required=True) accessId = _messages.StringField(2, required=True) class StorageProjectsHmacKeysGetRequest(_messages.Message): """A StorageProjectsHmacKeysGetRequest object. Fields: projectId: Project ID accessId: Name of the HMAC key for which the metadata is being requested. """ projectId = _messages.StringField(2, required=True) accessId = _messages.StringField(1, required=True) class StorageProjectsHmacKeysListRequest(_messages.Message): """A StorageProjectsHmacKeysListRequest object. Fields: projectId: Project ID serviceAccountEmail: If present only, keys for the given service account will be returned. showDeletedKeys: Whether or not to show keys in the DELETED state. maxResults: Maximum number of items to return in a single page of responses. The service will use this parameter or 1,000 items, whichever is smaller. pageToken: A previously-returned page token representing part of the larger set of results to view. """ projectId = _messages.StringField(1, required=True) serviceAccountEmail = _messages.StringField(2) showDeletedKeys = _messages.BooleanField(3) maxResults = _messages.IntegerField(4, variant=_messages.Variant.UINT32, default=1000) pageToken = _messages.StringField(5) class StorageProjectsHmacKeysUpdateRequest(_messages.Message): """A StorageProjectsHmacKeysUpdateRequest object. Fields: projectId: Project ID accessId: Name of the HMAC key being updated. """ projectId = _messages.StringField(1, required=True) accessId = _messages.StringField(2, required=True) resource = _messages.MessageField('HmacKeyMetadata', 3) class HmacKeyMetadata(_messages.Message): """An HMAC Key Metadata resource. Fields: id: The ID of the HMAC key, including the Project ID and the Access ID. selfLink: The link to this resource. accessId: The ID of the HMAC Key. projectId: Project ID owning the service account to which the key authenticates. serviceAccountEmail: The email address of the key's associated service account. state: The state of the key. Can be one of ACTIVE, INACTIVE, or DELETED. timeCreated: The creation time of the key in RFC 3339 format. updated: The modification time of the key in RFC 3339 format. etag: HTTP 1.1 Entity tag for the key. kind: The kind of item this is. For HMAC Key metadata, this is always storage#hmacKeyMetadata """ id = _messages.StringField(1) selfLink = _messages.StringField(2) accessId = _messages.StringField(3) projectId = _messages.StringField(4) serviceAccountEmail = _messages.StringField(5) state = _messages.StringField(6) timeCreated = _message_types.DateTimeField(7) updated = _message_types.DateTimeField(8) etag = _messages.StringField(9) kind = _messages.StringField(10, default=u'storage#hmacKeyMetadata') class HmacKey(_messages.Message): """An HMAC Key resource. Fields: metadata: Key metadata. secret: HMAC secret key material. kind: The kind of item this is. For HMAC keys, this is always storage#hmacKey. """ metadata = _messages.MessageField('HmacKeyMetadata', 1) secret = _messages.StringField(2) kind = _messages.StringField(3, default=u'storage#hmacKey') class HmacKeyMetadataList(_messages.Message): """A list of HMAC key metadata. Fields: items: The list of HMAC key metadata. kind: The kind of item this is. For a list of HMAC key, this is always storage#hmacKeysMetadata. nextPageToken: The continuation token, used to page through large result sets. Provide this value in a subsequent request to return the next page of results. """ items = _messages.MessageField('HmacKeyMetadata', 1, repeated=True) kind = _messages.StringField(2, default=u'storage#hmacKeysMetadata') nextPageToken = _messages.StringField(3) class HmacKeysDeleteResponse(_messages.Message): """An empty HmacKeysDelete response.""" third_party/storage_apitools/__init__.py 0000644 00000000000 15025013616 0014551 0 ustar 00 third_party/__init__.py 0000644 00000000000 15025013616 0011173 0 ustar 00 third_party/kms_apitools/__pycache__/cloudkms_v1_messages.cpython-39.pyc 0000644 00000122472 15025013616 0022567 0 ustar 00 a +(Wg�� � @ s� d Z ddlmZ ddlmZ ddlmZ dZG dd� dej�Z G dd � d ej�Z G d d � d ej�Z G d d � d ej�Z G dd� dej�Z G dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG dd� dej�ZG d d � d ej�ZG d d � d ej�ZG d d!� d!ej�ZG d"d#� d#ej�ZG d$d%� d%ej�ZG d&d'� d'ej�ZG d(d)� d)ej�ZG d*d+� d+ej�ZG d,d-� d-ej�Z G d.d/� d/ej�Z G d0d1� d1ej�Z G d2d3� d3ej�Z G d4d5� d5ej�Z G d6d7� d7ej�Z!G d8d9� d9ej�Z"G d:d;� d;ej�Z#G d<d=� d=ej�Z$G d>d?� d?ej�Z%G d@dA� dAej�Z&G dBdC� dCej�Z'G dDdE� dEej�Z(G dFdG� dGej�Z)G dHdI� dIej�Z*G dJdK� dKej�Z+G dLdM� dMej�Z,G dNdO� dOej�Z-G dPdQ� dQej�Z.G dRdS� dSej�Z/G dTdU� dUej�Z0G dVdW� dWej�Z1G dXdY� dYej�Z2G dZd[� d[ej�Z3G d\d]� d]ej�Z4G d^d_� d_ej�Z5G d`da� daej�Z6G dbdc� dcej�Z7G ddde� deej�Z8e�9e5dfdg� e�:e5j;dhdi� e�:e5j;djdk� dlS )mz�Generated message classes for cloudkms version v1. Manages encryption for your cloud services the same way you do on-premises. You can generate, use, rotate, and destroy AES256 encryption keys. � )�messages)�encoding)� extra_types�cloudkmsc @ s8 e Zd ZdZejdddd�Zejddd�Ze�d�Z dS ) � AuditConfiga� Specifies the audit configuration for a service. The configuration determines which permission types are logged, and what identities, if any, are exempted from logging. An AuditConfig must have one or more AuditLogConfigs. If there are AuditConfigs for both `allServices` and a specific service, the union of the two AuditConfigs is used for that service: the log_types specified in each AuditConfig are enabled, and the exempted_members in each AuditConfig are exempted. Example Policy with multiple AuditConfigs: { "audit_configs": [ { "service": "allServices" "audit_log_configs": [ { "log_type": "DATA_READ", "exempted_members": [ "user:foo@gmail.com" ] }, { "log_type": "DATA_WRITE", }, { "log_type": "ADMIN_READ", } ] }, { "service": "fooservice.googleapis.com" "audit_log_configs": [ { "log_type": "DATA_READ", }, { "log_type": "DATA_WRITE", "exempted_members": [ "user:bar@gmail.com" ] } ] } ] } For fooservice, this policy enables DATA_READ, DATA_WRITE and ADMIN_READ logging. It also exempts foo@gmail.com from DATA_READ logging, and bar@gmail.com from DATA_WRITE logging. Fields: auditLogConfigs: The configuration for logging of each type of permission. Next ID: 4 exemptedMembers: A string attribute. service: Specifies a service that will be enabled for audit logging. For example, `storage.googleapis.com`, `cloudsql.googleapis.com`. `allServices` is a special value that covers all services. �AuditLogConfig� T��repeated� � N) �__name__� __module__� __qualname__�__doc__� _messages� MessageFieldZauditLogConfigs� StringField�exemptedMembers�service� r r �B/opt/gsutil/gslib/third_party/kms_apitools/cloudkms_v1_messages.pyr s r c @ s< e Zd ZdZG dd� dej�Zejddd�Ze� dd�Z dS ) r a� Provides the configuration for logging a type of permissions. Example: { "audit_log_configs": [ { "log_type": "DATA_READ", "exempted_members": [ "user:foo@gmail.com" ] }, { "log_type": "DATA_WRITE", } ] } This enables 'DATA_READ' and 'DATA_WRITE' logging, while exempting foo@gmail.com from DATA_READ logging. Enums: LogTypeValueValuesEnum: The log type that this config enables. Fields: exemptedMembers: Specifies the identities that do not cause logging for this type of permission. Follows the same format of Binding.members. logType: The log type that this config enables. c @ s e Zd ZdZdZdZdZdZdS )z%AuditLogConfig.LogTypeValueValuesEnuma. The log type that this config enables. Values: LOG_TYPE_UNSPECIFIED: Default case. Should never be this. ADMIN_READ: Admin reads. Example: CloudIAM getIamPolicy DATA_WRITE: Data writes. Example: CloudSQL Users create DATA_READ: Data reads. Example: CloudSQL Users list r r r r N)r r r r ZLOG_TYPE_UNSPECIFIEDZ ADMIN_READZ DATA_WRITEZ DATA_READr r r r �LogTypeValueValuesEnumE s r r Tr r N) r r r r r �Enumr r r � EnumFieldZlogTyper r r r r 4 s r c @ s4 e Zd ZdZe�dd�Zejddd�Ze�d�Z dS ) �Bindinga� Associates `members` with a `role`. Fields: condition: The condition that is associated with this binding. NOTE: an unsatisfied condition will not allow user access via current binding. Different bindings, including their conditions, are examined independently. This field is GOOGLE_INTERNAL. members: Specifies the identities requesting access for a Cloud Platform resource. `members` can have the following values: * `allUsers`: A special identifier that represents anyone who is on the internet; with or without a Google account. * `allAuthenticatedUsers`: A special identifier that represents anyone who is authenticated with a Google account or a service account. * `user:{emailid}`: An email address that represents a specific Google account. For example, `alice@gmail.com` or `joe@example.com`. * `serviceAccount:{emailid}`: An email address that represents a service account. For example, `my-other- app@appspot.gserviceaccount.com`. * `group:{emailid}`: An email address that represents a Google group. For example, `admins@example.com`. * `domain:{domain}`: A Google Apps domain name that represents all the users of that domain. For example, `google.com` or `example.com`. role: Role that is assigned to `members`. For example, `roles/viewer`, `roles/editor`, or `roles/owner`. Required �Exprr r Tr r N) r r r r r r � conditionr �members�roler r r r r W s r c @ s e Zd ZdZejddd�ZdS )�#CloudkmsProjectsLocationsGetRequestzeA CloudkmsProjectsLocationsGetRequest object. Fields: name: Resource name for the location. r T��requiredN�r r r r r r �namer r r r r u s r c @ s4 e Zd ZdZe�dd�Ze�d�Zejddd�Z dS ) �.CloudkmsProjectsLocationsKeyRingsCreateRequesta� A CloudkmsProjectsLocationsKeyRingsCreateRequest object. Fields: keyRing: A KeyRing resource to be passed as the request body. keyRingId: Required. It must be unique within a location and match the regular expression `[a-zA-Z0-9_-]{1,63}` parent: Required. The resource name of the location associated with the KeyRings, in the format `projects/*/locations/*`. �KeyRingr r r Tr! N) r r r r r r �keyRingr � keyRingId�parentr r r r r% s r% c @ s4 e Zd ZdZe�dd�Ze�d�Zejddd�Z dS ) �8CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequesta_ A CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequest object. Fields: cryptoKey: A CryptoKey resource to be passed as the request body. cryptoKeyId: Required. It must be unique within a KeyRing and match the regular expression `[a-zA-Z0-9_-]{1,63}` parent: Required. The name of the KeyRing associated with the CryptoKeys. � CryptoKeyr r r Tr! N) r r r r r r � cryptoKeyr � cryptoKeyIdr) r r r r r* � s r* c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�ICloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsCreateRequesta A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsCreateRequest object. Fields: cryptoKeyVersion: A CryptoKeyVersion resource to be passed as the request body. parent: Required. The name of the CryptoKey associated with the CryptoKeyVersions. �CryptoKeyVersionr r Tr! N) r r r r r r �cryptoKeyVersionr r) r r r r r. � s r. c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�JCloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsDestroyRequesta A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsDestroyRequest object. Fields: destroyCryptoKeyVersionRequest: A DestroyCryptoKeyVersionRequest resource to be passed as the request body. name: The resource name of the CryptoKeyVersion to destroy. � DestroyCryptoKeyVersionRequestr r Tr! N) r r r r r r � destroyCryptoKeyVersionRequestr r$ r r r r r1 � s r1 c @ s e Zd ZdZejddd�ZdS )�FCloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsGetRequestz�A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsGetRequest object. Fields: name: The name of the CryptoKeyVersion to get. r Tr! Nr# r r r r r4 � s r4 c @ s: e Zd ZdZejdejjd�Ze� d�Z ej ddd�Z dS ) �GCloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsListRequesta� A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsListRequest object. Fields: pageSize: Optional limit on the number of CryptoKeyVersions to include in the response. Further CryptoKeyVersions can subsequently be obtained by including the ListCryptoKeyVersionsResponse.next_page_token in a subsequent request. If unspecified, the server will pick an appropriate default. pageToken: Optional pagination token, returned earlier via ListCryptoKeyVersionsResponse.next_page_token. parent: Required. The resource name of the CryptoKey to list, in the format `projects/*/locations/*/keyRings/*/cryptoKeys/*`. r ��variantr r Tr! N� r r r r r � IntegerField�Variant�INT32�pageSizer � pageTokenr) r r r r r5 � s r5 c @ s4 e Zd ZdZe�dd�Zejddd�Ze�d�Z dS ) �HCloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsPatchRequesta� A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsPatchRequest object. Fields: cryptoKeyVersion: A CryptoKeyVersion resource to be passed as the request body. name: Output only. The resource name for this CryptoKeyVersion in the format `projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*`. updateMask: Required list of fields to be updated in this request. r/ r r Tr! r N) r r r r r r r0 r r$ � updateMaskr r r r r> � s r> c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�JCloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsRestoreRequesta A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsRestoreRequest object. Fields: name: The resource name of the CryptoKeyVersion to restore. restoreCryptoKeyVersionRequest: A RestoreCryptoKeyVersionRequest resource to be passed as the request body. r Tr! � RestoreCryptoKeyVersionRequestr N) r r r r r r r$ r � restoreCryptoKeyVersionRequestr r r r r@ � s r@ c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�9CloudkmsProjectsLocationsKeyRingsCryptoKeysDecryptRequesta+ A CloudkmsProjectsLocationsKeyRingsCryptoKeysDecryptRequest object. Fields: decryptRequest: A DecryptRequest resource to be passed as the request body. name: Required. The resource name of the CryptoKey to use for decryption. The server will choose the appropriate version. �DecryptRequestr r Tr! N) r r r r r r �decryptRequestr r$ r r r r rC � s rC c @ s* e Zd ZdZe�dd�Zejddd�ZdS )�9CloudkmsProjectsLocationsKeyRingsCryptoKeysEncryptRequesta\ A CloudkmsProjectsLocationsKeyRingsCryptoKeysEncryptRequest object. Fields: encryptRequest: A EncryptRequest resource to be passed as the request body. name: Required. The resource name of the CryptoKey or CryptoKeyVersion to use for encryption. If a CryptoKey is specified, the server will use its primary version. �EncryptRequestr r Tr! N) r r r r r r �encryptRequestr r$ r r r r rF s rF c @ s e Zd ZdZejddd�ZdS )�>CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequestz�A CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being requested. See the operation documentation for the appropriate value for this field. r Tr! N�r r r r r r �resourcer r r r rI s rI c @ s e Zd ZdZejddd�ZdS )�5CloudkmsProjectsLocationsKeyRingsCryptoKeysGetRequestzyA CloudkmsProjectsLocationsKeyRingsCryptoKeysGetRequest object. Fields: name: The name of the CryptoKey to get. r Tr! Nr# r r r r rL ( s rL c @ s: e Zd ZdZejdejjd�Ze� d�Z ej ddd�Z dS ) �6CloudkmsProjectsLocationsKeyRingsCryptoKeysListRequesta[ A CloudkmsProjectsLocationsKeyRingsCryptoKeysListRequest object. Fields: pageSize: Optional limit on the number of CryptoKeys to include in the response. Further CryptoKeys can subsequently be obtained by including the ListCryptoKeysResponse.next_page_token in a subsequent request. If unspecified, the server will pick an appropriate default. pageToken: Optional pagination token, returned earlier via ListCryptoKeysResponse.next_page_token. parent: Required. The resource name of the KeyRing to list, in the format `projects/*/locations/*/keyRings/*`. r r6 r r Tr! Nr8 r r r r rM 2 s rM c @ s4 e Zd ZdZe�dd�Zejddd�Ze�d�Z dS ) �7CloudkmsProjectsLocationsKeyRingsCryptoKeysPatchRequesta^ A CloudkmsProjectsLocationsKeyRingsCryptoKeysPatchRequest object. Fields: cryptoKey: A CryptoKey resource to be passed as the request body. name: Output only. The resource name for this CryptoKey in the format `projects/*/locations/*/keyRings/*/cryptoKeys/*`. updateMask: Required list of fields to be updated in this request. r+ r r Tr! r N) r r r r r r r, r r$ r? r r r r rN E s rN c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�>CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequestaZ A CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being specified. See the operation documentation for the appropriate value for this field. setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the request body. r Tr! �SetIamPolicyRequestr N� r r r r r r rK r �setIamPolicyRequestr r r r rO T s rO c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�DCloudkmsProjectsLocationsKeyRingsCryptoKeysTestIamPermissionsRequestau A CloudkmsProjectsLocationsKeyRingsCryptoKeysTestIamPermissionsRequest object. Fields: resource: REQUIRED: The resource for which the policy detail is being requested. See the operation documentation for the appropriate value for this field. testIamPermissionsRequest: A TestIamPermissionsRequest resource to be passed as the request body. r Tr! �TestIamPermissionsRequestr N� r r r r r r rK r �testIamPermissionsRequestr r r r rS c s rS c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�FCloudkmsProjectsLocationsKeyRingsCryptoKeysUpdatePrimaryVersionRequesta A CloudkmsProjectsLocationsKeyRingsCryptoKeysUpdatePrimaryVersionRequest object. Fields: name: The resource name of the CryptoKey to update. updateCryptoKeyPrimaryVersionRequest: A UpdateCryptoKeyPrimaryVersionRequest resource to be passed as the request body. r Tr! �$UpdateCryptoKeyPrimaryVersionRequestr N) r r r r r r r$ r �$updateCryptoKeyPrimaryVersionRequestr r r r rW s s rW c @ s e Zd ZdZejddd�ZdS )�4CloudkmsProjectsLocationsKeyRingsGetIamPolicyRequestz�A CloudkmsProjectsLocationsKeyRingsGetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being requested. See the operation documentation for the appropriate value for this field. r Tr! NrJ r r r r rZ � s rZ c @ s e Zd ZdZejddd�ZdS )�+CloudkmsProjectsLocationsKeyRingsGetRequestzmA CloudkmsProjectsLocationsKeyRingsGetRequest object. Fields: name: The name of the KeyRing to get. r Tr! Nr# r r r r r[ � s r[ c @ s: e Zd ZdZejdejjd�Ze� d�Z ej ddd�Z dS ) �,CloudkmsProjectsLocationsKeyRingsListRequestaT A CloudkmsProjectsLocationsKeyRingsListRequest object. Fields: pageSize: Optional limit on the number of KeyRings to include in the response. Further KeyRings can subsequently be obtained by including the ListKeyRingsResponse.next_page_token in a subsequent request. If unspecified, the server will pick an appropriate default. pageToken: Optional pagination token, returned earlier via ListKeyRingsResponse.next_page_token. parent: Required. The resource name of the location associated with the KeyRings, in the format `projects/*/locations/*`. r r6 r r Tr! Nr8 r r r r r\ � s r\ c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�4CloudkmsProjectsLocationsKeyRingsSetIamPolicyRequestaP A CloudkmsProjectsLocationsKeyRingsSetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being specified. See the operation documentation for the appropriate value for this field. setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the request body. r Tr! rP r NrQ r r r r r] � s r] c @ s* e Zd ZdZejddd�Ze�dd�ZdS )�:CloudkmsProjectsLocationsKeyRingsTestIamPermissionsRequestai A CloudkmsProjectsLocationsKeyRingsTestIamPermissionsRequest object. Fields: resource: REQUIRED: The resource for which the policy detail is being requested. See the operation documentation for the appropriate value for this field. testIamPermissionsRequest: A TestIamPermissionsRequest resource to be passed as the request body. r Tr! rT r NrU r r r r r^ � s r^ c @ sD e Zd ZdZe�d�Zejddd�Zejdej j d�Z e�d�Z d S ) �$CloudkmsProjectsLocationsListRequesta A CloudkmsProjectsLocationsListRequest object. Fields: filter: The standard list filter. name: The resource that owns the locations collection, if applicable. pageSize: The standard list page size. pageToken: The standard list page token. r r Tr! r r6 � N) r r r r r r �filterr$ r9 r: r; r< r= r r r r r_ � s r_ c @ s� e Zd ZdZG dd� dej�Ze�d�G dd� dej ��Z e� d�Z e� dd�Ze� d �Ze� d �Ze� d d �Ze�dd �Ze� d�ZdS )r+ a> A CryptoKey represents a logical key that can be used for cryptographic operations. A CryptoKey is made up of one or more versions, which represent the actual key material used in cryptographic operations. Enums: PurposeValueValuesEnum: The immutable purpose of this CryptoKey. Currently, the only acceptable purpose is ENCRYPT_DECRYPT. Messages: LabelsValue: Labels with user defined metadata. Fields: createTime: Output only. The time at which this CryptoKey was created. labels: Labels with user defined metadata. name: Output only. The resource name for this CryptoKey in the format `projects/*/locations/*/keyRings/*/cryptoKeys/*`. nextRotationTime: At next_rotation_time, the Key Management Service will automatically: 1. Create a new version of this CryptoKey. 2. Mark the new version as primary. Key rotations performed manually via CreateCryptoKeyVersion and UpdateCryptoKeyPrimaryVersion do not affect next_rotation_time. primary: Output only. A copy of the "primary" CryptoKeyVersion that will be used by Encrypt when this CryptoKey is given in EncryptRequest.name. The CryptoKey's primary version can be updated via UpdateCryptoKeyPrimaryVersion. purpose: The immutable purpose of this CryptoKey. Currently, the only acceptable purpose is ENCRYPT_DECRYPT. rotationPeriod: next_rotation_time will be advanced by this period when the service automatically rotates a key. Must be at least one day. If rotation_period is set, next_rotation_time must also be set. c @ s e Zd ZdZdZdZdS )z CryptoKey.PurposeValueValuesEnuma The immutable purpose of this CryptoKey. Currently, the only acceptable purpose is ENCRYPT_DECRYPT. Values: CRYPTO_KEY_PURPOSE_UNSPECIFIED: Not specified. ENCRYPT_DECRYPT: CryptoKeys with this purpose may be used with Encrypt and Decrypt. r r N)r r r r Z CRYPTO_KEY_PURPOSE_UNSPECIFIED�ENCRYPT_DECRYPTr r r r �PurposeValueValuesEnum� s rc �additionalPropertiesc @ s2 e Zd ZdZG dd� dej�Zejdddd�ZdS )zCryptoKey.LabelsValuez�Labels with user defined metadata. Messages: AdditionalProperty: An additional property for a LabelsValue object. Fields: additionalProperties: Additional properties of type LabelsValue c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z(CryptoKey.LabelsValue.AdditionalProperty��An additional property for a LabelsValue object. Fields: key: Name of the additional property. value: A string attribute. r r N�r r r r r r �key�valuer r r r �AdditionalProperty s ri r Tr N� r r r r r �Messageri r rd r r r r � LabelsValue s rl r r r r` r/ � � � N)r r r r r r rc r �MapUnrecognizedFieldsrk rl r � createTimer �labelsr$ ZnextRotationTimeZprimaryr �purposeZrotationPeriodr r r r r+ � s r+ c @ sV e Zd ZdZG dd� dej�Ze�d�Ze�d�Z e�d�Z e�d�Z e� dd�Z d S ) r/ a� A CryptoKeyVersion represents an individual cryptographic key, and the associated key material. It can be used for cryptographic operations either directly, or via its parent CryptoKey, in which case the server will choose the appropriate version for the operation. For security reasons, the raw cryptographic key material represented by a CryptoKeyVersion can never be viewed or exported. It can only be used to encrypt or decrypt data when an authorized user or application invokes Cloud KMS. Enums: StateValueValuesEnum: The current state of the CryptoKeyVersion. Fields: createTime: Output only. The time at which this CryptoKeyVersion was created. destroyEventTime: Output only. The time this CryptoKeyVersion's key material was destroyed. Only present if state is DESTROYED. destroyTime: Output only. The time this CryptoKeyVersion's key material is scheduled for destruction. Only present if state is DESTROY_SCHEDULED. name: Output only. The resource name for this CryptoKeyVersion in the format `projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*`. state: The current state of the CryptoKeyVersion. c @ s$ e Zd ZdZdZdZdZdZdZdS )z%CryptoKeyVersion.StateValueValuesEnuma� The current state of the CryptoKeyVersion. Values: CRYPTO_KEY_VERSION_STATE_UNSPECIFIED: Not specified. ENABLED: This version may be used in Encrypt and Decrypt requests. DISABLED: This version may not be used, but the key material is still available, and the version can be placed back into the ENABLED state. DESTROYED: This version is destroyed, and the key material is no longer stored. A version may not leave this state once entered. DESTROY_SCHEDULED: This version is scheduled for destruction, and will be destroyed soon. Call RestoreCryptoKeyVersion to put it back into the DISABLED state. r r r r r` N) r r r r Z$CRYPTO_KEY_VERSION_STATE_UNSPECIFIEDZENABLED�DISABLEDZ DESTROYEDZDESTROY_SCHEDULEDr r r r �StateValueValuesEnum@ s ru r r r r` rm N)r r r r r r ru r rq ZdestroyEventTimeZ destroyTimer$ r �stater r r r r/ ' s r/ c @ s$ e Zd ZdZe�d�Ze�d�ZdS )rD a8 Request message for KeyManagementService.Decrypt. Fields: additionalAuthenticatedData: Optional data that must match the data originally supplied in EncryptRequest.additional_authenticated_data. ciphertext: Required. The encrypted data originally returned in EncryptResponse.ciphertext. r r N)r r r r r � BytesField�additionalAuthenticatedData� ciphertextr r r r rD [ s rD c @ s e Zd ZdZe�d�ZdS )�DecryptResponsez�Response message for KeyManagementService.Decrypt. Fields: plaintext: The decrypted data originally supplied in EncryptRequest.plaintext. r N)r r r r r rw � plaintextr r r r rz i s rz c @ s e Zd ZdZdS )r2 zARequest message for KeyManagementService.DestroyCryptoKeyVersion.N�r r r r r r r r r2 t s r2 c @ s$ e Zd ZdZe�d�Ze�d�ZdS )rG a] Request message for KeyManagementService.Encrypt. Fields: additionalAuthenticatedData: Optional data that, if specified, must also be provided during decryption through DecryptRequest.additional_authenticated_data. Must be no larger than 64KiB. plaintext: Required. The data to encrypt. Must be no larger than 64KiB. r r N)r r r r r rw rx r{ r r r r rG x s rG c @ s$ e Zd ZdZe�d�Ze�d�ZdS )�EncryptResponsez�Response message for KeyManagementService.Encrypt. Fields: ciphertext: The encrypted data. name: The resource name of the CryptoKeyVersion used in encryption. r r N) r r r r r rw ry r r$ r r r r r} � s r} c @ s8 e Zd ZdZe�d�Ze�d�Ze�d�Ze�d�Z dS )r a~ Represents an expression text. Example: title: "User account presence" description: "Determines whether the request has a user account" expression: "size(request.user) > 0" Fields: description: An optional description of the expression. This is a longer text which describes the expression, e.g. when hovered over it in a UI. expression: Textual representation of an expression in Common Expression Language syntax. The application context of the containing message determines which well-known feature set of CEL is supported. location: An optional string indicating the location of the expression for error reporting, e.g. a file name and a position in the file. title: An optional title for the expression, i.e. a short string describing its purpose. This can be used e.g. in UIs which allow to enter the expression. r r r r` N) r r r r r r � description� expression�location�titler r r r r � s r c @ s$ e Zd ZdZe�d�Ze�d�ZdS )r& a A KeyRing is a toplevel logical grouping of CryptoKeys. Fields: createTime: Output only. The time at which this KeyRing was created. name: Output only. The resource name for the KeyRing in the format `projects/*/locations/*/keyRings/*`. r r N)r r r r r r rq r$ r r r r r& � s r& c @ s< e Zd ZdZejdddd�Ze�d�Zej dej j d�Z d S ) � ListCryptoKeyVersionsResponsea{ Response message for KeyManagementService.ListCryptoKeyVersions. Fields: cryptoKeyVersions: The list of CryptoKeyVersions. nextPageToken: A token to retrieve next page of results. Pass this value in ListCryptoKeyVersionsRequest.page_token to retrieve the next page of results. totalSize: The total number of CryptoKeyVersions that matched the query. r/ r Tr r r r6 N) r r r r r r ZcryptoKeyVersionsr � nextPageTokenr9 r: r; � totalSizer r r r r� � s r� c @ s< e Zd ZdZejdddd�Ze�d�Zej dej j d�Z d S ) �ListCryptoKeysResponseaX Response message for KeyManagementService.ListCryptoKeys. Fields: cryptoKeys: The list of CryptoKeys. nextPageToken: A token to retrieve next page of results. Pass this value in ListCryptoKeysRequest.page_token to retrieve the next page of results. totalSize: The total number of CryptoKeys that matched the query. r+ r Tr r r r6 N) r r r r r r Z cryptoKeysr r� r9 r: r; r� r r r r r� � s r� c @ s< e Zd ZdZejdddd�Ze�d�Zej dej j d�Z d S ) �ListKeyRingsResponseaH Response message for KeyManagementService.ListKeyRings. Fields: keyRings: The list of KeyRings. nextPageToken: A token to retrieve next page of results. Pass this value in ListKeyRingsRequest.page_token to retrieve the next page of results. totalSize: The total number of KeyRings that matched the query. r& r Tr r r r6 N) r r r r r r ZkeyRingsr r� r9 r: r; r� r r r r r� � s r� c @ s* e Zd ZdZejdddd�Ze�d�ZdS )�ListLocationsResponsez�The response message for Locations.ListLocations. Fields: locations: A list of locations that matches the specified filter in the request. nextPageToken: The standard List next-page token. �Locationr Tr r N) r r r r r r � locationsr r� r r r r r� � s r� c @ st e Zd ZdZe�d�G dd� dej��Ze�d�G dd� dej��Z e� dd�Z e� d�Z e� dd �Ze� d �Zd S ) r� a A resource that represents Google Cloud Platform location. Messages: LabelsValue: Cross-service attributes for the location. For example {"cloud.googleapis.com/region": "us-east1"} MetadataValue: Service-specific metadata. For example the available capacity at the given location. Fields: labels: Cross-service attributes for the location. For example {"cloud.googleapis.com/region": "us-east1"} locationId: The canonical id for this location. For example: `"us-east1"`. metadata: Service-specific metadata. For example the available capacity at the given location. name: Resource name for the location, which may vary between implementations. For example: `"projects/example-project/locations/us- east1"` rd c @ s2 e Zd ZdZG dd� dej�Zejdddd�ZdS )zLocation.LabelsValuea Cross-service attributes for the location. For example {"cloud.googleapis.com/region": "us-east1"} Messages: AdditionalProperty: An additional property for a LabelsValue object. Fields: additionalProperties: Additional properties of type LabelsValue c @ s$ e Zd ZdZe�d�Ze�d�ZdS )z'Location.LabelsValue.AdditionalPropertyre r r Nrf r r r r ri s ri r Tr Nrj r r r r rl s rl c @ s2 e Zd ZdZG dd� dej�Zejdddd�ZdS )zLocation.MetadataValuea( Service-specific metadata. For example the available capacity at the given location. Messages: AdditionalProperty: An additional property for a MetadataValue object. Fields: additionalProperties: Properties of the object. Contains field @type with type URL. c @ s& e Zd ZdZe�d�Ze�dd�ZdS )z)Location.MetadataValue.AdditionalPropertyz�An additional property for a MetadataValue object. Fields: key: Name of the additional property. value: A extra_types.JsonValue attribute. r zextra_types.JsonValuer N) r r r r r r rg r rh r r r r ri . s ri r Tr Nrj r r r r � MetadataValue! s r� r r r r` N)r r r r r rp r rk rl r� r rr r Z locationId�metadatar$ r r r r r� � s r� c @ sV e Zd ZdZejdddd�Zejdddd�Ze�d�Z e� d �Z ej d ej jd �Zd S ) �PolicyaA Defines an Identity and Access Management (IAM) policy. It is used to specify access control policies for Cloud Platform resources. A `Policy` consists of a list of `bindings`. A `Binding` binds a list of `members` to a `role`, where the members can be user accounts, Google groups, Google domains, and service accounts. A `role` is a named list of permissions defined by IAM. **Example** { "bindings": [ { "role": "roles/owner", "members": [ "user:mike@example.com", "group:admins@example.com", "domain:google.com", "serviceAccount:my-other- app@appspot.gserviceaccount.com", ] }, { "role": "roles/viewer", "members": ["user:sean@example.com"] } ] } For a description of IAM and its features, see the [IAM developer's guide](https://cloud.google.com/iam). Fields: auditConfigs: Specifies cloud audit logging configuration for this policy. bindings: Associates a list of `members` to a `role`. `bindings` with no members will result in an error. etag: `etag` is used for optimistic concurrency control as a way to help prevent simultaneous updates of a policy from overwriting each other. It is strongly suggested that systems make use of the `etag` in the read- modify-write cycle to perform policy updates in order to avoid race conditions: An `etag` is returned in the response to `getIamPolicy`, and systems are expected to put that etag in the request to `setIamPolicy` to ensure that their change will be applied to the same version of the policy. If no `etag` is provided in the call to `setIamPolicy`, then the existing policy is overwritten blindly. iamOwned: A boolean attribute. version: Version of the `Policy`. The default version is 0. r r Tr r r r r` rm r6 N)r r r r r r Z auditConfigs�bindingsrw �etag� BooleanFieldZiamOwnedr9 r: r; �versionr r r r r� A s r� c @ s e Zd ZdZdS )rA zARequest message for KeyManagementService.RestoreCryptoKeyVersion.Nr| r r r r rA h s rA c @ s& e Zd ZdZe�dd�Ze�d�ZdS )rP aH Request message for `SetIamPolicy` method. Fields: policy: REQUIRED: The complete policy to be applied to the `resource`. The size of the policy is limited to a few 10s of KB. An empty policy is a valid policy but certain Cloud Platform services (such as Projects) might reject them. updateMask: OPTIONAL: A FieldMask specifying which fields of the policy to modify. Only the fields in the mask will be modified. If no mask is provided, the following default mask is used: paths: "bindings, etag" This field is only used by Cloud IAM. r� r r N) r r r r r r �policyr r? r r r r rP l s rP c @ s� e Zd ZdZG dd� dej�ZG dd� dej�Ze�dd�Z e� d�Z ejddd d �Z e� d �Z e� d �Ze� d �Ze� d�Ze� d�Zejddd �Zejddd �Ze� d�Ze� d�Ze� d�Ze� d�ZdS )�StandardQueryParametersay Query parameters accepted by all methods. Enums: FXgafvValueValuesEnum: V1 error format. AltValueValuesEnum: Data format for response. Fields: f__xgafv: V1 error format. access_token: OAuth access token. alt: Data format for response. bearer_token: OAuth bearer token. callback: JSONP fields: Selector specifying which fields to include in a partial response. key: API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token. oauth_token: OAuth 2.0 token for the current user. pp: Pretty-print response. prettyPrint: Returns response with indentations and line breaks. quotaUser: Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. trace: A tracing token of the form "token:<tokenid>" to include in api requests. uploadType: Legacy upload protocol for media (e.g. "media", "multipart"). upload_protocol: Upload protocol for media (e.g. "raw", "multipart"). c @ s e Zd ZdZdZdZdZdS )z*StandardQueryParameters.AltValueValuesEnumz�Data format for response. Values: json: Responses with Content-Type of application/json media: Media download with context-dependent Content-Type proto: Responses with Content-Type of application/x-protobuf r r r N)r r r r �json�media�protor r r r �AltValueValuesEnum� s r� c @ s e Zd ZdZdZdZdS )z-StandardQueryParameters.FXgafvValueValuesEnumzVV1 error format. Values: _1: v1 error format _2: v2 error format r r N)r r r r �_1�_2r r r r �FXgafvValueValuesEnum� s r� r r r r� )�defaultr` rm rn ro � � T� � � � � N)r r r r r r r� r� r �f__xgafvr � access_token�alt� bearer_token�callback�fieldsrg � oauth_tokenr� �pp� prettyPrint� quotaUser�trace� uploadType�upload_protocolr r r r r� ~ s" r� c @ s e Zd ZdZejddd�ZdS )rT a? Request message for `TestIamPermissions` method. Fields: permissions: The set of permissions to check for the `resource`. Permissions with wildcards (such as '*' or 'storage.*') are not allowed. For more information see [IAM Overview](https://cloud.google.com/iam/docs/overview#permissions). r Tr N�r r r r r r � permissionsr r r r rT � s rT c @ s e Zd ZdZejddd�ZdS )�TestIamPermissionsResponsez�Response message for `TestIamPermissions` method. Fields: permissions: A subset of `TestPermissionsRequest.permissions` that the caller is allowed. r Tr Nr� r r r r r� � s r� c @ s e Zd ZdZe�d�ZdS )rX z�Request message for KeyManagementService.UpdateCryptoKeyPrimaryVersion. Fields: cryptoKeyVersionId: The id of the child CryptoKeyVersion to use as primary. r N)r r r r r r ZcryptoKeyVersionIdr r r r rX � s rX r� z$.xgafvr� �1r� �2N)<r �apitools.base.protorpcliter r �apitools.base.pyr r �packagerk r r r r r% r* r. r1 r4 r5 r> r@ rC rF rI rL rM rN rO rS rW rZ r[ r\ r] r^ r_ r+ r/ rD rz r2 rG r} r r& r� r� r� r� r� r� rA rP r� rT r� rX �AddCustomJsonFieldMapping�AddCustomJsonEnumMappingr� r r r r �<module> sz $# N4 M'C ��� third_party/kms_apitools/__pycache__/cloudkms_v1_client.cpython-39.pyc 0000644 00000070074 15025013616 0022236 0 ustar 00 a +(Wgʂ � @ s� d Z ddlmZ ddlZddlZddlZddlmZ ddlm m m Z ddl Z ddlmZ ddlmZ G dd� dej�ZdS ) z1Generated client library for cloudkms version v1.� )�absolute_importN)�base_api)�MetricsCollector)� system_utilc s� e Zd ZdZeZdZdZdgZdZ dZ dZ de j e�� ejf Ze�� rhed7 Ze�� rhed e�� 7 Ze�� rzed 7 Zned 7 Zd ZdZd Zd � fdd� ZG dd� dej �Z G dd� dej �Z G dd� dej �Z G dd� dej �Z G dd� dej �Z!� Z"S ) � CloudkmsV1z9Generated client library for service cloudkms version v1.z https://cloudkms.googleapis.com/Zcloudkmsz.https://www.googleapis.com/auth/cloud-platform�v1�nomatterz!apitools gsutil/%s Python/%s (%s)z google-cloud-sdkz/%sz analytics/disabledz analytics/enabledN� TFc sn |p| j }tt| �j||||||||| | d� | �| �| _| �| �| _| �| �| _ | � | �| _ | � | �| _ dS )z Create a new cloudkms handle.) � credentials�get_credentials�http�model� log_request� log_response�credentials_args�default_global_params�additional_http_headersN)�BASE_URL�superr �__init__�;ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService�8projects_locations_keyRings_cryptoKeys_cryptoKeyVersions�*ProjectsLocationsKeyRingsCryptoKeysService�&projects_locations_keyRings_cryptoKeys� ProjectsLocationsKeyRingsService�projects_locations_keyRings�ProjectsLocationsService�projects_locations�ProjectsService�projects) �self�urlr r r r r r r r r �� __class__� �@/opt/gsutil/gslib/third_party/kms_apitools/cloudkms_v1_client.pyr - s � zCloudkmsV1.__init__c s� e Zd ZdZdZ� fdd�Zddd�Zdd � e_dd d �Zd d � e_dd d�Z dd � e _d dd�Z dd � e _d dd�Z dd � e _d dd�Z dd � e _� Z S ) zFCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsServicezXService class for the projects_locations_keyRings_cryptoKeys_cryptoKeyVersions resource.r c s t tj| ��|� i | _d S �N)r r r r �_upload_configs�r �clientr" r$ r% r F s zOCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService.__init__Nc C s | � d�}| j|||d�S )a� Create a new CryptoKeyVersion in a CryptoKey. The server will assign the next sequential id. If unset, state will be set to ENABLED. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsCreateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKeyVersion) The response message. �Create�� global_params��GetMethodConfig� _RunMethod�r �requestr, �configr$ r$ r% r* K s �zMCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService.Createc C s$ t jddddgdgg ddddd d � S ) N�rv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions�POSTzHcloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.create�parent� v1/{+parent}/cryptoKeyVersions�cryptoKeyVersionZICloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsCreateRequest�CryptoKeyVersionF� � flat_path� http_method� method_id�ordered_params� path_params� query_params� relative_path� request_field�request_type_name�response_type_name�supports_download�r � ApiMethodInfor$ r$ r$ r% �<lambda>\ s �zOCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService.<lambda>c C s | � d�}| j|||d�S )a� Schedule a CryptoKeyVersion for destruction. Upon calling this method, CryptoKeyVersion.state will be set to DESTROY_SCHEDULED and destroy_time will be set to a time 24 hours in the future, at which point the state will be changed to DESTROYED, and the key material will be irrevocably destroyed. Before the destroy_time is reached, RestoreCryptoKeyVersion may be called to reverse the process. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsDestroyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKeyVersion) The response message. �Destroyr+ r- r0 r$ r$ r% rH j s �zNCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService.Destroyc C s$ t jddddgdgg ddddd d � S ) Nz�v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions/{cryptoKeyVersionsId}:destroyr4 zIcloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.destroy�namezv1/{+name}:destroyZ destroyCryptoKeyVersionRequestZJCloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsDestroyRequestr8 Fr9 rE r$ r$ r$ r% rG � s �c C s | � d�}| j|||d�S )a; Returns metadata for a given CryptoKeyVersion. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKeyVersion) The response message. �Getr+ r- r0 r$ r$ r% rJ � s �zJCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService.Getc C s$ t jddddgdgg ddddd d � S ) N��v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions/{cryptoKeyVersionsId}�GETzEcloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.getrI � v1/{+name}r ZFCloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsGetRequestr8 Fr9 rE r$ r$ r$ r% rG � s �c C s | � d�}| j|||d�S )a3 Lists CryptoKeyVersions. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListCryptoKeyVersionsResponse) The response message. �Listr+ r- r0 r$ r$ r% rN � s �zKCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService.Listc C s( t jddddgdgddgddd d d d � S ) Nr3 rL zFcloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.listr5 �pageSize� pageTokenr6 r ZGCloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsListRequestZ ListCryptoKeyVersionsResponseFr9 rE r$ r$ r$ r% rG � s �c C s | � d�}| j|||d�S )a� Update a CryptoKeyVersion's metadata. state may be changed between ENABLED and DISABLED using this method. See DestroyCryptoKeyVersion and RestoreCryptoKeyVersion to move between other states. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKeyVersion) The response message. �Patchr+ r- r0 r$ r$ r% rQ � s �zLCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService.Patchc C s& t jddddgdgdgdddd d d � S ) NrK �PATCHzGcloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.patchrI � updateMaskrM r7 ZHCloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsPatchRequestr8 Fr9 rE r$ r$ r$ r% rG � s �c C s | � d�}| j|||d�S )a� Restore a CryptoKeyVersion in the. DESTROY_SCHEDULED, state. Upon restoration of the CryptoKeyVersion, state will be set to DISABLED, and destroy_time will be cleared. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsRestoreRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKeyVersion) The response message. �Restorer+ r- r0 r$ r$ r% rT � s �zNCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService.Restorec C s$ t jddddgdgg ddddd d � S ) Nz�v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions/{cryptoKeyVersionsId}:restorer4 zIcloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.restorerI zv1/{+name}:restoreZ restoreCryptoKeyVersionRequestZJCloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsRestoreRequestr8 Fr9 rE r$ r$ r$ r% rG � s �)N)N)N)N)N)N)�__name__� __module__� __qualname__�__doc__�_NAMEr r* � method_configrH rJ rN rQ rT � __classcell__r$ r$ r" r% r A s r c s� e Zd ZdZdZ� fdd�Zd%dd�Zdd � e_d&d d �Zd d � e_d'd d�Z dd � e _d(dd�Z dd � e _d)dd�Z dd � e _d*dd�Z dd � e _d+dd�Z dd � e _d,d d �Zd d � e_d-d d �Zd!d � e_d.d"d#�Zd$d � e_� ZS )/z5CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysServicezFService class for the projects_locations_keyRings_cryptoKeys resource.r c s t tj| ��|� i | _d S r& )r r r r r' r( r" r$ r% r s z>CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.__init__Nc C s | � d�}| j|||d�S )a@ Create a new CryptoKey within a KeyRing. CryptoKey.purpose is required. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKey) The response message. r* r+ r- r0 r$ r$ r% r* s �z<CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.Createc C s& t jddddgdgdgdddd d d � S ) N�Qv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeysr4 z6cloudkms.projects.locations.keyRings.cryptoKeys.creater5 � cryptoKeyId�v1/{+parent}/cryptoKeys� cryptoKey�8CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequest� CryptoKeyFr9 rE r$ r$ r$ r% rG ! s �z>CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.<lambda>c C s | � d�}| j|||d�S )a+ Decrypts data that was protected by Encrypt. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysDecryptRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (DecryptResponse) The response message. �Decryptr+ r- r0 r$ r$ r% rb / s �z=CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.Decryptc C s$ t jddddgdgg ddddd d � S ) Nzhv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:decryptr4 z7cloudkms.projects.locations.keyRings.cryptoKeys.decryptrI zv1/{+name}:decryptZdecryptRequestZ9CloudkmsProjectsLocationsKeyRingsCryptoKeysDecryptRequestZDecryptResponseFr9 rE r$ r$ r$ r% rG < s �c C s | � d�}| j|||d�S )aD Encrypts data, so that it can only be recovered by a call to Decrypt. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysEncryptRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (EncryptResponse) The response message. �Encryptr+ r- r0 r$ r$ r% rc J s �z=CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.Encryptc C s$ t jddddgdgg ddddd d � S ) Nzhv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:encryptr4 z7cloudkms.projects.locations.keyRings.cryptoKeys.encryptrI zv1/{+name}:encryptZencryptRequestZ9CloudkmsProjectsLocationsKeyRingsCryptoKeysEncryptRequestZEncryptResponseFr9 rE r$ r$ r$ r% rG W s �c C s | � d�}| j|||d�S )aF Returns metadata for a given CryptoKey, as well as its. primary CryptoKeyVersion. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKey) The response message. rJ r+ r- r0 r$ r$ r% rJ e s �z9CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.Getc C s$ t jddddgdgg ddddd d � S ) N�`v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}rL z3cloudkms.projects.locations.keyRings.cryptoKeys.getrI rM r Z5CloudkmsProjectsLocationsKeyRingsCryptoKeysGetRequestra Fr9 rE r$ r$ r$ r% rG s s �c C s | � d�}| j|||d�S )ax Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. � GetIamPolicyr+ r- r0 r$ r$ r% re � s �zBCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.GetIamPolicyc C s$ t jddddgdgg ddddd d � S ) Nzmv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:getIamPolicyrL z<cloudkms.projects.locations.keyRings.cryptoKeys.getIamPolicy�resource�v1/{+resource}:getIamPolicyr �>CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequest�PolicyFr9 rE r$ r$ r$ r% rG � s �c C s | � d�}| j|||d�S )a Lists CryptoKeys. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListCryptoKeysResponse) The response message. rN r+ r- r0 r$ r$ r% rN � s �z:CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.Listc C s( t jddddgdgddgddd d d d � S ) Nr\ rL z4cloudkms.projects.locations.keyRings.cryptoKeys.listr5 rO rP r^ r Z6CloudkmsProjectsLocationsKeyRingsCryptoKeysListRequestZListCryptoKeysResponseFr9 rE r$ r$ r$ r% rG � s �c C s | � d�}| j|||d�S )a Update a CryptoKey. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKey) The response message. rQ r+ r- r0 r$ r$ r% rQ � s �z;CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.Patchc C s& t jddddgdgdgdddd d d � S ) Nrd rR z5cloudkms.projects.locations.keyRings.cryptoKeys.patchrI rS rM r_ Z7CloudkmsProjectsLocationsKeyRingsCryptoKeysPatchRequestra Fr9 rE r$ r$ r$ r% rG � s �c C s | � d�}| j|||d�S )aS Sets the access control policy on the specified resource. Replaces any. existing policy. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. � SetIamPolicyr+ r- r0 r$ r$ r% rj � s �zBCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.SetIamPolicyc C s$ t jddddgdgg ddddd d � S ) Nzmv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:setIamPolicyr4 z<cloudkms.projects.locations.keyRings.cryptoKeys.setIamPolicyrf �v1/{+resource}:setIamPolicy�setIamPolicyRequest�>CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequestri Fr9 rE r$ r$ r$ r% rG � s �c C s | � d�}| j|||d�S )aq Returns permissions that a caller has on the specified resource. If the resource does not exist, this will return an empty set of permissions, not a NOT_FOUND error. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. �TestIamPermissionsr+ r- r0 r$ r$ r% rn � s �zHCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.TestIamPermissionsc C s$ t jddddgdgg ddddd d � S ) Nzsv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:testIamPermissionsr4 zBcloudkms.projects.locations.keyRings.cryptoKeys.testIamPermissionsrf �!v1/{+resource}:testIamPermissions�testIamPermissionsRequestZDCloudkmsProjectsLocationsKeyRingsCryptoKeysTestIamPermissionsRequest�TestIamPermissionsResponseFr9 rE r$ r$ r$ r% rG s �c C s | � d�}| j|||d�S )aE Update the version of a CryptoKey that will be used in Encrypt. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysUpdatePrimaryVersionRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKey) The response message. �UpdatePrimaryVersionr+ r- r0 r$ r$ r% rr s �zJCloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService.UpdatePrimaryVersionc C s$ t jddddgdgg ddddd d � S ) Nzuv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:updatePrimaryVersionr4 zDcloudkms.projects.locations.keyRings.cryptoKeys.updatePrimaryVersionrI z v1/{+name}:updatePrimaryVersionZ$updateCryptoKeyPrimaryVersionRequestZFCloudkmsProjectsLocationsKeyRingsCryptoKeysUpdatePrimaryVersionRequestra Fr9 rE r$ r$ r$ r% rG s �)N)N)N)N)N)N)N)N)N)N)rU rV rW rX rY r r* rZ rb rc rJ re rN rQ rj rn rr r[ r$ r$ r" r% r s. r c s� e Zd ZdZdZ� fdd�Zddd�Zdd � e_dd d �Zd d � e_dd d�Z dd � e _d dd�Z dd � e _d dd�Z dd � e _d dd�Z dd � e _� Z S ) z+CloudkmsV1.ProjectsLocationsKeyRingsServicez;Service class for the projects_locations_keyRings resource.r c s t tj| ��|� i | _d S r& )r r r r r' r( r" r$ r% r 1 s z4CloudkmsV1.ProjectsLocationsKeyRingsService.__init__Nc C s | � d�}| j|||d�S )a! Create a new KeyRing in a given Project and Location. Args: request: (CloudkmsProjectsLocationsKeyRingsCreateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (KeyRing) The response message. r* r+ r- r0 r$ r$ r% r* 6 s �z2CloudkmsV1.ProjectsLocationsKeyRingsService.Createc C s& t jddddgdgdgdddd d d � S ) N�9v1/projects/{projectsId}/locations/{locationsId}/keyRingsr4 z+cloudkms.projects.locations.keyRings.creater5 � keyRingId�v1/{+parent}/keyRings�keyRing�.CloudkmsProjectsLocationsKeyRingsCreateRequest�KeyRingFr9 rE r$ r$ r$ r% rG C s �z4CloudkmsV1.ProjectsLocationsKeyRingsService.<lambda>c C s | � d�}| j|||d�S )a Returns metadata for a given KeyRing. Args: request: (CloudkmsProjectsLocationsKeyRingsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (KeyRing) The response message. rJ r+ r- r0 r$ r$ r% rJ Q s �z/CloudkmsV1.ProjectsLocationsKeyRingsService.Getc C s$ t jddddgdgg ddddd d � S ) NzFv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}rL z(cloudkms.projects.locations.keyRings.getrI rM r Z+CloudkmsProjectsLocationsKeyRingsGetRequestrx Fr9 rE r$ r$ r$ r% rG ^ s �c C s | � d�}| j|||d�S )an Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set. Args: request: (CloudkmsProjectsLocationsKeyRingsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. re r+ r- r0 r$ r$ r% re l s �z8CloudkmsV1.ProjectsLocationsKeyRingsService.GetIamPolicyc C s$ t jddddgdgg ddddd d � S ) NzSv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}:getIamPolicyrL z1cloudkms.projects.locations.keyRings.getIamPolicyrf rg r Z4CloudkmsProjectsLocationsKeyRingsGetIamPolicyRequestri Fr9 rE r$ r$ r$ r% rG { s �c C s | � d�}| j|||d�S )a Lists KeyRings. Args: request: (CloudkmsProjectsLocationsKeyRingsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListKeyRingsResponse) The response message. rN r+ r- r0 r$ r$ r% rN � s �z0CloudkmsV1.ProjectsLocationsKeyRingsService.Listc C s( t jddddgdgddgddd d d d � S ) Nrs rL z)cloudkms.projects.locations.keyRings.listr5 rO rP ru r Z,CloudkmsProjectsLocationsKeyRingsListRequestZListKeyRingsResponseFr9 rE r$ r$ r$ r% rG � s �c C s | � d�}| j|||d�S )aI Sets the access control policy on the specified resource. Replaces any. existing policy. Args: request: (CloudkmsProjectsLocationsKeyRingsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. rj r+ r- r0 r$ r$ r% rj � s �z8CloudkmsV1.ProjectsLocationsKeyRingsService.SetIamPolicyc C s$ t jddddgdgg ddddd d � S ) NzSv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}:setIamPolicyr4 z1cloudkms.projects.locations.keyRings.setIamPolicyrf rk rl Z4CloudkmsProjectsLocationsKeyRingsSetIamPolicyRequestri Fr9 rE r$ r$ r$ r% rG � s �c C s | � d�}| j|||d�S )ag Returns permissions that a caller has on the specified resource. If the resource does not exist, this will return an empty set of permissions, not a NOT_FOUND error. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. Args: request: (CloudkmsProjectsLocationsKeyRingsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. rn r+ r- r0 r$ r$ r% rn � s �z>CloudkmsV1.ProjectsLocationsKeyRingsService.TestIamPermissionsc C s$ t jddddgdgg ddddd d � S ) NzYv1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}:testIamPermissionsr4 z7cloudkms.projects.locations.keyRings.testIamPermissionsrf ro rp Z:CloudkmsProjectsLocationsKeyRingsTestIamPermissionsRequestrq Fr9 rE r$ r$ r$ r% rG � s �)N)N)N)N)N)N)rU rV rW rX rY r r* rZ rJ re rN rj rn r[ r$ r$ r" r% r , s r c sL e Zd ZdZdZ� fdd�Zd dd�Zdd � e_dd d �Zd d � e_� Z S )z#CloudkmsV1.ProjectsLocationsServicez2Service class for the projects_locations resource.r c s t tj| ��|� i | _d S r& )r r r r r' r( r" r$ r% r � s z,CloudkmsV1.ProjectsLocationsService.__init__Nc C s | � d�}| j|||d�S )a Get information about a location. Args: request: (CloudkmsProjectsLocationsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Location) The response message. rJ r+ r- r0 r$ r$ r% rJ � s �z'CloudkmsV1.ProjectsLocationsService.Getc C s$ t jddddgdgg ddddd d � S ) Nz0v1/projects/{projectsId}/locations/{locationsId}rL z cloudkms.projects.locations.getrI rM r Z#CloudkmsProjectsLocationsGetRequest�LocationFr9 rE r$ r$ r$ r% rG � s �z,CloudkmsV1.ProjectsLocationsService.<lambda>c C s | � d�}| j|||d�S )a1 Lists information about the supported locations for this service. Args: request: (CloudkmsProjectsLocationsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListLocationsResponse) The response message. rN r+ r- r0 r$ r$ r% rN s �z(CloudkmsV1.ProjectsLocationsService.Listc C s( t jddddgdgg d�dddd d d � S ) Nz"v1/projects/{projectsId}/locationsrL z cloudkms.projects.locations.listrI )�filterrO rP zv1/{+name}/locationsr Z$CloudkmsProjectsLocationsListRequestZListLocationsResponseFr9 rE r$ r$ r$ r% rG s �)N)N) rU rV rW rX rY r rJ rZ rN r[ r$ r$ r" r% r � s r c s$ e Zd ZdZdZ� fdd�Z� ZS )zCloudkmsV1.ProjectsServicez(Service class for the projects resource.r c s t tj| ��|� i | _d S r& )r r r r r' r( r" r$ r% r & s z#CloudkmsV1.ProjectsService.__init__)rU rV rW rX rY r r[ r$ r$ r" r% r ! s r ) r NTNNFFNNN)#rU rV rW rX �messages�MESSAGES_MODULEr �_PACKAGE�_SCOPES�_VERSION� _CLIENT_ID�_CLIENT_SECRET�gslib�VERSION�platform�python_version�sys� _USER_AGENTr �InvokedViaCloudSdk�CloudSdkVersionr � IsDisabled�_CLIENT_CLASS_NAME� _URL_VERSION�_API_KEYr r �BaseApiServicer r r r r r[ r$ r$ r" r% r sH � � H & 6@r )rX � __future__r �osr� r� �apitools.base.pyr �3gslib.third_party.kms_apitools.cloudkms_v1_messages� third_party� kms_apitools�cloudkms_v1_messagesr{ r� Z gslib.metricsr � gslib.utilsr � BaseApiClientr r$ r$ r$ r% �<module> s third_party/kms_apitools/__pycache__/__init__.cpython-39.pyc 0000644 00000000221 15025013616 0020173 0 ustar 00 a +(Wg � @ s d S )N� r r r �6/opt/gsutil/gslib/third_party/kms_apitools/__init__.py�<module> � third_party/kms_apitools/cloudkms_v1_client.py 0000644 00000101312 15025013616 0015735 0 ustar 00 """Generated client library for cloudkms version v1.""" # NOTE: This file is autogenerated and should not be edited by hand. # NOTENOTE: This file has been gently hand-edited. Do not simply replace # with an autogenerated version - merge with this version! # For instance, the __future__ import immediately below... from __future__ import absolute_import import os import platform import sys from apitools.base.py import base_api import gslib.third_party.kms_apitools.cloudkms_v1_messages as messages import gslib from gslib.metrics import MetricsCollector from gslib.utils import system_util class CloudkmsV1(base_api.BaseApiClient): """Generated client library for service cloudkms version v1.""" MESSAGES_MODULE = messages BASE_URL = u'https://cloudkms.googleapis.com/' _PACKAGE = u'cloudkms' _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform'] _VERSION = u'v1' _CLIENT_ID = 'nomatter' _CLIENT_SECRET = 'nomatter' _USER_AGENT = 'apitools gsutil/%s Python/%s (%s)' % ( gslib.VERSION, platform.python_version(), sys.platform) if system_util.InvokedViaCloudSdk(): _USER_AGENT += ' google-cloud-sdk' if system_util.CloudSdkVersion(): _USER_AGENT += '/%s' % system_util.CloudSdkVersion() if MetricsCollector.IsDisabled(): _USER_AGENT += ' analytics/disabled' else: _USER_AGENT += ' analytics/enabled' _CLIENT_CLASS_NAME = u'CloudkmsV1' _URL_VERSION = u'v1' _API_KEY = None def __init__(self, url='', credentials=None, get_credentials=True, http=None, model=None, log_request=False, log_response=False, credentials_args=None, default_global_params=None, additional_http_headers=None): """Create a new cloudkms handle.""" url = url or self.BASE_URL super(CloudkmsV1, self).__init__( url, credentials=credentials, get_credentials=get_credentials, http=http, model=model, log_request=log_request, log_response=log_response, credentials_args=credentials_args, default_global_params=default_global_params, additional_http_headers=additional_http_headers) self.projects_locations_keyRings_cryptoKeys_cryptoKeyVersions = self.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService(self) self.projects_locations_keyRings_cryptoKeys = self.ProjectsLocationsKeyRingsCryptoKeysService(self) self.projects_locations_keyRings = self.ProjectsLocationsKeyRingsService(self) self.projects_locations = self.ProjectsLocationsService(self) self.projects = self.ProjectsService(self) class ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService(base_api.BaseApiService): """Service class for the projects_locations_keyRings_cryptoKeys_cryptoKeyVersions resource.""" _NAME = u'projects_locations_keyRings_cryptoKeys_cryptoKeyVersions' def __init__(self, client): super(CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsService, self).__init__(client) self._upload_configs = { } def Create(self, request, global_params=None): """Create a new CryptoKeyVersion in a CryptoKey. The server will assign the next sequential id. If unset, state will be set to ENABLED. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsCreateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKeyVersion) The response message. """ config = self.GetMethodConfig('Create') return self._RunMethod( config, request, global_params=global_params) Create.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.create', ordered_params=[u'parent'], path_params=[u'parent'], query_params=[], relative_path=u'v1/{+parent}/cryptoKeyVersions', request_field=u'cryptoKeyVersion', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsCreateRequest', response_type_name=u'CryptoKeyVersion', supports_download=False, ) def Destroy(self, request, global_params=None): """Schedule a CryptoKeyVersion for destruction. Upon calling this method, CryptoKeyVersion.state will be set to DESTROY_SCHEDULED and destroy_time will be set to a time 24 hours in the future, at which point the state will be changed to DESTROYED, and the key material will be irrevocably destroyed. Before the destroy_time is reached, RestoreCryptoKeyVersion may be called to reverse the process. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsDestroyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKeyVersion) The response message. """ config = self.GetMethodConfig('Destroy') return self._RunMethod( config, request, global_params=global_params) Destroy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions/{cryptoKeyVersionsId}:destroy', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.destroy', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}:destroy', request_field=u'destroyCryptoKeyVersionRequest', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsDestroyRequest', response_type_name=u'CryptoKeyVersion', supports_download=False, ) def Get(self, request, global_params=None): """Returns metadata for a given CryptoKeyVersion. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKeyVersion) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions/{cryptoKeyVersionsId}', http_method=u'GET', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.get', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}', request_field='', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsGetRequest', response_type_name=u'CryptoKeyVersion', supports_download=False, ) def List(self, request, global_params=None): """Lists CryptoKeyVersions. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListCryptoKeyVersionsResponse) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions', http_method=u'GET', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.list', ordered_params=[u'parent'], path_params=[u'parent'], query_params=[u'pageSize', u'pageToken'], relative_path=u'v1/{+parent}/cryptoKeyVersions', request_field='', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsListRequest', response_type_name=u'ListCryptoKeyVersionsResponse', supports_download=False, ) def Patch(self, request, global_params=None): """Update a CryptoKeyVersion's metadata. state may be changed between ENABLED and DISABLED using this method. See DestroyCryptoKeyVersion and RestoreCryptoKeyVersion to move between other states. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKeyVersion) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) Patch.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions/{cryptoKeyVersionsId}', http_method=u'PATCH', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.patch', ordered_params=[u'name'], path_params=[u'name'], query_params=[u'updateMask'], relative_path=u'v1/{+name}', request_field=u'cryptoKeyVersion', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsPatchRequest', response_type_name=u'CryptoKeyVersion', supports_download=False, ) def Restore(self, request, global_params=None): """Restore a CryptoKeyVersion in the. DESTROY_SCHEDULED, state. Upon restoration of the CryptoKeyVersion, state will be set to DISABLED, and destroy_time will be cleared. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsRestoreRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKeyVersion) The response message. """ config = self.GetMethodConfig('Restore') return self._RunMethod( config, request, global_params=global_params) Restore.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions/{cryptoKeyVersionsId}:restore', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.cryptoKeyVersions.restore', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}:restore', request_field=u'restoreCryptoKeyVersionRequest', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsRestoreRequest', response_type_name=u'CryptoKeyVersion', supports_download=False, ) class ProjectsLocationsKeyRingsCryptoKeysService(base_api.BaseApiService): """Service class for the projects_locations_keyRings_cryptoKeys resource.""" _NAME = u'projects_locations_keyRings_cryptoKeys' def __init__(self, client): super(CloudkmsV1.ProjectsLocationsKeyRingsCryptoKeysService, self).__init__(client) self._upload_configs = { } def Create(self, request, global_params=None): """Create a new CryptoKey within a KeyRing. CryptoKey.purpose is required. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKey) The response message. """ config = self.GetMethodConfig('Create') return self._RunMethod( config, request, global_params=global_params) Create.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.create', ordered_params=[u'parent'], path_params=[u'parent'], query_params=[u'cryptoKeyId'], relative_path=u'v1/{+parent}/cryptoKeys', request_field=u'cryptoKey', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequest', response_type_name=u'CryptoKey', supports_download=False, ) def Decrypt(self, request, global_params=None): """Decrypts data that was protected by Encrypt. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysDecryptRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (DecryptResponse) The response message. """ config = self.GetMethodConfig('Decrypt') return self._RunMethod( config, request, global_params=global_params) Decrypt.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:decrypt', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.decrypt', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}:decrypt', request_field=u'decryptRequest', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysDecryptRequest', response_type_name=u'DecryptResponse', supports_download=False, ) def Encrypt(self, request, global_params=None): """Encrypts data, so that it can only be recovered by a call to Decrypt. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysEncryptRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (EncryptResponse) The response message. """ config = self.GetMethodConfig('Encrypt') return self._RunMethod( config, request, global_params=global_params) Encrypt.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:encrypt', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.encrypt', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}:encrypt', request_field=u'encryptRequest', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysEncryptRequest', response_type_name=u'EncryptResponse', supports_download=False, ) def Get(self, request, global_params=None): """Returns metadata for a given CryptoKey, as well as its. primary CryptoKeyVersion. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKey) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}', http_method=u'GET', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.get', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}', request_field='', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysGetRequest', response_type_name=u'CryptoKey', supports_download=False, ) def GetIamPolicy(self, request, global_params=None): """Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('GetIamPolicy') return self._RunMethod( config, request, global_params=global_params) GetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:getIamPolicy', http_method=u'GET', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.getIamPolicy', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:getIamPolicy', request_field='', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def List(self, request, global_params=None): """Lists CryptoKeys. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListCryptoKeysResponse) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys', http_method=u'GET', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.list', ordered_params=[u'parent'], path_params=[u'parent'], query_params=[u'pageSize', u'pageToken'], relative_path=u'v1/{+parent}/cryptoKeys', request_field='', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysListRequest', response_type_name=u'ListCryptoKeysResponse', supports_download=False, ) def Patch(self, request, global_params=None): """Update a CryptoKey. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysPatchRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKey) The response message. """ config = self.GetMethodConfig('Patch') return self._RunMethod( config, request, global_params=global_params) Patch.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}', http_method=u'PATCH', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.patch', ordered_params=[u'name'], path_params=[u'name'], query_params=[u'updateMask'], relative_path=u'v1/{+name}', request_field=u'cryptoKey', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysPatchRequest', response_type_name=u'CryptoKey', supports_download=False, ) def SetIamPolicy(self, request, global_params=None): """Sets the access control policy on the specified resource. Replaces any. existing policy. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('SetIamPolicy') return self._RunMethod( config, request, global_params=global_params) SetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:setIamPolicy', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.setIamPolicy', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:setIamPolicy', request_field=u'setIamPolicyRequest', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def TestIamPermissions(self, request, global_params=None): """Returns permissions that a caller has on the specified resource. If the resource does not exist, this will return an empty set of permissions, not a NOT_FOUND error. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. """ config = self.GetMethodConfig('TestIamPermissions') return self._RunMethod( config, request, global_params=global_params) TestIamPermissions.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:testIamPermissions', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.testIamPermissions', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:testIamPermissions', request_field=u'testIamPermissionsRequest', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysTestIamPermissionsRequest', response_type_name=u'TestIamPermissionsResponse', supports_download=False, ) def UpdatePrimaryVersion(self, request, global_params=None): """Update the version of a CryptoKey that will be used in Encrypt. Args: request: (CloudkmsProjectsLocationsKeyRingsCryptoKeysUpdatePrimaryVersionRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (CryptoKey) The response message. """ config = self.GetMethodConfig('UpdatePrimaryVersion') return self._RunMethod( config, request, global_params=global_params) UpdatePrimaryVersion.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}/cryptoKeys/{cryptoKeysId}:updatePrimaryVersion', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.cryptoKeys.updatePrimaryVersion', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}:updatePrimaryVersion', request_field=u'updateCryptoKeyPrimaryVersionRequest', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCryptoKeysUpdatePrimaryVersionRequest', response_type_name=u'CryptoKey', supports_download=False, ) class ProjectsLocationsKeyRingsService(base_api.BaseApiService): """Service class for the projects_locations_keyRings resource.""" _NAME = u'projects_locations_keyRings' def __init__(self, client): super(CloudkmsV1.ProjectsLocationsKeyRingsService, self).__init__(client) self._upload_configs = { } def Create(self, request, global_params=None): """Create a new KeyRing in a given Project and Location. Args: request: (CloudkmsProjectsLocationsKeyRingsCreateRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (KeyRing) The response message. """ config = self.GetMethodConfig('Create') return self._RunMethod( config, request, global_params=global_params) Create.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.create', ordered_params=[u'parent'], path_params=[u'parent'], query_params=[u'keyRingId'], relative_path=u'v1/{+parent}/keyRings', request_field=u'keyRing', request_type_name=u'CloudkmsProjectsLocationsKeyRingsCreateRequest', response_type_name=u'KeyRing', supports_download=False, ) def Get(self, request, global_params=None): """Returns metadata for a given KeyRing. Args: request: (CloudkmsProjectsLocationsKeyRingsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (KeyRing) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}', http_method=u'GET', method_id=u'cloudkms.projects.locations.keyRings.get', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}', request_field='', request_type_name=u'CloudkmsProjectsLocationsKeyRingsGetRequest', response_type_name=u'KeyRing', supports_download=False, ) def GetIamPolicy(self, request, global_params=None): """Gets the access control policy for a resource. Returns an empty policy if the resource exists and does not have a policy set. Args: request: (CloudkmsProjectsLocationsKeyRingsGetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('GetIamPolicy') return self._RunMethod( config, request, global_params=global_params) GetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}:getIamPolicy', http_method=u'GET', method_id=u'cloudkms.projects.locations.keyRings.getIamPolicy', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:getIamPolicy', request_field='', request_type_name=u'CloudkmsProjectsLocationsKeyRingsGetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def List(self, request, global_params=None): """Lists KeyRings. Args: request: (CloudkmsProjectsLocationsKeyRingsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListKeyRingsResponse) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings', http_method=u'GET', method_id=u'cloudkms.projects.locations.keyRings.list', ordered_params=[u'parent'], path_params=[u'parent'], query_params=[u'pageSize', u'pageToken'], relative_path=u'v1/{+parent}/keyRings', request_field='', request_type_name=u'CloudkmsProjectsLocationsKeyRingsListRequest', response_type_name=u'ListKeyRingsResponse', supports_download=False, ) def SetIamPolicy(self, request, global_params=None): """Sets the access control policy on the specified resource. Replaces any. existing policy. Args: request: (CloudkmsProjectsLocationsKeyRingsSetIamPolicyRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Policy) The response message. """ config = self.GetMethodConfig('SetIamPolicy') return self._RunMethod( config, request, global_params=global_params) SetIamPolicy.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}:setIamPolicy', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.setIamPolicy', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:setIamPolicy', request_field=u'setIamPolicyRequest', request_type_name=u'CloudkmsProjectsLocationsKeyRingsSetIamPolicyRequest', response_type_name=u'Policy', supports_download=False, ) def TestIamPermissions(self, request, global_params=None): """Returns permissions that a caller has on the specified resource. If the resource does not exist, this will return an empty set of permissions, not a NOT_FOUND error. Note: This operation is designed to be used for building permission-aware UIs and command-line tools, not for authorization checking. This operation may "fail open" without warning. Args: request: (CloudkmsProjectsLocationsKeyRingsTestIamPermissionsRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (TestIamPermissionsResponse) The response message. """ config = self.GetMethodConfig('TestIamPermissions') return self._RunMethod( config, request, global_params=global_params) TestIamPermissions.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}/keyRings/{keyRingsId}:testIamPermissions', http_method=u'POST', method_id=u'cloudkms.projects.locations.keyRings.testIamPermissions', ordered_params=[u'resource'], path_params=[u'resource'], query_params=[], relative_path=u'v1/{+resource}:testIamPermissions', request_field=u'testIamPermissionsRequest', request_type_name=u'CloudkmsProjectsLocationsKeyRingsTestIamPermissionsRequest', response_type_name=u'TestIamPermissionsResponse', supports_download=False, ) class ProjectsLocationsService(base_api.BaseApiService): """Service class for the projects_locations resource.""" _NAME = u'projects_locations' def __init__(self, client): super(CloudkmsV1.ProjectsLocationsService, self).__init__(client) self._upload_configs = { } def Get(self, request, global_params=None): """Get information about a location. Args: request: (CloudkmsProjectsLocationsGetRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (Location) The response message. """ config = self.GetMethodConfig('Get') return self._RunMethod( config, request, global_params=global_params) Get.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations/{locationsId}', http_method=u'GET', method_id=u'cloudkms.projects.locations.get', ordered_params=[u'name'], path_params=[u'name'], query_params=[], relative_path=u'v1/{+name}', request_field='', request_type_name=u'CloudkmsProjectsLocationsGetRequest', response_type_name=u'Location', supports_download=False, ) def List(self, request, global_params=None): """Lists information about the supported locations for this service. Args: request: (CloudkmsProjectsLocationsListRequest) input message global_params: (StandardQueryParameters, default: None) global arguments Returns: (ListLocationsResponse) The response message. """ config = self.GetMethodConfig('List') return self._RunMethod( config, request, global_params=global_params) List.method_config = lambda: base_api.ApiMethodInfo( flat_path=u'v1/projects/{projectsId}/locations', http_method=u'GET', method_id=u'cloudkms.projects.locations.list', ordered_params=[u'name'], path_params=[u'name'], query_params=[u'filter', u'pageSize', u'pageToken'], relative_path=u'v1/{+name}/locations', request_field='', request_type_name=u'CloudkmsProjectsLocationsListRequest', response_type_name=u'ListLocationsResponse', supports_download=False, ) class ProjectsService(base_api.BaseApiService): """Service class for the projects resource.""" _NAME = u'projects' def __init__(self, client): super(CloudkmsV1.ProjectsService, self).__init__(client) self._upload_configs = { } third_party/kms_apitools/resources.py 0000644 00000004030 15025013616 0014161 0 ustar 00 # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Resource definitions for cloud platform apis.""" import enum BASE_URL = 'https://cloudkms.googleapis.com/v1/' class Collections(enum.Enum): """Collections for all supported apis.""" PROJECTS_LOCATIONS = ( 'projects.locations', '{+name}', { '': 'projects/{projectsId}/locations/{locationsId}', }, [u'name'] ) PROJECTS_LOCATIONS_KEYRINGS = ( 'projects.locations.keyRings', '{+name}', { '': 'projects/{projectsId}/locations/{locationsId}/keyRings/' '{keyRingsId}', }, [u'name'] ) PROJECTS_LOCATIONS_KEYRINGS_CRYPTOKEYS = ( 'projects.locations.keyRings.cryptoKeys', '{+name}', { '': 'projects/{projectsId}/locations/{locationsId}/keyRings/' '{keyRingsId}/cryptoKeys/{cryptoKeysId}', }, [u'name'] ) PROJECTS_LOCATIONS_KEYRINGS_CRYPTOKEYS_CRYPTOKEYVERSIONS = ( 'projects.locations.keyRings.cryptoKeys.cryptoKeyVersions', '{+name}', { '': 'projects/{projectsId}/locations/{locationsId}/keyRings/' '{keyRingsId}/cryptoKeys/{cryptoKeysId}/cryptoKeyVersions/' '{cryptoKeyVersionsId}', }, [u'name'] ) def __init__(self, collection_name, path, flat_paths, params): self.collection_name = collection_name self.path = path self.flat_paths = flat_paths self.params = params third_party/kms_apitools/cloudkms_v1_messages.py 0000644 00000112700 15025013616 0016271 0 ustar 00 """Generated message classes for cloudkms version v1. Manages encryption for your cloud services the same way you do on-premises. You can generate, use, rotate, and destroy AES256 encryption keys. """ # NOTE: This file is autogenerated and should not be edited by hand. from apitools.base.protorpclite import messages as _messages from apitools.base.py import encoding from apitools.base.py import extra_types package = 'cloudkms' class AuditConfig(_messages.Message): """Specifies the audit configuration for a service. The configuration determines which permission types are logged, and what identities, if any, are exempted from logging. An AuditConfig must have one or more AuditLogConfigs. If there are AuditConfigs for both `allServices` and a specific service, the union of the two AuditConfigs is used for that service: the log_types specified in each AuditConfig are enabled, and the exempted_members in each AuditConfig are exempted. Example Policy with multiple AuditConfigs: { "audit_configs": [ { "service": "allServices" "audit_log_configs": [ { "log_type": "DATA_READ", "exempted_members": [ "user:foo@gmail.com" ] }, { "log_type": "DATA_WRITE", }, { "log_type": "ADMIN_READ", } ] }, { "service": "fooservice.googleapis.com" "audit_log_configs": [ { "log_type": "DATA_READ", }, { "log_type": "DATA_WRITE", "exempted_members": [ "user:bar@gmail.com" ] } ] } ] } For fooservice, this policy enables DATA_READ, DATA_WRITE and ADMIN_READ logging. It also exempts foo@gmail.com from DATA_READ logging, and bar@gmail.com from DATA_WRITE logging. Fields: auditLogConfigs: The configuration for logging of each type of permission. Next ID: 4 exemptedMembers: A string attribute. service: Specifies a service that will be enabled for audit logging. For example, `storage.googleapis.com`, `cloudsql.googleapis.com`. `allServices` is a special value that covers all services. """ auditLogConfigs = _messages.MessageField('AuditLogConfig', 1, repeated=True) exemptedMembers = _messages.StringField(2, repeated=True) service = _messages.StringField(3) class AuditLogConfig(_messages.Message): """Provides the configuration for logging a type of permissions. Example: { "audit_log_configs": [ { "log_type": "DATA_READ", "exempted_members": [ "user:foo@gmail.com" ] }, { "log_type": "DATA_WRITE", } ] } This enables 'DATA_READ' and 'DATA_WRITE' logging, while exempting foo@gmail.com from DATA_READ logging. Enums: LogTypeValueValuesEnum: The log type that this config enables. Fields: exemptedMembers: Specifies the identities that do not cause logging for this type of permission. Follows the same format of Binding.members. logType: The log type that this config enables. """ class LogTypeValueValuesEnum(_messages.Enum): """The log type that this config enables. Values: LOG_TYPE_UNSPECIFIED: Default case. Should never be this. ADMIN_READ: Admin reads. Example: CloudIAM getIamPolicy DATA_WRITE: Data writes. Example: CloudSQL Users create DATA_READ: Data reads. Example: CloudSQL Users list """ LOG_TYPE_UNSPECIFIED = 0 ADMIN_READ = 1 DATA_WRITE = 2 DATA_READ = 3 exemptedMembers = _messages.StringField(1, repeated=True) logType = _messages.EnumField('LogTypeValueValuesEnum', 2) class Binding(_messages.Message): """Associates `members` with a `role`. Fields: condition: The condition that is associated with this binding. NOTE: an unsatisfied condition will not allow user access via current binding. Different bindings, including their conditions, are examined independently. This field is GOOGLE_INTERNAL. members: Specifies the identities requesting access for a Cloud Platform resource. `members` can have the following values: * `allUsers`: A special identifier that represents anyone who is on the internet; with or without a Google account. * `allAuthenticatedUsers`: A special identifier that represents anyone who is authenticated with a Google account or a service account. * `user:{emailid}`: An email address that represents a specific Google account. For example, `alice@gmail.com` or `joe@example.com`. * `serviceAccount:{emailid}`: An email address that represents a service account. For example, `my-other- app@appspot.gserviceaccount.com`. * `group:{emailid}`: An email address that represents a Google group. For example, `admins@example.com`. * `domain:{domain}`: A Google Apps domain name that represents all the users of that domain. For example, `google.com` or `example.com`. role: Role that is assigned to `members`. For example, `roles/viewer`, `roles/editor`, or `roles/owner`. Required """ condition = _messages.MessageField('Expr', 1) members = _messages.StringField(2, repeated=True) role = _messages.StringField(3) class CloudkmsProjectsLocationsGetRequest(_messages.Message): """A CloudkmsProjectsLocationsGetRequest object. Fields: name: Resource name for the location. """ name = _messages.StringField(1, required=True) class CloudkmsProjectsLocationsKeyRingsCreateRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCreateRequest object. Fields: keyRing: A KeyRing resource to be passed as the request body. keyRingId: Required. It must be unique within a location and match the regular expression `[a-zA-Z0-9_-]{1,63}` parent: Required. The resource name of the location associated with the KeyRings, in the format `projects/*/locations/*`. """ keyRing = _messages.MessageField('KeyRing', 1) keyRingId = _messages.StringField(2) parent = _messages.StringField(3, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequest object. Fields: cryptoKey: A CryptoKey resource to be passed as the request body. cryptoKeyId: Required. It must be unique within a KeyRing and match the regular expression `[a-zA-Z0-9_-]{1,63}` parent: Required. The name of the KeyRing associated with the CryptoKeys. """ cryptoKey = _messages.MessageField('CryptoKey', 1) cryptoKeyId = _messages.StringField(2) parent = _messages.StringField(3, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsCreateRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsCreateRequest object. Fields: cryptoKeyVersion: A CryptoKeyVersion resource to be passed as the request body. parent: Required. The name of the CryptoKey associated with the CryptoKeyVersions. """ cryptoKeyVersion = _messages.MessageField('CryptoKeyVersion', 1) parent = _messages.StringField(2, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsDestroyRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsDestroyRequest object. Fields: destroyCryptoKeyVersionRequest: A DestroyCryptoKeyVersionRequest resource to be passed as the request body. name: The resource name of the CryptoKeyVersion to destroy. """ destroyCryptoKeyVersionRequest = _messages.MessageField('DestroyCryptoKeyVersionRequest', 1) name = _messages.StringField(2, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsGetRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsGetRequest object. Fields: name: The name of the CryptoKeyVersion to get. """ name = _messages.StringField(1, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsListRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsListRequest object. Fields: pageSize: Optional limit on the number of CryptoKeyVersions to include in the response. Further CryptoKeyVersions can subsequently be obtained by including the ListCryptoKeyVersionsResponse.next_page_token in a subsequent request. If unspecified, the server will pick an appropriate default. pageToken: Optional pagination token, returned earlier via ListCryptoKeyVersionsResponse.next_page_token. parent: Required. The resource name of the CryptoKey to list, in the format `projects/*/locations/*/keyRings/*/cryptoKeys/*`. """ pageSize = _messages.IntegerField(1, variant=_messages.Variant.INT32) pageToken = _messages.StringField(2) parent = _messages.StringField(3, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsPatchRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsPatchRequest object. Fields: cryptoKeyVersion: A CryptoKeyVersion resource to be passed as the request body. name: Output only. The resource name for this CryptoKeyVersion in the format `projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*`. updateMask: Required list of fields to be updated in this request. """ cryptoKeyVersion = _messages.MessageField('CryptoKeyVersion', 1) name = _messages.StringField(2, required=True) updateMask = _messages.StringField(3) class CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsRestoreRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysCryptoKeyVersionsRestoreRequest object. Fields: name: The resource name of the CryptoKeyVersion to restore. restoreCryptoKeyVersionRequest: A RestoreCryptoKeyVersionRequest resource to be passed as the request body. """ name = _messages.StringField(1, required=True) restoreCryptoKeyVersionRequest = _messages.MessageField('RestoreCryptoKeyVersionRequest', 2) class CloudkmsProjectsLocationsKeyRingsCryptoKeysDecryptRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysDecryptRequest object. Fields: decryptRequest: A DecryptRequest resource to be passed as the request body. name: Required. The resource name of the CryptoKey to use for decryption. The server will choose the appropriate version. """ decryptRequest = _messages.MessageField('DecryptRequest', 1) name = _messages.StringField(2, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysEncryptRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysEncryptRequest object. Fields: encryptRequest: A EncryptRequest resource to be passed as the request body. name: Required. The resource name of the CryptoKey or CryptoKeyVersion to use for encryption. If a CryptoKey is specified, the server will use its primary version. """ encryptRequest = _messages.MessageField('EncryptRequest', 1) name = _messages.StringField(2, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being requested. See the operation documentation for the appropriate value for this field. """ resource = _messages.StringField(1, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysGetRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysGetRequest object. Fields: name: The name of the CryptoKey to get. """ name = _messages.StringField(1, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysListRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysListRequest object. Fields: pageSize: Optional limit on the number of CryptoKeys to include in the response. Further CryptoKeys can subsequently be obtained by including the ListCryptoKeysResponse.next_page_token in a subsequent request. If unspecified, the server will pick an appropriate default. pageToken: Optional pagination token, returned earlier via ListCryptoKeysResponse.next_page_token. parent: Required. The resource name of the KeyRing to list, in the format `projects/*/locations/*/keyRings/*`. """ pageSize = _messages.IntegerField(1, variant=_messages.Variant.INT32) pageToken = _messages.StringField(2) parent = _messages.StringField(3, required=True) class CloudkmsProjectsLocationsKeyRingsCryptoKeysPatchRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysPatchRequest object. Fields: cryptoKey: A CryptoKey resource to be passed as the request body. name: Output only. The resource name for this CryptoKey in the format `projects/*/locations/*/keyRings/*/cryptoKeys/*`. updateMask: Required list of fields to be updated in this request. """ cryptoKey = _messages.MessageField('CryptoKey', 1) name = _messages.StringField(2, required=True) updateMask = _messages.StringField(3) class CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being specified. See the operation documentation for the appropriate value for this field. setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the request body. """ resource = _messages.StringField(1, required=True) setIamPolicyRequest = _messages.MessageField('SetIamPolicyRequest', 2) class CloudkmsProjectsLocationsKeyRingsCryptoKeysTestIamPermissionsRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysTestIamPermissionsRequest object. Fields: resource: REQUIRED: The resource for which the policy detail is being requested. See the operation documentation for the appropriate value for this field. testIamPermissionsRequest: A TestIamPermissionsRequest resource to be passed as the request body. """ resource = _messages.StringField(1, required=True) testIamPermissionsRequest = _messages.MessageField('TestIamPermissionsRequest', 2) class CloudkmsProjectsLocationsKeyRingsCryptoKeysUpdatePrimaryVersionRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsCryptoKeysUpdatePrimaryVersionRequest object. Fields: name: The resource name of the CryptoKey to update. updateCryptoKeyPrimaryVersionRequest: A UpdateCryptoKeyPrimaryVersionRequest resource to be passed as the request body. """ name = _messages.StringField(1, required=True) updateCryptoKeyPrimaryVersionRequest = _messages.MessageField('UpdateCryptoKeyPrimaryVersionRequest', 2) class CloudkmsProjectsLocationsKeyRingsGetIamPolicyRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsGetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being requested. See the operation documentation for the appropriate value for this field. """ resource = _messages.StringField(1, required=True) class CloudkmsProjectsLocationsKeyRingsGetRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsGetRequest object. Fields: name: The name of the KeyRing to get. """ name = _messages.StringField(1, required=True) class CloudkmsProjectsLocationsKeyRingsListRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsListRequest object. Fields: pageSize: Optional limit on the number of KeyRings to include in the response. Further KeyRings can subsequently be obtained by including the ListKeyRingsResponse.next_page_token in a subsequent request. If unspecified, the server will pick an appropriate default. pageToken: Optional pagination token, returned earlier via ListKeyRingsResponse.next_page_token. parent: Required. The resource name of the location associated with the KeyRings, in the format `projects/*/locations/*`. """ pageSize = _messages.IntegerField(1, variant=_messages.Variant.INT32) pageToken = _messages.StringField(2) parent = _messages.StringField(3, required=True) class CloudkmsProjectsLocationsKeyRingsSetIamPolicyRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsSetIamPolicyRequest object. Fields: resource: REQUIRED: The resource for which the policy is being specified. See the operation documentation for the appropriate value for this field. setIamPolicyRequest: A SetIamPolicyRequest resource to be passed as the request body. """ resource = _messages.StringField(1, required=True) setIamPolicyRequest = _messages.MessageField('SetIamPolicyRequest', 2) class CloudkmsProjectsLocationsKeyRingsTestIamPermissionsRequest(_messages.Message): """A CloudkmsProjectsLocationsKeyRingsTestIamPermissionsRequest object. Fields: resource: REQUIRED: The resource for which the policy detail is being requested. See the operation documentation for the appropriate value for this field. testIamPermissionsRequest: A TestIamPermissionsRequest resource to be passed as the request body. """ resource = _messages.StringField(1, required=True) testIamPermissionsRequest = _messages.MessageField('TestIamPermissionsRequest', 2) class CloudkmsProjectsLocationsListRequest(_messages.Message): """A CloudkmsProjectsLocationsListRequest object. Fields: filter: The standard list filter. name: The resource that owns the locations collection, if applicable. pageSize: The standard list page size. pageToken: The standard list page token. """ filter = _messages.StringField(1) name = _messages.StringField(2, required=True) pageSize = _messages.IntegerField(3, variant=_messages.Variant.INT32) pageToken = _messages.StringField(4) class CryptoKey(_messages.Message): """A CryptoKey represents a logical key that can be used for cryptographic operations. A CryptoKey is made up of one or more versions, which represent the actual key material used in cryptographic operations. Enums: PurposeValueValuesEnum: The immutable purpose of this CryptoKey. Currently, the only acceptable purpose is ENCRYPT_DECRYPT. Messages: LabelsValue: Labels with user defined metadata. Fields: createTime: Output only. The time at which this CryptoKey was created. labels: Labels with user defined metadata. name: Output only. The resource name for this CryptoKey in the format `projects/*/locations/*/keyRings/*/cryptoKeys/*`. nextRotationTime: At next_rotation_time, the Key Management Service will automatically: 1. Create a new version of this CryptoKey. 2. Mark the new version as primary. Key rotations performed manually via CreateCryptoKeyVersion and UpdateCryptoKeyPrimaryVersion do not affect next_rotation_time. primary: Output only. A copy of the "primary" CryptoKeyVersion that will be used by Encrypt when this CryptoKey is given in EncryptRequest.name. The CryptoKey's primary version can be updated via UpdateCryptoKeyPrimaryVersion. purpose: The immutable purpose of this CryptoKey. Currently, the only acceptable purpose is ENCRYPT_DECRYPT. rotationPeriod: next_rotation_time will be advanced by this period when the service automatically rotates a key. Must be at least one day. If rotation_period is set, next_rotation_time must also be set. """ class PurposeValueValuesEnum(_messages.Enum): """The immutable purpose of this CryptoKey. Currently, the only acceptable purpose is ENCRYPT_DECRYPT. Values: CRYPTO_KEY_PURPOSE_UNSPECIFIED: Not specified. ENCRYPT_DECRYPT: CryptoKeys with this purpose may be used with Encrypt and Decrypt. """ CRYPTO_KEY_PURPOSE_UNSPECIFIED = 0 ENCRYPT_DECRYPT = 1 @encoding.MapUnrecognizedFields('additionalProperties') class LabelsValue(_messages.Message): """Labels with user defined metadata. Messages: AdditionalProperty: An additional property for a LabelsValue object. Fields: additionalProperties: Additional properties of type LabelsValue """ class AdditionalProperty(_messages.Message): """An additional property for a LabelsValue object. Fields: key: Name of the additional property. value: A string attribute. """ key = _messages.StringField(1) value = _messages.StringField(2) additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) createTime = _messages.StringField(1) labels = _messages.MessageField('LabelsValue', 2) name = _messages.StringField(3) nextRotationTime = _messages.StringField(4) primary = _messages.MessageField('CryptoKeyVersion', 5) purpose = _messages.EnumField('PurposeValueValuesEnum', 6) rotationPeriod = _messages.StringField(7) class CryptoKeyVersion(_messages.Message): """A CryptoKeyVersion represents an individual cryptographic key, and the associated key material. It can be used for cryptographic operations either directly, or via its parent CryptoKey, in which case the server will choose the appropriate version for the operation. For security reasons, the raw cryptographic key material represented by a CryptoKeyVersion can never be viewed or exported. It can only be used to encrypt or decrypt data when an authorized user or application invokes Cloud KMS. Enums: StateValueValuesEnum: The current state of the CryptoKeyVersion. Fields: createTime: Output only. The time at which this CryptoKeyVersion was created. destroyEventTime: Output only. The time this CryptoKeyVersion's key material was destroyed. Only present if state is DESTROYED. destroyTime: Output only. The time this CryptoKeyVersion's key material is scheduled for destruction. Only present if state is DESTROY_SCHEDULED. name: Output only. The resource name for this CryptoKeyVersion in the format `projects/*/locations/*/keyRings/*/cryptoKeys/*/cryptoKeyVersions/*`. state: The current state of the CryptoKeyVersion. """ class StateValueValuesEnum(_messages.Enum): """The current state of the CryptoKeyVersion. Values: CRYPTO_KEY_VERSION_STATE_UNSPECIFIED: Not specified. ENABLED: This version may be used in Encrypt and Decrypt requests. DISABLED: This version may not be used, but the key material is still available, and the version can be placed back into the ENABLED state. DESTROYED: This version is destroyed, and the key material is no longer stored. A version may not leave this state once entered. DESTROY_SCHEDULED: This version is scheduled for destruction, and will be destroyed soon. Call RestoreCryptoKeyVersion to put it back into the DISABLED state. """ CRYPTO_KEY_VERSION_STATE_UNSPECIFIED = 0 ENABLED = 1 DISABLED = 2 DESTROYED = 3 DESTROY_SCHEDULED = 4 createTime = _messages.StringField(1) destroyEventTime = _messages.StringField(2) destroyTime = _messages.StringField(3) name = _messages.StringField(4) state = _messages.EnumField('StateValueValuesEnum', 5) class DecryptRequest(_messages.Message): """Request message for KeyManagementService.Decrypt. Fields: additionalAuthenticatedData: Optional data that must match the data originally supplied in EncryptRequest.additional_authenticated_data. ciphertext: Required. The encrypted data originally returned in EncryptResponse.ciphertext. """ additionalAuthenticatedData = _messages.BytesField(1) ciphertext = _messages.BytesField(2) class DecryptResponse(_messages.Message): """Response message for KeyManagementService.Decrypt. Fields: plaintext: The decrypted data originally supplied in EncryptRequest.plaintext. """ plaintext = _messages.BytesField(1) class DestroyCryptoKeyVersionRequest(_messages.Message): """Request message for KeyManagementService.DestroyCryptoKeyVersion.""" class EncryptRequest(_messages.Message): """Request message for KeyManagementService.Encrypt. Fields: additionalAuthenticatedData: Optional data that, if specified, must also be provided during decryption through DecryptRequest.additional_authenticated_data. Must be no larger than 64KiB. plaintext: Required. The data to encrypt. Must be no larger than 64KiB. """ additionalAuthenticatedData = _messages.BytesField(1) plaintext = _messages.BytesField(2) class EncryptResponse(_messages.Message): """Response message for KeyManagementService.Encrypt. Fields: ciphertext: The encrypted data. name: The resource name of the CryptoKeyVersion used in encryption. """ ciphertext = _messages.BytesField(1) name = _messages.StringField(2) class Expr(_messages.Message): """Represents an expression text. Example: title: "User account presence" description: "Determines whether the request has a user account" expression: "size(request.user) > 0" Fields: description: An optional description of the expression. This is a longer text which describes the expression, e.g. when hovered over it in a UI. expression: Textual representation of an expression in Common Expression Language syntax. The application context of the containing message determines which well-known feature set of CEL is supported. location: An optional string indicating the location of the expression for error reporting, e.g. a file name and a position in the file. title: An optional title for the expression, i.e. a short string describing its purpose. This can be used e.g. in UIs which allow to enter the expression. """ description = _messages.StringField(1) expression = _messages.StringField(2) location = _messages.StringField(3) title = _messages.StringField(4) class KeyRing(_messages.Message): """A KeyRing is a toplevel logical grouping of CryptoKeys. Fields: createTime: Output only. The time at which this KeyRing was created. name: Output only. The resource name for the KeyRing in the format `projects/*/locations/*/keyRings/*`. """ createTime = _messages.StringField(1) name = _messages.StringField(2) class ListCryptoKeyVersionsResponse(_messages.Message): """Response message for KeyManagementService.ListCryptoKeyVersions. Fields: cryptoKeyVersions: The list of CryptoKeyVersions. nextPageToken: A token to retrieve next page of results. Pass this value in ListCryptoKeyVersionsRequest.page_token to retrieve the next page of results. totalSize: The total number of CryptoKeyVersions that matched the query. """ cryptoKeyVersions = _messages.MessageField('CryptoKeyVersion', 1, repeated=True) nextPageToken = _messages.StringField(2) totalSize = _messages.IntegerField(3, variant=_messages.Variant.INT32) class ListCryptoKeysResponse(_messages.Message): """Response message for KeyManagementService.ListCryptoKeys. Fields: cryptoKeys: The list of CryptoKeys. nextPageToken: A token to retrieve next page of results. Pass this value in ListCryptoKeysRequest.page_token to retrieve the next page of results. totalSize: The total number of CryptoKeys that matched the query. """ cryptoKeys = _messages.MessageField('CryptoKey', 1, repeated=True) nextPageToken = _messages.StringField(2) totalSize = _messages.IntegerField(3, variant=_messages.Variant.INT32) class ListKeyRingsResponse(_messages.Message): """Response message for KeyManagementService.ListKeyRings. Fields: keyRings: The list of KeyRings. nextPageToken: A token to retrieve next page of results. Pass this value in ListKeyRingsRequest.page_token to retrieve the next page of results. totalSize: The total number of KeyRings that matched the query. """ keyRings = _messages.MessageField('KeyRing', 1, repeated=True) nextPageToken = _messages.StringField(2) totalSize = _messages.IntegerField(3, variant=_messages.Variant.INT32) class ListLocationsResponse(_messages.Message): """The response message for Locations.ListLocations. Fields: locations: A list of locations that matches the specified filter in the request. nextPageToken: The standard List next-page token. """ locations = _messages.MessageField('Location', 1, repeated=True) nextPageToken = _messages.StringField(2) class Location(_messages.Message): """A resource that represents Google Cloud Platform location. Messages: LabelsValue: Cross-service attributes for the location. For example {"cloud.googleapis.com/region": "us-east1"} MetadataValue: Service-specific metadata. For example the available capacity at the given location. Fields: labels: Cross-service attributes for the location. For example {"cloud.googleapis.com/region": "us-east1"} locationId: The canonical id for this location. For example: `"us-east1"`. metadata: Service-specific metadata. For example the available capacity at the given location. name: Resource name for the location, which may vary between implementations. For example: `"projects/example-project/locations/us- east1"` """ @encoding.MapUnrecognizedFields('additionalProperties') class LabelsValue(_messages.Message): """Cross-service attributes for the location. For example {"cloud.googleapis.com/region": "us-east1"} Messages: AdditionalProperty: An additional property for a LabelsValue object. Fields: additionalProperties: Additional properties of type LabelsValue """ class AdditionalProperty(_messages.Message): """An additional property for a LabelsValue object. Fields: key: Name of the additional property. value: A string attribute. """ key = _messages.StringField(1) value = _messages.StringField(2) additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) @encoding.MapUnrecognizedFields('additionalProperties') class MetadataValue(_messages.Message): """Service-specific metadata. For example the available capacity at the given location. Messages: AdditionalProperty: An additional property for a MetadataValue object. Fields: additionalProperties: Properties of the object. Contains field @type with type URL. """ class AdditionalProperty(_messages.Message): """An additional property for a MetadataValue object. Fields: key: Name of the additional property. value: A extra_types.JsonValue attribute. """ key = _messages.StringField(1) value = _messages.MessageField('extra_types.JsonValue', 2) additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) labels = _messages.MessageField('LabelsValue', 1) locationId = _messages.StringField(2) metadata = _messages.MessageField('MetadataValue', 3) name = _messages.StringField(4) class Policy(_messages.Message): """Defines an Identity and Access Management (IAM) policy. It is used to specify access control policies for Cloud Platform resources. A `Policy` consists of a list of `bindings`. A `Binding` binds a list of `members` to a `role`, where the members can be user accounts, Google groups, Google domains, and service accounts. A `role` is a named list of permissions defined by IAM. **Example** { "bindings": [ { "role": "roles/owner", "members": [ "user:mike@example.com", "group:admins@example.com", "domain:google.com", "serviceAccount:my-other- app@appspot.gserviceaccount.com", ] }, { "role": "roles/viewer", "members": ["user:sean@example.com"] } ] } For a description of IAM and its features, see the [IAM developer's guide](https://cloud.google.com/iam). Fields: auditConfigs: Specifies cloud audit logging configuration for this policy. bindings: Associates a list of `members` to a `role`. `bindings` with no members will result in an error. etag: `etag` is used for optimistic concurrency control as a way to help prevent simultaneous updates of a policy from overwriting each other. It is strongly suggested that systems make use of the `etag` in the read- modify-write cycle to perform policy updates in order to avoid race conditions: An `etag` is returned in the response to `getIamPolicy`, and systems are expected to put that etag in the request to `setIamPolicy` to ensure that their change will be applied to the same version of the policy. If no `etag` is provided in the call to `setIamPolicy`, then the existing policy is overwritten blindly. iamOwned: A boolean attribute. version: Version of the `Policy`. The default version is 0. """ auditConfigs = _messages.MessageField('AuditConfig', 1, repeated=True) bindings = _messages.MessageField('Binding', 2, repeated=True) etag = _messages.BytesField(3) iamOwned = _messages.BooleanField(4) version = _messages.IntegerField(5, variant=_messages.Variant.INT32) class RestoreCryptoKeyVersionRequest(_messages.Message): """Request message for KeyManagementService.RestoreCryptoKeyVersion.""" class SetIamPolicyRequest(_messages.Message): """Request message for `SetIamPolicy` method. Fields: policy: REQUIRED: The complete policy to be applied to the `resource`. The size of the policy is limited to a few 10s of KB. An empty policy is a valid policy but certain Cloud Platform services (such as Projects) might reject them. updateMask: OPTIONAL: A FieldMask specifying which fields of the policy to modify. Only the fields in the mask will be modified. If no mask is provided, the following default mask is used: paths: "bindings, etag" This field is only used by Cloud IAM. """ policy = _messages.MessageField('Policy', 1) updateMask = _messages.StringField(2) class StandardQueryParameters(_messages.Message): """Query parameters accepted by all methods. Enums: FXgafvValueValuesEnum: V1 error format. AltValueValuesEnum: Data format for response. Fields: f__xgafv: V1 error format. access_token: OAuth access token. alt: Data format for response. bearer_token: OAuth bearer token. callback: JSONP fields: Selector specifying which fields to include in a partial response. key: API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token. oauth_token: OAuth 2.0 token for the current user. pp: Pretty-print response. prettyPrint: Returns response with indentations and line breaks. quotaUser: Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. trace: A tracing token of the form "token:<tokenid>" to include in api requests. uploadType: Legacy upload protocol for media (e.g. "media", "multipart"). upload_protocol: Upload protocol for media (e.g. "raw", "multipart"). """ class AltValueValuesEnum(_messages.Enum): """Data format for response. Values: json: Responses with Content-Type of application/json media: Media download with context-dependent Content-Type proto: Responses with Content-Type of application/x-protobuf """ json = 0 media = 1 proto = 2 class FXgafvValueValuesEnum(_messages.Enum): """V1 error format. Values: _1: v1 error format _2: v2 error format """ _1 = 0 _2 = 1 f__xgafv = _messages.EnumField('FXgafvValueValuesEnum', 1) access_token = _messages.StringField(2) alt = _messages.EnumField('AltValueValuesEnum', 3, default=u'json') bearer_token = _messages.StringField(4) callback = _messages.StringField(5) fields = _messages.StringField(6) key = _messages.StringField(7) oauth_token = _messages.StringField(8) pp = _messages.BooleanField(9, default=True) prettyPrint = _messages.BooleanField(10, default=True) quotaUser = _messages.StringField(11) trace = _messages.StringField(12) uploadType = _messages.StringField(13) upload_protocol = _messages.StringField(14) class TestIamPermissionsRequest(_messages.Message): """Request message for `TestIamPermissions` method. Fields: permissions: The set of permissions to check for the `resource`. Permissions with wildcards (such as '*' or 'storage.*') are not allowed. For more information see [IAM Overview](https://cloud.google.com/iam/docs/overview#permissions). """ permissions = _messages.StringField(1, repeated=True) class TestIamPermissionsResponse(_messages.Message): """Response message for `TestIamPermissions` method. Fields: permissions: A subset of `TestPermissionsRequest.permissions` that the caller is allowed. """ permissions = _messages.StringField(1, repeated=True) class UpdateCryptoKeyPrimaryVersionRequest(_messages.Message): """Request message for KeyManagementService.UpdateCryptoKeyPrimaryVersion. Fields: cryptoKeyVersionId: The id of the child CryptoKeyVersion to use as primary. """ cryptoKeyVersionId = _messages.StringField(1) encoding.AddCustomJsonFieldMapping( StandardQueryParameters, 'f__xgafv', '$.xgafv') encoding.AddCustomJsonEnumMapping( StandardQueryParameters.FXgafvValueValuesEnum, '_1', '1') encoding.AddCustomJsonEnumMapping( StandardQueryParameters.FXgafvValueValuesEnum, '_2', '2') third_party/kms_apitools/__init__.py 0000644 00000000000 15025013616 0013677 0 ustar 00 cs_api_map.py 0000644 00000006476 15025013616 0007225 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """API map classes used with the CloudApiDelegator class.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.boto_translation import BotoTranslation from gslib.gcs_json_api import GcsJsonApi class ApiSelector(object): """Enum class for API.""" XML = 'XML' JSON = 'JSON' class ApiMapConstants(object): """Enum class for API map entries.""" API_MAP = 'apiclass' SUPPORT_MAP = 'supported' DEFAULT_MAP = 'default' class GsutilApiClassMapFactory(object): """Factory for generating gsutil API class maps. A valid class map is defined as: { (key) Provider prefix used in URI strings. (value) { (key) ApiSelector describing the API format. (value) CloudApi child class that implements this API. } } """ @classmethod def GetClassMap(cls): """Returns the default gsutil class map.""" gs_class_map = { ApiSelector.XML: BotoTranslation, ApiSelector.JSON: GcsJsonApi } s3_class_map = { ApiSelector.XML: BotoTranslation, } class_map = { 'gs': gs_class_map, 's3': s3_class_map, } return class_map class GsutilApiMapFactory(object): """Factory the generates the default gsutil API map. The API map determines which Cloud API implementation is used for a given command. A valid API map is defined as: { (key) ApiMapConstants.API_MAP : (value) Gsutil API class map (as described in GsutilApiClassMapFactory comments). (key) ApiMapConstants.SUPPORT_MAP : (value) { (key) Provider prefix used in URI strings. (value) list of ApiSelectors supported by the command for this provider. } (key) ApiMapConstants.DEFAULT_MAP : (value) { (key) Provider prefix used in URI strings. (value) Default ApiSelector for this command and provider. } } """ @classmethod def GetApiMap(cls, gsutil_api_class_map_factory, support_map, default_map): """Creates a GsutilApiMap for use by the command from the inputs. Args: gsutil_api_class_map_factory: Factory defining a GetClassMap() function adhering to GsutilApiClassMapFactory semantics. support_map: Entries for ApiMapConstants.SUPPORT_MAP as described above. default_map: Entries for ApiMapConstants.DEFAULT_MAP as described above. Returns: GsutilApiMap generated from the inputs. """ return { ApiMapConstants.API_MAP: gsutil_api_class_map_factory.GetClassMap(), ApiMapConstants.SUPPORT_MAP: support_map, ApiMapConstants.DEFAULT_MAP: default_map } boto_resumable_upload.py 0000644 00000061303 15025013616 0011466 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2010 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Boto translation layer for resumable uploads. See https://cloud.google.com/storage/docs/resumable-uploads-xml for details. Resumable uploads will retry interrupted uploads, resuming at the byte count completed by the last upload attempt. If too many retries happen with no progress (per configurable num_retries param), the upload will be aborted in the current process. Unlike the boto implementation of resumable upload handler, this class does not directly interact with tracker files. Originally Google wrote and contributed this code to the boto project, then copied that code back into gsutil on the release of gsutil 4.0 which supports both boto and non-boto codepaths for resumable uploads. Any bug fixes made to this file should also be integrated to resumable_upload_handler.py in boto, where applicable. TODO: gsutil-beta: Add a similar comment to the boto code. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import errno import random import re import socket import time import six from six.moves import urllib from six.moves import http_client from boto import config from boto import UserAgent from boto.connection import AWSAuthConnection from boto.exception import ResumableTransferDisposition from boto.exception import ResumableUploadException from gslib.exception import InvalidUrlError from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNumRetries from gslib.utils.constants import XML_PROGRESS_CALLBACKS from gslib.utils.constants import UTF8 if six.PY3: long = int class BotoResumableUpload(object): """Upload helper class for resumable uploads via boto.""" BUFFER_SIZE = 8192 RETRYABLE_EXCEPTIONS = (http_client.HTTPException, IOError, socket.error, socket.gaierror) # (start, end) response indicating service has nothing (upload protocol uses # inclusive numbering). SERVICE_HAS_NOTHING = (0, -1) def __init__(self, tracker_callback, logger, resume_url=None, num_retries=None): """Constructor. Instantiate once for each uploaded file. Args: tracker_callback: Callback function that takes a string argument. Used by caller to track this upload across upload interruption. logger: logging.logger instance to use for debug messages. resume_url: If present, attempt to resume the upload at this URL. num_retries: Number of times to retry the upload making no progress. This count resets every time we make progress, so the upload can span many more than this number of retries. """ if resume_url: self._SetUploadUrl(resume_url) else: self.upload_url = None self.num_retries = num_retries self.service_has_bytes = 0 # Byte count at last service check. # Save upload_start_point in instance state so caller can find how # much was transferred by this ResumableUploadHandler (across retries). self.upload_start_point = None self.tracker_callback = tracker_callback self.logger = logger def _SetUploadUrl(self, url): """Saves URL and resets upload state. Called when we start a new resumable upload or get a new tracker URL for the upload. Args: url: URL string for the upload. Raises InvalidUrlError if URL is syntactically invalid. """ parse_result = urllib.parse.urlparse(url) if (parse_result.scheme.lower() not in ['http', 'https'] or not parse_result.netloc): raise InvalidUrlError('Invalid upload URL (%s)' % url) self.upload_url = url self.upload_url_host = (config.get('Credentials', 'gs_host', None) or parse_result.netloc) self.upload_url_path = '%s?%s' % (parse_result.path, parse_result.query) self.service_has_bytes = 0 def _BuildContentRangeHeader(self, range_spec='*', length_spec='*'): return 'bytes %s/%s' % (range_spec, length_spec) def _QueryServiceState(self, conn, file_length): """Queries service to find out state of given upload. Note that this method really just makes special case use of the fact that the upload service always returns the current start/end state whenever a PUT doesn't complete. Args: conn: HTTPConnection to use for the query. file_length: Total length of the file. Returns: HTTP response from sending request. Raises: ResumableUploadException if problem querying service. """ # Send an empty PUT so that service replies with this resumable # transfer's state. put_headers = { 'Content-Range': (self._BuildContentRangeHeader('*', file_length)), 'Content-Length': '0' } return AWSAuthConnection.make_request(conn, 'PUT', path=self.upload_url_path, auth_path=self.upload_url_path, headers=put_headers, host=self.upload_url_host) def _QueryServicePos(self, conn, file_length): """Queries service to find out what bytes it currently has. Args: conn: HTTPConnection to use for the query. file_length: Total length of the file. Returns: (service_start, service_end), where the values are inclusive. For example, (0, 2) would mean that the service has bytes 0, 1, *and* 2. Raises: ResumableUploadException if problem querying service. """ resp = self._QueryServiceState(conn, file_length) if resp.status == 200: # To handle the boundary condition where the service has the complete # file, we return (service_start, file_length-1). That way the # calling code can always simply read up through service_end. (If we # didn't handle this boundary condition here, the caller would have # to check whether service_end == file_length and read one fewer byte # in that case.) return (0, file_length - 1) # Completed upload. if resp.status != 308: # This means the service didn't have any state for the given # upload ID, which can happen (for example) if the caller saved # the upload URL to a file and then tried to restart the transfer # after that upload ID has gone stale. In that case we need to # start a new transfer (and the caller will then save the new # upload URL to the tracker file). raise ResumableUploadException( 'Got non-308 response (%s) from service state query' % resp.status, ResumableTransferDisposition.START_OVER) got_valid_response = False range_spec = resp.getheader('range') if range_spec: # Parse 'bytes=<from>-<to>' range_spec. m = re.search(r'bytes=(\d+)-(\d+)', range_spec) if m: service_start = long(m.group(1)) service_end = long(m.group(2)) got_valid_response = True else: # No Range header, which means the service does not yet have # any bytes. Note that the Range header uses inclusive 'from' # and 'to' values. Since Range 0-0 would mean that the service # has byte 0, omitting the Range header is used to indicate that # the service doesn't have any bytes. return self.SERVICE_HAS_NOTHING if not got_valid_response: raise ResumableUploadException( 'Couldn\'t parse upload service state query response (%s)' % str(resp.getheaders()), ResumableTransferDisposition.START_OVER) if conn.debug >= 1: self.logger.debug('Service has: Range: %d - %d.', service_start, service_end) return (service_start, service_end) def _StartNewResumableUpload(self, key, headers=None): """Starts a new resumable upload. Args: key: Boto Key representing the object to upload. headers: Headers to use in the upload requests. Raises: ResumableUploadException if any errors occur. """ conn = key.bucket.connection if conn.debug >= 1: self.logger.debug('Starting new resumable upload.') self.service_has_bytes = 0 # Start a new resumable upload by sending a POST request with an # empty body and the "X-Goog-Resumable: start" header. Include any # caller-provided headers (e.g., Content-Type) EXCEPT Content-Length # (and raise an exception if they tried to pass one, since it's # a semantic error to specify it at this point, and if we were to # include one now it would cause the service to expect that many # bytes; the POST doesn't include the actual file bytes We set # the Content-Length in the subsequent PUT, based on the uploaded # file size. post_headers = {} for k in headers: if k.lower() == 'content-length': raise ResumableUploadException( 'Attempt to specify Content-Length header (disallowed)', ResumableTransferDisposition.ABORT) post_headers[k] = headers[k] post_headers[conn.provider.resumable_upload_header] = 'start' resp = conn.make_request('POST', key.bucket.name, key.name, post_headers) # Get upload URL from response 'Location' header. body = resp.read() # Check for various status conditions. if resp.status in [429, 500, 503]: # Retry after a delay. raise ResumableUploadException( 'Got status %d from attempt to start resumable upload. ' 'Will wait/retry' % resp.status, ResumableTransferDisposition.WAIT_BEFORE_RETRY) elif resp.status != 200 and resp.status != 201: raise ResumableUploadException( 'Got status %d from attempt to start resumable upload. ' 'Aborting' % resp.status, ResumableTransferDisposition.ABORT) # Else we got 200 or 201 response code, indicating the resumable # upload was created. upload_url = resp.getheader('Location') if not upload_url: raise ResumableUploadException( 'No resumable upload URL found in resumable initiation ' 'POST response (%s)' % body, ResumableTransferDisposition.WAIT_BEFORE_RETRY) self._SetUploadUrl(upload_url) self.tracker_callback(upload_url) def _UploadFileBytes(self, conn, http_conn, fp, file_length, total_bytes_uploaded, cb, num_cb, headers): """Attempts to upload file bytes. Makes a single attempt using an existing resumable upload connection. Args: conn: HTTPConnection from the boto Key. http_conn: Separate HTTPConnection for the transfer. fp: File pointer containing bytes to upload. file_length: Total length of the file. total_bytes_uploaded: The total number of bytes uploaded. cb: Progress callback function that takes (progress, total_size). num_cb: Granularity of the callback (maximum number of times the callback will be called during the file transfer). If negative, perform callback with each buffer read. headers: Headers to be used in the upload requests. Returns: (etag, generation, metageneration) from service upon success. Raises: ResumableUploadException if any problems occur. """ buf = fp.read(self.BUFFER_SIZE) if cb: # The cb_count represents the number of full buffers to send between # cb executions. if num_cb > 2: cb_count = file_length / self.BUFFER_SIZE / (num_cb - 2) elif num_cb < 0: cb_count = -1 else: cb_count = 0 i = 0 cb(total_bytes_uploaded, file_length) # Build resumable upload headers for the transfer. Don't send a # Content-Range header if the file is 0 bytes long, because the # resumable upload protocol uses an *inclusive* end-range (so, sending # 'bytes 0-0/1' would actually mean you're sending a 1-byte file). put_headers = headers.copy() if headers else {} if file_length: if total_bytes_uploaded == file_length: range_header = self._BuildContentRangeHeader('*', file_length) else: range_header = self._BuildContentRangeHeader( '%d-%d' % (total_bytes_uploaded, file_length - 1), file_length) put_headers['Content-Range'] = range_header # Set Content-Length to the total bytes we'll send with this PUT. put_headers['Content-Length'] = str(file_length - total_bytes_uploaded) http_request = AWSAuthConnection.build_base_http_request( conn, 'PUT', path=self.upload_url_path, auth_path=None, headers=put_headers, host=self.upload_url_host) http_conn.putrequest('PUT', http_request.path) for k in put_headers: http_conn.putheader(k, put_headers[k]) http_conn.endheaders() # Turn off debug on http connection so upload content isn't included # in debug stream. http_conn.set_debuglevel(0) while buf: # Some code is duplicated here, but separating the PY2 and PY3 paths makes # this easier to remove PY2 blocks when we move to PY3 only. if six.PY2: http_conn.send(buf) total_bytes_uploaded += len(buf) else: if isinstance(buf, bytes): http_conn.send(buf) total_bytes_uploaded += len(buf) else: # Probably a unicode/str object, try encoding. buf_bytes = buf.encode(UTF8) http_conn.send(buf_bytes) total_bytes_uploaded += len(buf_bytes) if cb: i += 1 if i == cb_count or cb_count == -1: cb(total_bytes_uploaded, file_length) i = 0 buf = fp.read(self.BUFFER_SIZE) # Restore http connection debug level. http_conn.set_debuglevel(conn.debug) if cb: cb(total_bytes_uploaded, file_length) if total_bytes_uploaded != file_length: # Abort (and delete the tracker file) so if the user retries # they'll start a new resumable upload rather than potentially # attempting to pick back up later where we left off. raise ResumableUploadException( 'File changed during upload: EOF at %d bytes of %d byte file.' % (total_bytes_uploaded, file_length), ResumableTransferDisposition.ABORT) resp = http_conn.getresponse() if resp.status == 200: # Success. return (resp.getheader('etag'), resp.getheader('x-goog-generation'), resp.getheader('x-goog-metageneration')) # Retry timeout (408) and status 429, 500 and 503 errors after a delay. elif resp.status in [408, 429, 500, 503]: disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY else: # Catch all for any other error codes. disposition = ResumableTransferDisposition.ABORT raise ResumableUploadException( 'Got response code %d while attempting ' 'upload (%s)' % (resp.status, resp.reason), disposition) def _AttemptResumableUpload(self, key, fp, file_length, headers, cb, num_cb): """Attempts a resumable upload. Args: key: Boto key representing object to upload. fp: File pointer containing upload bytes. file_length: Total length of the upload. headers: Headers to be used in upload requests. cb: Progress callback function that takes (progress, total_size). num_cb: Granularity of the callback (maximum number of times the callback will be called during the file transfer). If negative, perform callback with each buffer read. Returns: (etag, generation, metageneration) from service upon success. Raises: ResumableUploadException if any problems occur. """ (service_start, service_end) = self.SERVICE_HAS_NOTHING conn = key.bucket.connection if self.upload_url: # Try to resume existing resumable upload. try: (service_start, service_end) = (self._QueryServicePos(conn, file_length)) self.service_has_bytes = service_start if conn.debug >= 1: self.logger.debug('Resuming transfer.') except ResumableUploadException as e: if conn.debug >= 1: self.logger.debug('Unable to resume transfer (%s).', e.message) self._StartNewResumableUpload(key, headers) else: self._StartNewResumableUpload(key, headers) # upload_start_point allows the code that instantiated the # ResumableUploadHandler to find out the point from which it started # uploading (e.g., so it can correctly compute throughput). if self.upload_start_point is None: self.upload_start_point = service_end total_bytes_uploaded = service_end + 1 # Start reading from the file based upon the number of bytes that the # server has so far. if total_bytes_uploaded < file_length: fp.seek(total_bytes_uploaded) conn = key.bucket.connection # Get a new HTTP connection (vs conn.get_http_connection(), which reuses # pool connections) because httplib requires a new HTTP connection per # transaction. (Without this, calling http_conn.getresponse() would get # "ResponseNotReady".) http_conn = conn.new_http_connection(self.upload_url_host, conn.port, conn.is_secure) http_conn.set_debuglevel(conn.debug) # Make sure to close http_conn at end so if a local file read # failure occurs partway through service will terminate current upload # and can report that progress on next attempt. try: return self._UploadFileBytes(conn, http_conn, fp, file_length, total_bytes_uploaded, cb, num_cb, headers) except (ResumableUploadException, socket.error): resp = self._QueryServiceState(conn, file_length) if resp.status == 400: raise ResumableUploadException( 'Got 400 response from service state query after failed resumable ' 'upload attempt. This can happen for various reasons, including ' 'specifying an invalid request (e.g., an invalid canned ACL) or ' 'if the file size changed between upload attempts', ResumableTransferDisposition.ABORT) else: raise finally: http_conn.close() def HandleResumableUploadException(self, e, debug): if e.disposition == ResumableTransferDisposition.ABORT_CUR_PROCESS: if debug >= 1: self.logger.debug( 'Caught non-retryable ResumableUploadException (%s); ' 'aborting but retaining tracker file', e.message) raise elif e.disposition == ResumableTransferDisposition.ABORT: if debug >= 1: self.logger.debug( 'Caught non-retryable ResumableUploadException (%s); ' 'aborting and removing tracker file', e.message) raise elif e.disposition == ResumableTransferDisposition.START_OVER: raise else: if debug >= 1: self.logger.debug('Caught ResumableUploadException (%s) - will retry', e.message) def TrackProgressLessIterations(self, service_had_bytes_before_attempt, debug=0): """Tracks the number of iterations without progress. Performs randomized exponential backoff. Args: service_had_bytes_before_attempt: Number of bytes the service had prior to this upload attempt. debug: debug level 0..3 """ # At this point we had a re-tryable failure; see if made progress. if self.service_has_bytes > service_had_bytes_before_attempt: self.progress_less_iterations = 0 # If progress, reset counter. else: self.progress_less_iterations += 1 if self.progress_less_iterations > self.num_retries: # Don't retry any longer in the current process. raise ResumableUploadException( 'Too many resumable upload attempts failed without ' 'progress. You might try this upload again later', ResumableTransferDisposition.ABORT_CUR_PROCESS) # Use binary exponential backoff to desynchronize client requests. sleep_time_secs = min(random.random() * (2**self.progress_less_iterations), GetMaxRetryDelay()) if debug >= 1: self.logger.debug( 'Got retryable failure (%d progress-less in a row).\n' 'Sleeping %3.1f seconds before re-trying', self.progress_less_iterations, sleep_time_secs) time.sleep(sleep_time_secs) def SendFile(self, key, fp, size, headers, canned_acl=None, cb=None, num_cb=XML_PROGRESS_CALLBACKS): """Upload a file to a key into a bucket on GS, resumable upload protocol. Args: key: `boto.s3.key.Key` or subclass representing the upload destination. fp: File pointer to upload size: Size of the file to upload. headers: The headers to pass along with the PUT request canned_acl: Optional canned ACL to apply to object. cb: Callback function that will be called to report progress on the upload. The callback should accept two integer parameters, the first representing the number of bytes that have been successfully transmitted to GS, and the second representing the total number of bytes that need to be transmitted. num_cb: (optional) If a callback is specified with the cb parameter, this parameter determines the granularity of the callback by defining the maximum number of times the callback will be called during the file transfer. Providing a negative integer will cause your callback to be called with each buffer read. Raises: ResumableUploadException if a problem occurs during the transfer. """ if not headers: headers = {} # If Content-Type header is present and set to None, remove it. # This is gsutil's way of asking boto to refrain from auto-generating # that header. content_type = 'Content-Type' if content_type in headers and headers[content_type] is None: del headers[content_type] if canned_acl: headers[key.provider.acl_header] = canned_acl headers['User-Agent'] = UserAgent file_length = size debug = key.bucket.connection.debug # Use num-retries from constructor if one was provided; else check # for a value specified in the boto config file; else default to 5. if self.num_retries is None: self.num_retries = GetNumRetries() self.progress_less_iterations = 0 while True: # Retry as long as we're making progress. service_had_bytes_before_attempt = self.service_has_bytes try: # Save generation and metageneration in class state so caller # can find these values, for use in preconditions of future # operations on the uploaded object. _, self.generation, self.metageneration = self._AttemptResumableUpload( key, fp, file_length, headers, cb, num_cb) key.generation = self.generation if debug >= 1: self.logger.debug('Resumable upload complete.') return except self.RETRYABLE_EXCEPTIONS as e: if debug >= 1: self.logger.debug('Caught exception (%s)', e.__repr__()) if isinstance(e, IOError) and e.errno == errno.EPIPE: # Broken pipe error causes httplib to immediately # close the socket (http://bugs.python.org/issue5542), # so we need to close the connection before we resume # the upload (which will cause a new connection to be # opened the next time an HTTP request is sent). key.bucket.connection.connection.close() except ResumableUploadException as e: self.HandleResumableUploadException(e, debug) self.TrackProgressLessIterations(service_had_bytes_before_attempt, debug=debug) context_config.py 0000644 00000020560 15025013616 0010131 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2020 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Manages device context mTLS certificates.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import atexit import json import os from boto import config import gslib from gslib import exception from gslib.utils import boto_util from gslib.utils import execution_util # Maintain a single context configuration. _singleton_config = None # Metadata JSON that stores information about the default certificate provider. _DEFAULT_METADATA_PATH = os.path.expanduser( os.path.join('~', '.secureConnect', 'context_aware_metadata.json')) _CERT_PROVIDER_COMMAND = "cert_provider_command" _CERT_PROVIDER_COMMAND_PASSPHRASE_OPTION = "--with_passphrase" class CertProvisionError(Exception): """Represents errors when provisioning a client certificate.""" pass class ContextConfigSingletonAlreadyExistsError(Exception): """Error for when create_context_config is called multiple times.""" pass def _is_pem_section_marker(line): """Returns (begin:bool, end:bool, name:str).""" if line.startswith('-----BEGIN ') and line.endswith('-----'): return True, False, line[11:-5] elif line.startswith('-----END ') and line.endswith('-----'): return False, True, line[9:-5] else: return False, False, '' def _split_pem_into_sections(contents, logger): """Returns dict with {name: section} by parsing contents in PEM format. A simple parser for PEM file. Please see RFC 7468 for the format of PEM file. Not using regex to improve performance catching nested matches. Note: This parser requires the post-encapsulation label of a section to match its pre-encapsulation label. It ignores a section without a matching label. Args: contents (str): Contents of a PEM file. logger (logging.logger): gsutil logger. Returns: A dict of the PEM file sections. """ result = {} pem_lines = [] pem_section_name = None for line in contents.splitlines(): line = line.strip() if not line: continue begin, end, name = _is_pem_section_marker(line) if begin: if pem_section_name: logger.warning('Section %s missing end line and will be ignored.' % pem_section_name) if name in result.keys(): logger.warning('Section %s already exists, and the older section will ' 'be ignored.' % name) pem_section_name = name pem_lines = [] elif end: if not pem_section_name: logger.warning( 'Section %s missing a beginning line and will be ignored.' % name) elif pem_section_name != name: logger.warning('Section %s missing a matching end line. Found: %s' % (pem_section_name, name)) pem_section_name = None if pem_section_name: pem_lines.append(line) if end: result[name] = '\n'.join(pem_lines) + '\n' pem_section_name = None if pem_section_name: logger.warning('Section %s missing an end line.' % pem_section_name) return result def _check_path(): """Checks for content aware metadata. If content aware metadata exists, return its absolute path; otherwise, returns None. Returns: str: Absolute path if exists. Otherwise, None. """ metadata_path = os.path.expanduser(_DEFAULT_METADATA_PATH) if not os.path.exists(metadata_path): return None return metadata_path def _read_metadata_file(metadata_path): """Loads context aware metadata from the given path. Returns: dict: The metadata JSON. Raises: CertProvisionError: If failed to parse metadata as JSON. """ try: with open(metadata_path) as f: return json.load(f) except ValueError as e: raise CertProvisionError(e) def _default_command(): """Loads default cert provider command. Returns: str: The default command. Raises: CertProvisionError: If command cannot be found. """ metadata_path = _check_path() if not metadata_path: raise CertProvisionError("Client certificate provider file not found.") metadata_json = _read_metadata_file(metadata_path) if _CERT_PROVIDER_COMMAND not in metadata_json: raise CertProvisionError("Client certificate provider command not found.") command = metadata_json[_CERT_PROVIDER_COMMAND] if (_CERT_PROVIDER_COMMAND_PASSPHRASE_OPTION not in command): command.append(_CERT_PROVIDER_COMMAND_PASSPHRASE_OPTION) return command class _ContextConfig(object): """Represents the configurations associated with context aware access. Only one instance of Config can be created for the program. """ def __init__(self, logger): """Initializes config. Args: logger (logging.logger): gsutil logger. """ self.logger = logger self.use_client_certificate = config.getbool('Credentials', 'use_client_certificate') self.client_cert_path = None if not self.use_client_certificate: # Don't spend time generating values gsutil won't need. return # Generates certificate and deletes it afterwards. atexit.register(self._unprovision_client_cert) self.client_cert_path = os.path.join(boto_util.GetGsutilStateDir(), 'caa_cert.pem') try: # Certs provisioned using endpoint verification are stored as a # single file holding both the public certificate and the private key. self._provision_client_cert(self.client_cert_path) except CertProvisionError as e: self.logger.error('Failed to provision client certificate: %s' % e) def _provision_client_cert(self, cert_path): """Executes certificate provider to obtain client certificate and keys.""" cert_command_string = config.get('Credentials', 'cert_provider_command', None) if cert_command_string: cert_command = cert_command_string.split(' ') else: # Load the default certificate provider if sit is not provided by user. cert_command = _default_command() try: command_stdout_string, _ = execution_util.ExecuteExternalCommand( cert_command) sections = _split_pem_into_sections(command_stdout_string, self.logger) with open(cert_path, 'w+') as f: f.write(sections['CERTIFICATE']) if 'ENCRYPTED PRIVATE KEY' in sections: f.write(sections['ENCRYPTED PRIVATE KEY']) self.client_cert_password = sections['PASSPHRASE'].splitlines()[1] else: f.write(sections['PRIVATE KEY']) self.client_cert_password = None except (exception.ExternalBinaryError, OSError) as e: raise CertProvisionError(e) except KeyError as e: raise CertProvisionError( 'Invalid output format from certificate provider, no %s' % e) def _unprovision_client_cert(self): """Cleans up any files or resources provisioned during config init.""" if self.client_cert_path is not None: try: os.remove(self.client_cert_path) self.logger.debug('Unprovisioned client cert: %s' % self.client_cert_path) except OSError as e: self.logger.error('Failed to remove client certificate: %s' % e) def create_context_config(logger): """Should be run once at gsutil startup. Creates global singleton. Args: logger (logging.logger): For logging during config functions. Returns: New ContextConfig singleton. Raises: Exception if singleton already exists. """ global _singleton_config if not _singleton_config: _singleton_config = _ContextConfig(logger) return _singleton_config raise ContextConfigSingletonAlreadyExistsError def get_context_config(): """Retrieves ContextConfig global singleton. Returns: ContextConfig or None if global singleton doesn't exist. """ return _singleton_config pubsub_api.py 0000644 00000026260 15025013616 0007254 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """JSON gsutil Cloud API implementation for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import logging import traceback from apitools.base.py import exceptions as apitools_exceptions from boto import config from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import ServiceException from gslib.gcs_json_credentials import SetUpJsonCredentialsAndCache from gslib.no_op_credentials import NoOpCredentials from gslib.third_party.pubsub_apitools import pubsub_v1_client as apitools_client from gslib.third_party.pubsub_apitools import pubsub_v1_messages as apitools_messages from gslib.utils import system_util from gslib.utils.boto_util import GetCertsFile from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNewHttp from gslib.utils.boto_util import GetNumRetries from gslib.utils.constants import UTF8 TRANSLATABLE_APITOOLS_EXCEPTIONS = (apitools_exceptions.HttpError) if system_util.InvokedViaCloudSdk(): _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gcloud auth login`') else: _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gsutil config`') class PubsubApi(object): """Wraps calls to the Cloud Pub/Sub v1 interface via apitools.""" def __init__(self, logger=None, credentials=None, debug=0): """Performs necessary setup for interacting with Google Cloud Pub/Sub. Args: logger: logging.logger for outputting log messages. credentials: Credentials to be used for interacting with Google Cloud Pub/Sub debug: Debug level for the API implementation (0..3). """ super(PubsubApi, self).__init__() self.logger = logger self.certs_file = GetCertsFile() self.http = GetNewHttp() self.http_base = 'https://' self.host_base = config.get('Credentials', 'gs_pubsub_host', 'pubsub.googleapis.com') gs_pubsub_port = config.get('Credentials', 'gs_pubsub_port', None) self.host_port = (':' + gs_pubsub_port) if gs_pubsub_port else '' self.url_base = (self.http_base + self.host_base + self.host_port) SetUpJsonCredentialsAndCache(self, logger, credentials=credentials) log_request = (debug >= 3) log_response = (debug >= 3) self.api_client = apitools_client.PubsubV1(url=self.url_base, http=self.http, log_request=log_request, log_response=log_response, credentials=self.credentials) self.num_retries = GetNumRetries() self.api_client.num_retries = self.num_retries self.max_retry_wait = GetMaxRetryDelay() self.api_client.max_retry_wait = self.max_retry_wait if isinstance(self.credentials, NoOpCredentials): # This API key is not secret and is used to identify gsutil during # anonymous requests. self.api_client.AddGlobalParam('key', 'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM') def GetTopic(self, topic_name): request = apitools_messages.PubsubProjectsTopicsGetRequest(topic=topic_name) try: return self.api_client.projects_topics.Get(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, topic_name=topic_name) def CreateTopic(self, topic_name): topic = apitools_messages.Topic(name=topic_name) try: return self.api_client.projects_topics.Create(topic) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, topic_name=topic_name) def DeleteTopic(self, topic_name): """Only used in system tests.""" request = apitools_messages.PubsubProjectsTopicsDeleteRequest( topic=topic_name) try: return self.api_client.projects_topics.Delete(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, topic_name=topic_name) def GetTopicIamPolicy(self, topic_name): request = apitools_messages.PubsubProjectsTopicsGetIamPolicyRequest( resource=topic_name) try: return self.api_client.projects_topics.GetIamPolicy(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, topic_name=topic_name) def SetTopicIamPolicy(self, topic_name, policy): policy_request = apitools_messages.SetIamPolicyRequest(policy=policy) request = apitools_messages.PubsubProjectsTopicsSetIamPolicyRequest( resource=topic_name, setIamPolicyRequest=policy_request) try: return self.api_client.projects_topics.SetIamPolicy(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, topic_name=topic_name) def _TranslateExceptionAndRaise(self, e, topic_name=None): """Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. topic_name: Optional topic name in request that caused the exception. Raises: Translated CloudApi exception, or the original exception if it was not translatable. """ if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('TranslateExceptionAndRaise: %s', traceback.format_exc()) translated_exception = self._TranslateApitoolsException( e, topic_name=topic_name) if translated_exception: raise translated_exception else: raise def _GetMessageFromHttpError(self, http_error): if isinstance(http_error, apitools_exceptions.HttpError): if getattr(http_error, 'content', None): try: json_obj = json.loads(http_error.content.decode(UTF8)) if 'error' in json_obj and 'message' in json_obj['error']: return json_obj['error']['message'] except Exception: # pylint: disable=broad-except # If we couldn't decode anything, just leave the message as None. pass def _GetAcceptableScopesFromHttpError(self, http_error): try: www_authenticate = http_error.response['www-authenticate'] # In the event of a scope error, the www-authenticate field of the HTTP # response should contain text of the form # # 'Bearer realm="https://oauth2.googleapis.com/", # error=insufficient_scope, # scope="${space separated list of acceptable scopes}"' # # Here we use a quick string search to find the scope list, just looking # for a substring with the form 'scope="${scopes}"'. scope_idx = www_authenticate.find('scope="') if scope_idx >= 0: scopes = www_authenticate[scope_idx:].split('"')[1] return 'Acceptable scopes: %s' % scopes except Exception: # pylint: disable=broad-except # Return None if we have any trouble parsing out the acceptable scopes. pass def _TranslateApitoolsException(self, e, topic_name=None): """Translates apitools exceptions into their gsutil equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. topic_name: Optional topic name in request that caused the exception. Returns: ServiceException for translatable exceptions, None otherwise. """ if isinstance(e, apitools_exceptions.HttpError): message = self._GetMessageFromHttpError(e) if e.status_code == 400: # It is possible that the Project ID is incorrect. Unfortunately the # JSON API does not give us much information about what part of the # request was bad. return BadRequestException(message or 'Bad Request', status=e.status_code) elif e.status_code == 401: if 'Login Required' in str(e): return AccessDeniedException(message or 'Access denied: login required.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) elif e.status_code == 403: if 'The account for the specified project has been disabled' in str(e): return AccessDeniedException(message or 'Account disabled.', status=e.status_code) elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e): return AccessDeniedException(message or 'Access denied: quota exceeded. ' 'Is your project ID valid?', status=e.status_code) elif 'User Rate Limit Exceeded' in str(e): return AccessDeniedException( 'Rate limit exceeded. Please retry this ' 'request later.', status=e.status_code) elif 'Access Not Configured' in str(e): return AccessDeniedException( 'Access Not Configured. Please go to the Google Cloud Platform ' 'Console (https://cloud.google.com/console#/project) for your ' 'project, select APIs and Auth and enable the ' 'Google Cloud Pub/Sub API.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) else: return AccessDeniedException(message or e.message, status=e.status_code) elif e.status_code == 404: return NotFoundException(message, status=e.status_code) elif e.status_code == 409 and topic_name: return ServiceException('The topic %s already exists.' % topic_name, status=e.status_code) elif e.status_code == 412: return PreconditionException(message, status=e.status_code) return ServiceException(message, status=e.status_code) lazy_wrapper.py 0000644 00000004407 15025013616 0007641 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Provides a LazyWrapper class to defer loading a value until it's needed.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class LazyWrapper(object): """Wrapper for lazily instantiated objects.""" def __init__(self, func): """The init method for LazyWrapper. Args: func: A function (lambda or otherwise) to lazily evaluate. """ self._func = func def __int__(self): try: return int(self._value) except AttributeError: self._value = self._func() return int(self._value) def __eq__(self, other): try: return self._value == other except AttributeError: self._value = self._func() return self._value == other def __repr__(self): try: return str(self._value) except AttributeError: self._value = self._func() return str(self._value) def __str__(self): try: return str(self._value) except AttributeError: self._value = self._func() return str(self._value) def __call__(self): """The call method for a LazyWrapper object.""" try: return self._value except AttributeError: self._value = self._func() return self._value def __len__(self): """The len method for a LazyWrapper object.""" try: return len(self._value) except AttributeError: self.__call__() return len(self._value) def __iter__(self): """The iter method for a LazyWrapper object.""" try: return self._value.__iter__() except AttributeError: self.__call__() return self._value.__iter__() thread_message.py 0000644 00000033327 15025013616 0010100 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Thread message classes. Messages are added to the status queue. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import threading from apitools.base.py.exceptions import Error as apitools_service_error from six.moves.http_client import error as six_service_error class StatusMessage(object): """General StatusMessage class. All Message classes inherit this StatusMessage class. """ def __init__(self, message_time, process_id=None, thread_id=None): """Creates a Message. Args: message_time: Time that this message was created (since Epoch). process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). """ self.time = message_time self.process_id = process_id or os.getpid() self.thread_id = thread_id or threading.current_thread().ident def __str__(self): """Returns a string with a valid constructor for this message.""" return ( '%s(%s, process_id=%s, thread_id=%s)' % (self.__class__.__name__, self.time, self.process_id, self.thread_id)) class RetryableErrorMessage(StatusMessage): """Message class for retryable errors encountered by the JSON API. This class contains information about the retryable error encountered to report to analytics collection and to display in the UI. """ def __init__(self, exception, message_time, num_retries=0, total_wait_sec=0, process_id=None, thread_id=None): """Creates a RetryableErrorMessage. Args: exception: The retryable error that was thrown. message_time: Float representing when message was created (seconds since Epoch). num_retries: The number of retries consumed so far. total_wait_sec: The total amount of time waited so far in retrying. process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). """ super(RetryableErrorMessage, self).__init__(message_time, process_id=process_id, thread_id=thread_id) self.error_type = exception.__class__.__name__ # The socket module error class names aren't descriptive enough, so we # make the error_type more specific. Standard Python uses the module name # 'socket' while PyPy uses '_socket' instead. if exception.__class__.__module__ in ('socket', '_socket'): self.error_type = 'Socket' + exception.__class__.__name__.capitalize() if (isinstance(exception, apitools_service_error) or isinstance(exception, six_service_error)): self.is_service_error = True else: self.is_service_error = False # The number of retries consumed to display to the user. self.num_retries = num_retries # The total amount of time waited on the request to display to the user. self.total_wait_sec = total_wait_sec def __str__(self): """Returns a string with a valid constructor for this message.""" return ('%s(%s(), num_retries=%s, total_wait_sec=%s, ' 'time=%s, process_id=%s, thread_id=%s)' % (self.__class__.__name__, self.error_type, self.num_retries, self.total_wait_sec, self.time, self.process_id, self.thread_id)) class FinalMessage(StatusMessage): """Creates a FinalMessage. A FinalMessage simply indicates that we have finished our operation. """ def __init__(self, message_time): """Creates a FinalMessage. Args: message_time: Float representing when message was created (seconds since Epoch). """ super(FinalMessage, self).__init__(message_time) class MetadataMessage(StatusMessage): """Creates a MetadataMessage. A MetadataMessage simply indicates that a metadata operation on a given object has been successfully done. The only passed argument is the time when such operation has finished. """ def __init__(self, message_time): """Creates a MetadataMessage. Args: message_time: Float representing when message was created (seconds since Epoch). """ super(MetadataMessage, self).__init__(message_time) class FileMessage(StatusMessage): """Marks start or end of an operation for a file, cloud object or component. This class contains general information about each file/component. With that, information such as total size and estimated time remaining may be calculated beforehand. """ # Enum message types FILE_DOWNLOAD = 1 FILE_UPLOAD = 2 FILE_CLOUD_COPY = 3 FILE_LOCAL_COPY = 4 FILE_DAISY_COPY = 5 FILE_REWRITE = 6 FILE_HASH = 7 COMPONENT_TO_UPLOAD = 8 # EXISTING_COMPONENT describes a component that already exists. The field # finished does not apply quite well for it, but the convention used by the UI # is to process the component alongside FileMessages from other components # when finished==False, and to ignore a FileMessage made for # EXISTING_COMPONENT when finished==True. EXISTING_COMPONENT = 9 COMPONENT_TO_DOWNLOAD = 10 EXISTING_OBJECT_TO_DELETE = 11 def __init__(self, src_url, dst_url, message_time, size=None, finished=False, component_num=None, message_type=None, bytes_already_downloaded=None, process_id=None, thread_id=None): """Creates a FileMessage. Args: src_url: FileUrl/CloudUrl representing the source file. dst_url: FileUrl/CloudUrl representing the destination file. message_time: Float representing when message was created (seconds since Epoch). size: Total size of this file/component, in bytes. finished: Boolean to indicate whether this is starting or finishing a file/component transfer. component_num: Component number, if dealing with a component. message_type: Type of the file/component. bytes_already_downloaded: Specific field for resuming downloads. When starting a component download, it tells how many bytes were already downloaded. process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). """ super(FileMessage, self).__init__(message_time, process_id=process_id, thread_id=thread_id) self.src_url = src_url self.dst_url = dst_url self.size = size self.component_num = component_num self.finished = finished self.message_type = message_type self.bytes_already_downloaded = bytes_already_downloaded def __str__(self): """Returns a string with a valid constructor for this message.""" return ('%s(\'%s\', \'%s\', %s, size=%s, finished=%s, component_num=%s, ' 'message_type=%s, bytes_already_downloaded=%s, process_id=%s, ' 'thread_id=%s)' % (self.__class__.__name__, self.src_url, self.dst_url, self.time, self.size, self.finished, self.component_num, self.message_type, self.bytes_already_downloaded, self.process_id, self.thread_id)) class ProgressMessage(StatusMessage): """Message class for a file/object/component progress. This class contains specific information about the current progress of a file, cloud object or single component. """ def __init__(self, size, processed_bytes, src_url, message_time, dst_url=None, component_num=None, operation_name=None, process_id=None, thread_id=None): """Creates a ProgressMessage. Args: size: Integer for total size of this file/component, in bytes. processed_bytes: Integer for number of bytes already processed from that specific component, which means processed_bytes <= size. src_url: FileUrl/CloudUrl representing the source file. message_time: Float representing when message was created (seconds since Epoch). dst_url: FileUrl/CloudUrl representing the destination file, or None for unary operations like hashing. component_num: Indicates the component number, if any. operation_name: Name of the operation that is being made over that component. process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). """ super(ProgressMessage, self).__init__(message_time) self.size = size self.processed_bytes = processed_bytes self.component_num = component_num self.src_url = src_url self.dst_url = dst_url self.finished = (size == processed_bytes) self.operation_name = operation_name def __str__(self): """Returns a string with a valid constructor for this message.""" # For a valid constructor, None should not be quoted. dst_url_string = ('\'%s\'' % self.dst_url) if self.dst_url else None operation_name_string = ( '\'%s\'' % self.operation_name) if self.operation_name else None return ('%s(%s, %s, \'%s\', %s, dst_url=%s, component_num=%s, ' 'operation_name=%s, process_id=%s, thread_id=%s)' % (self.__class__.__name__, self.size, self.processed_bytes, self.src_url, self.time, dst_url_string, self.component_num, operation_name_string, self.process_id, self.thread_id)) class SeekAheadMessage(StatusMessage): """Message class for results obtained by SeekAheadThread(). It estimates the number of objects and total size in case the task_queue cannot hold all tasks at once (only used in large operations). This class contains information about all the objects yet to be processed. """ def __init__(self, num_objects, size, message_time): """Creates a SeekAheadMessage. Args: num_objects: Number of total objects that the SeekAheadThread estimates. size: Total size corresponding to the sum of the size of objects iterated by SeekAheadThread. message_time: Float representing when message was created (seconds since Epoch). """ super(SeekAheadMessage, self).__init__(message_time) self.num_objects = num_objects self.size = size def __str__(self): """Returns a string with a valid constructor for this message.""" return ('%s(%s, %s, %s, process_id=%s, thread_id=%s)' % (self.__class__.__name__, self.num_objects, self.size, self.time, self.process_id, self.thread_id)) class ProducerThreadMessage(StatusMessage): """Message class for results obtained by calculations made on ProducerThread. It estimates the number of objects and total size currently dealty by task_queue. If the task_queue cannot support all objects at once, the SeekAheadThread will be responsible for sending an accurate message. """ def __init__(self, num_objects, size, message_time, finished=False): """Creates a SeekAheadMessage. Args: num_objects: Number of total objects that the task_queue has. size: Total size corresponding to the sum of the size of objects iterated by the task_queue message_time: Float representing when message was created (seconds since Epoch). finished: Boolean to indicate whether this is the final message from the ProducerThread. The difference is that this message displays the correct total size and number of objects, whereas the previous ones were periodic (on the number of files) updates. """ super(ProducerThreadMessage, self).__init__(message_time) self.num_objects = num_objects self.size = size self.finished = finished def __str__(self): """Returns a string with a valid constructor for this message.""" return ('%s(%s, %s, %s, finished=%s)' % (self.__class__.__name__, self.num_objects, self.size, self.time, self.finished)) class PerformanceSummaryMessage(StatusMessage): """Message class to log PerformanceSummary parameters. This class acts as a relay between a multiprocess/multithread situation and the global status queue, from which the PerformanceSummary info gets consumed. """ def __init__(self, message_time, uses_slice): """Creates a PerformanceSummaryMessage. Args: message_time: Float representing when message was created (seconds since Epoch). uses_slice: True if the command uses slice parallelism. """ super(PerformanceSummaryMessage, self).__init__(message_time, process_id=None, thread_id=None) self.uses_slice = uses_slice def __str__(self): """Returns a string with a valid constructor for this message.""" return ('%s(%s, %s)' % (self.__class__.__name__, self.time, self.uses_slice)) metrics.py 0000644 00000115755 15025013616 0006601 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Static data and helper functions for collecting user data.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import atexit from collections import defaultdict from functools import wraps import logging import os import pickle import platform import re import socket import subprocess import sys import tempfile import textwrap import time import uuid import six from six.moves import input from six.moves import urllib import boto from gslib import VERSION from gslib.metrics_tuple import Metric from gslib.utils import system_util from gslib.utils.unit_util import CalculateThroughput from gslib.utils.unit_util import HumanReadableToBytes _GA_ENDPOINT = 'https://ssl.google-analytics.com/collect' _GA_TID = 'UA-36037335-16' _GA_TID_TESTING = 'UA-36037335-17' _GA_COMMANDS_CATEGORY = 'Command' _GA_ERRORRETRY_CATEGORY = 'RetryableError' _GA_ERRORFATAL_CATEGORY = 'FatalError' _GA_PERFSUM_CATEGORY = 'PerformanceSummary' # This approximates whether the host is on a Google corporate network, # but since users choose their own hostnames, it is not guaranteed to be # accurate. _GOOGLE_CORP_HOST_RE = re.compile(r'.*google\.com$') _UUID_FILE_PATH = os.path.expanduser(os.path.join('~', '.gsutil/analytics-uuid')) # If this string is written to analytics-uuid, that means that the user said # 'no' to analytics, and it should thus be disabled. _DISABLED_TEXT = 'DISABLED' # Analytics collection uses the environment variable 'GSUTIL_TEST_ANALYTICS'. # - A value of '1' completely disables analytics collection. This is used during # non-analytics tests to avoid reporting analytics events during normal # testing. # - A value of '2' sets testing parameters and prevents the metrics_reporter.py # subprocess from writing to the metrics log file. This is used during # analytics tests in integration test subprocesses to disable reporting to GA # and avoid overwriting the metrics log file. In this scenario, the main # process’s root logger is enabled for debug output, so we still want # collection to occur so we can read metrics log messages from stderr. # - Any other value sets default behavior. # Map from descriptive labels to the key labels that GA recognizes. _GA_LABEL_MAP = { 'Event Category': 'ec', 'Event Action': 'ea', 'Event Label': 'el', 'Event Value': 'ev', 'Command Name': 'cd1', 'Global Options': 'cd2', 'Command-Level Options': 'cd3', 'Config': 'cd4', 'Command Alias': 'cd5', 'Fatal Error': 'cd6', 'Parallelism Strategy': 'cd7', 'Source URL Type': 'cd8', 'Provider Types': 'cd9', 'Timestamp': 'cd10', 'Execution Time': 'cm1', 'Retryable Errors': 'cm2', 'Is Google Corp User': 'cm3', 'Num Processes': 'cm4', 'Num Threads': 'cm5', 'Number of Files/Objects Transferred': 'cm6', 'Size of Files/Objects Transferred': 'cm7', 'Average Overall Throughput': 'cm8', 'Num Retryable Service Errors': 'cm9', 'Num Retryable Network Errors': 'cm10', 'Thread Idle Time Percent': 'cm11', 'Slowest Thread Throughput': 'cm12', 'Fastest Thread Throughput': 'cm13', 'Disk I/O Time': 'cm14', } class MetricsCollector(object): """A singleton class to handle metrics reporting to Google Analytics (GA). This class is not thread or process-safe, and logging directly to the MetricsCollector instance can only be done by a single thread. """ def __init__(self, ga_tid=_GA_TID, endpoint=_GA_ENDPOINT): """Initialize a new MetricsCollector. This should only be invoked through the GetCollector or StartTestCollector functions. Args: ga_tid: The Google Analytics tracking ID to use for metrics collection. endpoint: The URL to send requests to. """ self.start_time = _GetTimeInMillis() self.endpoint = endpoint self.logger = logging.getLogger() # Calculate parameters to send with every GA event, stored as ga_params. cid = MetricsCollector._GetCID() # gsutil developers should set this config value to true in order to hit the # testing GA property rather than the production property. use_test_property = boto.config.getbool('GSUtil', 'use_test_GA_property') if use_test_property: ga_tid = _GA_TID_TESTING # A string of non-PII config values. config_values = self._ValidateAndGetConfigValues() # Approximate if this is a Google corporate user. is_corp_user = 0 if _GOOGLE_CORP_HOST_RE.match(socket.gethostname()): is_corp_user = 1 self.ga_params = { 'v': '1', 'tid': ga_tid, 'cid': cid, 't': 'event', _GA_LABEL_MAP['Config']: config_values, _GA_LABEL_MAP['Is Google Corp User']: is_corp_user, } # Used by Google Analytics to track user OS. self.user_agent = '{system}/{release}'.format(system=platform.system(), release=platform.release()) # A list of collected, unsent Metrics. This list is currently bounded by # the number of retryable error types, and should not grow too large so that # we stay well within memory constraints. self._metrics = [] # Store a count of the number of each type of retryable error. self.retryable_errors = defaultdict(int) # This will be set to keep track of information necessary for a # PerformanceSummary if the cp or rsync commands are run. self.perf_sum_params = None _instance = None # Whether analytics collection is disabled or not. _disabled_cache = None def _ValidateAndGetConfigValues(self): """Parses the user's config file to aggregate non-PII config values. Returns: A comma-delimited string of config values explicitly set by the user in key:value pairs, sorted alphabetically by key. """ config_values = [] # If a user has an invalid config value set, we will mark it as such with # this value. If a user did not enter a value, we will not report it. invalid_value_string = 'INVALID' def GetAndValidateConfigValue(section, category, validation_fn): try: config_value = boto.config.get_value(section, category) if config_value and validation_fn(config_value): config_values.append((category, config_value)) # If the user entered a non-valid config value, store it as invalid. elif config_value: config_values.append((category, invalid_value_string)) # This function gets called during initialization of the MetricsCollector. # If any of the logic fails, we do not want to hinder the gsutil command # being run, and thus ignore any exceptions. except: # pylint: disable=bare-except config_values.append((category, invalid_value_string)) # Validate boolean values. for section, bool_category in (('Boto', 'https_validate_certificates'), ('GSUtil', 'disable_analytics_prompt'), ('GSUtil', 'use_magicfile'), ('GSUtil', 'tab_completion_time_logs')): GetAndValidateConfigValue(section=section, category=bool_category, validation_fn=lambda val: str(val).lower() in ('true', 'false')) # Define a threshold for some config values which should be reasonably low. small_int_threshold = 2000 # Validate small integers. for section, small_int_category in ( ('Boto', 'debug'), ('Boto', 'http_socket_timeout'), ('Boto', 'num_retries'), ('Boto', 'max_retry_delay'), ('GSUtil', 'default_api_version'), ('GSUtil', 'sliced_object_download_max_components'), ('GSUtil', 'parallel_process_count'), ('GSUtil', 'parallel_thread_count'), ('GSUtil', 'software_update_check_period'), ('GSUtil', 'tab_completion_timeout'), ('OAuth2', 'oauth2_refresh_retries'), ): GetAndValidateConfigValue(section=section, category=small_int_category, validation_fn=lambda val: str(val).isdigit() and int(val) < small_int_threshold) # Validate large integers. for section, large_int_category in (('GSUtil', 'resumable_threshold'), ('GSUtil', 'rsync_buffer_lines'), ('GSUtil', 'task_estimation_threshold')): GetAndValidateConfigValue(section=section, category=large_int_category, validation_fn=lambda val: str(val).isdigit()) # Validate data sizes. for section, data_size_category in ( ('GSUtil', 'parallel_composite_upload_component_size'), ('GSUtil', 'parallel_composite_upload_threshold'), ('GSUtil', 'sliced_object_download_component_size'), ('GSUtil', 'sliced_object_download_threshold'), ): config_value = boto.config.get_value(section, data_size_category) if config_value: try: size_in_bytes = HumanReadableToBytes(config_value) config_values.append((data_size_category, size_in_bytes)) except ValueError: config_values.append((data_size_category, invalid_value_string)) # Validate specific options. # pylint: disable=g-long-lambda GetAndValidateConfigValue( section='GSUtil', category='check_hashes', validation_fn=lambda val: val in ('if_fast_else_fail', 'if_fast_else_skip', 'always', 'never')) # pylint: enable=g-long-lambda GetAndValidateConfigValue( section='GSUtil', category='content_language', validation_fn=lambda val: val.isalpha() and len(val) <= 3) GetAndValidateConfigValue( section='GSUtil', category='json_api_version', validation_fn=lambda val: val[0].lower() == 'v' and val[1:].isdigit()) GetAndValidateConfigValue(section='GSUtil', category='prefer_api', validation_fn=lambda val: val in ('json', 'xml')) GetAndValidateConfigValue(section='OAuth2', category='token_cache', validation_fn=lambda val: val in ('file_system', 'in_memory')) return ','.join( sorted([ '{0}:{1}'.format(config[0], config[1]) for config in config_values ])) @staticmethod def GetCollector(ga_tid=_GA_TID): """Returns the singleton MetricsCollector instance or None if disabled.""" if MetricsCollector.IsDisabled(): return None if not MetricsCollector._instance: MetricsCollector._instance = MetricsCollector(ga_tid) return MetricsCollector._instance @staticmethod def IsDisabled(): """Returns whether metrics collection should be disabled.""" if MetricsCollector._disabled_cache is None: MetricsCollector._CheckAndSetDisabledCache() return MetricsCollector._disabled_cache @classmethod def _CheckAndSetDisabledCache(cls): """Sets _disabled_cache based on user opt-in or out.""" # Disable collection for a test case where no metrics should be collected. if os.environ.get('GSUTIL_TEST_ANALYTICS') == '1': cls._disabled_cache = True # Enable test collector for a subprocess integration test case where we # check the log output, which requires a test collector. elif os.environ.get('GSUTIL_TEST_ANALYTICS') == '2': cls._disabled_cache = False cls.StartTestCollector() # Non-testing cases involve checking for shim usage, the cloud SDK wrapper # and the analytics uuid file. elif boto.config.getbool('GSUtil', 'use_gcloud_storage', False): cls._disabled_cache = True elif system_util.InvokedViaCloudSdk(): cls._disabled_cache = not os.environ.get('GA_CID') elif os.path.exists(_UUID_FILE_PATH): with open(_UUID_FILE_PATH) as f: cls._disabled_cache = (f.read() == _DISABLED_TEXT) else: cls._disabled_cache = True @classmethod def StartTestCollector(cls, endpoint='https://example.com', user_agent='user-agent-007', ga_params=None): """Reset the singleton MetricsCollector with testing parameters. Should only be used for tests, where we want to change the default parameters. Args: endpoint: str, URL to post to user_agent: str, User-Agent string for header. ga_params: A list of two-dimensional string tuples to send as parameters. """ # Re-enable analytics for the duration of the testing. if cls.IsDisabled(): os.environ['GSUTIL_TEST_ANALYTICS'] = '0' cls._disabled_cache = False cls._instance = cls(_GA_TID_TESTING, endpoint) if ga_params is None: ga_params = {'a': 'b', 'c': 'd'} cls._instance.ga_params = ga_params cls._instance.user_agent = user_agent if os.environ['GSUTIL_TEST_ANALYTICS'] != '2': cls._instance.start_time = 0 @classmethod def StopTestCollector(cls, original_instance=None): """Reset the MetricsCollector with default parameters after testing. Args: original_instance: The original instance of the MetricsCollector so we can set the collector back to its original state. """ os.environ['GSUTIL_TEST_ANALYTICS'] = '1' cls._disabled_cache = None cls._instance = original_instance @staticmethod def _GetCID(): """Gets the client id from the UUID file or the SDK opt-in, or returns None. Returns: str, The hex string of the client id. """ if os.path.exists(_UUID_FILE_PATH): with open(_UUID_FILE_PATH) as f: cid = f.read() if cid: return cid # Returns CID from SDK. This value will be None if there is no opt-in from # the SDK. return os.environ.get('GA_CID') def ExtendGAParams(self, new_params): """Extends self.ga_params to include new parameters. This is only used to record parameters that are sent with every event type, such as global and command-level options. Args: new_params: A dictionary of key-value parameters to send. """ self.ga_params.update(new_params) def GetGAParam(self, param_name): """Convenience function for getting a ga_param of the collector. Args: param_name: The descriptive name of the param (e.g. 'Command Name'). Must be a key in _GA_LABEL_MAP. Returns: The GA parameter specified, or None. """ return self.ga_params.get(_GA_LABEL_MAP[param_name]) def CollectGAMetric(self, category, action, label=VERSION, value=0, execution_time=None, **custom_params): """Adds a GA metric with the given parameters to the metrics queue. Args: category: str, the GA Event category. action: str, the GA Event action. label: str, the GA Event label. value: int, the GA Event value. execution_time: int, the execution time to record in ms. **custom_params: A dictionary of key, value pairs containing custom metrics and dimensions to send with the GA Event. """ params = [('ec', category), ('ea', action), ('el', label), ('ev', value), (_GA_LABEL_MAP['Timestamp'], _GetTimeInMillis())] params.extend([ (k, v) for k, v in six.iteritems(custom_params) if v is not None ]) params.extend([ (k, v) for k, v in six.iteritems(self.ga_params) if v is not None ]) # Log how long after the start of the program this event happened. if execution_time is None: execution_time = _GetTimeInMillis() - self.start_time params.append((_GA_LABEL_MAP['Execution Time'], execution_time)) data = urllib.parse.urlencode(sorted(params)) self._metrics.append( Metric(endpoint=self.endpoint, method='POST', body=data, user_agent=self.user_agent)) # TODO: Collect CPU usage (Linux-only), latency to first byte, and slowest # thread process. class _PeformanceSummaryParams(object): """This class contains information to create a PerformanceSummary event.""" def __init__(self): self.num_processes = 0 self.num_threads = 0 self.num_retryable_service_errors = 0 self.num_retryable_network_errors = 0 self.provider_types = set() # Store the disk stats at the beginning of the command so we can calculate # time spent on disk I/O. if system_util.IS_LINUX: self.disk_counters_start = system_util.GetDiskCounters() # True if using fan parallelism, when the user specifies the -m option. self.uses_fan = False # True if the command uses slice parallelism. self.uses_slice = False # The total times in seconds spent idle and executing by threads. self.thread_idle_time = 0 self.thread_execution_time = 0 # This maps (process id, thread id) to a _ThreadThroughputInfo object, # keeping track of elapsed time and bytes processed. self.thread_throughputs = defaultdict(self._ThreadThroughputInformation) # Data transfer statistics. self.avg_throughput = None # This is the amount of time spent on the Apply call of cp and rsync. self.total_elapsed_time = None self.total_bytes_transferred = None self.num_objects_transferred = 0 # Information to determine the type of transfer. self.is_daisy_chain = False self.has_file_dst = False self.has_cloud_dst = False self.has_file_src = False self.has_cloud_src = False class _ThreadThroughputInformation(object): """A class to keep track of throughput information for a single thread.""" def __init__(self): self.total_bytes_transferred = 0 self.total_elapsed_time = 0 # Start time in seconds of the current task. None if the thread is not # currently executing a task. self.task_start_time = None # The size of the current task, if any. A FileMessage gives us size at # the beginning of a transfer, but we don't want to count these bytes # until the transfer is complete. self.task_size = None def LogTaskStart(self, start_time, bytes_to_transfer): self.task_start_time = start_time self.task_size = bytes_to_transfer def LogTaskEnd(self, end_time): self.total_elapsed_time += end_time - self.task_start_time self.total_bytes_transferred += self.task_size self.task_start_time = None self.task_size = None def GetThroughput(self): return CalculateThroughput(self.total_bytes_transferred, self.total_elapsed_time) def UpdatePerformanceSummaryParams(self, params): """Updates the _PeformanceSummaryParams object. Args: params: A dictionary of keyword arguments. - uses_fan: True if the command uses fan parallelism. - uses_slice: True if the command uses slice parallelism. - avg_throughput: The average throughput of the data transfer. - is_daisy_chain: True if the transfer uses the daisy-chain method. - has_file_dst: True if the transfer's destination is a file URL. - has_cloud_dst: True if the transfer's destination is in the cloud. - has_file_src: True if the transfer has a file URL as a source. - has_cloud_src: True if the transfer has a cloud URL as a source. - total_elapsed_time: The total amount of time spent on Apply. - total_bytes_transferred: The total number of bytes transferred. - thread_idle_time: The additional amount of time that threads spent idle in Apply. - thread_execution_time: The additional amount of time that threads spent executing in Apply. - num_retryable_service_errors: The additional number of retryable service errors that occurred. - num_retryable_network_errors: The additional number of retryable network errors that occurred. - num_processes: The number of processes used in a call to Apply. - num_threads: The number of threads used in a call to Apply. - num_objects_transferred: The total number of objects transferred, as specified by a ProducerThreadMessage. - provider_types: A list of additional provider types used. - file_message: A FileMessage used to calculate thread throughput and number of objects transferred in the non-parallel case. """ if self.GetGAParam('Command Name') not in ('cp', 'rsync'): return if self.perf_sum_params is None: self.perf_sum_params = self._PeformanceSummaryParams() # Most of the parameters are logged only one time, but FileMessages are much # more common, with a minimum of 2 per file copied. Since FileMessages are # logged on their own, we can avoid performing the other checks. if 'file_message' in params: self._ProcessFileMessage(file_message=params['file_message']) return for param_name, param in six.iteritems(params): # These parameters start in 0 or False state and can be updated to a # non-zero value or True. if param_name in ('uses_fan', 'uses_slice', 'avg_throughput', 'is_daisy_chain', 'has_file_dst', 'has_cloud_dst', 'has_file_src', 'has_cloud_src', 'total_elapsed_time', 'total_bytes_transferred', 'num_objects_transferred'): cur_value = getattr(self.perf_sum_params, param_name) if not cur_value: setattr(self.perf_sum_params, param_name, param) # These parameters need to be incremented. if param_name in ('thread_idle_time', 'thread_execution_time', 'num_retryable_service_errors', 'num_retryable_network_errors'): cur_value = getattr(self.perf_sum_params, param_name) setattr(self.perf_sum_params, param_name, cur_value + param) # Different calls to Apply might use different numbers of processes and # threads; we want the maximum of these. if param_name in ('num_processes', 'num_threads'): cur_value = getattr(self.perf_sum_params, param_name) if cur_value < param: setattr(self.perf_sum_params, param_name, param) # If we have new provider types, add them to our set. if param_name == 'provider_types': self.perf_sum_params.provider_types.update(param) def _ProcessFileMessage(self, file_message): """Processes FileMessages for thread throughput calculations. Update a thread's throughput based on the FileMessage, which marks the start or end of a file or component transfer. The FileMessage provides the number of bytes transferred as well as start and end time. Args: file_message: The FileMessage to process. """ thread_info = (self.perf_sum_params.thread_throughputs[( file_message.process_id, file_message.thread_id)]) if file_message.finished: # If this operation doesn't use parallelism, we manually update the # number of objects transferred rather than relying on # ProducerThreadMessages. if not (self.perf_sum_params.uses_slice or self.perf_sum_params.uses_fan): self.perf_sum_params.num_objects_transferred += 1 thread_info.LogTaskEnd(file_message.time) else: thread_info.LogTaskStart(file_message.time, file_message.size) def _CollectCommandAndErrorMetrics(self): """Aggregates command and error info and adds them to the metrics list.""" # Collect the command metric, including the number of retryable errors. command_name = self.GetGAParam('Command Name') if command_name: self.CollectGAMetric(category=_GA_COMMANDS_CATEGORY, action=command_name, **{ _GA_LABEL_MAP['Retryable Errors']: sum(self.retryable_errors.values()) }) # Collect the retryable errors. for error_type, num_errors in six.iteritems(self.retryable_errors): self.CollectGAMetric(category=_GA_ERRORRETRY_CATEGORY, action=error_type, **{_GA_LABEL_MAP['Retryable Errors']: num_errors}) # Collect the fatal error, if any. fatal_error_type = self.GetGAParam('Fatal Error') if fatal_error_type: self.CollectGAMetric(category=_GA_ERRORFATAL_CATEGORY, action=fatal_error_type) def _CollectPerformanceSummaryMetric(self): """Aggregates PerformanceSummary info and adds the metric to the list.""" if self.perf_sum_params is None: return custom_params = {} # These parameters need no further processing. for attr_name, label in ( ('num_processes', 'Num Processes'), ('num_threads', 'Num Threads'), ('num_retryable_service_errors', 'Num Retryable Service Errors'), ('num_retryable_network_errors', 'Num Retryable Network Errors'), ('avg_throughput', 'Average Overall Throughput'), ('num_objects_transferred', 'Number of Files/Objects Transferred'), ('total_bytes_transferred', 'Size of Files/Objects Transferred'), ): custom_params[_GA_LABEL_MAP[label]] = getattr(self.perf_sum_params, attr_name) # Calculate the disk stats again to calculate deltas of time spent on I/O. if system_util.IS_LINUX: disk_start = self.perf_sum_params.disk_counters_start disk_end = system_util.GetDiskCounters() # Read and write time are the 5th and 6th elements of the stat tuple. custom_params[_GA_LABEL_MAP['Disk I/O Time']] = ( sum([stat[4] + stat[5] for stat in disk_end.values()]) - sum([stat[4] + stat[5] for stat in disk_start.values()])) # Determine source URL type(s). if self.perf_sum_params.has_cloud_src: src_url_type = 'both' if self.perf_sum_params.has_file_src else 'cloud' else: src_url_type = 'file' custom_params[_GA_LABEL_MAP['Source URL Type']] = src_url_type # Determine the type of parallelism used, if any. if self.perf_sum_params.uses_fan: strategy = 'both' if self.perf_sum_params.uses_slice else 'fan' else: strategy = 'slice' if self.perf_sum_params.uses_slice else 'none' custom_params[_GA_LABEL_MAP['Parallelism Strategy']] = strategy # Determine the percentage of time that threads spent idle. total_time = (self.perf_sum_params.thread_idle_time + self.perf_sum_params.thread_execution_time) if total_time: custom_params[_GA_LABEL_MAP['Thread Idle Time Percent']] = ( float(self.perf_sum_params.thread_idle_time) / float(total_time)) # Determine the slowest and fastest thread throughputs. if self.perf_sum_params.thread_throughputs: throughputs = [ thread.GetThroughput() for thread in self.perf_sum_params.thread_throughputs.values() ] custom_params[_GA_LABEL_MAP['Slowest Thread Throughput']] = min( throughputs) custom_params[_GA_LABEL_MAP['Fastest Thread Throughput']] = max( throughputs) # Determine the provider(s) used. custom_params[_GA_LABEL_MAP['Provider Types']] = ','.join( sorted(self.perf_sum_params.provider_types)) # Determine the transfer types. # This maps a transfer type to whether the condition has been met for it. transfer_types = { 'CloudToCloud': self.perf_sum_params.has_cloud_src and self.perf_sum_params.has_cloud_dst, 'CloudToFile': self.perf_sum_params.has_cloud_src and self.perf_sum_params.has_file_dst, 'DaisyChain': self.perf_sum_params.is_daisy_chain, 'FileToCloud': self.perf_sum_params.has_file_src and self.perf_sum_params.has_cloud_dst, 'FileToFile': self.perf_sum_params.has_file_src and self.perf_sum_params.has_file_dst, } action = ','.join( sorted([ transfer_type for transfer_type, cond in six.iteritems(transfer_types) if cond ])) # Use the time spent on Apply rather than the total command execution time # for the execution time metric. This aligns more closely with throughput # and bytes transferred, and the corresponding Command event already tells # us the total time. If PerformanceSummary events are expanded, this may not # reflect one Apply call as commands like rm may call Apply twice. Currently # Apply is timed directly in the RunCommand methods of cp and rsync. apply_execution_time = _GetTimeInMillis( self.perf_sum_params.total_elapsed_time) self.CollectGAMetric(category=_GA_PERFSUM_CATEGORY, action=action, execution_time=apply_execution_time, **custom_params) def ReportMetrics(self, wait_for_report=False, log_level=None, log_file_path=None): """Reports the collected metrics using a separate async process. Args: wait_for_report: bool, True if the main process should wait for the subprocess to exit for testing purposes. log_level: int, The subprocess logger's level of debugging for testing purposes. log_file_path: str, The file that the metrics_reporter module should write its logs to. If not supplied, the metrics_reporter module will use a predetermined default path. This parameter is intended for use by tests that need to evaluate the contents of the file at this path. """ self._CollectCommandAndErrorMetrics() self._CollectPerformanceSummaryMetric() if not self._metrics: return if not log_level: log_level = self.logger.getEffectiveLevel() # If this a testing subprocess, we don't want to write to the log file. if os.environ.get('GSUTIL_TEST_ANALYTICS') == '2': log_level = logging.WARN temp_metrics_file = tempfile.NamedTemporaryFile(delete=False) temp_metrics_file_name = six.ensure_str(temp_metrics_file.name) with temp_metrics_file: pickle.dump(self._metrics, temp_metrics_file) logging.debug(self._metrics) self._metrics = [] if log_file_path is not None: # If the path is not None, we'll need to surround the path with quotes # so that the path is passed as a string to the metrics_reporter module. log_file_path = six.ensure_str('r"%s"' % log_file_path) reporting_code = six.ensure_str( 'from gslib.metrics_reporter import ReportMetrics; ' 'ReportMetrics(r"{0}", {1}, log_file_path={2})'.format( temp_metrics_file_name, log_level, log_file_path)) execution_args = [sys.executable, '-c', reporting_code] exec_env = os.environ.copy() exec_env['PYTHONPATH'] = os.pathsep.join(sys.path) # Ensuring submodule (sm) environment keys and values are all str. sm_env = dict() for k, v in six.iteritems(exec_env): sm_env[six.ensure_str(k)] = six.ensure_str(v) try: # In order for Popen to work correctly with Windows/Py3 shell needs # to be True. p = subprocess.Popen(execution_args, env=sm_env, shell=(six.PY3 and system_util.IS_WINDOWS)) self.logger.debug('Metrics reporting process started...') if wait_for_report: # NOTE: p.wait() can cause a deadlock. p.communicate() is recommended. # See python docs for more information. p.communicate() self.logger.debug('Metrics reporting process finished.') except OSError: # This can happen specifically if the Python executable moves between the # start of this process and now. self.logger.debug('Metrics reporting process failed to start.') # Delete the tempfile that would normally be cleaned up in the subprocess. try: os.unlink(temp_metrics_file.name) except: # pylint: disable=bare-except pass def CaptureAndLogException(func): """Function decorator to capture and log any exceptions. This is extra insurance that analytics collection will not hinder the command being run upon an error. Args: func: The function to wrap. Returns: The wrapped function. """ @wraps(func) def Wrapper(*args, **kwds): try: return func(*args, **kwds) except Exception as e: # pylint:disable=broad-except logger = logging.getLogger('metrics') logger.debug('Exception captured in %s during metrics collection: %s', func.__name__, e) return Wrapper def CaptureThreadStatException(func): """Function decorator to ignore an exception on collecting thread stats. An exception can happen if the thread_stats dictionary's manager gets shutdown before the thread's process is successfully killed. See _ThreadStat.AggregateStat for how we handle that case. Args: func: The function to wrap. Returns: The wrapped function. """ @wraps(func) def Wrapper(*args, **kwds): try: return func(*args, **kwds) except: # pylint:disable=bare-except # Don't surface the exception to the user. pass return Wrapper # Ordering of these wrappers is important, since both have exception handling. # CaptureAndLogException will swallow the exception. @atexit.register @CaptureAndLogException def Shutdown(): """Reports the metrics that were collected upon termination.""" collector = MetricsCollector.GetCollector() if collector: collector.ReportMetrics() @CaptureAndLogException def LogCommandParams(command_name=None, subcommands=None, global_opts=None, sub_opts=None, command_alias=None): """Logs info about the gsutil command being run. This only updates the collector's ga_params. The actual command metric will be collected once ReportMetrics() is called at shutdown. Args: command_name: str, The official command name (e.g. version instead of ver). subcommands: A list of subcommands as strings already validated by RunCommand. We do not log subcommands for the help or test commands. global_opts: A list of string tuples already parsed by __main__. sub_opts: A list of command-level options as string tuples already parsed by RunCommand. command_alias: str, The supported alias that the user inputed. """ collector = MetricsCollector.GetCollector() if not collector: return # Never re-log any parameter that's already there, as we should only be # collecting the global (user-run) command. if command_name and not collector.GetGAParam('Command Name'): collector.ExtendGAParams({_GA_LABEL_MAP['Command Name']: command_name}) if global_opts and not collector.GetGAParam('Global Options'): global_opts_string = ','.join( sorted([opt[0].strip('-') for opt in global_opts])) collector.ExtendGAParams( {_GA_LABEL_MAP['Global Options']: global_opts_string}) # Only log subcommands, suboptions, and command alias if a command has been # logged. command_name = collector.GetGAParam('Command Name') if not command_name: return if subcommands: full_command_name = '{0} {1}'.format(command_name, ' '.join(subcommands)) collector.ExtendGAParams({_GA_LABEL_MAP['Command Name']: full_command_name}) if sub_opts and not collector.GetGAParam('Command-Level Options'): sub_opts_string = ','.join(sorted([opt[0].strip('-') for opt in sub_opts])) collector.ExtendGAParams( {_GA_LABEL_MAP['Command-Level Options']: sub_opts_string}) if command_alias and not collector.GetGAParam('Command Alias'): collector.ExtendGAParams({_GA_LABEL_MAP['Command Alias']: command_alias}) @CaptureAndLogException def LogRetryableError(message): """Logs that a retryable error was caught for a gsutil command. Args: message: The RetryableErrorMessage posted to the global status queue. """ collector = MetricsCollector.GetCollector() if collector: # Update the retryable_errors defaultdict. collector.retryable_errors[message.error_type] += 1 # Update the service error or network error count. if message.is_service_error: LogPerformanceSummaryParams(num_retryable_service_errors=1) else: LogPerformanceSummaryParams(num_retryable_network_errors=1) @CaptureAndLogException def LogFatalError(exception): """Logs that a fatal error was caught for a gsutil command. Args: exception: The exception that the command failed on. """ collector = MetricsCollector.GetCollector() if collector: collector.ExtendGAParams( {_GA_LABEL_MAP['Fatal Error']: exception.__class__.__name__}) @CaptureAndLogException def LogPerformanceSummaryParams(**params_to_update): """Logs parameters necessary for a PerformanceSummary. gsutil periodically monitors its own threads; at the end of the execution of each cp/rsync command, it will present a performance summary of the command run. Args: **params_to_update: A dictionary. See UpdatePerformanceSummaryParams for details. The argument ambiguity at this level allows for flexibility in dealing with arguments that are processed similarly. """ collector = MetricsCollector.GetCollector() if collector: collector.UpdatePerformanceSummaryParams(params_to_update) @CaptureAndLogException def CheckAndMaybePromptForAnalyticsEnabling(): """Asks a user to opt-in to data collection if a UUID file does not exist. If the user agrees, generates a UUID file. Will not prompt if part of SDK. """ disable_prompt = boto.config.get_value('GSUtil', 'disable_analytics_prompt') if (not os.path.exists(_UUID_FILE_PATH) and not disable_prompt and not system_util.InvokedViaCloudSdk()): enable_analytics = input('\n' + textwrap.fill( 'gsutil developers rely on user feedback to make improvements to the ' 'tool. Would you like to send anonymous usage statistics to help ' 'improve gsutil? [y/N]') + ' ') text_to_write = _DISABLED_TEXT if enable_analytics.lower()[0] == 'y': text_to_write = uuid.uuid4().hex system_util.CreateDirIfNeeded(os.path.dirname(_UUID_FILE_PATH)) with open(_UUID_FILE_PATH, 'w') as f: f.write(text_to_write) def _GetTimeInMillis(time_in_sec=None): if time_in_sec is None: time_in_sec = time.time() return int(time_in_sec * 1000) addlhelp/__pycache__/__init__.cpython-39.pyc 0000644 00000000503 15025013616 0014716 0 ustar 00 a +(Wg � @ s8 d Z ddlmZ ddlmZ ddlmZ ddlmZ dS )zPackage marker file.� )�absolute_import)�print_function)�division)�unicode_literalsN)�__doc__� __future__r r r r � r r �&/opt/gsutil/gslib/addlhelp/__init__.py�<module> s addlhelp/prod.py 0000644 00000020167 15025013616 0007644 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about using gsutil for production tasks.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider from gslib.utils.constants import RESUMABLE_THRESHOLD_MIB _DETAILED_HELP_TEXT = (""" <B>OVERVIEW</B> If you use gsutil in large production tasks (such as uploading or downloading many GiBs of data each night), there are a number of things you can do to help ensure success. Specifically, this section discusses how to script large production tasks around gsutil's resumable transfer mechanism. <B>BACKGROUND ON RESUMABLE TRANSFERS</B> First, it's helpful to understand gsutil's resumable transfer mechanism, and how your script needs to be implemented around this mechanism to work reliably. gsutil uses resumable transfer support when you attempt to download a file of any size or to upload a file larger than a configurable threshold (by default, this threshold is %d MiB). If a transfer fails partway through (e.g., because of an intermittent network problem), gsutil uses a `truncated randomized binary exponential backoff-and-retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_ that by default retries transfers up to 23 times over a 10 minute period of time. If the transfer fails each of these attempts with no intervening progress, gsutil gives up on the transfer, but keeps a "tracker" file for it in a configurable location (the default location is ~/.gsutil/, in a file named by a combination of the SHA1 hash of the name of the bucket and object being transferred and the last 16 characters of the file name). When transfers fail in this fashion, you can rerun gsutil at some later time (e.g., after the networking problem has been resolved), and the resumable transfer picks up where it left off. <B>SCRIPTING DATA TRANSFER TASKS</B> To script large production data transfer tasks around this mechanism, you can implement a script that runs periodically, determines which file transfers have not yet succeeded, and runs gsutil to copy them. Below, we offer a number of suggestions about how this type of scripting should be implemented: 1. When resumable transfers fail without any progress 23 times in a row over the course of up to 10 minutes, it probably won't work to simply retry the transfer immediately. A more successful strategy would be to have a cron job that runs every 30 minutes, determines which transfers need to be run, and runs them. If the network experiences intermittent problems, the script picks up where it left off and will eventually succeed (once the network problem has been resolved). 2. If your business depends on timely data transfer, you should consider implementing some network monitoring. For example, you can implement a task that attempts a small download every few minutes and raises an alert if the attempt fails for several attempts in a row (or more or less frequently depending on your requirements), so that your IT staff can investigate problems promptly. As usual with monitoring implementations, you should experiment with the alerting thresholds, to avoid false positive alerts that cause your staff to begin ignoring the alerts. 3. There are a variety of ways you can determine what files remain to be transferred. We recommend that you avoid attempting to get a complete listing of a bucket containing many objects (e.g., tens of thousands or more). One strategy is to structure your object names in a way that represents your transfer process, and use gsutil prefix wildcards to request partial bucket listings. For example, if your periodic process involves downloading the current day's objects, you could name objects using a year-month-day-object-ID format and then find today's objects by using a command like gsutil ls "gs://bucket/2011-09-27-*". Note that it is more efficient to have a non-wildcard prefix like this than to use something like gsutil ls "gs://bucket/*-2011-09-27". The latter command actually requests a complete bucket listing and then filters in gsutil, while the former asks Google Storage to return the subset of objects whose names start with everything up to the "*". For data uploads, another technique would be to move local files from a "to be processed" area to a "done" area as your script successfully copies files to the cloud. You can do this in parallel batches by using a command like: gsutil -m cp -r to_upload/subdir_$i gs://bucket/subdir_$i where i is a shell loop variable. Make sure to check the shell $status variable is 0 after each gsutil cp command, to detect if some of the copies failed, and rerun the affected copies. With this strategy, the file system keeps track of all remaining work to be done. 4. If you have really large numbers of objects in a single bucket (say hundreds of thousands or more), you should consider tracking your objects in a database instead of using bucket listings to enumerate the objects. For example this database could track the state of your downloads, so you can determine what objects need to be downloaded by your periodic download script by querying the database locally instead of performing a bucket listing. 5. Make sure you don't delete partially downloaded temporary files after a transfer fails: gsutil picks up where it left off (and performs a hash of the final downloaded content to ensure data integrity), so deleting partially transferred files will cause you to lose progress and make more wasteful use of your network. 6. If you have a fast network connection, you can speed up the transfer of large numbers of files by using the gsutil -m (multi-threading / multi-processing) option. Be aware, however, that gsutil doesn't attempt to keep track of which files were downloaded successfully in cases where some files failed to download. For example, if you use multi-threaded transfers to download 100 files and 3 failed to download, it is up to your scripting process to determine which transfers didn't succeed, and retry them. A periodic check-and-run approach like outlined earlier would handle this case. If you use parallel transfers (gsutil -m) you might want to experiment with the number of threads being used (via the parallel_thread_count setting in the .boto config file). By default, gsutil uses 10 threads for Linux and 24 threads for other operating systems. Depending on your network speed, available memory, CPU load, and other conditions, this may or may not be optimal. Try experimenting with higher or lower numbers of threads to find the best number of threads for your environment. """ % RESUMABLE_THRESHOLD_MIB) class CommandOptions(HelpProvider): """Additional help about using gsutil for production tasks.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='prod', help_name_aliases=[ 'production', 'resumable', 'resumable upload', 'resumable transfer', 'resumable download', 'scripts', 'scripting', ], help_type='additional_help', help_one_line_summary='Scripting Production Transfers', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/naming.py 0000644 00000021520 15025013616 0010143 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about gsutil object and bucket naming.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>BUCKET NAME REQUIREMENTS</B> Google Cloud Storage has a single namespace, so you will not be allowed to create a bucket with a name already in use by another user. You can, however, carve out parts of the bucket name space corresponding to your company's domain name (see "DOMAIN NAMED BUCKETS"). Bucket names must conform to standard DNS naming conventions. This is because a bucket name can appear in a DNS record as part of a CNAME redirect. In addition to meeting DNS naming requirements, Google Cloud Storage imposes other requirements on bucket naming. At a minimum, your bucket names must meet the following requirements: - Bucket names must contain only lowercase letters, numbers, dashes (-), and dots (.). - Bucket names must start and end with a number or letter. - Bucket names must contain 3 to 63 characters. Names containing dots can contain up to 222 characters, but each dot-separated component can be no longer than 63 characters. - Bucket names cannot be represented as an IPv4 address in dotted-decimal notation (for example, 192.168.5.4). - Bucket names cannot begin with the "goog" prefix. - For DNS compliance, you should not have a period adjacent to another period or dash. For example, ".." or "-." or ".-" are not acceptable. <B>OBJECT NAME REQUIREMENTS</B> Object names can contain any sequence of Unicode characters, of length 1-1024 bytes when UTF-8 encoded. Object names must not contain CarriageReturn, CarriageReturnLineFeed, or the XML-disallowed surrogate blocks (xFFFE or xFFFF). We strongly recommend that you abide by the following object naming conventions: - Avoid using control characters that are illegal in XML 1.0 in your object names (#x7F-#x84 and #x86-#x9F). These characters will cause XML listing issues when you try to list your objects. - Avoid using "#" in your object names. gsutil interprets object names ending with #<numeric string> as version identifiers, so including "#" in object names can make it difficult or impossible to perform various operations on such objects using gsutil (see 'gsutil help versions'). - Avoid using "[", "]", "*", or "?" in your object names. gsutil interprets these characters as wildcards, so including any of these characters in object names can make it difficult or impossible to perform various wildcard operations using gsutil (see 'gsutil help wildcards'). See also 'gsutil help encoding' about file/object name encoding requirements and potential interoperability concerns. <B>DOMAIN NAMED BUCKETS</B> You can carve out parts of the Google Cloud Storage bucket name space by creating buckets with domain names (like "example.com"). Before you can create a bucket name containing one or more '.' characters, the following rules apply: - If the name is a syntactically valid DNS name ending with a currently-recognized top-level domain (such as .com), you will be required to verify domain ownership. - Otherwise you will be disallowed from creating the bucket. If your project needs to use a domain-named bucket, you need to have a team member both verify the domain and create the bucket. This is because Google Cloud Storage checks for domain ownership against the user who creates the bucket, so the user who creates the bucket must also be verified as an owner or manager of the domain. To verify as the owner or manager of a domain, use the Google Webmaster Tools verification process. The Webmaster Tools verification process provides three methods for verifying an owner or manager of a domain: 1. Adding a special Meta tag to a site's homepage. 2. Uploading a special HTML file to a site. 3. Adding a DNS TXT record to a domain's DNS configuration. Meta tag verification and HTML file verification are easier to perform and are probably adequate for most situations. DNS TXT record verification is a domain-based verification method that is useful in situations where a site wants to tightly control who can create domain-named buckets. Once a site creates a DNS TXT record to verify ownership of a domain, it takes precedence over meta tag and HTML file verification. For example, you might have two IT staff members who are responsible for managing your site, called "example.com." If they complete the DNS TXT record verification, only they would be able to create buckets called "example.com", "reports.example.com", "downloads.example.com", and other domain-named buckets. Site-Based Verification ----------------------- If you have administrative control over the HTML files that make up a site, you can use one of the site-based verification methods to verify that you control or own a site. When you do this, Google Cloud Storage lets you create buckets representing the verified site and any sub-sites - provided nobody has used the DNS TXT record method to verify domain ownership of a parent of the site. As an example, assume that nobody has used the DNS TXT record method to verify ownership of the following domains: abc.def.example.com, def.example.com, and example.com. In this case, Google Cloud Storage lets you create a bucket named abc.def.example.com if you verify that you own or control any of the following sites: http://abc.def.example.com http://def.example.com http://example.com Domain-Based Verification ------------------------- If you have administrative control over a domain's DNS configuration, you can use the DNS TXT record verification method to verify that you own or control a domain. When you use the domain-based verification method to verify that you own or control a domain, Google Cloud Storage lets you create buckets that represent any subdomain under the verified domain. Furthermore, Google Cloud Storage prevents anybody else from creating buckets under that domain unless you add their name to the list of verified domain owners or they have verified their domain ownership by using the DNS TXT record verification method. For example, if you use the DNS TXT record verification method to verify your ownership of the domain example.com, Google Cloud Storage will let you create bucket names that represent any subdomain under the example.com domain, such as abc.def.example.com, example.com/music/jazz, or abc.example.com/music/jazz. Using the DNS TXT record method to verify domain ownership supersedes verification by site-based verification methods. For example, if you use the Meta tag method or HTML file method to verify domain ownership of http://example.com, but someone else uses the DNS TXT record method to verify ownership of the example.com domain, Google Cloud Storage will not allow you to create a bucket named example.com. To create the bucket example.com, the domain owner who used the DNS TXT method to verify domain ownership must add you to the list of verified domain owners for example.com. The DNS TXT record verification method is particularly useful if you manage a domain for a large organization that has numerous subdomains because it lets you control who can create buckets representing those domain names. Note: If you use the DNS TXT record verification method to verify ownership of a domain, you cannot create a CNAME record for that domain. RFC 1034 disallows inclusion of any other resource records if there is a CNAME resource record present. If you want to create a CNAME resource record for a domain, you must use the Meta tag verification method or the HTML file verification method. """) class CommandOptions(HelpProvider): """Additional help about gsutil object and bucket naming.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='naming', help_name_aliases=['domain', 'limits', 'name', 'names'], help_type='additional_help', help_one_line_summary='Object and Bucket Naming', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/metadata.py 0000644 00000007367 15025013616 0010467 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about object metadata.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>OVERVIEW OF METADATA</B> Objects can have associated metadata, which control aspects of how GET requests are handled, including ``Content-Type``, ``Cache-Control``, ``Content-Disposition``, and ``Content-Encoding``. In addition, you can set custom ``key:value`` metadata for use by your applications. For a discussion of specific metadata properties, see the `metadata concept page <https://cloud.google.com/storage/docs/metadata>`_. There are two ways to set metadata on objects: - At upload time you can specify one or more metadata properties to associate with objects, using the ``gsutil -h option``. For example, the following command would cause gsutil to set the ``Content-Type`` and ``Cache-Control`` for each of the files being uploaded from a local directory named ``images``: gsutil -h "Content-Type:text/html" \\ -h "Cache-Control:public, max-age=3600" cp -r images \\ gs://bucket/images Note that -h is an option on the gsutil command, not the cp sub-command. - You can set or remove metadata fields from already uploaded objects using the ``gsutil setmeta`` command. See "gsutil help setmeta". <B>SETTABLE FIELDS; FIELD VALUES</B> You can't set some metadata fields, such as ETag and Content-Length. The fields you can set are: - ``Cache-Control`` - ``Content-Disposition`` - ``Content-Encoding`` - ``Content-Language`` - ``Content-Type`` - ``Custom-Time`` - Custom metadata Field names are case-insensitive. All fields and their values must consist only of ASCII characters, with the exception of values for ``x-goog-meta-`` fields, which may contain arbitrary Unicode values. Note that when setting metadata using the XML API, which sends custom metadata as HTTP headers, Unicode characters are encoded using UTF-8, then url-encoded to ASCII. For example: gsutil setmeta -h "x-goog-meta-foo: ã" gs://bucket/object stores the custom metadata key-value pair of ``foo`` and ``%C3%A3``. Subsequently, running ``ls -L`` using the JSON API to list the object's metadata prints ``%C3%A3``, while ``ls -L`` using the XML API url-decodes this value automatically, printing the character ``ã``. <B>VIEWING CURRENTLY SET METADATA</B> You can see what metadata is currently set on an object by using: gsutil ls -L gs://the_bucket/the_object """) class CommandOptions(HelpProvider): """Additional help about object metadata.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='metadata', help_name_aliases=[ 'cache-control', 'caching', 'content type', 'mime type', 'mime', 'type', ], help_type='additional_help', help_one_line_summary='Working With Object Metadata', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/creds.py 0000644 00000020123 15025013616 0007770 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about types of credentials and authentication.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>OVERVIEW</B> gsutil currently supports several types of credentials/authentication, as well as the ability to `access public data anonymously <https://cloud.google.com/storage/docs/access-public-data>`_. Each of these type of credentials is discussed in more detail below, along with information about configuring and using credentials via the Cloud SDK. <B>Configuring/Using Credentials via Cloud SDK Distribution of gsutil</B> When gsutil is installed/used via the Cloud SDK ("gcloud"), credentials are stored by Cloud SDK in a non-user-editable file located under ~/.config/gcloud (any manipulation of credentials should be done via the gcloud auth command). If you need to set up multiple credentials (e.g., one for an individual user account and a second for a service account), the gcloud auth command manages the credentials for you, and you switch between credentials using the gcloud auth command as well (for more details, see https://cloud.google.com/sdk/gcloud/reference/auth). Once credentials have been configured via ``gcloud auth``, those credentials are used regardless of whether the user has any boto configuration files (which are located at ~/.boto unless a different path is specified in the BOTO_CONFIG environment variable). However, gsutil still looks for credentials in the boto config file if a type of non-GCS credential is needed that's not stored in the gcloud credential store (e.g., an HMAC credential for an S3 account). <B>SUPPORTED CREDENTIAL TYPES</B> gsutil supports several types of credentials (the specific subset depends on which distribution of gsutil you are using; see above discussion). OAuth2 user account: This type of credential can be used for authenticating requests on behalf of a specific user (which is probably the most common use of gsutil). This is the default type of credential that is created when you run ``gcloud init``. This credential type is not supported for stand-alone versions of gsutil. For more details about OAuth2 authentication, see: https://developers.google.com/accounts/docs/OAuth2#scenarios HMAC: This type of credential can be used by programs that are implemented using HMAC authentication, which is an authentication mechanism supported by certain other cloud storage service providers. This type of credential can also be used for interactive use when moving data to/from service providers that support HMAC credentials. This is the type of credential that is created when you run ``gsutil config -a``. Note that it's possible to set up HMAC credentials for both Google Cloud Storage and another service provider; or to set up OAuth2 user account credentials for Google Cloud Storage and HMAC credentials for another service provider. To do so, after you run the ``gcloud init`` command, you can edit the generated ~/.boto config file and look for comments for where other credentials can be added. For more details about HMAC authentication, see `HMAC keys for Cloud Storage <https://cloud.google.com/storage/docs/authentication/hmackeys>`_. OAuth2 service account: This is a credential you can use when authenticating on behalf of a service or application (as opposed to a user). For example, if you intend to run gsutil out of a nightly cron job to upload/download data, using a service account means the cron job does not depend on credentials of an individual employee at your company. This is the type of credential that is configured when you run ``gcloud auth login`` with the ``--cred-file`` flag (or ``gsutil config -e`` when using stand-alone versions of gsutil). You should generally avoid this credential type because it requires storing highly-privileged credentials in your local environment, creating potential security risks. Instead, service account impersonation or workload identity federation are the recommended alternatives when authenticating on behalf of a service or application. It is important to note that a service account is considered an Editor by default for the purposes of API access, rather than an Owner. In particular, the fact that Editors have OWNER access in the default object and bucket ACLs, but the predefined ACL options remove OWNER access from Editors, can lead to unexpected results. The solution to this problem is to use "gsutil acl ch" instead of "gsutil acl set <predefined-ACL>" to change permissions on a bucket. To set up a service account for use with gsutil, see `Authorize a service account using a service account key <https://cloud.google.com/sdk/docs/authorizing#key>`_. For more details about OAuth2 service accounts, see `Using OAuth 2.0 for server to server applications <https://developers.google.com/identity/protocols/oauth2/service-account>`_. Compute Engine internal service account: This is the type of service account used for accounts hosted by App Engine or Compute Engine. Such credentials are created automatically for you on Compute Engine when you run the ``gcloud compute instances create`` command and the credentials can be controlled with the ``--scopes`` flag. For more details about using service account credentials for authenticating workloads on Compute Engine, see https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances. For more details about App Engine service accounts, see https://developers.google.com/appengine/docs/python/appidentity/overview Service account impersonation: Impersonating a service account is useful in scenarios where you need to grant short-term access to specific resources. For example, if you have a bucket of sensitive data that is typically read-only and want to temporarily grant write access through a trusted service account. You can specify which service account to use for impersonation by running ``gsutil -i``, ``gsutil config -e`` and editing the boto configuration file, or ``gcloud config set auth/impersonate_service_account [service_account_email_address]``. In order to impersonate, your original credentials need to be granted roles/iam.serviceAccountTokenCreator on the target service account. For more information see https://cloud.google.com/docs/authentication/use-service-account-impersonation. External account credentials (workload identity federation): Using workload identity federation, you can access Google Cloud resources from Amazon Web Services (AWS), Microsoft Azure or any identity provider that supports OpenID Connect (OIDC) or SAML 2.0. For more information see https://cloud.google.com/iam/docs/workload-identity-federation. """) class CommandOptions(HelpProvider): """Additional help about types of credentials and authentication.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='creds', help_name_aliases=['credentials', 'authentication', 'auth', 'gcloud'], help_type='additional_help', help_one_line_summary='Credential Types Supporting Various Use Cases', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/security.py 0000644 00000025335 15025013616 0010551 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about types of credentials and authentication.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>OVERVIEW</B> This help section provides details about various precautions taken by gsutil to protect data security, as well as recommendations for how customers should safeguard security. <B>TRANSPORT LAYER SECURITY</B> gsutil performs all operations using transport-layer encryption (HTTPS), to protect against data leakage over shared network links. This is also important because gsutil uses "bearer tokens" for authentication (OAuth2) as well as for resumable upload identifiers, and such tokens must be protected from being eavesdropped and reused. gsutil also supports the older HMAC style of authentication via the XML API (see `gsutil endpoints <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_). While HMAC authentication does not use bearer tokens (and thus is not subject to eavesdropping/replay attacks), it's still important to encrypt data traffic. To add an extra layer of security, gsutil supports mutual TLS (mTLS) for the Cloud Storage JSON API. With mTLS, the client verifies the server certificate, and the server also verifies the client. To find out more about how to enable mTLS, see the `install docs <https://cloud.google.com/storage/docs/gsutil_install>`_. <B>LOCAL FILE STORAGE SECURITY</B> gsutil takes a number of precautions to protect against security exploits in the files it stores locally: - When the ``gcloud init``, ``gsutil config -a``, or ``gsutil config -e`` commands run, they set file protection mode 600 ("-rw-------") on the .boto configuration file they generate, so only the user (or superuser) can read it. This is important because these files contain security-sensitive information, including credentials and proxy configuration. - These commands also use file protection mode 600 for the private key file stored locally when you create service account credentials. - The default level of logging output from gsutil commands does not include security-sensitive information, such as OAuth2 tokens and proxy configuration information. (See the "RECOMMENDED USER PRECAUTIONS" section below if you increase the level of debug output, using the gsutil -D option.) Note that protection modes are not supported on Windows, so if you use gsutil on Windows we recommend using an encrypted file system and strong account passwords. <B>SECURITY-SENSITIVE FILES WRITTEN TEMPORARILY TO DISK BY GSUTIL</B> gsutil buffers data in temporary files in several situations: - While compressing data being uploaded via gsutil cp -z/-Z, gsutil buffers the data in temporary files with protection 600, which it deletes after the upload is complete (similarly for downloading files that were uploaded with gsutil cp -z/-Z or some other process that sets the Content-Encoding to "gzip"). However, if you kill the gsutil process while the upload is under way the partially written file will be left in place. See the "CHANGING TEMP DIRECTORIES" section in "gsutil help cp" for details of where the temporary files are written and how to change the temp directory location. - When performing a resumable upload gsutil stores the upload ID (which, as noted above, is a bearer token and thus should be safe-guarded) in a file under ~/.gsutil/tracker-files with protection 600, and deletes this file after the upload is complete. However, if the upload doesn't complete successfully the tracker file is left in place so the resumable upload can be re-attempted later. Over time it's possible to accumulate these tracker files from aborted upload attempts, though resumable upload IDs are only valid for 1 week, so the security risk only exists for files less than that age. If you consider the risk of leaving aborted upload IDs in the tracker directory too high you could modify your upload scripts to delete the tracker files; or you could create a cron job to clear the tracker directory periodically. - The gsutil rsync command stores temporary files (with protection 600) containing the names, sizes, and checksums of source and destination directories/buckets, which it deletes after the rsync is complete. However, if you kill the gsutil process while the rsync is under way the listing files will be left in place. Note that gsutil deletes temporary files using the standard OS unlink system call, which does not perform `data wiping <https://en.wikipedia.org/wiki/Data_erasure>`_. Thus, the content of such temporary files can be recovered by a determined adversary. <B>ACCESS CONTROL LISTS</B> Unless you specify a different ACL (e.g., via the gsutil cp -a option), by default objects written to a bucket use the default object ACL on that bucket. Unless you modify that ACL (e.g., via the gsutil defacl command), by default it will allow all project editors write access to the object and read/write access to the object's metadata and will allow all project viewers read access to the object. The Google Cloud Storage access control system includes the ability to specify that objects are publicly readable. Make sure you intend for any objects you write with this permission to be public. Once "published", data on the Internet can be copied to many places, so it's effectively impossible to regain read control over an object written with this permission. The Google Cloud Storage access control system includes the ability to specify that buckets are publicly writable. While configuring a bucket this way can be convenient for various purposes, we recommend against using this permission - it can be abused for distributing illegal content, viruses, and other malware, and the bucket owner is legally and financially responsible for the content stored in their buckets. If you need to make content available to customers who don't have Google accounts consider instead using signed URLs (see "gsutil help signurl"). <B>SOFTWARE INTEGRITY AND UPDATES</B> gsutil is distributed as a part of the bundled Cloud SDK release. This distribution method takes a variety of security precautions to protect the integrity of the software. We strongly recommend against getting a copy of gsutil from any other sources (such as mirror sites). <B>PROXY USAGE</B> gsutil supports access via proxies, such as Squid and a number of commercial products. A full description of their capabilities is beyond the scope of this documentation, but proxies can be configured to support many security-related functions, including virus scanning, Data Leakage Prevention, control over which certificates/CA's are trusted, content type filtering, and many more capabilities. Some of these features can slow or block legitimate gsutil behavior. For example, virus scanning depends on decrypting file content, which in turn requires that the proxy terminate the gsutil connection and establish a new connection - and in some cases proxies will rewrite content in ways that result in checksum validation errors and other problems. For details on configuring proxies, see the proxy help text generated in your .boto configuration file by the ``gcloud init``, ``gsutil -a``, and ``gsutil -e`` commands. <B>ENCRYPTION AT REST</B> All Google Cloud Storage data are automatically stored in an encrypted state, but you can also provide your own encryption keys. For more information, see `Cloud Storage Encryption <https://cloud.google.com/storage/docs/encryption>`_. <B>DATA PRIVACY</B> Google will never ask you to share your credentials, password, or other security-sensitive information. Beware of potential phishing scams where someone attempts to impersonate Google and asks for such information. <B>MEASUREMENT DATA</B> The gsutil perfdiag command collects a variety of performance-related measurements and details about your local system and network environment, for use in troubleshooting performance problems. None of this information will be sent to Google unless you choose to send it. <B>RECOMMENDED USER PRECAUTIONS</B> The first and foremost precaution is: Never share your credentials. Each user should have distinct credentials. If you run gsutil -D (to generate debugging output) it will include OAuth2 refresh and access tokens in the output. Make sure to redact this information before sending this debug output to anyone during troubleshooting/tech support interactions. If you run gsutil --trace-token (to send a trace directly to Google), sensitive information like OAuth2 tokens and the contents of any files accessed during the trace may be included in the content of the trace. Customer-supplied encryption key information in the .boto configuration is security sensitive. The proxy configuration information in the .boto configuration is security-sensitive, especially if your proxy setup requires user and password information. Even if your proxy setup doesn't require user and password, the host and port number for your proxy is often considered security-sensitive. Protect access to your .boto configuration file. If you are using gsutil from a production environment (e.g., via a cron job running on a host in your data center), use service account credentials rather than individual user account credentials. These credentials were designed for such use and, for example, protect you from losing access when an employee leaves your company. """) class CommandOptions(HelpProvider): """Additional help about security and privacy considerations using gsutil.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='security', help_name_aliases=['protection', 'privacy', 'proxies', 'proxy'], help_type='additional_help', help_one_line_summary='Security and Privacy Considerations', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/command_opts.py 0000644 00000017612 15025013616 0011364 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about gsutil command-level options.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>DESCRIPTION</B> gsutil supports separate options for the top-level gsutil command and the individual sub-commands (like cp, rm, etc.) The top-level options control behavior of gsutil that apply across commands. For example, in the command: gsutil -m cp -p file gs://bucket/obj the -m option applies to gsutil, while the -p option applies to the cp sub-command. <B>OPTIONS</B> -D Shows HTTP requests/headers and additional debug info needed when posting support requests, including exception stack traces. CAUTION: The output from using this flag includes authentication credentials. Before including this flag in your command, be sure you understand how the command's output is used, and, if necessary, remove or redact sensitive information. -DD Same as -D, plus HTTP upstream payload. -h Allows you to specify certain HTTP headers, for example: gsutil -h "Cache-Control:public,max-age=3600" \\ -h "Content-Type:text/html" cp ... Note that you need to quote the headers/values that contain spaces (such as "Content-Disposition: attachment; filename=filename.ext"), to avoid having the shell split them into separate arguments. The following headers are stored as object metadata and used in future requests on the object: Cache-Control Content-Disposition Content-Encoding Content-Language Content-Type The following headers are used to check data integrity: Content-MD5 gsutil also supports custom metadata headers with a matching Cloud Storage Provider prefix, such as: x-goog-meta- Note that for gs:// URLs, the Cache Control header is specific to the API being used. The XML API accepts any cache control headers and returns them during object downloads. The JSON API respects only the public, private, no-cache, max-age, and no-transform cache control headers. See "gsutil help setmeta" for the ability to set metadata fields on objects after they have been uploaded. -i Allows you to use the configured credentials to impersonate a service account, for example: gsutil -i "service-account@google.com" ls gs://pub Note that this setting will be ignored by the XML API and S3. See 'gsutil help creds' for more information on impersonating service accounts. -m Causes supported operations (acl ch, acl set, cp, mv, rm, rsync, and setmeta) to run in parallel. This can significantly improve performance if you are performing operations on a large number of files over a reasonably fast network connection. gsutil performs the specified operation using a combination of multi-threading and multi-processing. The number of threads and processors are determined by ``parallel_thread_count`` and ``parallel_process_count``, respectively. These values are set in the .boto configuration file or specified in individual requests with the ``-o`` top-level flag. Because gsutil has no built-in support for throttling requests, you should experiment with these values. The optimal values can vary based on a number of factors, including network speed, number of CPUs, and available memory. Using the -m option can consume a significant amount of network bandwidth and cause problems or make your performance worse if you use a slower network. For example, if you start a large rsync operation over a network link that's also used by a number of other important jobs, there could be degraded performance in those jobs. Similarly, the -m option can make your performance worse, especially for cases that perform all operations locally, because it can "thrash" your local disk. To prevent such issues, reduce the values for ``parallel_thread_count`` and ``parallel_process_count``, or stop using the -m option entirely. One tool that you can use to limit how much I/O capacity gsutil consumes and prevent it from monopolizing your local disk is `ionice <http://www.tutorialspoint.com/unix_commands/ionice.htm>`_ (built in to many Linux systems). For example, the following command reduces the I/O priority of gsutil so it doesn't monopolize your local disk: ionice -c 2 -n 7 gsutil -m rsync -r ./dir gs://some bucket If a download or upload operation using parallel transfer fails before the entire transfer is complete (e.g. failing after 300 of 1000 files have been transferred), you must restart the entire transfer. Also, although most commands normally fail upon encountering an error when the -m flag is disabled, all commands continue to try all operations when -m is enabled with multiple threads or processes, and the number of failed operations (if any) are reported as an exception at the end of the command's execution. -o Override values in the `boto configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_ for the current command, in the format ``<section>:<name>=<value>``. For example, ``gsutil -o "GSUtil:parallel_thread_count=4" ...``. This does not pass the option to gsutil integration tests and does not change the values that are saved in the boto configuration file. -q Causes gsutil to perform operations quietly, i.e., without reporting progress indicators of files being copied or removed, etc. Errors are still reported. This option can be useful for running gsutil from a cron job that logs its output to a file, for which the only information desired in the log is failures. -u Allows you to specify the ID or number of a user project to be billed for the request. For example: gsutil -u "bill-this-project" cp ... """) class CommandOptions(HelpProvider): """Additional help about gsutil command-level options.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='options', help_name_aliases=['arg', 'args', 'cli', 'opt', 'opts'], help_type='additional_help', help_one_line_summary='Global Command Line Options', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/shim.py 0000644 00000021224 15025013616 0007633 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about the shim for running gcloud storage.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>OVERVIEW</B> Cloud SDK includes a new CLI, gcloud storage, that can be considerably faster than gsutil when performing uploads and downloads with less parameter tweaking. This new CLI has a syntax and command structure that is familiar to gsutil users but is fundamentally different in many important ways. To ease transition to this new CLI, gsutil provides a shim that translates your gsutil commands to gcloud storage commands if an equivalent exists, and falls back to gsutil's usual behavior if an equivalent does not exist. <B> PREREQUISITES </B> You should be running gsutil binary installed via Cloud SDK tool. To verify this, run ``gsutil ver -l`` and you should be able to see ``pass cloud sdk credentials to gsutil: True``. If it is ``False``, try running ``gcloud config set pass_credentials_to_gsutil True``. If you do not have ``gcloud`` installed, follow the `Cloud SDK installation instructions. <https://cloud.google.com/sdk/docs/install>`_. <B>TO ENABLE</B> Set ``use_gcloud_storage=True`` in the ``.boto`` config file under the ``[GSUtil]`` section: [GSUtil] use_gcloud_storage=True You can also set the flag for individual commands using the top-level ``-o`` flag: gsutil -o "GSUtil:use_gcloud_storage=True" -m cp -p file gs://bucket/obj <B>AVAILABLE COMMANDS</B> The gcloud storage CLI only supports a subset of gsutil commands. What follows is a list of commands supported by the shim with any differences in behavior noted. acl ------------------------ - The ``ch`` subcommand is not supported. autoclass ------------------------ - Works as expected. bucketpolicyonly ------------------------ - Works as expected. cat ------------------------ - Prints object data for a second object even if the first object is invalid. compose ------------------------ - Works as expected. cors ------------------------ - ``get`` subcommand prints "[]" instead of "gs://[bucket name] has no CORS configuration". cp ------------------------ - Copies a second object even if the first object is invalid. - Does not support file to file copies. - Supports copying objects cloud-to-cloud with trailing slashes in the name. - The all-version flag (``-A``) silently enables sequential execution rather than raising an error. defacl ------------------------ - The ``ch`` subcommand is not supported. defstorageclass ------------------------ - Works as expected. hash ------------------------ - In gsutil, the ``-m`` and ``-c`` flags that affect which hashes are displayed are ignored for cloud objects. This behavior is fixed for the shim and gcloud storage. iam ------------------------ - The ``ch`` subcommand is not supported. - The ``-f`` flag will continue on any error, not just API errors. kms ------------------------ - The authorize subcommand returns informational messages in a different format. - The encryption subcommand returns informational messages in a different format. labels ------------------------ - ``get`` subcommand prints "[]" instead of "gs://[bucket name] has no labels configuration." lifecycle ------------------------ - Works as expected. logging ------------------------ - The get subcommand has different JSON spacing and doesn't print an informational message if no configuration is found. ls ------------------------ - `ls -L` output uses spaces instead of tabs. mb ------------------------ - Works as expected. mv ------------------------ - See notes on cp. notification ------------------------ - The list subcommand prints configuration information as YAML. - The delete subcommand offers progress tracking and parallelization. pap ------------------------ - Works as expected. rb ------------------------ - Works as expected. requesterpays ------------------------ - Works as expected. rewrite ------------------------ - The -k flag does not throw an error if called without a new key. In both the shim and unshimmed cases, the old key is maintained. rm ------------------------ - ``$folder$`` delete markers are not supported. rpo ------------------------ - Works as expected. setmeta ------------------------ - Does not throw an error if no headers are changed. stat ------------------------ - Includes a field "Storage class update time:" which may throw off tabbing. ubla ------------------------ - Works as expected. versioning ------------------------ - Works as expected. web ------------------------ - The get subcommand has different JSON spacing and doesn't print an informational message if no configuration is found. <B>BOTO CONFIGURATION</B> Configuration found in the boto file is mapped 1:1 to gcloud environment variables where appropriate. Important: The ``gs_oauth2_refresh_token`` and ``gs_service_key_file`` fields are ignored by the shim and authentication is taken care by ``gcloud``. [Credentials] ------------------------ - aws_access_key_id: AWS_ACCESS_KEY_ID - aws_secret_access_key: AWS_SECRET_ACCESS_KEY - use_client_certificate: CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE [Boto] ------------------------ - proxy: CLOUDSDK_PROXY_ADDRESS - proxy_type: CLOUDSDK_PROXY_TYPE - proxy_port: CLOUDSDK_PROXY_PORT - proxy_user: CLOUDSDK_PROXY_USERNAME - proxy_pass: CLOUDSDK_PROXY_PASSWORD - proxy_rdns: CLOUDSDK_PROXY_RDNS - http_socket_timeout: CLOUDSDK_CORE_HTTP_TIMEOUT - ca_certificates_file: CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILE - max_retry_delay: CLOUDSDK_STORAGE_BASE_RETRY_DELAY - num_retries: CLOUDSDK_STORAGE_MAX_RETRIES [GSUtil] ------------------------ - check_hashes: CLOUDSDK_STORAGE_CHECK_HASHES - default_project_id: CLOUDSDK_CORE_PROJECT - disable_analytics_prompt: CLOUDSDK_CORE_DISABLE_USAGE_REPORTING - use_magicfile: CLOUDSDK_STORAGE_USE_MAGICFILE - parallel_composite_upload_threshold: CLOUDSDK_STORAGE_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD - resumable_threshold: CLOUDSDK_STORAGE_RESUMABLE_THRESHOLD [OAuth2] ------------------------ - client_id: CLOUDSDK_AUTH_CLIENT_ID - client_secret: CLOUDSDK_AUTH_CLIENT_SECRET - provider_authorization_uri: CLOUDSDK_AUTH_AUTH_HOST - provider_token_uri: CLOUDSDK_AUTH_TOKEN_HOST <B>GENERAL COMPATIBILITY NOTES</B> - Due to its compatibility across all major platforms, multiprocessing is enabled for all commands by default (equivalent to the -m option always being included in gsutil). - A sequence of asterisks greater than 2 (i.e. ``***``) are always treated as a single asterisk. - Unlike gsutil, gcloud is not designed to be used in parallel invocations, and doing so (i.e. running the shim from 2 terminals at once) can lead to unpredictable behavior. - Assuming a bucket contains an object ``gs://bucket/nested/foo.txt``, gsutil's wildcard iterator will match ``foo.txt`` given a URL like ``gs://bucket/*/nested/*``. The shim will not match ``foo.txt`` given the same URL. - This will be updated as new commands are supported by both gcloud storage and the shim. - If Unicode is having issues, try setting the environment variable ``PYTHONUTF8`` to ``1``. Specifically, this may help on the Windows command-line (CMD). """) class CommandOptions(HelpProvider): """Additional help about the shim for running gcloud storage.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='shim', help_name_aliases=['gcloudstorage', 'gcloud storage'], help_type='additional_help', help_one_line_summary='Shim for Running gcloud storage', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/support.py 0000644 00000010765 15025013616 0010417 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about technical and billing support.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>TECHNICAL SUPPORT</B> If you have any questions or encounter any problems with Google Cloud Storage, please first read the `FAQ <https://cloud.google.com/storage/docs/faq>`_. If you still have questions please use one of the following methods as appropriate, providing the details noted below: A) For API, tool usage, or other software development-related questions, please search for and post questions on Stack Overflow, using the official `google-cloud-storage tag <http://stackoverflow.com/questions/tagged/google-cloud-storage>`_. Our support team actively monitors questions to this tag and we'll do our best to respond. B) For gsutil bugs or feature requests, please check if there is already a `existing GitHub issue <https://github.com/GoogleCloudPlatform/gsutil/issues>`_ that covers your request. If not, create a `new GitHub issue <https://github.com/GoogleCloudPlatform/gsutil/issues/new>`_. To help us diagnose any issues you encounter, when creating a new issue please provide these details in addition to the description of your problem: - The resource you are attempting to access (bucket name, object name), assuming they are not sensitive. - The operation you attempted (GET, PUT, etc.) - The time and date (including timezone) at which you encountered the problem - If you can use gsutil to reproduce your issue, specify the -D option to display your request's HTTP details, and provide these details in the issue. Warning: The gsutil -d, -D, and -DD options will also print the authentication header with authentication credentials for your Google Cloud Storage account. Make sure to remove any "Authorization:" headers before you post HTTP details to the issue. Note also that if you upload files large enough to use resumable uploads, the resumable upload IDs are security-sensitive while an upload is not yet complete, so should not be posted on public forums. If you make any local modifications to gsutil, please make sure to use a released copy of gsutil (instead of your locally modified copy) when providing the gsutil -D output noted above. We cannot support versions of gsutil that include local modifications. (However, we're open to user contributions; see "gsutil help dev".) <B>BILLING AND ACCOUNT QUESTIONS</B> A) For billing documentation, please visit https://cloud.google.com/storage/pricing. If you want to cancel billing, follow the instructions at `Cloud Storage FAQ <https://cloud.google.com/storage/docs/faq#disablebilling>`_. Caution: When you disable billing, you also disable the Google Cloud Storage service. Make sure you want to disable the Google Cloud Storage service before you disable billing. B) For support regarding billing, please see `billing support <https://support.google.com/cloud/contact/cloud_platform_billing>`_. For other questions regarding your account, Terms Of Service, Google Cloud Console, or other administration-related questions please see `Google Cloud Platform support <https://support.google.com/cloud/answer/6282346#gcp>`_. """) class CommandOptions(HelpProvider): """Additional help about technical and billing support.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='support', help_name_aliases=[ 'techsupport', 'tech support', 'technical support', 'billing', 'faq', 'questions', ], help_type='additional_help', help_one_line_summary='Google Cloud Storage Support', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/dev.py 0000644 00000014204 15025013616 0007451 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about contributing code to gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>OVERVIEW</B> We're open to incorporating gsutil code changes authored by users. Here are some guidelines: 1. Before we can accept code submissions, we have to jump a couple of legal hurdles. Please fill out either the individual or corporate Contributor License Agreement: - If you are an individual writing original source code and you're sure you own the intellectual property, then you'll need to sign an individual CLA (https://cla.developers.google.com/about/google-individual). - If you work for a company that wants to allow you to contribute your work to gsutil, then you'll need to sign a corporate CLA (https://cla.developers.google.com/about/google-corporate) Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll add you to the official list of contributors and be able to accept your patches. 2. If you found a bug or have an idea for a feature enhancement, we suggest you check https://github.com/GoogleCloudPlatform/gsutil/issues to see if it has already been reported by another user. From there you can also subscribe to updates to the issue. If a GitHub issue doesn't already exist, create one about your idea before sending actual code. Often we can discuss the idea and help propose things that could save you later revision work. 3. We tend to avoid adding command line options that are of use to only a very small fraction of users, especially if there's some other way to accommodate such needs. Adding such options complicates the code and also adds overhead to users having to read through an "alphabet soup" list of option documentation. 4. While gsutil has a number of features specific to Google Cloud Storage, it can also be used with other cloud storage providers. We're open to including changes for making gsutil support features specific to other providers, as long as those changes don't make gsutil work worse for Google Cloud Storage. If you do make such changes we recommend including someone with knowledge of the specific provider as a code reviewer (see below). 5. You can check out the gsutil code from the GitHub repository: https://github.com/GoogleCloudPlatform/gsutil To clone a read-only copy of the repository: git clone git://github.com/GoogleCloudPlatform/gsutil.git To push your own changes to GitHub, click the Fork button on the repository page and clone the repository from your own fork. 6. The gsutil git repository uses git submodules to pull in external modules. After checking out the repository, make sure to also pull the submodules by entering into the gsutil top-level directory and run: git submodule update --init --recursive 7. Please make sure to run all tests against your modified code. To do this, change directories into the gsutil top-level directory and run: ./gsutil test The above tests take a long time to run because they send many requests to the production service. The gsutil test command has a -u argument that will only run unit tests. These run quickly, as they are executed with an in-memory mock storage service implementation. To run only the unit tests, run: ./gsutil test -u If you made changes to boto, please run the boto tests. For these tests you need to use HMAC credentials (from gsutil config -a), because the boto test suite doesn't import the OAuth2 handler. You'll also need to install some Python modules. Change directories into the boto root directory at third_party/boto and run: pip install -r requirements.txt (You probably need to run this command using sudo.) Make sure each of the individual installations succeeded. If they don't you may need to run the install command again. Then ensure your .boto file has HMAC credentials defined, and then change directories into boto's tests directory and run: python test.py unit python test.py -t s3 -t gs -t ssl 8. Please consider contributing test code for your change, especially if the change impacts any of the core gsutil code (like the gsutil cp command). 9. Please run the yapf linter with the config files in the root of the GitHub repository: yapf -irp . 10. When it's time to send us code, please submit a PR to the `gsutil GitHub repository <https://github.com/GoogleCloudPlatform/gsutil>`_. For help on making GitHub PRs, please refer to this `GitHub help document <https://help.github.com/en/articles/about-pull-requests>`_. """) class CommandOptions(HelpProvider): """Additional help about contributing code to gsutil.""" # TODO: gsutil-beta: Add lint .rc file and linting instructions. # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='dev', help_name_aliases=[ 'development', 'developer', 'code', 'mods', 'software', ], help_type='additional_help', help_one_line_summary='Contributing Code to gsutil', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/versions.py 0000644 00000007671 15025013616 0010555 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about object versioning.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>OVERVIEW</B> Versioning-enabled buckets maintain noncurrent versions of objects, providing a way to un-delete data that you accidentally deleted, or to retrieve older versions of your data. Noncurrent objects are ignored by gsutil commands unless you indicate it should do otherwise by setting a relevant command flag or by including a specific generation number in your command. For example, wildcards like ``*`` and ``**`` do not, by themselves, act on noncurrent object versions. When using gsutil cp, you cannot specify a version-specific URL as the destination, because writes to Cloud Storage always create a new version. Trying to specify a version-specific URL as the destination of ``gsutil cp`` results in an error. When you specify a noncurrent object as a source in a copy command, you always create a new object version and retain the original (even when using the command to restore a live version). You can use the ``gsutil mv`` command to simultaneously restore an object version and remove the noncurrent copy that was used as the source. You can turn versioning on or off for a bucket at any time. Turning versioning off leaves existing object versions in place and simply causes the bucket to delete the existing live version of the object whenever a new version is uploaded. Regardless of whether you have enabled versioning on a bucket, every object has two associated positive integer fields: - the generation, which is updated when a new object replaces an existing object with the same name. Note that there is no guarantee that generation numbers increase for successive versions, only that each new version has a unique generation number. - the metageneration, which identifies the metadata generation. It starts at 1; is updated every time the metadata (e.g., ACL or Content-Type) for a given content generation is updated; and gets reset when the generation number changes. Of these two integers, only the generation is used when working with versioned data. Both generation and metageneration can be used with concurrency control. To learn more about versioning and concurrency, see the following documentation: - `Overview of Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_ - `Guide for using Object Versioning <https://cloud.google.com/storage/docs/using-object-versioning>`_ - The `reference page for the gsutil versioning command <https://cloud.google.com/storage/docs/gsutil/commands/versioning>`_ - `Request preconditions <https://cloud.google.com/storage/docs/request-preconditions>`_ """) class CommandOptions(HelpProvider): """Additional help about object versioning.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='versions', help_name_aliases=['concurrency', 'concurrency control'], help_type='additional_help', help_one_line_summary='Object Versioning and Concurrency Control', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/crc32c.py 0000644 00000015002 15025013616 0007747 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about CRC32C and installing crcmod.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>OVERVIEW</B> Google Cloud Storage provides a cyclic redundancy check (CRC) header that allows clients to verify the integrity of object contents. For non-composite objects Google Cloud Storage also provides an MD5 header to allow clients to verify object integrity, but for composite objects only the CRC is available. gsutil automatically performs integrity checks on all uploads and downloads. Additionally, you can use the ``gsutil hash`` command to calculate a CRC for any local file. The CRC variant used by Google Cloud Storage is called CRC32C (Castagnoli), which is not available in the standard Python distribution. The implementation of CRC32C used by gsutil is provided by a third-party Python module called `crcmod <https://pypi.python.org/pypi/crcmod>`_. The crcmod module contains a pure-Python implementation of CRC32C, but using it results in slow checksum computation and subsequently very poor performance. A Python C extension is also provided by crcmod, which requires compiling into a binary module for use. gsutil ships with a precompiled crcmod C extension for macOS; for other platforms, see the installation instructions below. At the end of each copy operation, the ``gsutil cp``, ``gsutil mv``, and ``gsutil rsync`` commands validate that the checksum of the source file/object matches the checksum of the destination file/object. If the checksums do not match, gsutil will delete the invalid copy and print a warning message. This very rarely happens, but if it does, you should retry the operation. <B>CONFIGURATION</B> To determine if the compiled version of crcmod is available in your Python environment, you can inspect the output of the ``gsutil version`` command for the "compiled crcmod" entry: $ gsutil version -l ... compiled crcmod: True ... If your crcmod library is compiled to a native binary, this value will be True. If using the pure-Python version, the value will be False. To control gsutil's behavior in response to crcmod's status, you can set the ``check_hashes`` variable in your `boto configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_. For details on this variable, see the surrounding comments in your boto configuration file. If ``check_hashes`` is not present in your configuration file, regenerate the file by running ``gsutil config`` with the appropriate ``-e`` or ``-a`` flag. <B>INSTALLATION</B> These installation instructions assume that: - You have ``pip`` installed. Consult the `pip installation instructions <https://pip.pypa.io/en/stable/installing/>`_ for details on how to install ``pip``. - Your installation of ``pip`` can be found in your ``PATH`` environment variable. If it cannot, you may need to replace ``pip3`` in the commands below with the full path to the executable. - You are installing the crcmod package for use with your system installation of Python, and thus use the ``sudo`` command. If installing crcmod for a different Python environment (e.g. in a virtualenv), you should omit ``sudo`` from the commands below. - You are using a Python 3 version with gsutil. You can determine which Python version gsutil is using by running ``gsutil version -l`` and looking for the ``python version: 2.x.x`` or ``python version: 3.x.x`` line. CentOS, RHEL, and Fedora ------------------------ To compile and install crcmod: yum install gcc python3-devel python3-setuptools redhat-rpm-config sudo pip3 uninstall crcmod sudo pip3 install --no-cache-dir -U crcmod Debian and Ubuntu ----------------- To compile and install crcmod: sudo apt-get install gcc python3-dev python3-setuptools sudo pip3 uninstall crcmod sudo pip3 install --no-cache-dir -U crcmod Enterprise SUSE ----------------- To compile and install crcmod when using Enterprise SUSE for SAP 12: sudo zypper install gcc python-devel sudo pip uninstall crcmod sudo pip install --no-cache-dir -U crcmod To compile and install crcmod when using Enterprise SUSE for SAP 15: sudo zypper install gcc python3-devel sudo pip uninstall crcmod sudo pip install --no-cache-dir -U crcmod macOS ----- gsutil distributes a pre-compiled version of crcmod for macOS, so you shouldn't need to compile and install it yourself. If for some reason the pre-compiled version is not being detected, please let the Google Cloud Storage team know (see ``gsutil help support``). To compile manually on macOS, you will first need to install `Xcode <https://developer.apple.com/xcode/>`_ and then run: pip3 install -U crcmod Windows ------- An installer is available for the compiled version of crcmod from the Python Package Index (PyPi) at the following URL: https://pypi.python.org/pypi/crcmod/1.7 NOTE: If you have installed crcmod and gsutil hasn't detected it, it may have been installed to the wrong directory. It should be located at <python_dir>\\files\\Lib\\site-packages\\crcmod\\ In some cases the installer will incorrectly install to <python_dir>\\Lib\\site-packages\\crcmod\\ Manually copying the crcmod directory to the correct location should resolve the issue. """) class CommandOptions(HelpProvider): """Additional help about CRC32C and installing crcmod.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='crc32c', help_name_aliases=['crc32', 'crc', 'crcmod'], help_type='additional_help', help_one_line_summary='CRC32C and Installing crcmod', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/encoding.py 0000644 00000020527 15025013616 0010466 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about CRC32C and installing crcmod.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>OVERVIEW</B> To reduce the chance for `filename encoding interoperability problems <https://en.wikipedia.org/wiki/Filename#Encoding_indication_interoperability>`_ gsutil uses `UTF-8 <https://en.wikipedia.org/wiki/UTF-8>`_ character encoding when uploading and downloading files. Because UTF-8 is in widespread (and growing) use, for most users nothing needs to be done to use UTF-8. Users with files stored in other encodings (such as `Latin 1 <https://en.wikipedia.org/wiki/ISO/IEC_8859-1>`_) must convert those filenames to UTF-8 before attempting to upload the files. The most common place where users who have filenames that use some other encoding encounter a gsutil error is while uploading files using the recursive (-R) option on the gsutil cp , mv, or rsync commands. When this happens you'll get an error like this: CommandException: Invalid Unicode path encountered ('dir1/dir2/file_name_with_\\xf6n_bad_chars'). gsutil cannot proceed with such files present. Please remove or rename this file and try again. Note that the invalid Unicode characters have been hex-encoded in this error message because otherwise trying to print them would result in another error. If you encounter such an error you can either remove the problematic file(s) or try to rename them and re-run the command. If you have a modest number of such files the simplest thing to do is to think of a different name for the file and manually rename the file (using local filesystem tools). If you have too many files for that to be practical, you can use a bulk rename tool or script. Unicode errors for valid Unicode filepaths can be caused by lack of Python locale configuration on Linux and Mac OSes. If your file paths are Unicode and you get encoding errors, ensure the LANG environment variable is set correctly. Typically, the LANG variable should be set to something like "en_US.UTF-8" or "de_DE.UTF-8". Note also that there's no restriction on the character encoding used in file content - it can be UTF-8, a different encoding, or non-character data (like audio or video content). The gsutil UTF-8 character encoding requirement applies only to filenames. <B>USING UNICODE FILENAMES ON WINDOWS</B> Windows support for Unicode in the command shell (cmd.exe or powershell) is somewhat painful, because Windows uses a Windows-specific character encoding called `cp1252 <https://en.wikipedia.org/wiki/Windows-1252>`_. To use Unicode characters you need to run this command in the command shell before the first time you use gsutil in that shell: chcp 65001 If you neglect to do this before using gsutil, the progress messages while uploading files with Unicode names or listing buckets with Unicode object names will look garbled (i.e., with different glyphs than you expect in the output). If you simply run the chcp command and re-run the gsutil command, the output should no longer look garbled. gsutil attempts to translate between cp1252 encoding and UTF-8 in the main places that Unicode encoding/decoding problems have been encountered to date (traversing the local file system while uploading files, and printing Unicode names while listing buckets). However, because gsutil must perform translation, it is likely there are other erroneous edge cases when using Windows with Unicode. If you encounter problems, you might consider instead using cygwin (on Windows) or Linux or macOS - all of which support Unicode. <B>USING UNICODE FILENAMES ON MACOS</B> macOS stores filenames in decomposed form (also known as `NFD normalization <https://en.wikipedia.org/wiki/Unicode_equivalence>`_). For example, if a filename contains an accented "e" character, that character will be converted to an "e" followed by an accent before being saved to the filesystem. As a consequence, it's possible to have different name strings for files uploaded from an operating system that doesn't enforce decomposed form (like Ubuntu) from one that does (like macOS). The following example shows how this behavior could lead to unexpected results. Say you create a file with non-ASCII characters on Ubuntu. Ubuntu stores that filename in its composed form. When you upload the file to the cloud, it is stored as named. But if you use gsutil rysnc to bring the file to a macOS machine and edit the file, then when you use gsutil rsync to bring this version back to the cloud, you end up with two different objects, instead of replacing the original. This is because macOS converted the filename to a decomposed form, and Cloud Storage sees this as a different object name. <B>CROSS-PLATFORM ENCODING PROBLEMS OF WHICH TO BE AWARE</B> Using UTF-8 for all object names and filenames will ensure that gsutil doesn't encounter character encoding errors while operating on the files. Unfortunately, it's still possible that files uploaded / downloaded this way can have interoperability problems, for a number of reasons unrelated to gsutil. For example: - Windows filenames are case-insensitive, while Google Cloud Storage, Linux, and macOS are not. Thus, for example, if you have two filenames on Linux differing only in case and upload both to Google Cloud Storage and then subsequently download them to Windows, you will end up with just one file whose contents came from the last of these files to be written to the filesystem. - macOS performs character encoding decomposition based on tables stored in the OS, and the tables change between Unicode versions. Thus the encoding used by an external library may not match that performed by the OS. It is possible that two object names may translate to a single local filename. - Windows console support for Unicode is difficult to use correctly. For a more thorough list of such issues see `this presentation <http://www.i18nguy.com/unicode/filename-issues-iuc33.pdf>`_ These problems mostly arise when sharing data across platforms (e.g., uploading data from a Windows machine to Google Cloud Storage, and then downloading from Google Cloud Storage to a machine running macOS). Unfortunately these problems are a consequence of the lack of a filename encoding standard, and users need to be aware of the kinds of problems that can arise when copying filenames across platforms. There is one precaution users can exercise to prevent some of these problems: When using the Windows console specify wildcards or folders (using the -R option) rather than explicitly named individual files. <B>CONVERTING FILENAMES TO UNICODE</B> Open-source tools are available to convert filenames for non-Unicode files. For example, to convert from latin1 (a common Windows encoding) to Unicode, you can use `Windows iconv <http://gnuwin32.sourceforge.net/packages/libiconv.htm>`_. For Unix-based systems, you can use `libiconv <https://www.gnu.org/software/libiconv/>`_. """) class CommandOptions(HelpProvider): """Additional help about filename encoding and interoperability problems.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='encoding', help_name_aliases=[ 'encodings', 'utf8', 'utf-8', 'latin1', 'unicode', 'interoperability', ], help_type='additional_help', help_one_line_summary='Filename encoding and interoperability problems', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) addlhelp/__init__.py 0000644 00000001435 15025013616 0010434 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Package marker file.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals addlhelp/acls.py 0000644 00000023224 15025013616 0007617 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Additional help about Access Control Lists.""" # TODO(iam-beta): Revise this to include new IAM syntax. from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.help_provider import HelpProvider _DETAILED_HELP_TEXT = (""" <B>OVERVIEW</B> Access Control Lists (ACLs) allow you to control who can read and write your data, and who can read and write the ACLs themselves. If not specified at the time an object is uploaded (e.g., via the gsutil cp -a option), objects will be created with a default object ACL set on the bucket (see "gsutil help defacl"). You can replace the ACL on an object or bucket using the "gsutil acl set" command, or modify the existing ACL using the "gsutil acl ch" command (see "gsutil help acl"). <B>BUCKET VS OBJECT ACLS</B> In Google Cloud Storage, the bucket ACL works as follows: - Users granted READ access are allowed to list the bucket contents and read bucket metadata other than its ACL. - Users granted WRITE access are allowed READ access and also are allowed to write and delete objects in that bucket, including overwriting previously written objects. - Users granted OWNER access are allowed WRITE access and also are allowed to read and write the bucket's ACL. The object ACL works as follows: - Users granted READ access are allowed to read the object's data and metadata. - Users granted OWNER access are allowed READ access and also are allowed to read and write the object's ACL. A couple of points are worth noting, that sometimes surprise users: 1. There is no WRITE access for objects; attempting to set an ACL with WRITE permission for an object will result in an error. 2. The bucket ACL plays no role in determining who can read objects; only the object ACL matters for that purpose. This is different from how things work in Linux file systems, where both the file and directory permission control file read access. It also means, for example, that someone with OWNER over the bucket may not have read access to objects in the bucket. This is by design, and supports useful cases. For example, you might want to set up bucket ownership so that a small group of administrators have OWNER on the bucket (with the ability to delete data to control storage costs), but not grant those users read access to the object data (which might be sensitive data that should only be accessed by a different specific group of users). <B>ACCESSING PUBLIC OBJECTS</B> Objects with public READ access can be accessed anonymously by gsutil, via a browser, or via Cloud Storage APIs. For more details on accessing public objects, see: https://cloud.google.com/storage/docs/access-public-data <B>CANNED ACLS</B> The simplest way to set an ACL on a bucket or object is using a "canned ACL". The available canned ACLs are: project-private Gives permission to the project team based on their roles. Anyone who is part of the team has READ permission, and project owners and project editors have OWNER permission. This is the default ACL for newly created buckets. This is also the default ACL for newly created objects unless the default object ACL for that bucket has been changed. For more details see "gsutil help projects". private Gives the requester (and only the requester) OWNER permission for a bucket or object. public-read Gives all users (whether logged in or anonymous) READ permission. When you apply this to an object, anyone on the Internet can read the object without authenticating. NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see 'gsutil help setmeta'. NOTE: Setting a bucket ACL to public-read will remove all OWNER and WRITE permissions from everyone except the project owner group. Setting an object ACL to public-read will remove all OWNER and WRITE permissions from everyone except the object owner. For this reason, we recommend using the "acl ch" command to make these changes; see "gsutil help acl ch" for details. public-read-write Gives all users READ and WRITE permission. This ACL applies only to buckets. NOTE: Setting a bucket to public-read-write will allow anyone on the Internet to upload anything to your bucket. You will be responsible for this content. NOTE: Setting a bucket ACL to public-read-write will remove all OWNER permissions from everyone except the project owner group. Setting an object ACL to public-read-write will remove all OWNER permissions from everyone except the object owner. For this reason, we recommend using the "acl ch" command to make these changes; see "gsutil help acl ch" for details. authenticated-read Gives the requester OWNER permission and gives all authenticated Google account holders READ permission. bucket-owner-read Gives the requester OWNER permission and gives the bucket owner READ permission. This is used only with objects. bucket-owner-full-control Gives the requester OWNER permission and gives the bucket owner OWNER permission. This is used only with objects. <B>ACL JSON</B> When you use a canned ACL, it is translated into an JSON representation that can later be retrieved and edited to specify more fine-grained detail about who can read and write buckets and objects. By running the "gsutil acl get" command you can retrieve the ACL JSON, and edit it to customize the permissions. As an example, if you create an object in a bucket that has no default object ACL set and then retrieve the ACL on the object, it will look something like this: [ { "entity": "group-00b4903a9740e42c29800f53bd5a9a62a2f96eb3f64a4313a115df3f3a776bf7", "entityId": "00b4903a9740e42c29800f53bd5a9a62a2f96eb3f64a4313a115df3f3a776bf7", "role": "OWNER" }, { "entity": "group-00b4903a977fd817e9da167bc81306489181a110456bb635f466d71cf90a0d51", "entityId": "00b4903a977fd817e9da167bc81306489181a110456bb635f466d71cf90a0d51", "role": "OWNER" }, { "entity": "00b4903a974898cc8fc309f2f2835308ba3d3df1b889d3fc7e33e187d52d8e71", "entityId": "00b4903a974898cc8fc309f2f2835308ba3d3df1b889d3fc7e33e187d52d8e71", "role": "READER" } ] The ACL consists collection of elements, each of which specifies an Entity and a Role. Entities are the way you specify an individual or group of individuals, and Roles specify what access they're permitted. This particular ACL grants OWNER to two groups (which means members of those groups are allowed to read the object and read and write the ACL), and READ permission to a third group. The project groups are (in order) the project owners group, editors group, and viewers group. The 64 digit hex identifiers (following any prefixes like "group-") used in this ACL are called canonical IDs. They are used to identify predefined groups associated with the project that owns the bucket: the Project Owners, Project Editors, and All Project Team Members groups. For more information the permissions and roles of these project groups, see "gsutil help projects". Here's an example of an ACL specified using the group-by-email and group-by-domain entities: [ { "entity": "group-travel-companion-owners@googlegroups.com" "email": "travel-companion-owners@googlegroups.com", "role": "OWNER", } { "domain": "example.com", "entity": "domain-example.com" "role": "READER", }, ] This ACL grants members of an email group OWNER, and grants READ access to any user in a domain (which must be a Google Apps for Business domain). By applying email group grants to a collection of objects you can edit access control for large numbers of objects at once via http://groups.google.com. That way, for example, you can easily and quickly change access to a group of company objects when employees join and leave your company (i.e., without having to individually change ACLs across potentially millions of objects). <B>SHARING SCENARIOS</B> For more detailed examples how to achieve various useful sharing use cases see https://cloud.google.com/storage/docs/collaboration """) class CommandOptions(HelpProvider): """Additional help about Access Control Lists.""" # Help specification. See help_provider.py for documentation. help_spec = HelpProvider.HelpSpec( help_name='acls', help_name_aliases=[ 'ACL', 'access control', 'access control list', 'authorization', 'canned', 'canned acl', ], help_type='additional_help', help_one_line_summary='Working With Access Control Lists', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) daisy_chain_wrapper.py 0000644 00000034614 15025013616 0011140 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Wrapper for use in daisy-chained copies.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals import collections import contextlib import os import threading import time from gslib.cloud_api import BadRequestException from gslib.cloud_api import CloudApi from gslib.utils import constants from gslib.utils import parallelism_framework_util from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey # This controls the amount of bytes downloaded per download request. # We do not buffer this many bytes in memory at a time - that is controlled by # DaisyChainWrapper.max_buffer_size. This is the upper bound of bytes that may # be unnecessarily downloaded if there is a break in the resumable upload. _DEFAULT_DOWNLOAD_CHUNK_SIZE = 1024 * 1024 * 100 class BufferWrapper(object): """Wraps the download file pointer to use our in-memory buffer.""" def __init__(self, daisy_chain_wrapper, mode='b'): """Provides a buffered write interface for a file download. Args: daisy_chain_wrapper: DaisyChainWrapper instance to use for buffer and locking. """ self.daisy_chain_wrapper = daisy_chain_wrapper if hasattr(daisy_chain_wrapper, 'mode'): self.mode = daisy_chain_wrapper.mode else: self.mode = mode def write(self, data): # pylint: disable=invalid-name """Waits for space in the buffer, then writes data to the buffer.""" while True: with self.daisy_chain_wrapper.lock: if (self.daisy_chain_wrapper.bytes_buffered < self.daisy_chain_wrapper.max_buffer_size): break # Buffer was full, yield thread priority so the upload can pull from it. time.sleep(0) data_len = len(data) if data_len: with self.daisy_chain_wrapper.lock: self.daisy_chain_wrapper.buffer.append(data) self.daisy_chain_wrapper.bytes_buffered += data_len @contextlib.contextmanager def AcquireLockWithTimeout(lock, timeout): # pylint: disable=invalid-name result = lock.acquire(timeout=timeout) yield result if result: lock.release() class DaisyChainWrapper(object): """Wrapper class for daisy-chaining a cloud download to an upload. This class instantiates a BufferWrapper object to buffer the download into memory, consuming a maximum of max_buffer_size. It implements intelligent behavior around read and seek that allow for all of the operations necessary to copy a file. This class is coupled with the XML and JSON implementations in that it expects that small buffers (maximum of constants.TRANSFER_BUFFER_SIZE) in size will be used. """ def __init__(self, src_url, src_obj_size, gsutil_api, compressed_encoding=False, progress_callback=None, download_chunk_size=_DEFAULT_DOWNLOAD_CHUNK_SIZE, decryption_key=None): """Initializes the daisy chain wrapper. Args: src_url: Source CloudUrl to copy from. src_obj_size: Size of source object. gsutil_api: gsutil Cloud API to use for the copy. compressed_encoding: If true, source object has content-encoding: gzip. progress_callback: Optional callback function for progress notifications for the download thread. Receives calls with arguments (bytes_transferred, total_size). download_chunk_size: Integer number of bytes to download per GetObjectMedia request. This is the upper bound of bytes that may be unnecessarily downloaded if there is a break in the resumable upload. decryption_key: Base64-encoded decryption key for the source object, if any. Raises: Exception: if the download thread doesn't start within 60 seconds """ # Current read position for the upload file pointer. self.position = 0 self.buffer = collections.deque() self.bytes_buffered = 0 # Maximum amount of bytes in memory at a time. self.max_buffer_size = 1024 * 1024 # 1 MiB self._download_chunk_size = download_chunk_size # We save one buffer's worth of data as a special case for boto, # which seeks back one buffer and rereads to compute hashes. This is # unnecessary because we can just compare cloud hash digests at the end, # but it allows this to work without modfiying boto. self.last_position = 0 self.last_data = None # Protects buffer, position, bytes_buffered, last_position, and last_data. self.lock = parallelism_framework_util.CreateLock() # Protects download_exception. self.download_exception_lock = parallelism_framework_util.CreateLock() self.src_obj_size = src_obj_size self.src_url = src_url self.compressed_encoding = compressed_encoding self.decryption_tuple = CryptoKeyWrapperFromKey(decryption_key) # This is safe to use the upload and download thread because the download # thread calls only GetObjectMedia, which creates a new HTTP connection # independent of gsutil_api. Thus, it will not share an HTTP connection # with the upload. self.gsutil_api = gsutil_api # If self.download_thread dies due to an exception, it is saved here so # that it can also be raised in the upload thread. self.download_exception = None self.download_thread = None self.progress_callback = progress_callback self.download_started = threading.Event() self.stop_download = threading.Event() self.StartDownloadThread(progress_callback=self.progress_callback) if not self.download_started.wait(60): raise Exception('Could not start download thread after 60 seconds.') self._unused_data_from_previous_read = b'' def StartDownloadThread(self, start_byte=0, progress_callback=None): # pylint: disable=invalid-name """Starts the download thread for the source object (from start_byte).""" def PerformDownload(start_byte, progress_callback): # pylint: disable=invalid-name """Downloads the source object in chunks. This function checks the stop_download event and exits early if it is set. It should be set when there is an error during the daisy-chain upload, then this function can be called again with the upload's current position as start_byte. Args: start_byte: Byte from which to begin the download. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). """ # TODO: Support resumable downloads. This would require the BufferWrapper # object to support seek() and tell() which requires coordination with # the upload. self.download_started.set() try: while start_byte + self._download_chunk_size < self.src_obj_size: self.gsutil_api.GetObjectMedia( self.src_url.bucket_name, self.src_url.object_name, BufferWrapper(self), compressed_encoding=self.compressed_encoding, start_byte=start_byte, end_byte=start_byte + self._download_chunk_size - 1, generation=self.src_url.generation, object_size=self.src_obj_size, download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, provider=self.src_url.scheme, progress_callback=progress_callback, decryption_tuple=self.decryption_tuple) if self.stop_download.is_set(): # Download thread needs to be restarted, so exit. self.stop_download.clear() return start_byte += self._download_chunk_size self.gsutil_api.GetObjectMedia( self.src_url.bucket_name, self.src_url.object_name, BufferWrapper(self), compressed_encoding=self.compressed_encoding, start_byte=start_byte, generation=self.src_url.generation, object_size=self.src_obj_size, download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, provider=self.src_url.scheme, progress_callback=progress_callback, decryption_tuple=self.decryption_tuple) # We catch all exceptions here because we want to store them. except Exception as e: # pylint: disable=broad-except # Save the exception so that it can be seen in the upload thread. with self.download_exception_lock: self.download_exception = e raise # TODO: If we do gzip encoding transforms mid-transfer, this will fail. self.download_thread = threading.Thread(target=PerformDownload, args=(start_byte, progress_callback)) self.download_thread.start() def read(self, amt=None): # pylint: disable=invalid-name """Exposes a stream from the in-memory buffer to the upload.""" if self.position == self.src_obj_size or amt == 0: # If there is no data left or 0 bytes were requested, return an empty # string so callers can call still call len() and read(0). return '' if amt is None or amt > constants.TRANSFER_BUFFER_SIZE: raise BadRequestException( 'Invalid HTTP read size %s during daisy chain operation, ' 'expected <= %s.' % (amt, constants.TRANSFER_BUFFER_SIZE)) if self._unused_data_from_previous_read: valid_data = self._unused_data_from_previous_read[:amt] self._unused_data_from_previous_read = ( self._unused_data_from_previous_read[amt:]) else: valid_data = b'' while True: with self.lock: if self.buffer: break if AcquireLockWithTimeout(self.download_exception_lock, 30): if self.download_exception: # Download thread died, so we will never recover. Raise the # exception that killed it. raise self.download_exception # pylint: disable=raising-bad-type else: if not self.download_thread.is_alive(): raise Exception('Download thread died suddenly.') # Buffer was empty, yield thread priority so the download thread can fill. time.sleep(0) with self.lock: # In a rare case where len(valid_data) < amt, and there are # elements in the buffer, e.g let's say valid_data = 'foo' and amt = 5, # and we have 'barbaz' in self.buffer, instead of returning 'fooba' # in the current call, we return 'foo', and in the subsequent calls # we will return 'barba' and 'z'. Given that we set # TRANSFER_BUFFER_SIZE as a multiple of the amt (8KiB), # we can safely assume that this will never happen practically. # This assumption helps us keep the logic simple and avoid # string concatenations and splits ('foo' + 'ba' in the above case). if not valid_data: data = self.buffer.popleft() valid_data = data[:amt] self._unused_data_from_previous_read = data[amt:] self.last_position = self.position self.last_data = valid_data data_len = len(valid_data) self.position += data_len self.bytes_buffered -= data_len if data_len > amt: raise BadRequestException( 'Invalid read during daisy chain operation, got data of size ' '%s, expected size %s.' % (data_len, amt)) return valid_data def tell(self): # pylint: disable=invalid-name with self.lock: return self.position def seek(self, offset, whence=os.SEEK_SET): # pylint: disable=invalid-name """Sets current read position of the stream.""" restart_download = False if whence == os.SEEK_END: if offset: raise IOError( 'Invalid seek during daisy chain operation. Non-zero offset %s ' 'from os.SEEK_END is not supported' % offset) with self.lock: self.last_position = self.position self.last_data = None # Safe because we check position against src_obj_size in read. self.position = self.src_obj_size elif whence == os.SEEK_SET: with self.lock: if offset == self.position: pass elif offset == self.last_position: self.position = self.last_position if self.last_data: # If we seek to end and then back, we won't have last_data; we'll # get it on the next call to read. self.buffer.appendleft(self.last_data) self.bytes_buffered += len(self.last_data) else: # Once a download is complete, boto seeks to 0 and re-reads to # compute the hash if an md5 isn't already present (for example a GCS # composite object), so we have to re-download the whole object. # Also, when daisy-chaining to a resumable upload, on error the # service may have received any number of the bytes; the download # needs to be restarted from that point. restart_download = True if restart_download: self.stop_download.set() # Consume any remaining bytes in the download thread so that # the thread can exit, then restart the thread at the desired position. while self.download_thread.is_alive(): with self.lock: while self.bytes_buffered: self.bytes_buffered -= len(self.buffer.popleft()) time.sleep(0) with self.lock: self.position = offset self.buffer = collections.deque() self.bytes_buffered = 0 self.last_position = 0 self.last_data = None self.stop_download.clear() self.StartDownloadThread(start_byte=offset, progress_callback=self.progress_callback) else: raise IOError('Daisy-chain download wrapper does not support ' 'seek mode %s' % whence) def seekable(self): # pylint: disable=invalid-name return True command_argument.py 0000644 00000010543 15025013616 0010440 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains classes related to argparse-based argument parsing.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from .tab_complete import CompleterType class CommandArgument(object): """Argparse style argument.""" def __init__(self, *args, **kwargs): """Constructs an argparse argument with the given data. See add_argument in argparse for description of the options. The only deviation from the argparse arguments is the 'completer' parameter. If 'completer' is present, it's used as the argcomplete completer for the argument. Args: *args: Position args to pass to argparse add_argument **kwargs: Named args to pass to argparse add_argument """ completer = None if 'completer' in kwargs: completer = kwargs['completer'] del kwargs['completer'] self.args = args self.kwargs = kwargs self.completer = completer @staticmethod def MakeZeroOrMoreCloudURLsArgument(): """Constructs an argument that takes 0 or more Cloud URLs as parameters.""" return CommandArgument('file', nargs='*', completer=CompleterType.CLOUD_OBJECT) @staticmethod def MakeZeroOrMoreCloudBucketURLsArgument(): """Constructs an argument that takes 0+ Cloud bucket URLs as parameters.""" return CommandArgument('file', nargs='*', completer=CompleterType.CLOUD_BUCKET) @staticmethod def MakeNCloudBucketURLsArgument(n): """Constructs an argument that takes N Cloud bucket URLs as parameters.""" return CommandArgument('file', nargs=n, completer=CompleterType.CLOUD_BUCKET) @staticmethod def MakeNCloudURLsArgument(n): """Constructs an argument that takes N Cloud URLs as parameters.""" return CommandArgument('file', nargs=n, completer=CompleterType.CLOUD_OBJECT) @staticmethod def MakeZeroOrMoreCloudOrFileURLsArgument(): """Constructs an argument that takes 0 or more Cloud or File URLs.""" return CommandArgument('file', nargs='*', completer=CompleterType.CLOUD_OR_LOCAL_OBJECT) @staticmethod def MakeNCloudOrFileURLsArgument(n): """Constructs an argument that takes N Cloud or File URLs as parameters.""" return CommandArgument('file', nargs=n, completer=CompleterType.CLOUD_OR_LOCAL_OBJECT) @staticmethod def MakeZeroOrMoreFileURLsArgument(): """Constructs an argument that takes 0 or more File URLs as parameters.""" return CommandArgument('file', nargs='*', completer=CompleterType.LOCAL_OBJECT) @staticmethod def MakeNFileURLsArgument(n): """Constructs an argument that takes N File URLs as parameters.""" return CommandArgument('file', nargs=n, completer=CompleterType.LOCAL_OBJECT) @staticmethod def MakeFileURLOrCannedACLArgument(): """Constructs an argument that takes a File URL or a canned ACL.""" return CommandArgument('file', nargs=1, completer=CompleterType.LOCAL_OBJECT_OR_CANNED_ACL) @staticmethod def MakeFreeTextArgument(): """Constructs an argument that takes arbitrary text.""" return CommandArgument('text', completer=CompleterType.NO_OP) @staticmethod def MakeOneOrMoreBindingsArgument(): """Constructs an argument that takes multiple bindings.""" return CommandArgument('binding', nargs='+', completer=CompleterType.NO_OP) parallel_tracker_file.py 0000644 00000030217 15025013616 0011426 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for composite upload tracker file functionality.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import namedtuple import errno import json import random import six import gslib from gslib.exception import CommandException from gslib.tracker_file import (WriteJsonDataToTrackerFile, RaiseUnwritableTrackerFileException) from gslib.utils.constants import UTF8 ObjectFromTracker = namedtuple('ObjectFromTracker', 'object_name generation') class _CompositeUploadTrackerEntry(object): """Enum class for composite upload tracker file JSON keys.""" COMPONENTS_LIST = 'components' COMPONENT_NAME = 'component_name' COMPONENT_GENERATION = 'component_generation' ENC_SHA256 = 'encryption_key_sha256' PREFIX = 'prefix' def ReadParallelUploadTrackerFile(tracker_file_name, logger): """Read the tracker file from the last parallel composite upload attempt. If it exists, the tracker file is of the format described in WriteParallelUploadTrackerFile or a legacy format. If the file doesn't exist or is formatted incorrectly, then the upload will start from the beginning. This function is not thread-safe and must be protected by a lock if called within Command.Apply. Args: tracker_file_name: The name of the tracker file to read parse. logger: logging.Logger for outputting log messages. Returns: enc_key_sha256: Encryption key SHA256 used to encrypt the existing components, or None if an encryption key was not used. component_prefix: String prefix used in naming the existing components, or None if no prefix was found. existing_components: A list of ObjectFromTracker objects representing the set of files that have already been uploaded. """ enc_key_sha256 = None prefix = None existing_components = [] tracker_file = None # If we already have a matching tracker file, get the serialization data # so that we can resume the upload. try: tracker_file = open(tracker_file_name, 'r') tracker_data = tracker_file.read() tracker_json = json.loads(tracker_data) enc_key_sha256 = tracker_json[_CompositeUploadTrackerEntry.ENC_SHA256] prefix = tracker_json[_CompositeUploadTrackerEntry.PREFIX] for component in tracker_json[_CompositeUploadTrackerEntry.COMPONENTS_LIST]: existing_components.append( ObjectFromTracker( component[_CompositeUploadTrackerEntry.COMPONENT_NAME], component[_CompositeUploadTrackerEntry.COMPONENT_GENERATION])) except IOError as e: # Ignore non-existent file (happens first time a upload is attempted on an # object, or when re-starting an upload after a # ResumableUploadStartOverException), but warn user for other errors. if e.errno != errno.ENOENT: logger.warn( 'Couldn\'t read upload tracker file (%s): %s. Restarting ' 'parallel composite upload from scratch.', tracker_file_name, e.strerror) except (KeyError, ValueError) as e: # Legacy format did not support user-supplied encryption. enc_key_sha256 = None (prefix, existing_components) = _ParseLegacyTrackerData(tracker_data) finally: if tracker_file: tracker_file.close() return (enc_key_sha256, prefix, existing_components) def _ParseLegacyTrackerData(tracker_data): """Parses a legacy parallel composite upload tracker file. Args: tracker_data: Legacy tracker file contents. Returns: component_prefix: The prefix used in naming the existing components, or None if no prefix was found. existing_components: A list of ObjectFromTracker objects representing the set of files that have already been uploaded. """ # Old tracker files used a non-JSON format. # The first line represents the prefix, followed by line pairs of object_name # and generation. Discard the last blank line. old_tracker_data = tracker_data.split('\n')[:-1] prefix = None existing_components = [] if old_tracker_data: prefix = old_tracker_data[0] i = 1 while i < len(old_tracker_data) - 1: (name, generation) = (old_tracker_data[i], old_tracker_data[i + 1]) if not generation: # Cover the '' case. generation = None existing_components.append(ObjectFromTracker(name, generation)) i += 2 return (prefix, existing_components) def ValidateParallelCompositeTrackerData(tracker_file_name, existing_enc_sha256, existing_prefix, existing_components, current_enc_key_sha256, bucket_url, command_obj, logger, delete_func, delete_exc_handler): """Validates that tracker data matches the current encryption key. If the data does not match, makes a best-effort attempt to delete existing temporary component objects encrypted with the old key. Args: tracker_file_name: String file name of tracker file. existing_enc_sha256: Encryption key SHA256 used to encrypt the existing components, or None if an encryption key was not used. existing_prefix: String prefix used in naming the existing components, or None if no prefix was found. existing_components: A list of ObjectFromTracker objects representing the set of files that have already been uploaded. current_enc_key_sha256: Current Encryption key SHA256 that should be used to encrypt objects. bucket_url: Bucket URL in which the components exist. command_obj: Command class for calls to Apply. logger: logging.Logger for outputting log messages. delete_func: command.Apply-callable function for deleting objects. delete_exc_handler: Exception handler for delete_func. Returns: prefix: existing_prefix, or None if the encryption key did not match. existing_components: existing_components, or empty list if the encryption key did not match. """ if six.PY3: if isinstance(existing_enc_sha256, str): existing_enc_sha256 = existing_enc_sha256.encode(UTF8) if isinstance(current_enc_key_sha256, str): current_enc_key_sha256 = current_enc_key_sha256.encode(UTF8) if existing_prefix and existing_enc_sha256 != current_enc_key_sha256: try: logger.warn( 'Upload tracker file (%s) does not match current encryption ' 'key. Deleting old components and restarting upload from ' 'scratch with a new tracker file that uses the current ' 'encryption key.', tracker_file_name) components_to_delete = [] for component in existing_components: url = bucket_url.Clone() url.object_name = component.object_name url.generation = component.generation command_obj.Apply( delete_func, components_to_delete, delete_exc_handler, arg_checker=gslib.command.DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SPEED) except: # pylint: disable=bare-except # Regardless of why we can't clean up old components, need to proceed # with the user's original intent to upload the file, so merely warn. component_names = [ component.object_name for component in existing_components ] logger.warn( 'Failed to delete some of the following temporary objects:\n%s\n' '(Continuing on to re-upload components from scratch.)', '\n'.join(component_names)) # Encryption keys have changed, so the old components and prefix # cannot be used. return (None, []) return (existing_prefix, existing_components) def GenerateComponentObjectPrefix(encryption_key_sha256=None): """Generates a random prefix for component objects. Args: encryption_key_sha256: Encryption key SHA256 that will be used to encrypt the components. This is hashed into the prefix to avoid collision during resumption with a different encryption key. Returns: String prefix for use in the composite upload. """ return str( (random.randint(1, (10**10) - 1) + hash(encryption_key_sha256)) % 10**10) def WriteComponentToParallelUploadTrackerFile(tracker_file_name, tracker_file_lock, component, logger, encryption_key_sha256=None): """Rewrites an existing tracker file with info about the uploaded component. Follows the format described in _CreateParallelUploadTrackerFile. Args: tracker_file_name: Tracker file to append to. tracker_file_lock: Thread and process-safe Lock protecting the tracker file. component: ObjectFromTracker describing the object that was uploaded. logger: logging.Logger for outputting log messages. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. """ with tracker_file_lock: (existing_enc_key_sha256, prefix, existing_components) = (ReadParallelUploadTrackerFile( tracker_file_name, logger)) if existing_enc_key_sha256 != encryption_key_sha256: raise CommandException( 'gsutil client error: encryption key SHA256 (%s) in tracker file ' 'does not match encryption key SHA256 (%s) of component %s' % (existing_enc_key_sha256, encryption_key_sha256, component.object_name)) newly_completed_components = [component] completed_components = existing_components + newly_completed_components WriteParallelUploadTrackerFile(tracker_file_name, prefix, completed_components, encryption_key_sha256=encryption_key_sha256) def WriteParallelUploadTrackerFile(tracker_file_name, prefix, components, encryption_key_sha256=None): """Writes information about components that were successfully uploaded. The tracker file is serialized JSON of the form: { "encryption_key_sha256": sha256 hash of encryption key (or null), "prefix": Prefix used for the component objects, "components": [ { "component_name": Component object name, "component_generation": Component object generation (or null), }, ... ] } where N is the number of components that have been successfully uploaded. This function is not thread-safe and must be protected by a lock if called within Command.Apply. Args: tracker_file_name: The name of the parallel upload tracker file. prefix: The generated prefix that used for uploading any existing components. components: A list of ObjectFromTracker objects that were uploaded. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. """ if six.PY3: if isinstance(encryption_key_sha256, bytes): encryption_key_sha256 = encryption_key_sha256.decode('ascii') tracker_components = [] for component in components: tracker_components.append({ _CompositeUploadTrackerEntry.COMPONENT_NAME: component.object_name, _CompositeUploadTrackerEntry.COMPONENT_GENERATION: component.generation }) tracker_file_data = { _CompositeUploadTrackerEntry.COMPONENTS_LIST: tracker_components, _CompositeUploadTrackerEntry.ENC_SHA256: encryption_key_sha256, _CompositeUploadTrackerEntry.PREFIX: prefix } WriteJsonDataToTrackerFile(tracker_file_name, tracker_file_data) tab_complete.py 0000644 00000025430 15025013616 0007557 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shell tab completion.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import itertools import json import threading import time import boto from boto.gs.acl import CannedACLStrings from gslib.storage_url import IsFileUrlString from gslib.storage_url import StorageUrlFromString from gslib.storage_url import StripOneSlash from gslib.utils.boto_util import GetTabCompletionCacheFilename from gslib.utils.boto_util import GetTabCompletionLogFilename from gslib.wildcard_iterator import CreateWildcardIterator TAB_COMPLETE_CACHE_TTL = 15 _TAB_COMPLETE_MAX_RESULTS = 1000 _TIMEOUT_WARNING = """ Tab completion aborted (took >%ss), you may complete the command manually. The timeout can be adjusted in the gsutil configuration file. """.rstrip() class CompleterType(object): CLOUD_BUCKET = 'cloud_bucket' CLOUD_OBJECT = 'cloud_object' CLOUD_OR_LOCAL_OBJECT = 'cloud_or_local_object' LOCAL_OBJECT = 'local_object' LOCAL_OBJECT_OR_CANNED_ACL = 'local_object_or_canned_acl' NO_OP = 'no_op' class LocalObjectCompleter(object): """Completer object for local files.""" def __init__(self): # This is only safe to import if argcomplete is present in the install # (which happens for Cloud SDK installs), so import on usage, not on load. # pylint: disable=g-import-not-at-top from argcomplete.completers import FilesCompleter self.files_completer = FilesCompleter() def __call__(self, prefix, **kwargs): return self.files_completer(prefix, **kwargs) class LocalObjectOrCannedACLCompleter(object): """Completer object for local files and canned ACLs. Currently, only Google Cloud Storage canned ACL names are supported. """ def __init__(self): self.local_object_completer = LocalObjectCompleter() def __call__(self, prefix, **kwargs): local_objects = self.local_object_completer(prefix, **kwargs) canned_acls = [acl for acl in CannedACLStrings if acl.startswith(prefix)] return local_objects + canned_acls class TabCompletionCache(object): """Cache for tab completion results.""" def __init__(self, prefix, results, timestamp, partial_results): self.prefix = prefix self.results = results self.timestamp = timestamp self.partial_results = partial_results @staticmethod def LoadFromFile(filename): """Instantiates the cache from a file. Args: filename: The file to load. Returns: TabCompletionCache instance with loaded data or an empty cache if the file cannot be loaded """ try: with open(filename, 'r') as fp: cache_dict = json.loads(fp.read()) prefix = cache_dict['prefix'] results = cache_dict['results'] timestamp = cache_dict['timestamp'] partial_results = cache_dict['partial-results'] except Exception: # pylint: disable=broad-except # Guarding against incompatible format changes in the cache file. # Erring on the side of not breaking tab-completion in case of cache # issues. prefix = None results = [] timestamp = 0 partial_results = False return TabCompletionCache(prefix, results, timestamp, partial_results) def GetCachedResults(self, prefix): """Returns the cached results for prefix or None if not in cache.""" current_time = time.time() if current_time - self.timestamp >= TAB_COMPLETE_CACHE_TTL: return None results = None if prefix == self.prefix: results = self.results elif (not self.partial_results and prefix.startswith(self.prefix) and prefix.count('/') == self.prefix.count('/')): results = [x for x in self.results if x.startswith(prefix)] if results is not None: # Update cache timestamp to make sure the cache entry does not expire if # the user is performing multiple completions in a single # bucket/subdirectory since we can answer these requests from the cache. # e.g. gs://prefix<tab> -> gs://prefix-mid<tab> -> gs://prefix-mid-suffix self.timestamp = time.time() return results def UpdateCache(self, prefix, results, partial_results): """Updates the in-memory cache with the results for the given prefix.""" self.prefix = prefix self.results = results self.partial_results = partial_results self.timestamp = time.time() def WriteToFile(self, filename): """Writes out the cache to the given file.""" json_str = json.dumps({ 'prefix': self.prefix, 'results': self.results, 'partial-results': self.partial_results, 'timestamp': self.timestamp, }) try: with open(filename, 'w') as fp: fp.write(json_str) except IOError: pass class CloudListingRequestThread(threading.Thread): """Thread that performs a listing request for the given URL string.""" def __init__(self, wildcard_url_str, gsutil_api): """Instantiates Cloud listing request thread. Args: wildcard_url_str: The URL to list. gsutil_api: gsutil Cloud API instance to use. """ super(CloudListingRequestThread, self).__init__() self.daemon = True self._wildcard_url_str = wildcard_url_str self._gsutil_api = gsutil_api self.results = None def run(self): it = CreateWildcardIterator( self._wildcard_url_str, self._gsutil_api).IterAll(bucket_listing_fields=['name']) self.results = [ str(c) for c in itertools.islice(it, _TAB_COMPLETE_MAX_RESULTS) ] class TimeoutError(Exception): pass class CloudObjectCompleter(object): """Completer object for Cloud URLs.""" def __init__(self, gsutil_api, bucket_only=False): """Instantiates completer for Cloud URLs. Args: gsutil_api: gsutil Cloud API instance to use. bucket_only: Whether the completer should only match buckets. """ self._gsutil_api = gsutil_api self._bucket_only = bucket_only def _PerformCloudListing(self, wildcard_url, timeout): """Perform a remote listing request for the given wildcard URL. Args: wildcard_url: The wildcard URL to list. timeout: Time limit for the request. Returns: Cloud resources matching the given wildcard URL. Raises: TimeoutError: If the listing does not finish within the timeout. """ request_thread = CloudListingRequestThread(wildcard_url, self._gsutil_api) request_thread.start() request_thread.join(timeout) if request_thread.is_alive(): # This is only safe to import if argcomplete is present in the install # (which happens for Cloud SDK installs), so import on usage, not on load. # pylint: disable=g-import-not-at-top import argcomplete argcomplete.warn(_TIMEOUT_WARNING % timeout) raise TimeoutError() results = request_thread.results return results def __call__(self, prefix, **kwargs): if not prefix: prefix = 'gs://' elif IsFileUrlString(prefix): return [] wildcard_url = prefix + '*' url = StorageUrlFromString(wildcard_url) if self._bucket_only and not url.IsBucket(): return [] timeout = boto.config.getint('GSUtil', 'tab_completion_timeout', 5) if timeout == 0: return [] start_time = time.time() cache = TabCompletionCache.LoadFromFile(GetTabCompletionCacheFilename()) cached_results = cache.GetCachedResults(prefix) timing_log_entry_type = '' if cached_results is not None: results = cached_results timing_log_entry_type = ' (from cache)' else: try: results = self._PerformCloudListing(wildcard_url, timeout) if self._bucket_only and len(results) == 1: results = [StripOneSlash(results[0])] partial_results = (len(results) == _TAB_COMPLETE_MAX_RESULTS) cache.UpdateCache(prefix, results, partial_results) except TimeoutError: timing_log_entry_type = ' (request timeout)' results = [] cache.WriteToFile(GetTabCompletionCacheFilename()) end_time = time.time() num_results = len(results) elapsed_seconds = end_time - start_time _WriteTimingLog( '%s results%s in %.2fs, %.2f results/second for prefix: %s\n' % (num_results, timing_log_entry_type, elapsed_seconds, num_results / elapsed_seconds, prefix)) return results class CloudOrLocalObjectCompleter(object): """Completer object for Cloud URLs or local files. Invokes the Cloud object completer if the input looks like a Cloud URL and falls back to local file completer otherwise. """ def __init__(self, gsutil_api): self.cloud_object_completer = CloudObjectCompleter(gsutil_api) self.local_object_completer = LocalObjectCompleter() def __call__(self, prefix, **kwargs): if IsFileUrlString(prefix): completer = self.local_object_completer else: completer = self.cloud_object_completer return completer(prefix, **kwargs) class NoOpCompleter(object): """Completer that always returns 0 results.""" def __call__(self, unused_prefix, **unused_kwargs): return [] def MakeCompleter(completer_type, gsutil_api): """Create a completer instance of the given type. Args: completer_type: The type of completer to create. gsutil_api: gsutil Cloud API instance to use. Returns: A completer instance. Raises: RuntimeError: if completer type is not supported. """ if completer_type == CompleterType.CLOUD_OR_LOCAL_OBJECT: return CloudOrLocalObjectCompleter(gsutil_api) elif completer_type == CompleterType.LOCAL_OBJECT: return LocalObjectCompleter() elif completer_type == CompleterType.LOCAL_OBJECT_OR_CANNED_ACL: return LocalObjectOrCannedACLCompleter() elif completer_type == CompleterType.CLOUD_BUCKET: return CloudObjectCompleter(gsutil_api, bucket_only=True) elif completer_type == CompleterType.CLOUD_OBJECT: return CloudObjectCompleter(gsutil_api) elif completer_type == CompleterType.NO_OP: return NoOpCompleter() else: raise RuntimeError('Unknown completer "%s"' % completer_type) def _WriteTimingLog(message): """Write an entry to the tab completion timing log, if it's enabled.""" if boto.config.getbool('GSUtil', 'tab_completion_time_logs', False): with open(GetTabCompletionLogFilename(), 'ab') as fp: fp.write(message) file_part.py 0000644 00000006623 15025013616 0007071 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FilePart implementation for representing part of a file.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import io class FilePart(io.IOBase): """Subclass of the file API for representing part of a file. This class behaves as a contiguous subset of a given file (e.g., this object will behave as though the desired part of the file was written to another file, and the second file was opened). """ # pylint: disable=super-init-not-called def __init__(self, filename, offset, length): """Initializes the FilePart. Args: filename: The name of the existing file, of which this object represents a part. offset: The position (in bytes) in the original file that corresponds to the first byte of the FilePart. length: The total number of bytes in the FilePart. """ self._fp = open(filename, 'rb') self.length = length self._start = offset self._end = self._start + self.length self._fp.seek(self._start) def __enter__(self): pass # pylint: disable=redefined-builtin def __exit__(self, type, value, traceback): self.close() def tell(self): return self._fp.tell() - self._start def read(self, size=-1): if size < 0: size = self.length size = min(size, self._end - self._fp.tell()) # Only read to our EOF return self._fp.read(max(0, size)) def seek(self, offset, whence=os.SEEK_SET): if whence == os.SEEK_END: return self._fp.seek(offset + self._end) elif whence == os.SEEK_CUR: return self._fp.seek(offset, whence) else: return self._fp.seek(self._start + offset) def close(self): self._fp.close() def flush(self, size=None): raise NotImplementedError('flush is not implemented in FilePart.') def fileno(self, size=None): raise NotImplementedError('fileno is not implemented in FilePart.') def isatty(self, size=None): raise NotImplementedError('isatty is not implemented in FilePart.') def next(self, size=None): raise NotImplementedError('next is not implemented in FilePart.') def readline(self, size=None): raise NotImplementedError('readline is not implemented in FilePart.') def readlines(self, size=None): raise NotImplementedError('readlines is not implemented in FilePart.') def xreadlines(self, size=None): raise NotImplementedError('xreadlines is not implemented in FilePart.') def truncate(self, size=None): raise NotImplementedError('truncate is not implemented in FilePart.') def write(self, size=None): raise NotImplementedError('write is not implemented in FilePart.') def writelines(self, size=None): raise NotImplementedError('writelines is not implemented in FilePart.') name_expansion.py 0000644 00000073735 15025013616 0010140 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Name expansion iterator and result classes. Name expansion support for the various ways gsutil lets users refer to collections of data (via explicit wildcarding as well as directory, bucket, and bucket subdir implicit wildcarding). This class encapsulates the various rules for determining how these expansions are done. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import logging import os import sys import six from apitools.base.py import encoding import gslib from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_GENERIC from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.seek_ahead_thread import SeekAheadResult from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages import gslib.wildcard_iterator from gslib.wildcard_iterator import StorageUrlFromString class NameExpansionResult(object): """Holds one fully expanded result from iterating over NameExpansionIterator. The member data in this class need to be pickleable because NameExpansionResult instances are passed through Multiprocessing.Queue. In particular, don't include any boto state like StorageUri, since that pulls in a big tree of objects, some of which aren't pickleable (and even if they were, pickling/unpickling such a large object tree would result in significant overhead). The state held in this object is needed for handling the various naming cases (e.g., copying from a single source URL to a directory generates different dest URL names than copying multiple URLs to a directory, to be consistent with naming rules used by the Unix cp command). For more details see comments in _NameExpansionIterator. """ def __init__(self, source_storage_url, is_multi_source_request, is_multi_top_level_source_request, names_container, expanded_storage_url, expanded_result): """Instantiates a result from name expansion. Args: source_storage_url: StorageUrl that was being expanded. is_multi_source_request: bool indicator whether multiple input URLs or src_url_str expanded to more than one BucketListingRef. is_multi_top_level_source_request: same as is_multi_source_request but measured before recursion. names_container: Bool indicator whether src_url names a container. expanded_storage_url: StorageUrl that was expanded. expanded_result: cloud object metadata in MessageToJson form (for pickleability), if any was iterated; None otherwise. Consumers must call JsonToMessage to get an apitools Object. """ self.source_storage_url = source_storage_url self.is_multi_source_request = is_multi_source_request self.is_multi_top_level_source_request = is_multi_top_level_source_request self.names_container = names_container self.expanded_storage_url = expanded_storage_url self.expanded_result = encoding.MessageToJson( expanded_result) if expanded_result else None def __repr__(self): return '%s' % self.expanded_storage_url class _NameExpansionIterator(object): """Class that iterates over all source URLs passed to the iterator. See details in __iter__ function doc. """ def __init__(self, command_name, debug, logger, gsutil_api, url_strs, recursion_requested, all_versions=False, cmd_supports_recursion=True, project_id=None, ignore_symlinks=False, continue_on_error=False, bucket_listing_fields=None): """Creates a NameExpansionIterator. Args: command_name: name of command being run. debug: Debug level to pass to underlying iterators (range 0..3). logger: logging.Logger object. gsutil_api: Cloud storage interface. Settable for testing/mocking. url_strs: PluralityCheckableIterator of URL strings needing expansion. recursion_requested: True if -r specified on command-line. If so, listings will be flattened so mapped-to results contain objects spanning subdirectories. all_versions: Bool indicating whether to iterate over all object versions. cmd_supports_recursion: Bool indicating whether this command supports a '-r' flag. Useful for printing helpful error messages. project_id: Project id to use for bucket retrieval. ignore_symlinks: If True, ignore symlinks during iteration. continue_on_error: If true, yield no-match exceptions encountered during iteration instead of raising them. bucket_listing_fields: Iterable fields to include in expanded results. Ex. ['name', 'acl']. Underyling iterator is responsible for converting these to list-style format ['items/name', 'items/acl']. If this is None, only the object name is included in the result. Examples of _NameExpansionIterator with recursion_requested=True: - Calling with one of the url_strs being 'gs://bucket' will enumerate all top-level objects, as will 'gs://bucket/' and 'gs://bucket/*'. - 'gs://bucket/**' will enumerate all objects in the bucket. - 'gs://bucket/abc' will enumerate either the single object abc or, if abc is a subdirectory, all objects under abc and any of its subdirectories. - 'gs://bucket/abc/**' will enumerate all objects under abc or any of its subdirectories. - 'file:///tmp' will enumerate all files under /tmp, as will 'file:///tmp/*' - 'file:///tmp/**' will enumerate all files under /tmp or any of its subdirectories. Example if recursion_requested=False: calling with gs://bucket/abc/* lists matching objects or subdirs, but not sub-subdirs or objects beneath subdirs. Note: In step-by-step comments below we give examples assuming there's a gs://bucket with object paths: abcd/o1.txt abcd/o2.txt xyz/o1.txt xyz/o2.txt and a directory file://dir with file paths: dir/a.txt dir/b.txt dir/c/ """ self.command_name = command_name self.debug = debug self.logger = logger self.gsutil_api = gsutil_api self.url_strs = url_strs self.recursion_requested = recursion_requested self.all_versions = all_versions # Check self.url_strs.HasPlurality() at start because its value can change # if url_strs is itself an iterator. self.url_strs.has_plurality = self.url_strs.HasPlurality() self.cmd_supports_recursion = cmd_supports_recursion self.project_id = project_id self.ignore_symlinks = ignore_symlinks self.continue_on_error = continue_on_error self.bucket_listing_fields = (set(['name']) if not bucket_listing_fields else bucket_listing_fields) # Map holding wildcard strings to use for flat vs subdir-by-subdir listings. # (A flat listing means show all objects expanded all the way down.) self._flatness_wildcard = {True: '**', False: '*'} def __iter__(self): """Iterates over all source URLs passed to the iterator. For each src url, expands wildcards, object-less bucket names, subdir bucket names, and directory names, and generates a flat listing of all the matching objects/files. You should instantiate this object using the static factory function NameExpansionIterator, because consumers of this iterator need the PluralityCheckableIterator wrapper built by that function. Yields: gslib.name_expansion.NameExpansionResult. Raises: CommandException: if errors encountered. """ for url_str in self.url_strs: storage_url = StorageUrlFromString(url_str) if (storage_url.IsFileUrl() and (storage_url.IsStream() or storage_url.IsFifo())): if self.url_strs.has_plurality: raise CommandException('Multiple URL strings are not supported ' 'with streaming ("-") URLs or named pipes.') yield NameExpansionResult(source_storage_url=storage_url, is_multi_source_request=False, is_multi_top_level_source_request=False, names_container=False, expanded_storage_url=storage_url, expanded_result=None) continue # Step 1: Expand any explicitly specified wildcards. The output from this # step is an iterator of BucketListingRef. # Starting with gs://buck*/abc* this step would expand to gs://bucket/abcd src_names_bucket = False if (storage_url.IsCloudUrl() and storage_url.IsBucket() and not self.recursion_requested): # UNIX commands like rm and cp will omit directory references. # If url_str refers only to buckets and we are not recursing, # then produce references of type BUCKET, because they are guaranteed # to pass through Step 2 and be omitted in Step 3. post_step1_iter = PluralityCheckableIterator( self.WildcardIterator(url_str).IterBuckets(bucket_fields=['id'])) else: # Get a list of objects and prefixes, expanding the top level for # any listed buckets. If our source is a bucket, however, we need # to treat all of the top level expansions as names_container=True. post_step1_iter = PluralityCheckableIterator( self.WildcardIterator(url_str).IterAll( bucket_listing_fields=self.bucket_listing_fields, expand_top_level_buckets=True)) if storage_url.IsCloudUrl() and storage_url.IsBucket(): src_names_bucket = True src_url_expands_to_multi = post_step1_iter.HasPlurality() is_multi_top_level_source_request = (self.url_strs.has_plurality or src_url_expands_to_multi) # Step 2: Expand bucket subdirs. The output from this # step is an iterator of (names_container, BucketListingRef). # Starting with gs://bucket/abcd this step would expand to: # iter([(True, abcd/o1.txt), (True, abcd/o2.txt)]). subdir_exp_wildcard = self._flatness_wildcard[self.recursion_requested] if self.recursion_requested: post_step2_iter = _ImplicitBucketSubdirIterator( self, post_step1_iter, subdir_exp_wildcard, self.bucket_listing_fields) else: post_step2_iter = _NonContainerTuplifyIterator(post_step1_iter) post_step2_iter = PluralityCheckableIterator(post_step2_iter) # Because we actually perform and check object listings here, this will # raise if url_args includes a non-existent object. However, # plurality_checkable_iterator will buffer the exception for us, not # raising it until the iterator is actually asked to yield the first # result. if post_step2_iter.IsEmpty(): if self.continue_on_error: try: raise CommandException(NO_URLS_MATCHED_TARGET % url_str) except CommandException as e: # Yield a specialized tuple of (exception, stack_trace) to # the wrapping PluralityCheckableIterator. yield (e, sys.exc_info()[2]) else: raise CommandException(NO_URLS_MATCHED_TARGET % url_str) # Step 3. Omit any directories, buckets, or bucket subdirectories for # non-recursive expansions. post_step3_iter = PluralityCheckableIterator( _OmitNonRecursiveIterator(post_step2_iter, self.recursion_requested, self.command_name, self.cmd_supports_recursion, self.logger)) src_url_expands_to_multi = post_step3_iter.HasPlurality() is_multi_source_request = (self.url_strs.has_plurality or src_url_expands_to_multi) # Step 4. Expand directories and buckets. This step yields the iterated # values. Starting with gs://bucket this step would expand to: # [abcd/o1.txt, abcd/o2.txt, xyz/o1.txt, xyz/o2.txt] # Starting with file://dir this step would expand to: # [dir/a.txt, dir/b.txt, dir/c/] for (names_container, blr) in post_step3_iter: src_names_container = src_names_bucket or names_container if blr.IsObject(): yield NameExpansionResult( source_storage_url=storage_url, is_multi_source_request=is_multi_source_request, is_multi_top_level_source_request= is_multi_top_level_source_request, names_container=src_names_container, expanded_storage_url=blr.storage_url, expanded_result=blr.root_object) else: # Use implicit wildcarding to do the enumeration. # At this point we are guaranteed that: # - Recursion has been requested because non-object entries are # filtered in step 3 otherwise. # - This is a prefix or bucket subdirectory because only # non-recursive iterations product bucket references. expanded_url = StorageUrlFromString(blr.url_string) if expanded_url.IsFileUrl(): # Convert dir to implicit recursive wildcard. url_to_iterate = '%s%s%s' % (blr, os.sep, subdir_exp_wildcard) else: # Convert subdir to implicit recursive wildcard. url_to_iterate = expanded_url.CreatePrefixUrl( wildcard_suffix=subdir_exp_wildcard) wc_iter = PluralityCheckableIterator( self.WildcardIterator(url_to_iterate).IterObjects( bucket_listing_fields=self.bucket_listing_fields)) src_url_expands_to_multi = (src_url_expands_to_multi or wc_iter.HasPlurality()) is_multi_source_request = (self.url_strs.has_plurality or src_url_expands_to_multi) # This will be a flattened listing of all underlying objects in the # subdir. for blr in wc_iter: yield NameExpansionResult( source_storage_url=storage_url, is_multi_source_request=is_multi_source_request, is_multi_top_level_source_request=( is_multi_top_level_source_request), names_container=True, expanded_storage_url=blr.storage_url, expanded_result=blr.root_object) def WildcardIterator(self, url_string): """Helper to instantiate gslib.WildcardIterator. Args are same as gslib.WildcardIterator interface, but this method fills in most of the values from instance state. Args: url_string: URL string naming wildcard objects to iterate. Returns: Wildcard iterator over URL string. """ return gslib.wildcard_iterator.CreateWildcardIterator( url_string, self.gsutil_api, all_versions=self.all_versions, project_id=self.project_id, ignore_symlinks=self.ignore_symlinks, logger=self.logger) class SeekAheadNameExpansionIterator(object): """Creates and wraps a _NameExpansionIterator and yields SeekAheadResults. Unlike the NameExpansionIterator, which can make API calls upon __init__ to check for plurality, this iterator does no work until the first iteration occurs. """ def __init__(self, command_name, debug, gsutil_api, url_strs, recursion_requested, all_versions=False, cmd_supports_recursion=True, project_id=None, ignore_symlinks=False, file_size_will_change=False): """Initializes a _NameExpansionIterator with the inputs.""" # Count data bytes only will be transferred/rewritten. # Note that the rsync command uses a different iterator, thus it is not # included here. self.count_data_bytes = (command_name in ('cp', 'mv', 'rewrite') and not file_size_will_change) # Only query the file size if we are counting data bytes, as this may # result in stat'ing files, which is more expensive. bucket_listing_fields = ['size'] if self.count_data_bytes else None self.name_expansion_iterator = _NameExpansionIterator( command_name, debug, logging.getLogger('dummy'), gsutil_api, PluralityCheckableIterator(url_strs), recursion_requested, all_versions=all_versions, cmd_supports_recursion=cmd_supports_recursion, project_id=project_id, ignore_symlinks=ignore_symlinks, continue_on_error=True, bucket_listing_fields=bucket_listing_fields) def __iter__(self): for name_expansion_result in self.name_expansion_iterator: if self.count_data_bytes and name_expansion_result.expanded_result: iterated_metadata = encoding.JsonToMessage( apitools_messages.Object, name_expansion_result.expanded_result) iterated_size = iterated_metadata.size or 0 yield SeekAheadResult(data_bytes=iterated_size) else: yield SeekAheadResult() def NameExpansionIterator(command_name, debug, logger, gsutil_api, url_strs, recursion_requested, all_versions=False, cmd_supports_recursion=True, project_id=None, ignore_symlinks=False, continue_on_error=False, bucket_listing_fields=None): """Static factory function for instantiating _NameExpansionIterator. This wraps the resulting iterator in a PluralityCheckableIterator and checks that it is non-empty. Also, allows url_strs to be either an array or an iterator. Args: command_name: name of command being run. debug: Debug level to pass to underlying iterators (range 0..3). logger: logging.Logger object. gsutil_api: Cloud storage interface. Settable for testing/mocking. url_strs: Iterable URL strings needing expansion. recursion_requested: True if -r specified on command-line. If so, listings will be flattened so mapped-to results contain objects spanning subdirectories. all_versions: Bool indicating whether to iterate over all object versions. cmd_supports_recursion: Bool indicating whether this command supports a '-r' flag. Useful for printing helpful error messages. project_id: Project id to use for the current command. ignore_symlinks: If True, ignore symlinks during iteration. continue_on_error: If true, yield no-match exceptions encountered during iteration instead of raising them. bucket_listing_fields: Iterable fields to include in expanded results. Ex. ['name', 'acl']. Underyling iterator is responsible for converting these to list-style format ['items/name', 'items/acl']. If this is None, only the object name is included in the result. Raises: CommandException if underlying iterator is empty. Returns: Name expansion iterator instance. For example semantics, see comments in NameExpansionIterator.__init__. """ url_strs = PluralityCheckableIterator(url_strs) name_expansion_iterator = _NameExpansionIterator( command_name, debug, logger, gsutil_api, url_strs, recursion_requested, all_versions=all_versions, cmd_supports_recursion=cmd_supports_recursion, project_id=project_id, ignore_symlinks=ignore_symlinks, continue_on_error=continue_on_error, bucket_listing_fields=bucket_listing_fields) name_expansion_iterator = PluralityCheckableIterator(name_expansion_iterator) if name_expansion_iterator.IsEmpty(): raise CommandException(NO_URLS_MATCHED_GENERIC) return name_expansion_iterator class _NonContainerTuplifyIterator(object): """Iterator that produces the tuple (False, blr) for each iterated value. Used for cases where blr_iter iterates over a set of BucketListingRefs known not to name containers. """ def __init__(self, blr_iter): """Instantiates iterator. Args: blr_iter: iterator of BucketListingRef. """ self.blr_iter = blr_iter def __iter__(self): for blr in self.blr_iter: yield (False, blr) class _OmitNonRecursiveIterator(object): """Iterator wrapper for that omits certain values for non-recursive requests. This iterates over tuples of (names_container, BucketListingReference) and omits directories, prefixes, and buckets from non-recurisve requests so that we can properly calculate whether the source URL expands to multiple URLs. For example, if we have a bucket containing two objects: bucket/foo and bucket/foo/bar and we do a non-recursive iteration, only bucket/foo will be yielded. """ def __init__(self, tuple_iter, recursion_requested, command_name, cmd_supports_recursion, logger): """Instanties the iterator. Args: tuple_iter: Iterator over names_container, BucketListingReference from step 2 in the NameExpansionIterator recursion_requested: If false, omit buckets, dirs, and subdirs command_name: Command name for user messages cmd_supports_recursion: Command recursion support for user messages logger: Log object for user messages """ self.tuple_iter = tuple_iter self.recursion_requested = recursion_requested self.command_name = command_name self.cmd_supports_recursion = cmd_supports_recursion self.logger = logger def __iter__(self): for (names_container, blr) in self.tuple_iter: if not self.recursion_requested and not blr.IsObject(): # At this point we either have a bucket or a prefix, # so if recursion is not requested, we're going to omit it. expanded_url = StorageUrlFromString(blr.url_string) if expanded_url.IsFileUrl(): desc = 'directory' else: desc = blr.type_name if self.cmd_supports_recursion: self.logger.info('Omitting %s "%s". (Did you mean to do %s -r?)', desc, blr.url_string, self.command_name) else: self.logger.info('Omitting %s "%s".', desc, blr.url_string) else: yield (names_container, blr) class _ImplicitBucketSubdirIterator(object): """Iterator wrapper that performs implicit bucket subdir expansion. Each iteration yields tuple (names_container, expanded BucketListingRefs) where names_container is true if URL names a directory, bucket, or bucket subdir. For example, iterating over [BucketListingRef("gs://abc")] would expand to: [BucketListingRef("gs://abc/o1"), BucketListingRef("gs://abc/o2")] if those subdir objects exist, and [BucketListingRef("gs://abc") otherwise. """ def __init__(self, name_exp_instance, blr_iter, subdir_exp_wildcard, bucket_listing_fields): """Instantiates the iterator. Args: name_exp_instance: calling instance of NameExpansion class. blr_iter: iterator over BucketListingRef prefixes and objects. subdir_exp_wildcard: wildcard for expanding subdirectories; expected values are ** if the mapped-to results should contain objects spanning subdirectories, or * if only one level should be listed. bucket_listing_fields: Fields requested in enumerated results. """ self.blr_iter = blr_iter self.name_exp_instance = name_exp_instance self.subdir_exp_wildcard = subdir_exp_wildcard self.bucket_listing_fields = bucket_listing_fields def __iter__(self): for blr in self.blr_iter: if blr.IsPrefix(): # This is a bucket subdirectory, list objects according to the wildcard. prefix_url = StorageUrlFromString(blr.url_string).CreatePrefixUrl( wildcard_suffix=self.subdir_exp_wildcard) implicit_subdir_iterator = PluralityCheckableIterator( self.name_exp_instance.WildcardIterator(prefix_url).IterAll( bucket_listing_fields=self.bucket_listing_fields)) if not implicit_subdir_iterator.IsEmpty(): for exp_blr in implicit_subdir_iterator: yield (True, exp_blr) else: # Prefix that contains no objects, for example in the $folder$ case # or an empty filesystem directory. yield (False, blr) elif blr.IsObject(): yield (False, blr) else: raise CommandException( '_ImplicitBucketSubdirIterator got a bucket reference %s' % blr) class CopyObjectInfo(object): """Represents the information needed for copying a single object. """ def __init__(self, name_expansion_result, exp_dst_url, have_existing_dst_container): """Instantiates the object info from name expansion result and destination. Args: name_expansion_result: StorageUrl that was being expanded. exp_dst_url: StorageUrl of the destination. have_existing_dst_container: Whether exp_url names an existing directory, bucket, or bucket subdirectory. """ self.source_storage_url = name_expansion_result.source_storage_url self.is_multi_source_request = name_expansion_result.is_multi_source_request self.is_multi_top_level_source_request = ( name_expansion_result.is_multi_top_level_source_request) self.names_container = name_expansion_result.names_container self.expanded_storage_url = name_expansion_result.expanded_storage_url self.expanded_result = name_expansion_result.expanded_result self.exp_dst_url = exp_dst_url self.have_existing_dst_container = have_existing_dst_container # Describes the destination information resulted from ExpandUrlToSingleBlr. DestinationInfo = collections.namedtuple( 'DestinationInfo', [ # The expanded destination StorageURL. 'exp_dst_url', # Bool indicating whether the expanded destination names an existing # directory, bucket, or bucket subdirectory. 'have_existing_dst_container', ]) # Describes (NameExpansionIterator, DestinationInfo) tuple. NameExpansionIteratorDestinationTuple = collections.namedtuple( 'NameExpansionIteratorDestinationTuple', [ 'name_expansion_iter', 'destination', ]) class CopyObjectsIterator(six.Iterator): """Iterator wrapper for copying objects and keeping track of source URL types. This is used in the cp command for copying from multiple source to multiple destinations. It takes a list of NameExpansionIteratorDestinationTuple. It wraps them and return CopyObjectInfo objects that wraps NameExpansionResult with the destination. It's used also for collecting analytics PerformanceSummary info, because there may be multiple source URLs and we want to know if any of them are file URLs, if any of them are cloud URLs, if any of them require daisy chain operations, and if any use different providers. The source URL type information will be aggregated at the end of _SequentialApply or _ParallelApply. """ def __init__(self, name_expansion_dest_iter, is_daisy_chain): """Instantiates the iterator. Args: name_expansion_dest_iter: NameExpansionIteratorDestinationTuple iterator. is_daisy_chain: The -D option in cp might have already been specified, in which case we do not need to check again for daisy chain operations. """ self.is_daisy_chain = is_daisy_chain self.has_file_src = False self.has_cloud_src = False self.provider_types = [] self.name_expansion_dest_iter = name_expansion_dest_iter name_expansion_dest_tuple = next(self.name_expansion_dest_iter) self.current_expansion_iter = name_expansion_dest_tuple.name_expansion_iter self.current_destination = name_expansion_dest_tuple.destination def __iter__(self): return self def __next__(self): """Keeps track of URL types as the command iterates over arguments.""" try: name_expansion_result = next(self.current_expansion_iter) except StopIteration: name_expansion_dest_tuple = next(self.name_expansion_dest_iter) self.current_expansion_iter = ( name_expansion_dest_tuple.name_expansion_iter) self.current_destination = name_expansion_dest_tuple.destination return self.__next__() elt = CopyObjectInfo(name_expansion_result, self.current_destination.exp_dst_url, self.current_destination.have_existing_dst_container) # Check if we've seen a file source. if not self.has_file_src and elt.source_storage_url.IsFileUrl(): self.has_file_src = True # Check if we've seen a cloud source. if not self.has_cloud_src and elt.source_storage_url.IsCloudUrl(): self.has_cloud_src = True # Check if we've seen a daisy-chain condition. if self.current_destination.exp_dst_url.IsCloudUrl(): dst_url_scheme = self.current_destination.exp_dst_url.scheme else: dst_url_scheme = None if (not self.is_daisy_chain and dst_url_scheme is not None and elt.source_storage_url.IsCloudUrl() and elt.source_storage_url.scheme != dst_url_scheme): self.is_daisy_chain = True # Check if we've seen a new provider type. if elt.source_storage_url.scheme not in self.provider_types: self.provider_types.append(elt.source_storage_url.scheme) return elt storage_uri_builder.py 0000644 00000004026 15025013616 0011150 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Class that holds state for instantiating StorageUri objects. The StorageUri func defined in this class uses that state (bucket_storage_uri_class and debug) needed plus gsutil default flag values to instantiate this frequently constructed object with just one param for most cases. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import boto class StorageUriBuilder(object): """Class for instantiating StorageUri objects.""" def __init__(self, debug, bucket_storage_uri_class): """Initializes the builder. Args: debug: Debug level to pass in to boto connection (range 0..3). bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. """ self.bucket_storage_uri_class = bucket_storage_uri_class self.debug = debug def StorageUri(self, uri_str): """Instantiates StorageUri using class state and gsutil default flag values. Args: uri_str: StorageUri naming bucket or object. Returns: boto.StorageUri for given uri_str. Raises: InvalidUriError: if uri_str not valid. """ return boto.storage_uri( uri_str, 'file', debug=self.debug, validate=False, bucket_storage_uri_class=self.bucket_storage_uri_class, suppress_consec_slashes=False) ui_controller.py 0000644 00000136655 15025013616 0010015 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Base classes for gsutil UI controller, UIThread and MainThreadUIQueue.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import deque import sys import threading import time from six.moves import queue as Queue from gslib.metrics import LogPerformanceSummaryParams from gslib.metrics import LogRetryableError from gslib.thread_message import FileMessage from gslib.thread_message import FinalMessage from gslib.thread_message import MetadataMessage from gslib.thread_message import PerformanceSummaryMessage from gslib.thread_message import ProducerThreadMessage from gslib.thread_message import ProgressMessage from gslib.thread_message import RetryableErrorMessage from gslib.thread_message import SeekAheadMessage from gslib.thread_message import StatusMessage from gslib.utils import parallelism_framework_util from gslib.utils.unit_util import DecimalShort from gslib.utils.unit_util import HumanReadableWithDecimalPlaces from gslib.utils.unit_util import MakeHumanReadable from gslib.utils.unit_util import PrettyTime _ZERO_TASKS_TO_DO_ARGUMENT = ( parallelism_framework_util.ZERO_TASKS_TO_DO_ARGUMENT) class EstimationSource(object): """enum for total size source.""" # Integer to indicate total size came from the final ProducerThreadMessage. # It has priority over all other total_size sources. PRODUCER_THREAD_FINAL = 1 # Integer to indicate total size came from SeekAheadThread. # It has priority over self.SEEK_AHEAD_THREAD and over # self.INDIVIDUAL_MESSAGES. SEEK_AHEAD_THREAD = 2 # Integer to indicate total size came from a ProducerThread estimation. # It has priority over self.INDIVIDUAL_MESSAGES. PRODUCER_THREAD_ESTIMATE = 3 # Stores the actual source from total_size. We start from FileMessages or # MetadataMessages. INDIVIDUAL_MESSAGES = 4 # Note: this priority based model was used in case we add new sources for # total_size in the future. It also allows us to search for smaller numbers # (larger priorities) rather than having to list those with higher priority. def BytesToFixedWidthString(num_bytes, decimal_places=1): """Adjusts proper width for printing num_bytes in readable format. Args: num_bytes: The number of bytes we must display. decimal_places: The standard number of decimal places. Returns: String of fixed width representing num_bytes. """ human_readable = HumanReadableWithDecimalPlaces(num_bytes, decimal_places=decimal_places) number_format = human_readable.split() if int(round(float(number_format[0]))) >= 1000: # If we are in the [1000:1024) range for the whole part of the number, # we must remove the decimal part. last_character = len(number_format[0]) - decimal_places - 1 number_format[0] = number_format[0][:last_character] return '%9s' % (' '.join(number_format)) class StatusMessageManager(object): """General manager for common functions shared by data and metadata managers. This subclass has the responsibility of having a common constructor and the same handler for SeekAheadMessages and ProducerThreadMessages. """ class _ThroughputInformation(object): """Class that contains all information needed for throughput calculation. This _ThroughputInformation is used to track progress and time at several points of our operation. """ def __init__(self, progress, report_time): """Constructor of _ThroughputInformation. Args: progress: The current progress, in bytes/second or objects/second. report_time: Float representing when progress was reported (seconds since Epoch). """ self.progress = progress self.time = report_time def __init__(self, update_message_period=1, update_spinner_period=0.6, sliding_throughput_period=5, first_throughput_latency=10, quiet_mode=False, custom_time=None, verbose=False, console_width=80): """Instantiates a StatusMessageManager. Args: update_message_period: Minimum period for refreshing and displaying new information. A non-positive value will ignore any time restrictions imposed by this field, but it will affect throughput and time remaining estimations. update_spinner_period: Minimum period for refreshing and displaying the spinner. A non-positive value will ignore any time restrictions imposed by this field. sliding_throughput_period: Sliding period for throughput calculation. A non-positive value will make it impossible to calculate the throughput. first_throughput_latency: Minimum waiting time before actually displaying throughput info. A non-positive value will ignore any time restrictions imposed by this field. quiet_mode: If True, do not print status messages (but still process them for analytics reporting as necessary). custom_time: If a custom start_time is desired. Used for testing. verbose: Tells whether or not the operation is on verbose mode. console_width: Width to display on console. This should not adjust the visual output, just the space padding. For proper visualization, we recommend setting this field to at least 80. """ self.update_message_period = update_message_period self.update_spinner_period = update_spinner_period self.sliding_throughput_period = sliding_throughput_period self.first_throughput_latency = first_throughput_latency self.quiet_mode = quiet_mode self.custom_time = custom_time self.verbose = verbose self.console_width = console_width # Initial estimation source for number of objects and total size # is through individual FileMessages or individual MetadataMessages, # depending on the StatusMessageManager superclass. self.num_objects_source = EstimationSource.INDIVIDUAL_MESSAGES self.total_size_source = EstimationSource.INDIVIDUAL_MESSAGES self.num_objects = 0 # Only used on data operations. Will remain 0 for metadata operations. self.total_size = 0 # Time at last info update displayed. self.refresh_message_time = (self.custom_time if self.custom_time else time.time()) self.start_time = self.refresh_message_time # Time at last spinner update. self.refresh_spinner_time = self.refresh_message_time # Measured in objects/second or bytes/second, depending on the superclass. self.throughput = 0.0 # Deque of _ThroughputInformation to help with throughput calculation. self.old_progress = deque() self.last_progress_time = 0 self.spinner_char_list = ['/', '-', '\\', '|'] self.current_spinner_index = 0 self.objects_finished = 0 self.num_objects = 0 # Number of objects being processed # This overrides time constraints for updating and displaying # important information, such as having finished to process an object. self.object_report_change = False self.final_message = False def GetSpinner(self): """Returns the current spinner character. Returns: char_to_print: Char to be printed as the spinner """ return self.spinner_char_list[self.current_spinner_index] def UpdateSpinner(self): """Updates the current spinner character.""" self.current_spinner_index = ((self.current_spinner_index + 1) % len(self.spinner_char_list)) def _HandleProducerThreadMessage(self, status_message): """Handles a ProducerThreadMessage. Args: status_message: The ProducerThreadMessage to be processed. """ if status_message.finished: # This means this was a final ProducerThreadMessage. if self.num_objects_source >= EstimationSource.PRODUCER_THREAD_FINAL: self.num_objects_source = EstimationSource.PRODUCER_THREAD_FINAL self.num_objects = status_message.num_objects if (self.total_size_source >= EstimationSource.PRODUCER_THREAD_FINAL and status_message.size): self.total_size_source = EstimationSource.PRODUCER_THREAD_FINAL self.total_size = status_message.size return if self.num_objects_source >= EstimationSource.PRODUCER_THREAD_ESTIMATE: self.num_objects_source = EstimationSource.PRODUCER_THREAD_ESTIMATE self.num_objects = status_message.num_objects if (self.total_size_source >= EstimationSource.PRODUCER_THREAD_ESTIMATE and status_message.size): self.total_size_source = EstimationSource.PRODUCER_THREAD_ESTIMATE self.total_size = status_message.size def _HandleSeekAheadMessage(self, status_message, stream): """Handles a SeekAheadMessage. Args: status_message: The SeekAheadMessage to be processed. stream: Stream to print messages. """ estimate_message = ('Estimated work for this command: objects: %s' % status_message.num_objects) if status_message.size: estimate_message += (', total size: %s' % MakeHumanReadable(status_message.size)) if self.total_size_source >= EstimationSource.SEEK_AHEAD_THREAD: self.total_size_source = EstimationSource.SEEK_AHEAD_THREAD self.total_size = status_message.size if self.num_objects_source >= EstimationSource.SEEK_AHEAD_THREAD: self.num_objects_source = EstimationSource.SEEK_AHEAD_THREAD self.num_objects = status_message.num_objects estimate_message += '\n' if not self.quiet_mode: stream.write(estimate_message) def _HandlePerformanceSummaryMessage(self, status_message): """Handles a PerformanceSummaryMessage. Args: status_message: The PerformanceSummaryMessage to be processed. """ LogPerformanceSummaryParams(uses_slice=status_message.uses_slice) def ShouldTrackThroughput(self, cur_time): """Decides whether enough time has passed to start tracking throughput. Args: cur_time: current time. Returns: Whether or not we should track the throughput. """ return cur_time - self.start_time >= self.first_throughput_latency def ShouldPrintProgress(self, cur_time): """Decides whether or not it is time for printing a new progress. Args: cur_time: current time. Returns: Whether or not we should print the progress. """ sufficient_time_elapsed = (cur_time - self.refresh_message_time >= self.update_message_period) # Don't report if we aren't actually going to do anything (for example, # an rsync that will sync 0 objects). nonzero_report = self.num_objects return (sufficient_time_elapsed or self.object_report_change) and (nonzero_report) def ShouldPrintSpinner(self, cur_time): """Decides whether or not it is time for updating the spinner character. Args: cur_time: Current time. Returns: Whether or not we should update and print the spinner. """ return (cur_time - self.refresh_spinner_time > self.update_spinner_period and self.total_size) def PrintSpinner(self, stream=sys.stderr): """Prints a spinner character. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. """ self.UpdateSpinner() if not self.quiet_mode: stream.write(self.GetSpinner() + '\r') def UpdateThroughput(self, cur_time, cur_progress): """Updates throughput if the required period for calculation has passed. The throughput is calculated by taking all the progress (objects or bytes) processed within the last sliding_throughput_period seconds, and dividing that by the time period between the oldest progress time within that range and the last progress measurement, which are defined by oldest_progress[1] and last_progress_time, respectively. Among the pros of this approach, a connection break or a sudden change in throughput is quickly noticeable. Furthermore, using the last throughput measurement rather than the current time allows us to have a better estimation of the actual throughput. Args: cur_time: Current time to check whether or not it is time for a new throughput measurement. cur_progress: The current progress, in number of objects finished or in bytes. """ while (len(self.old_progress) > 1 and cur_time - self.old_progress[0].time > self.sliding_throughput_period): self.old_progress.popleft() if not self.old_progress: return oldest_progress = self.old_progress[0] if self.last_progress_time == oldest_progress.time: self.throughput = 0 return # If old-progress is not empty and the time of oldest_progress does not # match the last_progress_time, we can safely calculate the throughput. self.throughput = ((cur_progress - oldest_progress.progress) / (self.last_progress_time - oldest_progress.time)) # Just to avoid -0.00 B/s. self.throughput = max(0, self.throughput) def PrintFinalSummaryMessage(self, stream=sys.stderr): """Prints a final message to indicate operation succeeded. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. """ string_to_print = ('Operation completed over %s objects' % DecimalShort(self.num_objects)) if self.total_size: string_to_print += ('/%s' % HumanReadableWithDecimalPlaces(self.total_size)) remaining_width = self.console_width - len(string_to_print) if not self.quiet_mode: stream.write(('\n' + string_to_print + '.' + (max(remaining_width, 0) * ' ') + '\n')) class MetadataManager(StatusMessageManager): """Manages shared state for metadata operations. This manager is specific for metadata operations. Among its main functions, it receives incoming StatusMessages, storing all necessary data about the current and past states of the system necessary to display to the UI. It also provides methods for calculating metrics such as throughput and estimated time remaining. Finally, it provides methods for displaying messages to the UI. """ def __init__(self, update_message_period=1, update_spinner_period=0.6, sliding_throughput_period=5, first_throughput_latency=10, quiet_mode=False, custom_time=None, verbose=False, console_width=80): # pylint: disable=g-doc-args """Instantiates a MetadataManager. See argument documentation in StatusMessageManager base class. """ # pylint: enable=g-doc-args super(MetadataManager, self).__init__(update_message_period=update_message_period, update_spinner_period=update_spinner_period, sliding_throughput_period=sliding_throughput_period, first_throughput_latency=first_throughput_latency, quiet_mode=quiet_mode, custom_time=custom_time, verbose=verbose, console_width=console_width) def GetProgress(self): """Gets the progress for a MetadataManager. Returns: The number of finished objects. """ return self.objects_finished def _HandleMetadataMessage(self, status_message): """Handles a MetadataMessage. Args: status_message: The MetadataMessage to be processed. """ self.objects_finished += 1 if self.num_objects_source >= EstimationSource.INDIVIDUAL_MESSAGES: self.num_objects_source = EstimationSource.INDIVIDUAL_MESSAGES self.num_objects += 1 # Ensures we print periodic progress, and that we send a final message. self.object_report_change = True self.last_progress_time = status_message.time if (self.objects_finished == self.num_objects and self.num_objects_source == EstimationSource.PRODUCER_THREAD_FINAL): self.final_message = True def ProcessMessage(self, status_message, stream): """Processes a message from _MainThreadUIQueue or _UIThread. Args: status_message: The StatusMessage item to be processed. stream: Stream to print messages. """ self.object_report_change = False if isinstance(status_message, SeekAheadMessage): self._HandleSeekAheadMessage(status_message, stream) elif isinstance(status_message, ProducerThreadMessage): self._HandleProducerThreadMessage(status_message) elif isinstance(status_message, MetadataMessage): self._HandleMetadataMessage(status_message) elif isinstance(status_message, RetryableErrorMessage): LogRetryableError(status_message) elif isinstance(status_message, PerformanceSummaryMessage): self._HandlePerformanceSummaryMessage(status_message) self.old_progress.append( self._ThroughputInformation(self.objects_finished, status_message.time)) def PrintProgress(self, stream=sys.stderr): """Prints progress and throughput/time estimation. Prints total number of objects and number of finished objects with the percentage of work done, potentially including the throughput (in objects/second) and estimated time remaining. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. """ # Time to update all information total_remaining = self.num_objects - self.objects_finished if self.throughput: time_remaining = total_remaining / self.throughput else: time_remaining = None char_to_print = self.GetSpinner() if self.num_objects_source <= EstimationSource.SEEK_AHEAD_THREAD: # An example of objects_completed here would be ' [2/3 objects]'. objects_completed = ('[' + DecimalShort(self.objects_finished) + '/' + DecimalShort(self.num_objects) + ' objects]') if self.num_objects == self.objects_finished: percentage = '100' else: percentage = ( '%3d' % min(99, int(100 * float(self.objects_finished) / self.num_objects))) percentage_completed = percentage + '% Done' else: # An example of objects_completed here would be ' [2 objects]'. objects_completed = ('[' + DecimalShort(self.objects_finished) + ' objects]') percentage_completed = '' if (self.refresh_message_time - self.start_time > self.first_throughput_latency): # Should also include throughput. # An example of throughput here would be '2 objects/s' throughput = '%.2f objects/s' % self.throughput if (self.num_objects_source <= EstimationSource.PRODUCER_THREAD_ESTIMATE and self.throughput): # Should also include time remaining. # An example of time remaining would be ' ETA 00:00:11'. time_remaining_str = 'ETA ' + PrettyTime(time_remaining) else: time_remaining_str = '' else: throughput = '' time_remaining_str = '' format_str = ('{char_to_print} {objects_completed} {percentage_completed}' ' {throughput} {time_remaining_str}') string_to_print = format_str.format( char_to_print=char_to_print, objects_completed=objects_completed, percentage_completed=percentage_completed, throughput=throughput, time_remaining_str=time_remaining_str) remaining_width = self.console_width - len(string_to_print) if not self.quiet_mode: stream.write(string_to_print + (max(remaining_width, 0) * ' ') + '\r') def CanHandleMessage(self, status_message): """Determines whether this manager is suitable for handling status_message. Args: status_message: The StatusMessage object to be analyzed. Returns: True if this message can be properly handled by this manager, False otherwise. """ if isinstance( status_message, (SeekAheadMessage, ProducerThreadMessage, MetadataMessage, FinalMessage, RetryableErrorMessage, PerformanceSummaryMessage)): return True return False class DataManager(StatusMessageManager): """Manages shared state for data operations. This manager is specific for data operations. Among its main functions, it receives incoming StatusMessages, storing all necessary data about the current and past states of the system necessary to display to the UI. It also provides methods for calculating metrics such as throughput and estimated time remaining. Finally, it provides methods for displaying messages to the UI. """ class _ProgressInformation(object): """Class that contains all progress information needed for a given file. This _ProgressInformation is used as the value associated with a file_name in the dict that stores the information about all processed files. """ def __init__(self, size): """Constructor of _ProgressInformation. Args: size: The total size of the file. """ # Sum of all progress obtained in this operation. self.new_progress_sum = 0 # Sum of all progress from previous operations (mainly for resuming # uploads or resuming downloads). self.existing_progress_sum = 0 # Dict for tracking the progress for each individual component. Key is # of the form (component_num, dst_url) and correspondent element is a # tuple which stores the current progress obtained from this operation, # and the progress obtained from previous operations. self.dict = {} # The total size for the file self.size = size def __init__(self, update_message_period=1, update_spinner_period=0.6, sliding_throughput_period=5, first_throughput_latency=10, quiet_mode=False, custom_time=None, verbose=False, console_width=None): # pylint: disable=g-doc-args """Instantiates a DataManager. See argument documentation in StatusMessageManager base class. """ # pylint: disable=g-doc-args super(DataManager, self).__init__(update_message_period=update_message_period, update_spinner_period=update_spinner_period, sliding_throughput_period=sliding_throughput_period, first_throughput_latency=first_throughput_latency, quiet_mode=quiet_mode, custom_time=custom_time, verbose=verbose, console_width=console_width) self.first_item = True self.total_progress = 0 # Sum of progress for all threads. self.new_progress = 0 self.existing_progress = 0 # Dict containing individual progress for each file. Key is filename # (from src_url). It maps to a _ProgressInformation object. self.individual_file_progress = {} self.component_total = 0 self.finished_components = 0 self.existing_components = 0 def GetProgress(self): """Gets the progress for a DataManager. Returns: The number of processed bytes in this operation. """ return self.new_progress def _HandleFileDescription(self, status_message): """Handles a FileMessage that describes a file. Args: status_message: the FileMessage to be processed. """ if not status_message.finished: # File started. if self.first_item and not self.custom_time: # Set initial time. self.refresh_message_time = status_message.time self.start_time = self.refresh_message_time self.last_throughput_time = self.refresh_message_time self.first_item = False # Gets file name (from src_url). file_name = status_message.src_url.url_string status_message.size = status_message.size if status_message.size else 0 # Creates a new entry on individual_file_progress. self.individual_file_progress[file_name] = self._ProgressInformation( status_message.size) if self.num_objects_source >= EstimationSource.INDIVIDUAL_MESSAGES: # This ensures the file has not been counted on SeekAheadThread or # in ProducerThread. self.num_objects_source = EstimationSource.INDIVIDUAL_MESSAGES self.num_objects += 1 if self.total_size_source >= EstimationSource.INDIVIDUAL_MESSAGES: # This ensures the file size has not been counted on SeekAheadThread or # in ProducerThread. self.total_size_source = EstimationSource.INDIVIDUAL_MESSAGES self.total_size += status_message.size self.object_report_change = True else: # File finished. self.objects_finished += 1 file_name = status_message.src_url.url_string file_progress = self.individual_file_progress[file_name] total_bytes_transferred = (file_progress.new_progress_sum + file_progress.existing_progress_sum) # Ensures total_progress has the right value. self.total_progress += file_progress.size - total_bytes_transferred self.new_progress += file_progress.size - total_bytes_transferred self.last_progress_time = status_message.time # Deleting _ProgressInformation object to save memory. del self.individual_file_progress[file_name] self.object_report_change = True if (self.objects_finished == self.num_objects and self.num_objects_source == EstimationSource.PRODUCER_THREAD_FINAL): self.final_message = True def _IsFile(self, file_message): """Tells whether or not this FileMessage represent a file. This is needed because FileMessage is used by both files and components. Args: file_message: The FileMessage to be analyzed. Returns: Whether or not this represents a file. """ message_type = file_message.message_type return (message_type == FileMessage.FILE_DOWNLOAD or message_type == FileMessage.FILE_UPLOAD or message_type == FileMessage.FILE_CLOUD_COPY or message_type == FileMessage.FILE_DAISY_COPY or message_type == FileMessage.FILE_LOCAL_COPY or message_type == FileMessage.FILE_REWRITE or message_type == FileMessage.FILE_HASH) def _HandleComponentDescription(self, status_message): """Handles a FileMessage that describes a component. Args: status_message: The FileMessage to be processed. """ if (status_message.message_type == FileMessage.EXISTING_COMPONENT and not status_message.finished): # Existing component: have to ensure total_progress accounts for it. self.existing_components += 1 file_name = status_message.src_url.url_string file_progress = self.individual_file_progress[file_name] key = (status_message.component_num, status_message.dst_url) file_progress.dict[key] = (0, status_message.size) file_progress.existing_progress_sum += status_message.size self.total_progress += status_message.size self.existing_progress += status_message.size elif ((status_message.message_type == FileMessage.COMPONENT_TO_UPLOAD or status_message.message_type == FileMessage.COMPONENT_TO_DOWNLOAD)): if not status_message.finished: # Component started. self.component_total += 1 if status_message.message_type == FileMessage.COMPONENT_TO_DOWNLOAD: file_name = status_message.src_url.url_string file_progress = self.individual_file_progress[file_name] file_progress.existing_progress_sum += ( status_message.bytes_already_downloaded) key = (status_message.component_num, status_message.dst_url) file_progress.dict[key] = (0, status_message.bytes_already_downloaded) self.total_progress += status_message.bytes_already_downloaded self.existing_progress += status_message.bytes_already_downloaded else: # Component finished. self.finished_components += 1 file_name = status_message.src_url.url_string file_progress = self.individual_file_progress[file_name] key = (status_message.component_num, status_message.dst_url) last_update = (file_progress.dict[key] if key in file_progress.dict else (0, 0)) self.total_progress += status_message.size - sum(last_update) self.new_progress += status_message.size - sum(last_update) self.last_progress_time = status_message.time file_progress.new_progress_sum += (status_message.size - sum(last_update)) file_progress.dict[key] = (status_message.size - last_update[1], last_update[1]) def _HandleProgressMessage(self, status_message): """Handles a ProgressMessage that tracks progress of a file or component. Args: status_message: The ProgressMessage to be processed. """ # Retrieving index and dict for this file. file_name = status_message.src_url.url_string file_progress = self.individual_file_progress[file_name] # Retrieves last update ((0,0) if no previous update) for this file or # component. To ensure uniqueness (among components), # we use a (component_num, dst_url) tuple as our key. key = (status_message.component_num, status_message.dst_url) last_update = (file_progress.dict[key] if key in file_progress.dict else (0, 0)) status_message.processed_bytes -= last_update[1] file_progress.new_progress_sum += (status_message.processed_bytes - last_update[0]) # Updates total progress with new update from component. self.total_progress += status_message.processed_bytes - last_update[0] self.new_progress += status_message.processed_bytes - last_update[0] # Updates file_progress.dict on component's key. file_progress.dict[key] = (status_message.processed_bytes, last_update[1]) self.last_progress_time = status_message.time def ProcessMessage(self, status_message, stream): """Processes a message from _MainThreadUIQueue or _UIThread. Args: status_message: The StatusMessage item to be processed. stream: Stream to print messages. Here only for SeekAheadThread """ self.object_report_change = False if isinstance(status_message, ProducerThreadMessage): # ProducerThread info. self._HandleProducerThreadMessage(status_message) elif isinstance(status_message, SeekAheadMessage): # SeekAheadThread info. self._HandleSeekAheadMessage(status_message, stream) elif isinstance(status_message, FileMessage): if self._IsFile(status_message): # File info. self._HandleFileDescription(status_message) else: # Component info. self._HandleComponentDescription(status_message) LogPerformanceSummaryParams(file_message=status_message) elif isinstance(status_message, ProgressMessage): # Progress info. self._HandleProgressMessage(status_message) elif isinstance(status_message, RetryableErrorMessage): LogRetryableError(status_message) elif isinstance(status_message, PerformanceSummaryMessage): self._HandlePerformanceSummaryMessage(status_message) self.old_progress.append( self._ThroughputInformation(self.new_progress, status_message.time)) def PrintProgress(self, stream=sys.stderr): """Prints progress and throughput/time estimation. If a ProducerThreadMessage or SeekAheadMessage has been provided, it outputs the number of files completed, number of total files, the current progress, the total size, and the percentage it represents. If none of those have been provided, it only includes the number of files completed, the current progress and total size (which might be updated), with no percentage as we do not know if more files are coming. It may also include time estimation (available only given ProducerThreadMessage or SeekAheadMessage provided) and throughput. For that to happen, there is an extra condition of at least first_throughput_latency seconds having been passed since the UIController started, and that either the ProducerThread or the SeekAheadThread have estimated total number of files and total size. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. """ # Time to update all information. total_remaining = self.total_size - self.total_progress if self.throughput: time_remaining = total_remaining / self.throughput else: time_remaining = None char_to_print = self.GetSpinner() if self.num_objects_source <= EstimationSource.SEEK_AHEAD_THREAD: # An example of objects_completed here would be ' [2/3 files]'. objects_completed = ('[' + DecimalShort(self.objects_finished) + '/' + DecimalShort(self.num_objects) + ' files]') else: # An example of objects_completed here would be ' [2 files]'. objects_completed = '[' + DecimalShort(self.objects_finished) + ' files]' # An example of bytes_progress would be '[101.0 MiB/1.0 GiB]'. bytes_progress = ('[%s/%s]' % (BytesToFixedWidthString( self.total_progress), BytesToFixedWidthString(self.total_size))) if self.total_size_source <= EstimationSource.SEEK_AHEAD_THREAD: if self.num_objects == self.objects_finished: percentage = '100' else: percentage = ( '%3d' % min(99, int(100 * float(self.total_progress) / self.total_size))) percentage_completed = percentage + '% Done' else: percentage_completed = '' if (self.refresh_message_time - self.start_time > self.first_throughput_latency): # Should also include throughput. # An example of throughput here would be ' 82.3 MiB/s' throughput = BytesToFixedWidthString(self.throughput) + '/s' if (self.total_size_source <= EstimationSource.PRODUCER_THREAD_ESTIMATE and self.throughput): # Should also include time remaining. # An example of time remaining would be ' ETA 00:00:11'. time_remaining_str = 'ETA ' + PrettyTime(time_remaining) else: time_remaining_str = '' else: throughput = '' time_remaining_str = '' format_str = ('{char_to_print} {objects_completed}{bytes_progress}' ' {percentage_completed} {throughput} {time_remaining_str}') string_to_print = format_str.format( char_to_print=char_to_print, objects_completed=objects_completed, bytes_progress=bytes_progress, percentage_completed=percentage_completed, throughput=throughput, time_remaining_str=time_remaining_str) remaining_width = self.console_width - len(string_to_print) if not self.quiet_mode: stream.write(string_to_print + (max(remaining_width, 0) * ' ') + '\r') def CanHandleMessage(self, status_message): """Determines whether this manager is suitable for handling status_message. Args: status_message: The StatusMessage object to be analyzed. Returns: True if this message can be properly handled by this manager, False otherwise. """ if isinstance(status_message, ( SeekAheadMessage, ProducerThreadMessage, FileMessage, ProgressMessage, FinalMessage, RetryableErrorMessage, PerformanceSummaryMessage, )): return True return False class UIController(object): """Controller UI class to integrate _MainThreadUIQueue and _UIThread. This class receives messages from _MainThreadUIQueue and _UIThread and send them to an appropriate manager, which will then processes and store data about them. """ def __init__(self, update_message_period=1, update_spinner_period=0.6, sliding_throughput_period=5, first_throughput_latency=10, quiet_mode=False, custom_time=None, verbose=False, dump_status_messages_file=None): """Instantiates a UIController. Args: update_message_period: Minimum period for refreshing and displaying new information. A non-positive value will ignore any time restrictions imposed by this field. update_spinner_period: Minimum period for refreshing and displaying the spinner. A non-positive value will ignore any time restrictions imposed by this field. sliding_throughput_period: Sliding period for throughput calculation. A non-positive value will make it impossible to calculate the throughput. first_throughput_latency: Minimum waiting time before actually displaying throughput info. A non-positive value will ignore any time restrictions imposed by this field. quiet_mode: If True, do not print status messages (but still process them for analytics reporting as necessary). custom_time: If a custom start_time is desired. Used for testing. verbose: Tells whether or not the operation is on verbose mode. dump_status_messages_file: File path for logging all received status messages, for debugging purposes. """ self.verbose = verbose self.update_message_period = update_message_period self.update_spinner_period = update_spinner_period self.sliding_throughput_period = sliding_throughput_period self.first_throughput_latency = first_throughput_latency self.manager = None self.quiet_mode = quiet_mode self.custom_time = custom_time self.console_width = 80 # Console width. Passed to manager. # List storing all estimation messages from SeekAheadThread or # ProducerThread. This is used when we still do not know which manager to # use. self.early_estimation_messages = [] self.printed_final_message = False self.dump_status_message_fp = None if dump_status_messages_file: self.dump_status_message_fp = open(dump_status_messages_file, 'ab') def _HandleMessage(self, status_message, stream, cur_time=None): """Processes a message, updates throughput and prints progress. Args: status_message: Message to be processed. Could be None if UIThread cannot retrieve message from status_queue. stream: stream to print messages. Usually sys.stderr, but customizable for testing. cur_time: Message time. Used to determine if it is time to refresh output, or calculate throughput. """ self.manager.ProcessMessage(status_message, stream) if self.manager.ShouldPrintProgress(cur_time): if self.manager.ShouldTrackThroughput(cur_time): self.manager.UpdateThroughput(cur_time, self.manager.GetProgress()) self.manager.PrintProgress(stream) self.manager.refresh_message_time = cur_time if self.manager.ShouldPrintSpinner(cur_time): self.manager.PrintSpinner(stream) self.manager.refresh_spinner_time = cur_time if ((isinstance(status_message, FinalMessage) or self.manager.final_message) and self.manager.num_objects and not self.printed_final_message): self.printed_final_message = True LogPerformanceSummaryParams( num_objects_transferred=self.manager.num_objects) self.manager.PrintFinalSummaryMessage(stream) def Call(self, status_message, stream, cur_time=None): """Coordinates UI manager and calls appropriate function to handle message. Args: status_message: Message to be processed. Could be None if UIThread cannot retrieve message from status_queue. stream: Stream to print messages. Usually sys.stderr, but customizable for testing. cur_time: Message time. Used to determine if it is time to refresh output, or calculate throughput. """ if not isinstance(status_message, StatusMessage): if status_message == _ZERO_TASKS_TO_DO_ARGUMENT and not self.manager: # Create a manager to handle early estimation messages before returning. self.manager = (DataManager( update_message_period=self.update_message_period, update_spinner_period=self.update_spinner_period, sliding_throughput_period=self.sliding_throughput_period, first_throughput_latency=self.first_throughput_latency, quiet_mode=self.quiet_mode, custom_time=self.custom_time, verbose=self.verbose, console_width=self.console_width)) for estimation_message in self.early_estimation_messages: self._HandleMessage(estimation_message, stream, cur_time=estimation_message.time) return if self.dump_status_message_fp: # TODO: Add Unicode support to string methods on message classes. # Currently, dump will fail with a UnicodeEncodeErorr if the message # class contains a Unicode attribute. self.dump_status_message_fp.write(str(status_message)) self.dump_status_message_fp.write('\n') if not cur_time: cur_time = status_message.time if not self.manager: if (isinstance(status_message, SeekAheadMessage) or isinstance(status_message, ProducerThreadMessage)): self.early_estimation_messages.append(status_message) return elif isinstance(status_message, MetadataMessage): self.manager = (MetadataManager( update_message_period=self.update_message_period, update_spinner_period=self.update_spinner_period, sliding_throughput_period=self.sliding_throughput_period, first_throughput_latency=self.first_throughput_latency, quiet_mode=self.quiet_mode, custom_time=self.custom_time, verbose=self.verbose, console_width=self.console_width)) for estimation_message in self.early_estimation_messages: self._HandleMessage(estimation_message, stream, cur_time) else: self.manager = (DataManager( update_message_period=self.update_message_period, update_spinner_period=self.update_spinner_period, sliding_throughput_period=self.sliding_throughput_period, first_throughput_latency=self.first_throughput_latency, quiet_mode=self.quiet_mode, custom_time=self.custom_time, verbose=self.verbose, console_width=self.console_width)) for estimation_message in self.early_estimation_messages: self._HandleMessage(estimation_message, stream, cur_time) if not self.manager.CanHandleMessage(status_message): if (isinstance(status_message, FileMessage) or isinstance(status_message, ProgressMessage)): # We have to create a DataManager to handle this data message. This is # to avoid a possible race condition where MetadataMessages are sent # before data messages. As such, this means that the DataManager has # priority, and whenever a data message is received, we ignore the # MetadataManager if one exists, and start a DataManager from scratch. # This can be done because we do not need any MetadataMessages to # properly handle a data operation. It could be useful to send the # early estimation messages, if those are available. self.manager = (DataManager( update_message_period=self.update_message_period, update_spinner_period=self.update_spinner_period, sliding_throughput_period=self.sliding_throughput_period, first_throughput_latency=self.first_throughput_latency, custom_time=self.custom_time, verbose=self.verbose, console_width=self.console_width)) for estimation_message in self.early_estimation_messages: self._HandleMessage(estimation_message, stream, cur_time) else: # No need to handle this message. return self._HandleMessage(status_message, stream, cur_time) class MainThreadUIQueue(object): """Handles status display and processing in the main thread / master process. This class emulates a queue to cover main-thread activity before or after Apply, as well as for the single-threaded, single-process case, i.e., _SequentialApply. When multiple threads or processes are used during calls to Apply, the main thread is waiting for work to complete, and this queue must remain unused until Apply returns. Code producing arguments for Apply (such as the NameExpansionIterator) must not post messages to this queue to avoid race conditions with the UIThread. This class sends the messages it receives to UIController, which decides the correct course of action. """ def __init__(self, stream, ui_controller): """Instantiates a _MainThreadUIQueue. Args: stream: Stream for printing messages. ui_controller: UIController to manage messages. """ super(MainThreadUIQueue, self).__init__() self.ui_controller = ui_controller self.stream = stream # pylint: disable=invalid-name, unused-argument def put(self, status_message, timeout=None): self.ui_controller.Call(status_message, self.stream) # pylint: enable=invalid-name, unused-argument class UIThread(threading.Thread): """Responsible for centralized printing across multiple processes/threads. This class pulls status messages that are posted to the centralized status queue and coordinates displaying status and progress to the user. It is used only during calls to _ParallelApply, which in turn is called only when multiple threads and/or processes are used. This class sends the messages it receives to UIController, which decides the correct course of action. """ def __init__(self, status_queue, stream, ui_controller, timeout=1): """Instantiates a _UIThread. Args: status_queue: Queue for reporting status updates. stream: Stream for printing messages. ui_controller: UI controller to manage messages. timeout: Timeout for getting a message. """ super(UIThread, self).__init__() self.status_queue = status_queue self.stream = stream self.timeout = timeout self.ui_controller = ui_controller self.start() def run(self): try: while True: try: status_message = self.status_queue.get(timeout=self.timeout) except Queue.Empty: status_message = None continue self.ui_controller.Call(status_message, self.stream) if status_message == _ZERO_TASKS_TO_DO_ARGUMENT: # Item from MainThread to indicate we are done. break except Exception as e: # pylint:disable=broad-except self.stream.write('Exception in UIThread: %s\n' % e) seek_ahead_thread.py 0000644 00000012164 15025013616 0010521 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Threading code for estimating total work of long-running gsutil commands.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import threading import time from gslib import thread_message from gslib.utils import constants from gslib.utils import parallelism_framework_util _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout class SeekAheadResult(object): """Result class for seek_ahead_iterator results. A class is used instead of a namedtuple, making it easier to document and use default keyword arguments. """ def __init__(self, est_num_ops=1, data_bytes=0): """Create a SeekAheadResult. Args: est_num_ops: Number of operations the iterated result represents. Operation is loosely defined as a single API call for a single object. The total number of API calls may not be known at the time of iteration, so this number is approximate. data_bytes: Number of data bytes that will be transferred (uploaded, downloaded, or rewritten) for this iterated result. """ self.est_num_ops = est_num_ops self.data_bytes = data_bytes class SeekAheadThread(threading.Thread): """Thread to estimate total work to be performed by all processes and threads. Because the ProducerThread can only buffer a certain number of tasks on the global task queue, it cannot reliably provide the total count or size of iterated results for operations involving many iterated arguments until it nears the end of iteration. This thread consumes an iterator that should be functionally identical to the ProducerThread, but iterates to the end without adding tasks to the global task queue in an effort to estimate the amount of total work that the call to Apply will perform. It should be used only for large operations, and thus it is created by the main ProducerThread only when the number of iterated arguments exceeds a threshold. This thread may produce an inaccurate estimate if its iterator produces different results than the iterator used by the ProducerThread. This can happen due to eventual listing consistency or due to the source being modified as iteration occurs. This thread estimates operations for top-level objects only; sub-operations (such as a parallel composite upload) should be reported via the iterator as a single object including the total number of bytes affected. """ def __init__(self, seek_ahead_iterator, cancel_event, status_queue): """Initializes the seek ahead thread. Args: seek_ahead_iterator: Iterator matching the ProducerThread's args_iterator, but returning only object name and/or size in the result. cancel_event: threading.Event for signaling the seek-ahead iterator to terminate. status_queue: Status queue for posting summary of fully iterated results. """ super(SeekAheadThread, self).__init__() self.status_queue = status_queue self.seek_ahead_iterator = seek_ahead_iterator self.cancel_event = cancel_event # For unit-testing only; use cancel_event to stop the thread. self.terminate = False self.start() def run(self): num_objects = 0 num_data_bytes = 0 try: for seek_ahead_result in self.seek_ahead_iterator: if self.terminate: return # Periodically check to see if the ProducerThread has actually # completed, at which point providing an estimate is no longer useful. if (num_objects % constants.NUM_OBJECTS_PER_LIST_PAGE) == 0: if self.cancel_event.isSet(): return num_objects += seek_ahead_result.est_num_ops num_data_bytes += seek_ahead_result.data_bytes except OSError as e: # This can happen because the seek_ahead_iterator races with the command # being run (e.g., a gsutil mv command, which iterates over and moves # files while the estimator is concurrently iterating over the files to # count them). If this happens return here without calling # _PutToQueueWithTimeout, so we don't signal the UI thread that seek ahead # work has completed its estimation. This will cause no estimate to be # printed, but the command will continue to execute as usual. return if self.cancel_event.isSet(): return _PutToQueueWithTimeout( self.status_queue, thread_message.SeekAheadMessage(num_objects, num_data_bytes, time.time())) storage_url.py 0000644 00000040241 15025013616 0007444 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """File and Cloud URL representation classes.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import re import stat import sys from gslib.exception import CommandException from gslib.exception import InvalidUrlError from gslib.utils import system_util from gslib.utils import text_util # Matches provider strings of the form 'gs://' PROVIDER_REGEX = re.compile(r'(?P<provider>[^:]*)://$') # Matches bucket strings of the form 'gs://bucket' BUCKET_REGEX = re.compile(r'(?P<provider>[^:]*)://(?P<bucket>[^/]*)/{0,1}$') # Matches object strings of the form 'gs://bucket/obj' OBJECT_REGEX = re.compile( r'(?P<provider>[^:]*)://(?P<bucket>[^/]*)/(?P<object>.*)') # Matches versioned object strings of the form 'gs://bucket/obj#1234' GS_GENERATION_REGEX = re.compile(r'(?P<object>.+)#(?P<generation>[0-9]+)$') # Matches versioned object strings of the form 's3://bucket/obj#NULL' S3_VERSION_REGEX = re.compile(r'(?P<object>.+)#(?P<version_id>.+)$') # Matches file strings of the form 'file://dir/filename' FILE_OBJECT_REGEX = re.compile(r'([^:]*://)(?P<filepath>.*)') # Regex to determine if a string contains any wildcards. WILDCARD_REGEX = re.compile(r'[*?\[\]]') RELATIVE_PATH_SYMBOLS = frozenset(['.', '..']) class StorageUrl(object): """Abstract base class for file and Cloud Storage URLs.""" def Clone(self): raise NotImplementedError('Clone not overridden') def IsFileUrl(self): raise NotImplementedError('IsFileUrl not overridden') def IsCloudUrl(self): raise NotImplementedError('IsCloudUrl not overridden') def IsStream(): raise NotImplementedError('IsStream not overridden') def IsFifo(self): raise NotImplementedError('IsFifo not overridden') def CreatePrefixUrl(self, wildcard_suffix=None): """Returns a prefix of this URL that can be used for iterating. Args: wildcard_suffix: If supplied, this wildcard suffix will be appended to the prefix with a trailing slash before being returned. Returns: A prefix of this URL that can be used for iterating. If this URL contains a trailing slash, it will be stripped to create the prefix. This helps avoid infinite looping when prefixes are iterated, but preserves other slashes so that objects with '/' in the name are handled properly. For example, when recursively listing a bucket with the following contents: gs://bucket// <-- object named slash gs://bucket//one-dir-deep a top-level expansion with '/' as a delimiter will result in the following URL strings: 'gs://bucket//' : OBJECT 'gs://bucket//' : PREFIX If we right-strip all slashes from the prefix entry and add a wildcard suffix, we will get 'gs://bucket/*' which will produce identical results (and infinitely recurse). Example return values: ('gs://bucket/subdir/', '*') becomes 'gs://bucket/subdir/*' ('gs://bucket/', '*') becomes 'gs://bucket/*' ('gs://bucket/', None) becomes 'gs://bucket' ('gs://bucket/subdir//', '*') becomes 'gs://bucket/subdir//*' ('gs://bucket/subdir///', '**') becomes 'gs://bucket/subdir///**' ('gs://bucket/subdir/', '*') where 'subdir/' is an object becomes 'gs://bucket/subdir/*', but iterating on this will return 'subdir/' as a BucketListingObject, so we will not recurse on it as a subdir during listing. """ raise NotImplementedError('CreatePrefixUrl not overridden') def _WarnIfUnsupportedDoubleWildcard(self): """Warn if ** use may lead to undefined results.""" # Accepted 'url_string' values with '**', where '^' = start, and '$' = end. # - ^**$ # - ^**/ # - /**$ # - /**/ if not self.object_name: return delimiter_bounded_url = self.delim + self.object_name + self.delim split_url = delimiter_bounded_url.split( '{delim}**{delim}'.format(delim=self.delim)) removed_correct_double_wildcards_url_string = ''.join(split_url) if '**' in removed_correct_double_wildcards_url_string: # Found a center '**' not in the format '/**/'. # Not using logger.warning b/c it's too much overhead to pass the logger # object to every StorageUrl. sys.stderr.write( '** behavior is undefined if directly preceeded or followed by' ' with characters other than / in the cloud and {} locally.'.format( os.sep)) @property def url_string(self): raise NotImplementedError('url_string not overridden') @property def versionless_url_string(self): raise NotImplementedError('versionless_url_string not overridden') def __eq__(self, other): return isinstance(other, StorageUrl) and self.url_string == other.url_string def __hash__(self): return hash(self.url_string) class _FileUrl(StorageUrl): """File URL class providing parsing and convenience methods. This class assists with usage and manipulation of an (optionally wildcarded) file URL string. Depending on the string contents, this class represents one or more directories or files. For File URLs, scheme is always file, bucket_name is always blank, and object_name contains the file/directory path. """ def __init__(self, url_string, is_stream=False, is_fifo=False): self.scheme = 'file' self.delim = os.sep self.bucket_name = '' # If given a URI that starts with "<scheme>://", the object name should not # include that prefix. match = FILE_OBJECT_REGEX.match(url_string) if match and match.lastindex == 2: self.object_name = match.group(2) else: self.object_name = url_string # On Windows, the pathname component separator is "\" instead of "/". If we # find an occurrence of "/", replace it with "\" so that other logic can # rely on being able to split pathname components on `os.sep`. if system_util.IS_WINDOWS: self.object_name = self.object_name.replace('/', '\\') self.generation = None self.is_stream = is_stream self.is_fifo = is_fifo self._WarnIfUnsupportedDoubleWildcard() def Clone(self): return _FileUrl(self.url_string) def IsFileUrl(self): return True def IsCloudUrl(self): return False def IsStream(self): return self.is_stream def IsFifo(self): return self.is_fifo def IsDirectory(self): return (not self.IsStream() and not self.IsFifo() and os.path.isdir(self.object_name)) def CreatePrefixUrl(self, wildcard_suffix=None): return self.url_string @property def url_string(self): return '%s://%s' % (self.scheme, self.object_name) @property def versionless_url_string(self): return self.url_string def __str__(self): return self.url_string class _CloudUrl(StorageUrl): """Cloud URL class providing parsing and convenience methods. This class assists with usage and manipulation of an (optionally wildcarded) cloud URL string. Depending on the string contents, this class represents a provider, bucket(s), or object(s). This class operates only on strings. No cloud storage API calls are made from this class. """ def __init__(self, url_string): self.scheme = None self.delim = '/' self.bucket_name = None self.object_name = None self.generation = None provider_match = PROVIDER_REGEX.match(url_string) bucket_match = BUCKET_REGEX.match(url_string) if provider_match: self.scheme = provider_match.group('provider') elif bucket_match: self.scheme = bucket_match.group('provider') self.bucket_name = bucket_match.group('bucket') else: object_match = OBJECT_REGEX.match(url_string) if object_match: self.scheme = object_match.group('provider') self.bucket_name = object_match.group('bucket') self.object_name = object_match.group('object') if self.object_name == '.' or self.object_name == '..': raise InvalidUrlError('%s is an invalid root-level object name' % self.object_name) if self.scheme == 'gs': generation_match = GS_GENERATION_REGEX.match(self.object_name) if generation_match: self.object_name = generation_match.group('object') self.generation = generation_match.group('generation') elif self.scheme == 's3': version_match = S3_VERSION_REGEX.match(self.object_name) if version_match: self.object_name = version_match.group('object') self.generation = version_match.group('version_id') else: raise InvalidUrlError( 'CloudUrl: URL string %s did not match URL regex' % url_string) if url_string[(len(self.scheme) + len('://')):].startswith(self.delim): raise InvalidUrlError( 'Cloud URL scheme should be followed by colon and two slashes: "://".' ' Found: "{}"'.format(url_string)) self._WarnIfUnsupportedDoubleWildcard() def Clone(self): return _CloudUrl(self.url_string) def IsFileUrl(self): return False def IsCloudUrl(self): return True def IsStream(self): raise NotImplementedError('IsStream not supported on CloudUrl') def IsFifo(self): raise NotImplementedError('IsFifo not supported on CloudUrl') def IsBucket(self): return bool(self.bucket_name and not self.object_name) def IsObject(self): return bool(self.bucket_name and self.object_name) def HasGeneration(self): return bool(self.generation) def IsProvider(self): return bool(self.scheme and not self.bucket_name) def CreatePrefixUrl(self, wildcard_suffix=None): prefix = StripOneSlash(self.versionless_url_string) if wildcard_suffix: prefix = '%s/%s' % (prefix, wildcard_suffix) return prefix @property def bucket_url_string(self): return '%s://%s/' % (self.scheme, self.bucket_name) @property def url_string(self): url_str = self.versionless_url_string if self.HasGeneration(): url_str += '#%s' % self.generation return url_str @property def versionless_url_string(self): if self.IsProvider(): return '%s://' % self.scheme elif self.IsBucket(): return self.bucket_url_string return '%s://%s/%s' % (self.scheme, self.bucket_name, self.object_name) def __str__(self): return self.url_string def GetSchemeFromUrlString(url_str): """Returns scheme component of a URL string.""" end_scheme_idx = url_str.find('://') if end_scheme_idx == -1: # File is the default scheme. return 'file' else: return url_str[0:end_scheme_idx].lower() def IsKnownUrlScheme(scheme_str): return scheme_str in ('file', 's3', 'gs') def _GetPathFromUrlString(url_str): """Returns path component of a URL string.""" end_scheme_idx = url_str.find('://') if end_scheme_idx == -1: return url_str else: return url_str[end_scheme_idx + 3:] def ContainsWildcard(url_string): """Checks whether url_string contains a wildcard. Args: url_string: URL string to check. Returns: bool indicator. """ return bool(WILDCARD_REGEX.search(url_string)) def GenerationFromUrlAndString(url, generation): """Decodes a generation from a StorageURL and a generation string. This is used to represent gs and s3 versioning. Args: url: StorageUrl representing the object. generation: Long or string representing the object's generation or version. Returns: Valid generation string for use in URLs. """ if url.scheme == 's3' and generation: return text_util.DecodeLongAsString(generation) return generation def HaveFileUrls(args_to_check): """Checks whether args_to_check contain any file URLs. Args: args_to_check: Command-line argument subset to check. Returns: True if args_to_check contains any file URLs. """ for url_str in args_to_check: storage_url = StorageUrlFromString(url_str) if storage_url.IsFileUrl(): return True return False def HaveProviderUrls(args_to_check): """Checks whether args_to_check contains any provider URLs (like 'gs://'). Args: args_to_check: Command-line argument subset to check. Returns: True if args_to_check contains any provider URLs. """ for url_str in args_to_check: storage_url = StorageUrlFromString(url_str) if storage_url.IsCloudUrl() and storage_url.IsProvider(): return True return False def IsCloudSubdirPlaceholder(url, blr=None): """Determines if a StorageUrl is a cloud subdir placeholder. This function is needed because GUI tools (like the GCS cloud console) allow users to create empty "folders" by creating a placeholder object; and parts of gsutil need to treat those placeholder objects specially. For example, gsutil rsync needs to avoid downloading those objects because they can cause conflicts (see comments in rsync command for details). We currently detect two cases: - Cloud objects whose name ends with '_$folder$' - Cloud objects whose name ends with '/' Args: url: (gslib.storage_url.StorageUrl) The URL to be checked. blr: (gslib.BucketListingRef or None) The blr to check, or None if not available. If `blr` is None, size won't be checked. Returns: (bool) True if the URL is a cloud subdir placeholder, otherwise False. """ if not url.IsCloudUrl(): return False url_str = url.url_string if url_str.endswith('_$folder$'): return True if blr and blr.IsObject(): size = blr.root_object.size else: size = 0 return size == 0 and url_str.endswith('/') def IsFileUrlString(url_str): """Returns whether a string is a file URL.""" return GetSchemeFromUrlString(url_str) == 'file' def StorageUrlFromString(url_str): """Static factory function for creating a StorageUrl from a string.""" scheme = GetSchemeFromUrlString(url_str) if not IsKnownUrlScheme(scheme): raise InvalidUrlError('Unrecognized scheme "%s"' % scheme) if scheme == 'file': path = _GetPathFromUrlString(url_str) is_stream = (path == '-') is_fifo = False try: is_fifo = stat.S_ISFIFO(os.stat(path).st_mode) except OSError: pass return _FileUrl(url_str, is_stream=is_stream, is_fifo=is_fifo) return _CloudUrl(url_str) def StripOneSlash(url_str): if url_str and url_str.endswith('/'): return url_str[:-1] return url_str def UrlsAreForSingleProvider(url_args): """Tests whether the URLs are all for a single provider. Args: url_args: (Iterable[str]) Collection of strings to check. Returns: True if all URLs are for single provider; False if `url_args` was empty (as this would not result in a single unique provider) or URLs targeted multiple unique providers. """ provider = None url = None for url_str in url_args: url = StorageUrlFromString(url_str) if not provider: provider = url.scheme elif url.scheme != provider: return False return provider is not None def UrlsAreMixOfBucketsAndObjects(urls): """Tests whether the URLs are a mix of buckets and objects. Args: url_args: (Iterable[gslib.storage_url.StorageUrl]) Collection of URLs to check. Returns: True if URLs are a mix of buckets and objects. False if URLs are all buckets or all objects. None if invalid Cloud URLs are included. """ if all(url.IsCloudUrl() for url in urls): are_buckets = list(map(lambda x: x.IsBucket(), urls)) return any(are_buckets) and not all(are_buckets) def RaiseErrorIfUrlsAreMixOfBucketsAndObjects(urls, recursion_requested): """Raises error if mix of buckets and objects adjusted for recursion.""" if UrlsAreMixOfBucketsAndObjects(urls) and not recursion_requested: raise CommandException('Cannot operate on a mix of buckets and objects.') README 0000644 00000000572 15025013616 0005427 0 ustar 00 This directory contains library code used by gsutil. Users are cautioned not to write programs that call the internal interfaces defined in here; these interfaces were defined only for use by gsutil, and are subject to change without notice. Moreover, Google supports this library only when used by gsutil, not when the library interfaces are called directly by other programs. project_id.py 0000644 00000003300 15025013616 0007233 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper module for Google Cloud Storage project ids.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import boto from gslib.cloud_api import ProjectIdException GOOG_PROJ_ID_HDR = 'x-goog-project-id' UNIT_TEST_PROJECT_ID = None # Set to a string value when running unit tests. def PopulateProjectId(project_id=None): """Returns the project_id from the boto config file if one is not provided.""" # Provided project_id takes highest precedence. if project_id: return project_id # If present, return the default project id supplied in the boto config file. default_id = boto.config.get_value('GSUtil', 'default_project_id') if default_id: return default_id # Unit tests shouldn't interact with projects, but some functionality # depends on this method to return a project id. If the default project id # value has been set by the unit test setup, return that id. if UNIT_TEST_PROJECT_ID: return UNIT_TEST_PROJECT_ID raise ProjectIdException('MissingProjectId') devshell_auth_plugin.py 0000644 00000003306 15025013616 0011324 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Developer Shell auth bridge. This enables Boto API auth in Developer Shell environment. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from boto.auth_handler import AuthHandler from boto.auth_handler import NotReadyToAuthenticate import oauth2client.contrib.devshell as devshell class DevshellAuth(AuthHandler): """Developer Shell authorization plugin class.""" capability = ['s3'] def __init__(self, path, config, provider): # Provider here is a boto.provider.Provider object (as opposed to the # provider attribute of CloudApi objects, which is a string). if provider.name != 'google': # Devshell credentials are valid for Google only and can't be used for s3. raise NotReadyToAuthenticate() try: self.creds = devshell.DevshellCredentials() except: raise NotReadyToAuthenticate() def add_auth(self, http_request): http_request.headers['Authorization'] = ('Bearer %s' % self.creds.access_token) data/cacerts.txt 0000644 00000716762 15025013616 0007664 0 ustar 00 # Issuer: C=AT, O=e-commerce monitoring GmbH, CN=GLOBALTRUST 2020 # Subject: C=AT, O=e-commerce monitoring GmbH, CN=GLOBALTRUST 2020 # SHA256 Fingerprint: 9A:29:6A:51:82:D1:D4:51:A2:E3:7F:43:9B:74:DA:AF:A2:67:52:33:29:F9:0F:9A:0D:20:07:C3:34:E2:3C:9A -----BEGIN CERTIFICATE----- MIIFgjCCA2qgAwIBAgILWku9WvtPilv6ZeUwDQYJKoZIhvcNAQELBQAwTTELMAkG A1UEBhMCQVQxIzAhBgNVBAoTGmUtY29tbWVyY2UgbW9uaXRvcmluZyBHbWJIMRkw FwYDVQQDExBHTE9CQUxUUlVTVCAyMDIwMB4XDTIwMDIxMDAwMDAwMFoXDTQwMDYx MDAwMDAwMFowTTELMAkGA1UEBhMCQVQxIzAhBgNVBAoTGmUtY29tbWVyY2UgbW9u aXRvcmluZyBHbWJIMRkwFwYDVQQDExBHTE9CQUxUUlVTVCAyMDIwMIICIjANBgkq hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAri5WrRsc7/aVj6B3GyvTY4+ETUWiD59b RatZe1E0+eyLinjF3WuvvcTfk0Uev5E4C64OFudBc/jbu9G4UeDLgztzOG53ig9Z YybNpyrOVPu44sB8R85gfD+yc/LAGbaKkoc1DZAoouQVBGM+uq/ufF7MpotQsjj3 QWPKzv9pj2gOlTblzLmMCcpL3TGQlsjMH/1WljTbjhzqLL6FLmPdqqmV0/0plRPw yJiT2S0WR5ARg6I6IqIoV6Lr/sCMKKCmfecqQjuCgGOlYx8ZzHyyZqjC0203b+J+ BlHZRYQfEs4kUmSFC0iAToexIiIwquuuvuAC4EDosEKAA1GqtH6qRNdDYfOiaxaJ SaSjpCuKAsR49GiKweR6NrFvG5Ybd0mN1MkGco/PU+PcF4UgStyYJ9ORJitHHmkH r96i5OTUawuzXnzUJIBHKWk7buis/UDr2O1xcSvy6Fgd60GXIsUf1DnQJ4+H4xj0 4KlGDfV0OoIu0G4skaMxXDtG6nsEEFZegB31pWXogvziB4xiRfUg3kZwhqG8k9Me dKZssCz3AwyIDMvUclOGvGBG85hqwvG/Q/lwIHfKN0F5VVJjjVsSn8VoxIidrPIw q7ejMZdnrY8XD2zHc+0klGvIg5rQmjdJBKuxFshsSUktq6HQjJLyQUp5ISXbY9e2 nKd+Qmn7OmMCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC AQYwHQYDVR0OBBYEFNwuH9FhN3nkq9XVsxJxaD1qaJwiMB8GA1UdIwQYMBaAFNwu H9FhN3nkq9XVsxJxaD1qaJwiMA0GCSqGSIb3DQEBCwUAA4ICAQCR8EICaEDuw2jA VC/f7GLDw56KoDEoqoOOpFaWEhCGVrqXctJUMHytGdUdaG/7FELYjQ7ztdGl4wJC XtzoRlgHNQIw4Lx0SsFDKv/bGtCwr2zD/cuz9X9tAy5ZVp0tLTWMstZDFyySCstd 6IwPS3BD0IL/qMy/pJTAvoe9iuOTe8aPmxadJ2W8esVCgmxcB9CpwYhgROmYhRZf +I/KARDOJcP5YBugxZfD0yyIMaK9MOzQ0MAS8cE54+X1+NZK3TTN+2/BT+MAi1bi kvcoskJ3ciNnxz8RFbLEAwW+uxF7Cr+obuf/WEPPm2eggAe2HcqtbepBEX4tdJP7 wry+UUTF72glJ4DjyKDUEuzZpTcdN3y0kcra1LGWge9oXHYQSa9+pTeAsRxSvTOB TI/53WXZFM2KJVj04sWDpQmQ1GwUY7VA3+vA/MRYfg0UFodUJ25W5HCEuGwyEn6C MUO+1918oa2u1qsgEu8KwxCMSZY13At1XrFP1U80DhEgB3VDRemjEdqso5nCtnkn 4rnvyOL2NSl6dPrFf4IFYqYK6miyeUcGbvJXqBUzxvd4Sj1Ce2t+/vdG6tHrju+I aFvowdlxfv1k7/9nR4hYJS8+hge9+6jlgqispdNpQ80xiEmEU5LAsTkbOYMBMMTy qfrQA71yN2BWHzZ8vTmR9W0Nv3vXkg== -----END CERTIFICATE----- # Issuer: C=BE, O=GlobalSign nv-sa, CN=GlobalSign Root E46 # Subject: C=BE, O=GlobalSign nv-sa, CN=GlobalSign Root E46 # SHA256 Fingerprint: CB:B9:C4:4D:84:B8:04:3E:10:50:EA:31:A6:9F:51:49:55:D7:BF:D2:E2:C6:B4:93:01:01:9A:D6:1D:9F:50:58 -----BEGIN CERTIFICATE----- MIICCzCCAZGgAwIBAgISEdK7ujNu1LzmJGjFDYQdmOhDMAoGCCqGSM49BAMDMEYx CzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRwwGgYDVQQD ExNHbG9iYWxTaWduIFJvb3QgRTQ2MB4XDTE5MDMyMDAwMDAwMFoXDTQ2MDMyMDAw MDAwMFowRjELMAkGA1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2Ex HDAaBgNVBAMTE0dsb2JhbFNpZ24gUm9vdCBFNDYwdjAQBgcqhkjOPQIBBgUrgQQA IgNiAAScDrHPt+ieUnd1NPqlRqetMhkytAepJ8qUuwzSChDH2omwlwxwEwkBjtjq R+q+soArzfwoDdusvKSGN+1wCAB16pMLey5SnCNoIwZD7JIvU4Tb+0cUB+hflGdd yXqBPCCjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud DgQWBBQxCpCPtsad0kRLgLWi5h+xEk8blTAKBggqhkjOPQQDAwNoADBlAjEA31SQ 7Zvvi5QCkxeCmb6zniz2C5GMn0oUsfZkvLtoURMMA/cVi4RguYv/Uo7njLwcAjA8 +RHUjE7AwWHCFUyqqx0LMV87HOIAl0Qx5v5zli/altP+CAezNIm8BZ/3Hobui3A= -----END CERTIFICATE----- # Issuer: C=BE, O=GlobalSign nv-sa, CN=GlobalSign Root R46 # Subject: C=BE, O=GlobalSign nv-sa, CN=GlobalSign Root R46 # SHA256 Fingerprint: 4F:A3:12:6D:8D:3A:11:D1:C4:85:5A:4F:80:7C:BA:D6:CF:91:9D:3A:5A:88:B0:3B:EA:2C:63:72:D9:3C:40:C9 -----BEGIN CERTIFICATE----- MIIFWjCCA0KgAwIBAgISEdK7udcjGJ5AXwqdLdDfJWfRMA0GCSqGSIb3DQEBDAUA MEYxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMRwwGgYD VQQDExNHbG9iYWxTaWduIFJvb3QgUjQ2MB4XDTE5MDMyMDAwMDAwMFoXDTQ2MDMy MDAwMDAwMFowRjELMAkGA1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYt c2ExHDAaBgNVBAMTE0dsb2JhbFNpZ24gUm9vdCBSNDYwggIiMA0GCSqGSIb3DQEB AQUAA4ICDwAwggIKAoICAQCsrHQy6LNl5brtQyYdpokNRbopiLKkHWPd08EsCVeJ OaFV6Wc0dwxu5FUdUiXSE2te4R2pt32JMl8Nnp8semNgQB+msLZ4j5lUlghYruQG vGIFAha/r6gjA7aUD7xubMLL1aa7DOn2wQL7Id5m3RerdELv8HQvJfTqa1VbkNud 316HCkD7rRlr+/fKYIje2sGP1q7Vf9Q8g+7XFkyDRTNrJ9CG0Bwta/OrffGFqfUo 0q3v84RLHIf8E6M6cqJaESvWJ3En7YEtbWaBkoe0G1h6zD8K+kZPTXhc+CtI4wSE y132tGqzZfxCnlEmIyDLPRT5ge1lFgBPGmSXZgjPjHvjK8Cd+RTyG/FWaha/LIWF zXg4mutCagI0GIMXTpRW+LaCtfOW3T3zvn8gdz57GSNrLNRyc0NXfeD412lPFzYE +cCQYDdF3uYM2HSNrpyibXRdQr4G9dlkbgIQrImwTDsHTUB+JMWKmIJ5jqSngiCN I/onccnfxkF0oE32kRbcRoxfKWMxWXEM2G/CtjJ9++ZdU6Z+Ffy7dXxd7Pj2Fxzs x2sZy/N78CsHpdlseVR2bJ0cpm4O6XkMqCNqo98bMDGfsVR7/mrLZqrcZdCinkqa ByFrgY/bxFn63iLABJzjqls2k+g9vXqhnQt2sQvHnf3PmKgGwvgqo6GDoLclcqUC 4wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV HQ4EFgQUA1yrc4GHqMywptWU4jaWSf8FmSwwDQYJKoZIhvcNAQEMBQADggIBAHx4 7PYCLLtbfpIrXTncvtgdokIzTfnvpCo7RGkerNlFo048p9gkUbJUHJNOxO97k4Vg JuoJSOD1u8fpaNK7ajFxzHmuEajwmf3lH7wvqMxX63bEIaZHU1VNaL8FpO7XJqti 2kM3S+LGteWygxk6x9PbTZ4IevPuzz5i+6zoYMzRx6Fcg0XERczzF2sUyQQCPtIk pnnpHs6i58FZFZ8d4kuaPp92CC1r2LpXFNqD6v6MVenQTqnMdzGxRBF6XLE+0xRF FRhiJBPSy03OXIPBNvIQtQ6IbbjhVp+J3pZmOUdkLG5NrmJ7v2B0GbhWrJKsFjLt rWhV/pi60zTe9Mlhww6G9kuEYO4Ne7UyWHmRVSyBQ7N0H3qqJZ4d16GLuc1CLgSk ZoNNiTW2bKg2SnkheCLQQrzRQDGQob4Ez8pn7fXwgNNgyYMqIgXQBztSvwyeqiv5 u+YfjyW6hY0XHgL+XVAEV8/+LbzvXMAaq7afJMbfc2hIkCwU9D9SGuTSyxTDYWnP 4vkYxboznxSjBF25cfe1lNj2M8FawTSLfJvdkzrnE6JwYZ+vj+vYxXX4M2bUdGc6 N3ec592kD3ZDZopD8p/7DEJ4Y9HiD2971KE9dJeFt0g5QdYg/NA6s/rob8SKunE3 vouXsXgxT7PntgMTzlSdriVZzH81Xwj3QEUxeCp6 -----END CERTIFICATE----- # Issuer: C=BE, O=GlobalSign nv-sa, OU=Root CA, CN=GlobalSign Root CA # Subject: C=BE, O=GlobalSign nv-sa, OU=Root CA, CN=GlobalSign Root CA # SHA256 Fingerprint: EB:D4:10:40:E4:BB:3E:C7:42:C9:E3:81:D3:1E:F2:A4:1A:48:B6:68:5C:96:E7:CE:F3:C1:DF:6C:D4:33:1C:99 -----BEGIN CERTIFICATE----- MIIDdTCCAl2gAwIBAgILBAAAAAABFUtaw5QwDQYJKoZIhvcNAQEFBQAwVzELMAkG A1UEBhMCQkUxGTAXBgNVBAoTEEdsb2JhbFNpZ24gbnYtc2ExEDAOBgNVBAsTB1Jv b3QgQ0ExGzAZBgNVBAMTEkdsb2JhbFNpZ24gUm9vdCBDQTAeFw05ODA5MDExMjAw MDBaFw0yODAxMjgxMjAwMDBaMFcxCzAJBgNVBAYTAkJFMRkwFwYDVQQKExBHbG9i YWxTaWduIG52LXNhMRAwDgYDVQQLEwdSb290IENBMRswGQYDVQQDExJHbG9iYWxT aWduIFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDaDuaZ jc6j40+Kfvvxi4Mla+pIH/EqsLmVEQS98GPR4mdmzxzdzxtIK+6NiY6arymAZavp xy0Sy6scTHAHoT0KMM0VjU/43dSMUBUc71DuxC73/OlS8pF94G3VNTCOXkNz8kHp 1Wrjsok6Vjk4bwY8iGlbKk3Fp1S4bInMm/k8yuX9ifUSPJJ4ltbcdG6TRGHRjcdG snUOhugZitVtbNV4FpWi6cgKOOvyJBNPc1STE4U6G7weNLWLBYy5d4ux2x8gkasJ U26Qzns3dLlwR5EiUWMWea6xrkEmCMgZK9FGqkjWZCrXgzT/LCrBbBlDSgeF59N8 9iFo7+ryUp9/k5DPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8E BTADAQH/MB0GA1UdDgQWBBRge2YaRQ2XyolQL30EzTSo//z9SzANBgkqhkiG9w0B AQUFAAOCAQEA1nPnfE920I2/7LqivjTFKDK1fPxsnCwrvQmeU79rXqoRSLblCKOz yj1hTdNGCbM+w6DjY1Ub8rrvrTnhQ7k4o+YviiY776BQVvnGCv04zcQLcFGUl5gE 38NflNUVyRRBnMRddWQVDf9VMOyGj/8N7yy5Y0b2qvzfvGn9LhJIZJrglfCm7ymP AbEVtQwdpf5pLGkkeB6zpxxxYu7KyJesF12KwvhHhm4qxFYxldBniYUr+WymXUad DKqC5JlR3XC321Y9YeRq4VzW9v493kHMB65jUr9TU/Qr6cf9tveCX4XSQRjbgbME HMUfpIBvFSDJ3gyICh3WZlXi/EjJKSZp4A== -----END CERTIFICATE----- # Issuer: C=BM, O=QuoVadis Limited, CN=QuoVadis Root CA 1 G3 # Subject: C=BM, O=QuoVadis Limited, CN=QuoVadis Root CA 1 G3 # SHA256 Fingerprint: 8A:86:6F:D1:B2:76:B5:7E:57:8E:92:1C:65:82:8A:2B:ED:58:E9:F2:F2:88:05:41:34:B7:F1:F4:BF:C9:CC:74 -----BEGIN CERTIFICATE----- MIIFYDCCA0igAwIBAgIUeFhfLq0sGUvjNwc1NBMotZbUZZMwDQYJKoZIhvcNAQEL BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMSBHMzAeFw0xMjAxMTIxNzI3NDRaFw00 MjAxMTIxNzI3NDRaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDEgRzMwggIiMA0GCSqG SIb3DQEBAQUAA4ICDwAwggIKAoICAQCgvlAQjunybEC0BJyFuTHK3C3kEakEPBtV wedYMB0ktMPvhd6MLOHBPd+C5k+tR4ds7FtJwUrVu4/sh6x/gpqG7D0DmVIB0jWe rNrwU8lmPNSsAgHaJNM7qAJGr6Qc4/hzWHa39g6QDbXwz8z6+cZM5cOGMAqNF341 68Xfuw6cwI2H44g4hWf6Pser4BOcBRiYz5P1sZK0/CPTz9XEJ0ngnjybCKOLXSoh 4Pw5qlPafX7PGglTvF0FBM+hSo+LdoINofjSxxR3W5A2B4GbPgb6Ul5jxaYA/qXp UhtStZI5cgMJYr2wYBZupt0lwgNm3fME0UDiTouG9G/lg6AnhF4EwfWQvTA9xO+o abw4m6SkltFi2mnAAZauy8RRNOoMqv8hjlmPSlzkYZqn0ukqeI1RPToV7qJZjqlc 3sX5kCLliEVx3ZGZbHqfPT2YfF72vhZooF6uCyP8Wg+qInYtyaEQHeTTRCOQiJ/G KubX9ZqzWB4vMIkIG1SitZgj7Ah3HJVdYdHLiZxfokqRmu8hqkkWCKi9YSgxyXSt hfbZxbGL0eUQMk1fiyA6PEkfM4VZDdvLCXVDaXP7a3F98N/ETH3Goy7IlXnLc6KO Tk0k+17kBL5yG6YnLUlamXrXXAkgt3+UuU/xDRxeiEIbEbfnkduebPRq34wGmAOt zCjvpUfzUwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB BjAdBgNVHQ4EFgQUo5fW816iEOGrRZ88F2Q87gFwnMwwDQYJKoZIhvcNAQELBQAD ggIBABj6W3X8PnrHX3fHyt/PX8MSxEBd1DKquGrX1RUVRpgjpeaQWxiZTOOtQqOC MTaIzen7xASWSIsBx40Bz1szBpZGZnQdT+3Btrm0DWHMY37XLneMlhwqI2hrhVd2 cDMT/uFPpiN3GPoajOi9ZcnPP/TJF9zrx7zABC4tRi9pZsMbj/7sPtPKlL92CiUN qXsCHKnQO18LwIE6PWThv6ctTr1NxNgpxiIY0MWscgKCP6o6ojoilzHdCGPDdRS5 YCgtW2jgFqlmgiNR9etT2DGbe+m3nUvriBbP+V04ikkwj+3x6xn0dxoxGE1nVGwv b2X52z3sIexe9PSLymBlVNFxZPT5pqOBMzYzcfCkeF9OrYMh3jRJjehZrJ3ydlo2 8hP0r+AJx2EqbPfgna67hkooby7utHnNkDPDs3b69fBsnQGQ+p6Q9pxyz0fawx/k NSBT8lTR32GDpgLiJTjehTItXnOQUl1CxM49S+H5GYQd1aJQzEH7QRTDvdbJWqNj ZgKAvQU6O0ec7AAmTPWIUb+oI38YB7AL7YsmoWTTYUrrXJ/es69nA7Mf3W1daWhp q1467HxpvMc7hU6eFbm0FU/DlXpY18ls6Wy58yljXrQs8C097Vpl4KlbQMJImYFt nh8GKjwStIsPm6Ik8KaN1nrgS7ZklmOVhMJKzRwuJIczYOXD -----END CERTIFICATE----- # Issuer: C=BM, O=QuoVadis Limited, CN=QuoVadis Root CA 2 G3 # Subject: C=BM, O=QuoVadis Limited, CN=QuoVadis Root CA 2 G3 # SHA256 Fingerprint: 8F:E4:FB:0A:F9:3A:4D:0D:67:DB:0B:EB:B2:3E:37:C7:1B:F3:25:DC:BC:DD:24:0E:A0:4D:AF:58:B4:7E:18:40 -----BEGIN CERTIFICATE----- MIIFYDCCA0igAwIBAgIURFc0JFuBiZs18s64KztbpybwdSgwDQYJKoZIhvcNAQEL BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMiBHMzAeFw0xMjAxMTIxODU5MzJaFw00 MjAxMTIxODU5MzJaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDIgRzMwggIiMA0GCSqG SIb3DQEBAQUAA4ICDwAwggIKAoICAQChriWyARjcV4g/Ruv5r+LrI3HimtFhZiFf qq8nUeVuGxbULX1QsFN3vXg6YOJkApt8hpvWGo6t/x8Vf9WVHhLL5hSEBMHfNrMW n4rjyduYNM7YMxcoRvynyfDStNVNCXJJ+fKH46nafaF9a7I6JaltUkSs+L5u+9ym c5GQYaYDFCDy54ejiK2toIz/pgslUiXnFgHVy7g1gQyjO/Dh4fxaXc6AcW34Sas+ O7q414AB+6XrW7PFXmAqMaCvN+ggOp+oMiwMzAkd056OXbxMmO7FGmh77FOm6RQ1 o9/NgJ8MSPsc9PG/Srj61YxxSscfrf5BmrODXfKEVu+lV0POKa2Mq1W/xPtbAd0j IaFYAI7D0GoT7RPjEiuA3GfmlbLNHiJuKvhB1PLKFAeNilUSxmn1uIZoL1NesNKq IcGY5jDjZ1XHm26sGahVpkUG0CM62+tlXSoREfA7T8pt9DTEceT/AFr2XK4jYIVz 8eQQsSWu1ZK7E8EM4DnatDlXtas1qnIhO4M15zHfeiFuuDIIfR0ykRVKYnLP43eh vNURG3YBZwjgQQvD6xVu+KQZ2aKrr+InUlYrAoosFCT5v0ICvybIxo/gbjh9Uy3l 7ZizlWNof/k19N+IxWA1ksB8aRxhlRbQ694Lrz4EEEVlWFA4r0jyWbYW8jwNkALG cC4BrTwV1wIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB BjAdBgNVHQ4EFgQU7edvdlq/YOxJW8ald7tyFnGbxD0wDQYJKoZIhvcNAQELBQAD ggIBAJHfgD9DCX5xwvfrs4iP4VGyvD11+ShdyLyZm3tdquXK4Qr36LLTn91nMX66 AarHakE7kNQIXLJgapDwyM4DYvmL7ftuKtwGTTwpD4kWilhMSA/ohGHqPHKmd+RC roijQ1h5fq7KpVMNqT1wvSAZYaRsOPxDMuHBR//47PERIjKWnML2W2mWeyAMQ0Ga W/ZZGYjeVYg3UQt4XAoeo0L9x52ID8DyeAIkVJOviYeIyUqAHerQbj5hLja7NQ4n lv1mNDthcnPxFlxHBlRJAHpYErAK74X9sbgzdWqTHBLmYF5vHX/JHyPLhGGfHoJE +V+tYlUkmlKY7VHnoX6XOuYvHxHaU4AshZ6rNRDbIl9qxV6XU/IyAgkwo1jwDQHV csaxfGl7w/U2Rcxhbl5MlMVerugOXou/983g7aEOGzPuVBj+D77vfoRrQ+NwmNtd dbINWQeFFSM51vHfqSYP1kjHs6Yi9TM3WpVHn3u6GBVv/9YUZINJ0gpnIdsPNWNg KCLjsZWDzYWm3S8P52dSbrsvhXz1SnPnxT7AvSESBT/8twNJAlvIJebiVDj1eYeM HVOyToV7BjjHLPj4sHKNJeV3UvQDHEimUF+IIDBu8oJDqz2XhOdT+yHBTw8imoa4 WSr2Rz0ZiC3oheGe7IUIarFsNMkd7EgrO3jtZsSOeWmD3n+M -----END CERTIFICATE----- # Issuer: C=BM, O=QuoVadis Limited, CN=QuoVadis Root CA 2 # Subject: C=BM, O=QuoVadis Limited, CN=QuoVadis Root CA 2 # SHA256 Fingerprint: 85:A0:DD:7D:D7:20:AD:B7:FF:05:F8:3D:54:2B:20:9D:C7:FF:45:28:F7:D6:77:B1:83:89:FE:A5:E5:C4:9E:86 -----BEGIN CERTIFICATE----- MIIFtzCCA5+gAwIBAgICBQkwDQYJKoZIhvcNAQEFBQAwRTELMAkGA1UEBhMCQk0x GTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxGzAZBgNVBAMTElF1b1ZhZGlzIFJv b3QgQ0EgMjAeFw0wNjExMjQxODI3MDBaFw0zMTExMjQxODIzMzNaMEUxCzAJBgNV BAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1pdGVkMRswGQYDVQQDExJRdW9W YWRpcyBSb290IENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCa GMpLlA0ALa8DKYrwD4HIrkwZhR0In6spRIXzL4GtMh6QRr+jhiYaHv5+HBg6XJxg Fyo6dIMzMH1hVBHL7avg5tKifvVrbxi3Cgst/ek+7wrGsxDp3MJGF/hd/aTa/55J WpzmM+Yklvc/ulsrHHo1wtZn/qtmUIttKGAr79dgw8eTvI02kfN/+NsRE8Scd3bB rrcCaoF6qUWD4gXmuVbBlDePSHFjIuwXZQeVikvfj8ZaCuWw419eaxGrDPmF60Tp +ARz8un+XJiM9XOva7R+zdRcAitMOeGylZUtQofX1bOQQ7dsE/He3fbE+Ik/0XX1 ksOR1YqI0JDs3G3eicJlcZaLDQP9nL9bFqyS2+r+eXyt66/3FsvbzSUr5R/7mp/i Ucw6UwxI5g69ybR2BlLmEROFcmMDBOAENisgGQLodKcftslWZvB1JdxnwQ5hYIiz PtGo/KPaHbDRsSNU30R2be1B2MGyIrZTHN81Hdyhdyox5C315eXbyOD/5YDXC2Og /zOhD7osFRXql7PSorW+8oyWHhqPHWykYTe5hnMz15eWniN9gqRMgeKh0bpnX5UH oycR7hYQe7xFSkyyBNKr79X9DFHOUGoIMfmR2gyPZFwDwzqLID9ujWc9Otb+fVuI yV77zGHcizN300QyNQliBJIWENieJ0f7OyHj+OsdWwIDAQABo4GwMIGtMA8GA1Ud EwEB/wQFMAMBAf8wCwYDVR0PBAQDAgEGMB0GA1UdDgQWBBQahGK8SEwzJQTU7tD2 A8QZRtGUazBuBgNVHSMEZzBlgBQahGK8SEwzJQTU7tD2A8QZRtGUa6FJpEcwRTEL MAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxGzAZBgNVBAMT ElF1b1ZhZGlzIFJvb3QgQ0EgMoICBQkwDQYJKoZIhvcNAQEFBQADggIBAD4KFk2f BluornFdLwUvZ+YTRYPENvbzwCYMDbVHZF34tHLJRqUDGCdViXh9duqWNIAXINzn g/iN/Ae42l9NLmeyhP3ZRPx3UIHmfLTJDQtyU/h2BwdBR5YM++CCJpNVjP4iH2Bl fF/nJrP3MpCYUNQ3cVX2kiF495V5+vgtJodmVjB3pjd4M1IQWK4/YY7yarHvGH5K WWPKjaJW1acvvFYfzznB4vsKqBUsfU16Y8Zsl0Q80m/DShcK+JDSV6IZUaUtl0Ha B0+pUNqQjZRG4T7wlP0QADj1O+hA4bRuVhogzG9Yje0uRY/W6ZM/57Es3zrWIozc hLsib9D45MY56QSIPMO661V6bYCZJPVsAfv4l7CUW+v90m/xd2gNNWQjrLhVoQPR TUIZ3Ph1WVaj+ahJefivDrkRoHy3au000LYmYjgahwz46P0u05B/B5EqHdZ+XIWD mbA4CD/pXvk1B+TJYm5Xf6dQlfe6yJvmjqIBxdZmv3lh8zwc4bmCXF2gw+nYSL0Z ohEUGW6yhhtoPkg3Goi3XZZenMfvJ2II4pEZXNLxId26F0KCl3GBUzGpn/Z9Yr9y 4aOTHcyKJloJONDO1w2AFrR4pTqHTI2KpdVGl/IsELm8VCLAAVBpQ570su9t+Oza 8eOx79+Rj1QqCyXBJhnEUhAFZdWCEOrCMc0u -----END CERTIFICATE----- # Issuer: C=BM, O=QuoVadis Limited, CN=QuoVadis Root CA 3 G3 # Subject: C=BM, O=QuoVadis Limited, CN=QuoVadis Root CA 3 G3 # SHA256 Fingerprint: 88:EF:81:DE:20:2E:B0:18:45:2E:43:F8:64:72:5C:EA:5F:BD:1F:C2:D9:D2:05:73:07:09:C5:D8:B8:69:0F:46 -----BEGIN CERTIFICATE----- MIIFYDCCA0igAwIBAgIULvWbAiin23r/1aOp7r0DoM8Sah0wDQYJKoZIhvcNAQEL BQAwSDELMAkGA1UEBhMCQk0xGTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxHjAc BgNVBAMTFVF1b1ZhZGlzIFJvb3QgQ0EgMyBHMzAeFw0xMjAxMTIyMDI2MzJaFw00 MjAxMTIyMDI2MzJaMEgxCzAJBgNVBAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBM aW1pdGVkMR4wHAYDVQQDExVRdW9WYWRpcyBSb290IENBIDMgRzMwggIiMA0GCSqG SIb3DQEBAQUAA4ICDwAwggIKAoICAQCzyw4QZ47qFJenMioKVjZ/aEzHs286IxSR /xl/pcqs7rN2nXrpixurazHb+gtTTK/FpRp5PIpM/6zfJd5O2YIyC0TeytuMrKNu FoM7pmRLMon7FhY4futD4tN0SsJiCnMK3UmzV9KwCoWdcTzeo8vAMvMBOSBDGzXR U7Ox7sWTaYI+FrUoRqHe6okJ7UO4BUaKhvVZR74bbwEhELn9qdIoyhA5CcoTNs+c ra1AdHkrAj80//ogaX3T7mH1urPnMNA3I4ZyYUUpSFlob3emLoG+B01vr87ERROR FHAGjx+f+IdpsQ7vw4kZ6+ocYfx6bIrc1gMLnia6Et3UVDmrJqMz6nWB2i3ND0/k A9HvFZcba5DFApCTZgIhsUfei5pKgLlVj7WiL8DWM2fafsSntARE60f75li59wzw eyuxwHApw0BiLTtIadwjPEjrewl5qW3aqDCYz4ByA4imW0aucnl8CAMhZa634Ryl sSqiMd5mBPfAdOhx3v89WcyWJhKLhZVXGqtrdQtEPREoPHtht+KPZ0/l7DxMYIBp VzgeAVuNVejH38DMdyM0SXV89pgR6y3e7UEuFAUCf+D+IOs15xGsIs5XPd7JMG0Q A4XN8f+MFrXBsj6IbGB/kE+V9/YtrQE5BwT6dYB9v0lQ7e/JxHwc64B+27bQ3RP+ ydOc17KXqQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB BjAdBgNVHQ4EFgQUxhfQvKjqAkPyGwaZXSuQILnXnOQwDQYJKoZIhvcNAQELBQAD ggIBADRh2Va1EodVTd2jNTFGu6QHcrxfYWLopfsLN7E8trP6KZ1/AvWkyaiTt3px KGmPc+FSkNrVvjrlt3ZqVoAh313m6Tqe5T72omnHKgqwGEfcIHB9UqM+WXzBusnI FUBhynLWcKzSt/Ac5IYp8M7vaGPQtSCKFWGafoaYtMnCdvvMujAWzKNhxnQT5Wvv oxXqA/4Ti2Tk08HS6IT7SdEQTXlm66r99I0xHnAUrdzeZxNMgRVhvLfZkXdxGYFg u/BYpbWcC/ePIlUnwEsBbTuZDdQdm2NnL9DuDcpmvJRPpq3t/O5jrFc/ZSXPsoaP 0Aj/uHYUbt7lJ+yreLVTubY/6CD50qi+YUbKh4yE8/nxoGibIh6BJpsQBJFxwAYf 3KDTuVan45gtf4Od34wrnDKOMpTwATwiKp9Dwi7DmDkHOHv8XgBCH/MyJnmDhPbl 8MFREsALHgQjDFSlTC9JxUrRtm5gDWv8a4uFJGS3iQ6rJUdbPM9+Sb3H6QrG2vd+ DhcI00iX0HGS8A85PjRqHH3Y8iKuu2n0M7SmSFXRDw4m6Oy2Cy2nhTXN/VnIn9HN PlopNLk9hM6xZdRZkZFWdSHBd575euFgndOtBBj0fOtek49TSiIp+EgrPk2GrFt/ ywaZWWDYWGWVjUTR939+J399roD1B0y2PpxxVJkES/1Y+Zj0 -----END CERTIFICATE----- # Issuer: C=BM, O=QuoVadis Limited, CN=QuoVadis Root CA 3 # Subject: C=BM, O=QuoVadis Limited, CN=QuoVadis Root CA 3 # SHA256 Fingerprint: 18:F1:FC:7F:20:5D:F8:AD:DD:EB:7F:E0:07:DD:57:E3:AF:37:5A:9C:4D:8D:73:54:6B:F4:F1:FE:D1:E1:8D:35 -----BEGIN CERTIFICATE----- MIIGnTCCBIWgAwIBAgICBcYwDQYJKoZIhvcNAQEFBQAwRTELMAkGA1UEBhMCQk0x GTAXBgNVBAoTEFF1b1ZhZGlzIExpbWl0ZWQxGzAZBgNVBAMTElF1b1ZhZGlzIFJv b3QgQ0EgMzAeFw0wNjExMjQxOTExMjNaFw0zMTExMjQxOTA2NDRaMEUxCzAJBgNV BAYTAkJNMRkwFwYDVQQKExBRdW9WYWRpcyBMaW1pdGVkMRswGQYDVQQDExJRdW9W YWRpcyBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDM V0IWVJzmmNPTTe7+7cefQzlKZbPoFog02w1ZkXTPkrgEQK0CSzGrvI2RaNggDhoB 4hp7Thdd4oq3P5kazethq8Jlph+3t723j/z9cI8LoGe+AaJZz3HmDyl2/7FWeUUr H556VOijKTVopAFPD6QuN+8bv+OPEKhyq1hX51SGyMnzW9os2l2ObjyjPtr7guXd 8lyyBTNvijbO0BNO/79KDDRMpsMhvVAEVeuxu537RR5kFd5VAYwCdrXLoT9Cabwv vWhDFlaJKjdhkf2mrk7AyxRllDdLkgbvBNDInIjbC3uBr7E9KsRlOni27tyAsdLT mZw67mtaa7ONt9XOnMK+pUsvFrGeaDsGb659n/je7Mwpp5ijJUMv7/FfJuGITfhe btfZFG4ZM2mnO4SJk8RTVROhUXhA+LjJou57ulJCg54U7QVSWllWp5f8nT8KKdjc T5EOE7zelaTfi5m+rJsziO+1ga8bxiJTyPbH7pcUsMV8eFLI8M5ud2CEpukqdiDt WAEXMJPpGovgc2PZapKUSU60rUqFxKMiMPwJ7Wgic6aIDFUhWMXhOp8q3crhkODZ c6tsgLjoC2SToJyMGf+z0gzskSaHirOi4XCPLArlzW1oUevaPwV/izLmE1xr/l9A 4iLItLRkT9a6fUg+qGkM17uGcclzuD87nSVL2v9A6wIDAQABo4IBlTCCAZEwDwYD VR0TAQH/BAUwAwEB/zCB4QYDVR0gBIHZMIHWMIHTBgkrBgEEAb5YAAMwgcUwgZMG CCsGAQUFBwICMIGGGoGDQW55IHVzZSBvZiB0aGlzIENlcnRpZmljYXRlIGNvbnN0 aXR1dGVzIGFjY2VwdGFuY2Ugb2YgdGhlIFF1b1ZhZGlzIFJvb3QgQ0EgMyBDZXJ0 aWZpY2F0ZSBQb2xpY3kgLyBDZXJ0aWZpY2F0aW9uIFByYWN0aWNlIFN0YXRlbWVu dC4wLQYIKwYBBQUHAgEWIWh0dHA6Ly93d3cucXVvdmFkaXNnbG9iYWwuY29tL2Nw czALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFPLAE+CCQz777i9nMpY1XNu4ywLQMG4G A1UdIwRnMGWAFPLAE+CCQz777i9nMpY1XNu4ywLQoUmkRzBFMQswCQYDVQQGEwJC TTEZMBcGA1UEChMQUXVvVmFkaXMgTGltaXRlZDEbMBkGA1UEAxMSUXVvVmFkaXMg Um9vdCBDQSAzggIFxjANBgkqhkiG9w0BAQUFAAOCAgEAT62gLEz6wPJv92ZVqyM0 7ucp2sNbtrCD2dDQ4iH782CnO11gUyeim/YIIirnv6By5ZwkajGxkHon24QRiSem d1o417+shvzuXYO8BsbRd2sPbSQvS3pspweWyuOEn62Iix2rFo1bZhfZFvSLgNLd +LJ2w/w4E6oM3kJpK27zPOuAJ9v1pkQNn1pVWQvVDVJIxa6f8i+AxeoyUDUSly7B 4f/xI4hROJ/yZlZ25w9Rl6VSDE1JUZU2Pb+iSwwQHYaZTKrzchGT5Or2m9qoXadN t54CrnMAyNojA+j56hl0YgCUyyIgvpSnWbWCar6ZeXqp8kokUvd0/bpO5qgdAm6x DYBEwa7TIzdfu4V8K5Iu6H6li92Z4b8nby1dqnuH/grdS/yO9SbkbnBCbjPsMZ57 k8HkyWkaPcBrTiJt7qtYTcbQQcEr6k8Sh17rRdhs9ZgC06DYVYoGmRmioHfRMJ6s zHXug/WwYjnPbFfiTNKRCw51KBuav/0aQ/HKd/s7j2G4aSgWQgRecCocIdiP4b0j Wy10QJLZYxkNc91pvGJHvOB0K7Lrfb5BG7XARsWhIstfTsEokt4YutUqKLsRixeT mJlglFwjz1onl14LBQaTNx47aTbrqZ5hHY8y2o4M1nQ+ewkk2gF3R8Q7zTSMmfXK 4SVhM7JZG+Ju1zdXtg2pEto= -----END CERTIFICATE----- # Issuer: C=CH, O=SwissSign AG, CN=SwissSign Gold CA - G2 # Subject: C=CH, O=SwissSign AG, CN=SwissSign Gold CA - G2 # SHA256 Fingerprint: 62:DD:0B:E9:B9:F5:0A:16:3E:A0:F8:E7:5C:05:3B:1E:CA:57:EA:55:C8:68:8F:64:7C:68:81:F2:C8:35:7B:95 -----BEGIN CERTIFICATE----- MIIFujCCA6KgAwIBAgIJALtAHEP1Xk+wMA0GCSqGSIb3DQEBBQUAMEUxCzAJBgNV BAYTAkNIMRUwEwYDVQQKEwxTd2lzc1NpZ24gQUcxHzAdBgNVBAMTFlN3aXNzU2ln biBHb2xkIENBIC0gRzIwHhcNMDYxMDI1MDgzMDM1WhcNMzYxMDI1MDgzMDM1WjBF MQswCQYDVQQGEwJDSDEVMBMGA1UEChMMU3dpc3NTaWduIEFHMR8wHQYDVQQDExZT d2lzc1NpZ24gR29sZCBDQSAtIEcyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC CgKCAgEAr+TufoskDhJuqVAtFkQ7kpJcyrhdhJJCEyq8ZVeCQD5XJM1QiyUqt2/8 76LQwB8CJEoTlo8jE+YoWACjR8cGp4QjK7u9lit/VcyLwVcfDmJlD909Vopz2q5+ bbqBHH5CjCA12UNNhPqE21Is8w4ndwtrvxEvcnifLtg+5hg3Wipy+dpikJKVyh+c 6bM8K8vzARO/Ws/BtQpgvd21mWRTuKCWs2/iJneRjOBiEAKfNA+k1ZIzUd6+jbqE emA8atufK+ze3gE/bk3lUIbLtK/tREDFylqM2tIrfKjuvqblCqoOpd8FUrdVxyJd MmqXl2MT28nbeTZ7hTpKxVKJ+STnnXepgv9VHKVxaSvRAiTysybUa9oEVeXBCsdt MDeQKuSeFDNeFhdVxVu1yzSJkvGdJo+hB9TGsnhQ2wwMC3wLjEHXuendjIj3o02y MszYF9rNt85mndT9Xv+9lz4pded+p2JYryU0pUHHPbwNUMoDAw8IWh+Vc3hiv69y FGkOpeUDDniOJihC8AcLYiAQZzlG+qkDzAQ4embvIIO1jEpWjpEA/I5cgt6IoMPi aG59je883WX0XaxR7ySArqpWl2/5rX3aYT+YdzylkbYcjCbaZaIJbcHiVOO5ykxM gI93e2CaHt+28kgeDrpOVG2Y4OGiGqJ3UM/EY5LsRxmd6+ZrzsECAwEAAaOBrDCB qTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUWyV7 lqRlUX64OfPAeGZe6Drn8O4wHwYDVR0jBBgwFoAUWyV7lqRlUX64OfPAeGZe6Drn 8O4wRgYDVR0gBD8wPTA7BglghXQBWQECAQEwLjAsBggrBgEFBQcCARYgaHR0cDov L3JlcG9zaXRvcnkuc3dpc3NzaWduLmNvbS8wDQYJKoZIhvcNAQEFBQADggIBACe6 45R88a7A3hfm5djV9VSwg/S7zV4Fe0+fdWavPOhWfvxyeDgD2StiGwC5+OlgzczO UYrHUDFu4Up+GC9pWbY9ZIEr44OE5iKHjn3g7gKZYbge9LgriBIWhMIxkziWMaa5 O1M/wySTVltpkuzFwbs4AOPsF6m43Md8AYOfMke6UiI0HTJ6CVanfCU2qT1L2sCC bwq7EsiHSycR+R4tx5M/nttfJmtS2S6K8RTGRI0Vqbe/vd6mGu6uLftIdxf+u+yv GPUqUfA5hJeVbG4bwyvEdGB5JbAKJ9/fXtI5z0V9QkvfsywexcZdylU6oJxpmo/a 77KwPJ+HbBIrZXAVUjEaJM9vMSNQH4xPjyPDdEFjHFWoFN0+4FFQz/EbMFYOkrCC hdiDyyJkvC24JdVUorgG6q2SpCSgwYa1ShNqR88uC1aVVMvOmttqtKay20EIhid3 92qgQmwLOM7XdVAyksLfKzAiSNDVQTglXaTpXZ/GlHXQRf0wl0OPkKsKx4ZzYEpp Ld6leNcG2mqeSz53OiATIgHQv2ieY2BrNU0LbbqhPcCT4H8js1WtciVORvnSFu+w ZMEBnunKoGqYDs/YYPIvSbjkQuE4NRb0yG5P94FW6LqjviOvrv1vA+ACOzB2+htt Qc8Bsem4yWb02ybzOqR08kkkW8mw0FfB+j564ZfJ -----END CERTIFICATE----- # Issuer: C=CH, O=SwissSign AG, CN=SwissSign Silver CA - G2 # Subject: C=CH, O=SwissSign AG, CN=SwissSign Silver CA - G2 # SHA256 Fingerprint: BE:6C:4D:A2:BB:B9:BA:59:B6:F3:93:97:68:37:42:46:C3:C0:05:99:3F:A9:8F:02:0D:1D:ED:BE:D4:8A:81:D5 -----BEGIN CERTIFICATE----- MIIFvTCCA6WgAwIBAgIITxvUL1S7L0swDQYJKoZIhvcNAQEFBQAwRzELMAkGA1UE BhMCQ0gxFTATBgNVBAoTDFN3aXNzU2lnbiBBRzEhMB8GA1UEAxMYU3dpc3NTaWdu IFNpbHZlciBDQSAtIEcyMB4XDTA2MTAyNTA4MzI0NloXDTM2MTAyNTA4MzI0Nlow RzELMAkGA1UEBhMCQ0gxFTATBgNVBAoTDFN3aXNzU2lnbiBBRzEhMB8GA1UEAxMY U3dpc3NTaWduIFNpbHZlciBDQSAtIEcyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8A MIICCgKCAgEAxPGHf9N4Mfc4yfjDmUO8x/e8N+dOcbpLj6VzHVxumK4DV644N0Mv Fz0fyM5oEMF4rhkDKxD6LHmD9ui5aLlV8gREpzn5/ASLHvGiTSf5YXu6t+WiE7br YT7QbNHm+/pe7R20nqA1W6GSy/BJkv6FCgU+5tkL4k+73JU3/JHpMjUi0R86TieF nbAVlDLaYQ1HTWBCrpJH6INaUFjpiou5XaHc3ZlKHzZnu0jkg7Y360g6rw9njxcH 6ATK72oxh9TAtvmUcXtnZLi2kUpCe2UuMGoM9ZDulebyzYLs2aFK7PayS+VFheZt eJMELpyCbTapxDFkH4aDCyr0NQp4yVXPQbBH6TCfmb5hqAaEuSh6XzjZG6k4sIN/ c8HDO0gqgg8hm7jMqDXDhBuDsz6+pJVpATqJAHgE2cn0mRmrVn5bi4Y5FZGkECwJ MoBgs5PAKrYYC51+jUnyEEp/+dVGLxmSo5mnJqy7jDzmDrxHB9xzUfFwZC8I+bRH HTBsROopN4WSaGa8gzj+ezku01DwH/teYLappvonQfGbGHLy9YR0SslnxFSuSGTf jNFusB3hB48IHpmccelM2KX3RxIfdNFRnobzwqIjQAtz20um53MGjMGg6cFZrEb6 5i/4z3GcRm25xBWNOHkDRUjvxF3XCO6HOSKGsg0PWEP3calILv3q1h8CAwEAAaOB rDCBqTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU F6DNweRBtjpbO8tFnb0cwpj6hlgwHwYDVR0jBBgwFoAUF6DNweRBtjpbO8tFnb0c wpj6hlgwRgYDVR0gBD8wPTA7BglghXQBWQEDAQEwLjAsBggrBgEFBQcCARYgaHR0 cDovL3JlcG9zaXRvcnkuc3dpc3NzaWduLmNvbS8wDQYJKoZIhvcNAQEFBQADggIB AHPGgeAn0i0P4JUw4ppBf1AsX19iYamGamkYDHRJ1l2E6kFSGG9YrVBWIGrGvShp WJHckRE1qTodvBqlYJ7YH39FkWnZfrt4csEGDyrOj4VwYaygzQu4OSlWhDJOhrs9 xCrZ1x9y7v5RoSJBsXECYxqCsGKrXlcSH9/L3XWgwF15kIwb4FDm3jH+mHtwX6WQ 2K34ArZv02DdQEsixT2tOnqfGhpHkXkzuoLcMmkDlm4fS/Bx/uNncqCxv1yL5PqZ IseEuRuNI5c/7SXgz2W79WEE790eslpBIlqhn10s6FvJbakMDHiqYMZWjwFaDGi8 aRl5xB9+lwW/xekkUV7U1UtT7dkjWjYDZaPBA61BMPNGG4WQr2W11bHkFlt4dR2X em1ZqSqPe97Dh4kQmUlzeMg9vVE1dCrV8X5pGyq7O70luJpaPXJhkGaH7gzWTdQR dAtq/gsD/KNVV4n+SsuuWxcFyPKNIzFTONItaj+CuY0IavdeQXRuwxF+B6wpYJE/ OMpXEA29MC/HpeZBoNquBYeaoKRlbEwJDIm6uNO5wJOKMPqN5ZprFQFOZ6raYlY+ hAhm0sQ2fac+EPyI4NSA5QC9qvNOBqN6avlicuMJT+ubDgEj8Z+7fNzcbBGXJbLy tGMU0gYqZ4yD9c7qB9iaah7s5Aq7KkzrCWA5zspi2C5u -----END CERTIFICATE----- # Issuer: C=CH, O=WISeKey, OU=OISTE Foundation Endorsed, CN=OISTE WISeKey Global Root GB CA # Subject: C=CH, O=WISeKey, OU=OISTE Foundation Endorsed, CN=OISTE WISeKey Global Root GB CA # SHA256 Fingerprint: 6B:9C:08:E8:6E:B0:F7:67:CF:AD:65:CD:98:B6:21:49:E5:49:4A:67:F5:84:5E:7B:D1:ED:01:9F:27:B8:6B:D6 -----BEGIN CERTIFICATE----- MIIDtTCCAp2gAwIBAgIQdrEgUnTwhYdGs/gjGvbCwDANBgkqhkiG9w0BAQsFADBt MQswCQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUg Rm91bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9i YWwgUm9vdCBHQiBDQTAeFw0xNDEyMDExNTAwMzJaFw0zOTEyMDExNTEwMzFaMG0x CzAJBgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBG b3VuZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2Jh bCBSb290IEdCIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2Be3 HEokKtaXscriHvt9OO+Y9bI5mE4nuBFde9IllIiCFSZqGzG7qFshISvYD06fWvGx WuR51jIjK+FTzJlFXHtPrby/h0oLS5daqPZI7H17Dc0hBt+eFf1Biki3IPShehtX 1F1Q/7pn2COZH8g/497/b1t3sWtuuMlk9+HKQUYOKXHQuSP8yYFfTvdv37+ErXNk u7dCjmn21HYdfp2nuFeKUWdy19SouJVUQHMD9ur06/4oQnc/nSMbsrY9gBQHTC5P 99UKFg29ZkM3fiNDecNAhvVMKdqOmq0NpQSHiB6F4+lT1ZvIiwNjeOvgGUpuuy9r M2RYk61pv48b74JIxwIDAQABo1EwTzALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUw AwEB/zAdBgNVHQ4EFgQUNQ/INmNe4qPs+TtmFc5RUuORmj0wEAYJKwYBBAGCNxUB BAMCAQAwDQYJKoZIhvcNAQELBQADggEBAEBM+4eymYGQfp3FsLAmzYh7KzKNbrgh cViXfa43FK8+5/ea4n32cZiZBKpDdHij40lhPnOMTZTg+XHEthYOU3gf1qKHLwI5 gSk8rxWYITD+KJAAjNHhy/peyP34EEY7onhCkRd0VQreUGdNZtGn//3ZwLWoo4rO ZvUPQ82nK1d7Y0Zqqi5S2PTt4W2tKZB4SLrhI6qjiey1q5bAtEuiHZeeevJuQHHf aPFlTc58Bd9TZaml8LGXBHAVRgOY1NK/VLSgWH1Sb9pWJmLU2NuJMW8c8CLC02Ic Nc1MaRVUGpCY3useX8p3x8uOPUNpnJpY0CQ73xtAln41rYHHTnG6iBM= -----END CERTIFICATE----- # Issuer: C=CH, O=WISeKey, OU=OISTE Foundation Endorsed, CN=OISTE WISeKey Global Root GC CA # Subject: C=CH, O=WISeKey, OU=OISTE Foundation Endorsed, CN=OISTE WISeKey Global Root GC CA # SHA256 Fingerprint: 85:60:F9:1C:36:24:DA:BA:95:70:B5:FE:A0:DB:E3:6F:F1:1A:83:23:BE:94:86:85:4F:B3:F3:4A:55:71:19:8D -----BEGIN CERTIFICATE----- MIICaTCCAe+gAwIBAgIQISpWDK7aDKtARb8roi066jAKBggqhkjOPQQDAzBtMQsw CQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUgRm91 bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwg Um9vdCBHQyBDQTAeFw0xNzA1MDkwOTQ4MzRaFw00MjA1MDkwOTU4MzNaMG0xCzAJ BgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBGb3Vu ZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2JhbCBS b290IEdDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAETOlQwMYPchi82PG6s4ni eUqjFqdrVCTbUf/q9Akkwwsin8tqJ4KBDdLArzHkdIJuyiXZjHWd8dvQmqJLIX4W p2OQ0jnUsYd4XxiWD1AbNTcPasbc2RNNpI6QN+a9WzGRo1QwUjAOBgNVHQ8BAf8E BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUSIcUrOPDnpBgOtfKie7T rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV 57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 -----END CERTIFICATE----- # Issuer: C=CN, O=China Financial Certification Authority, CN=CFCA EV ROOT # Subject: C=CN, O=China Financial Certification Authority, CN=CFCA EV ROOT # SHA256 Fingerprint: 5C:C3:D7:8E:4E:1D:5E:45:54:7A:04:E6:87:3E:64:F9:0C:F9:53:6D:1C:CC:2E:F8:00:F3:55:C4:C5:FD:70:FD -----BEGIN CERTIFICATE----- MIIFjTCCA3WgAwIBAgIEGErM1jANBgkqhkiG9w0BAQsFADBWMQswCQYDVQQGEwJD TjEwMC4GA1UECgwnQ2hpbmEgRmluYW5jaWFsIENlcnRpZmljYXRpb24gQXV0aG9y aXR5MRUwEwYDVQQDDAxDRkNBIEVWIFJPT1QwHhcNMTIwODA4MDMwNzAxWhcNMjkx MjMxMDMwNzAxWjBWMQswCQYDVQQGEwJDTjEwMC4GA1UECgwnQ2hpbmEgRmluYW5j aWFsIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MRUwEwYDVQQDDAxDRkNBIEVWIFJP T1QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDXXWvNED8fBVnVBU03 sQ7smCuOFR36k0sXgiFxEFLXUWRwFsJVaU2OFW2fvwwbwuCjZ9YMrM8irq93VCpL TIpTUnrD7i7es3ElweldPe6hL6P3KjzJIx1qqx2hp/Hz7KDVRM8Vz3IvHWOX6Jn5 /ZOkVIBMUtRSqy5J35DNuF++P96hyk0g1CXohClTt7GIH//62pCfCqktQT+x8Rgp 7hZZLDRJGqgG16iI0gNyejLi6mhNbiyWZXvKWfry4t3uMCz7zEasxGPrb382KzRz EpR/38wmnvFyXVBlWY9ps4deMm/DGIq1lY+wejfeWkU7xzbh72fROdOXW3NiGUgt hxwG+3SYIElz8AXSG7Ggo7cbcNOIabla1jj0Ytwli3i/+Oh+uFzJlU9fpy25IGvP a931DfSCt/SyZi4QKPaXWnuWFo8BGS1sbn85WAZkgwGDg8NNkt0yxoekN+kWzqot aK8KgWU6cMGbrU1tVMoqLUuFG7OA5nBFDWteNfB/O7ic5ARwiRIlk9oKmSJgamNg TnYGmE69g60dWIolhdLHZR4tjsbftsbhf4oEIRUpdPA+nJCdDC7xij5aqgwJHsfV PKPtl8MeNPo4+QgO48BdK4PRVmrJtqhUUy54Mmc9gn900PvhtgVguXDbjgv5E1hv cWAQUhC5wUEJ73IfZzF4/5YFjQIDAQABo2MwYTAfBgNVHSMEGDAWgBTj/i39KNAL tbq2osS/BqoFjJP7LzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAd BgNVHQ4EFgQU4/4t/SjQC7W6tqLEvwaqBYyT+y8wDQYJKoZIhvcNAQELBQADggIB ACXGumvrh8vegjmWPfBEp2uEcwPenStPuiB/vHiyz5ewG5zz13ku9Ui20vsXiObT ej/tUxPQ4i9qecsAIyjmHjdXNYmEwnZPNDatZ8POQQaIxffu2Bq41gt/UP+TqhdL jOztUmCypAbqTuv0axn96/Ua4CUqmtzHQTb3yHQFhDmVOdYLO6Qn+gjYXB74BGBS ESgoA//vU2YApUo0FmZ8/Qmkrp5nGm9BC2sGE5uPhnEFtC+NiWYzKXZUmhH4J/qy P5Hgzg0b8zAarb8iXRvTvyUFTeGSGn+ZnzxEk8rUQElsgIfXBDrDMlI1Dlb4pd19 xIsNER9Tyx6yF7Zod1rg1MvIB671Oi6ON7fQAUtDKXeMOZePglr4UeWJoBjnaH9d Ci77o0cOPaYjesYBx4/IXr9tgFa+iiS6M+qf4TIRnvHST4D2G0CvOJ4RUHlzEhLN 5mydLIhyPDCBBpEi6lmt2hkuIsKNuYyH4Ga8cyNfIWRjgEj1oDwYPZTISEEdQLpe /v5WOaHIz16eGWRGENoXkbcFgKyLmZJ956LYBws2J+dIeWCKw9cTXPhyQN9Ky8+Z AAoACxGV2lZFA4gKn2fQ1XmxqI1AbQ3CekD6819kR5LLU7m7Wc5P/dAVUwHY3+vZ 5nbv0CO7O6l5s9UCKc2Jo5YPSjXnTkLAdc0Hz+Ys63su -----END CERTIFICATE----- # Issuer: C=CN, O=GUANG DONG CERTIFICATE AUTHORITY CO.,LTD., CN=GDCA TrustAUTH R5 ROOT # Subject: C=CN, O=GUANG DONG CERTIFICATE AUTHORITY CO.,LTD., CN=GDCA TrustAUTH R5 ROOT # SHA256 Fingerprint: BF:FF:8F:D0:44:33:48:7D:6A:8A:A6:0C:1A:29:76:7A:9F:C2:BB:B0:5E:42:0F:71:3A:13:B9:92:89:1D:38:93 -----BEGIN CERTIFICATE----- MIIFiDCCA3CgAwIBAgIIfQmX/vBH6nowDQYJKoZIhvcNAQELBQAwYjELMAkGA1UE BhMCQ04xMjAwBgNVBAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZ IENPLixMVEQuMR8wHQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMB4XDTE0 MTEyNjA1MTMxNVoXDTQwMTIzMTE1NTk1OVowYjELMAkGA1UEBhMCQ04xMjAwBgNV BAoMKUdVQU5HIERPTkcgQ0VSVElGSUNBVEUgQVVUSE9SSVRZIENPLixMVEQuMR8w HQYDVQQDDBZHRENBIFRydXN0QVVUSCBSNSBST09UMIICIjANBgkqhkiG9w0BAQEF AAOCAg8AMIICCgKCAgEA2aMW8Mh0dHeb7zMNOwZ+Vfy1YI92hhJCfVZmPoiC7XJj Dp6L3TQsAlFRwxn9WVSEyfFrs0yw6ehGXTjGoqcuEVe6ghWinI9tsJlKCvLriXBj TnnEt1u9ol2x8kECK62pOqPseQrsXzrj/e+APK00mxqriCZ7VqKChh/rNYmDf1+u KU49tm7srsHwJ5uu4/Ts765/94Y9cnrrpftZTqfrlYwiOXnhLQiPzLyRuEH3FMEj qcOtmkVEs7LXLM3GKeJQEK5cy4KOFxg2fZfmiJqwTTQJ9Cy5WmYqsBebnh52nUpm MUHfP/vFBu8btn4aRjb3ZGM74zkYI+dndRTVdVeSN72+ahsmUPI2JgaQxXABZG12 ZuGR224HwGGALrIuL4xwp9E7PLOR5G62xDtw8mySlwnNR30YwPO7ng/Wi64HtloP zgsMR6flPri9fcebNaBhlzpBdRfMK5Z3KpIhHtmVdiBnaM8Nvd/WHwlqmuLMc3Gk L30SgLdTMEZeS1SZD2fJpcjyIMGC7J0R38IC+xo70e0gmu9lZJIQDSri3nDxGGeC jGHeuLzRL5z7D9Ar7Rt2ueQ5Vfj4oR24qoAATILnsn8JuLwwoC8N9VKejveSswoA HQBUlwbgsQfZxw9cZX08bVlX5O2ljelAU58VS6Bx9hoh49pwBiFYFIeFd3mqgnkC AwEAAaNCMEAwHQYDVR0OBBYEFOLJQJ9NzuiaoXzPDj9lxSmIahlRMA8GA1UdEwEB /wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQDRSVfg p8xoWLoBDysZzY2wYUWsEe1jUGn4H3++Fo/9nesLqjJHdtJnJO29fDMylyrHBYZm DRd9FBUb1Ov9H5r2XpdptxolpAqzkT9fNqyL7FeoPueBihhXOYV0GkLH6VsTX4/5 COmSdI31R9KrO9b7eGZONn356ZLpBN79SWP8bfsUcZNnL0dKt7n/HipzcEYwv1ry L3ml4Y0M2fmyYzeMN2WFcGpcWwlyua1jPLHd+PwyvzeG5LuOmCd+uh8W4XAR8gPf JWIyJyYYMoSf/wA6E7qaTfRPuBRwIrHKK5DOKcFw9C+df/KQHtZa37dG/OaG+svg IHZ6uqbL9XzeYqWxi+7egmaKTjowHz+Ay60nugxe19CxVsp3cbK1daFQqUBDF8Io 2c9Si1vIY9RCPqAzekYu9wogRlR+ak8x8YF+QnQ4ZXMn7sZ8uI7XpTrXmKGcjBBV 09tL7ECQ8s1uV9JiDnxXk7Gnbc2dg7sq5+W2O3FYrf3RRbxake5TFW/TRQl1brqQ XR4EzzffHqhmsYzmIGrv/EhOdJhCrylvLmrH+33RZjEizIYAfmaDDEL0vTSSwxrq T8p+ck0LcIymSLumoRT2+1hEmRSuqguTaaApJUqlyyvdimYHFngVV3Eb7PVHhPOe MTd61X8kreS8/f3MboPoDKi3QWwH3b08hpcv0g== -----END CERTIFICATE----- # Issuer: C=CN, O=UniTrust, CN=UCA Extended Validation Root # Subject: C=CN, O=UniTrust, CN=UCA Extended Validation Root # SHA256 Fingerprint: D4:3A:F9:B3:54:73:75:5C:96:84:FC:06:D7:D8:CB:70:EE:5C:28:E7:73:FB:29:4E:B4:1E:E7:17:22:92:4D:24 -----BEGIN CERTIFICATE----- MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs 1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp 4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj 2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax -----END CERTIFICATE----- # Issuer: C=CN, O=UniTrust, CN=UCA Global G2 Root # Subject: C=CN, O=UniTrust, CN=UCA Global G2 Root # SHA256 Fingerprint: 9B:EA:11:C9:76:FE:01:47:64:C1:BE:56:A6:F9:14:B5:A5:60:31:7A:BD:99:88:39:33:82:E5:16:1A:A0:49:3C -----BEGIN CERTIFICATE----- MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl 1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI +Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX UB+K+wb1whnw0A== -----END CERTIFICATE----- # Issuer: C=CN, O=iTrusChina Co.,Ltd., CN=vTrus ECC Root CA # Subject: C=CN, O=iTrusChina Co.,Ltd., CN=vTrus ECC Root CA # SHA256 Fingerprint: 30:FB:BA:2C:32:23:8E:2A:98:54:7A:F9:79:31:E5:50:42:8B:9B:3F:1C:8E:EB:66:33:DC:FA:86:C5:B2:7D:D3 -----BEGIN CERTIFICATE----- MIICDzCCAZWgAwIBAgIUbmq8WapTvpg5Z6LSa6Q75m0c1towCgYIKoZIzj0EAwMw RzELMAkGA1UEBhMCQ04xHDAaBgNVBAoTE2lUcnVzQ2hpbmEgQ28uLEx0ZC4xGjAY BgNVBAMTEXZUcnVzIEVDQyBSb290IENBMB4XDTE4MDczMTA3MjY0NFoXDTQzMDcz MTA3MjY0NFowRzELMAkGA1UEBhMCQ04xHDAaBgNVBAoTE2lUcnVzQ2hpbmEgQ28u LEx0ZC4xGjAYBgNVBAMTEXZUcnVzIEVDQyBSb290IENBMHYwEAYHKoZIzj0CAQYF K4EEACIDYgAEZVBKrox5lkqqHAjDo6LN/llWQXf9JpRCux3NCNtzslt188+cToL0 v/hhJoVs1oVbcnDS/dtitN9Ti72xRFhiQgnH+n9bEOf+QP3A2MMrMudwpremIFUd e4BdS49nTPEQo0IwQDAdBgNVHQ4EFgQUmDnNvtiyjPeyq+GtJK97fKHbH88wDwYD VR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwCgYIKoZIzj0EAwMDaAAwZQIw V53dVvHH4+m4SVBrm2nDb+zDfSXkV5UTQJtS0zvzQBm8JsctBp61ezaf9SXUY2sA AjEA6dPGnlaaKsyh2j/IZivTWJwghfqrkYpwcBE4YGQLYgmRWAD5Tfs0aNoJrSEG GJTO -----END CERTIFICATE----- # Issuer: C=CN, O=iTrusChina Co.,Ltd., CN=vTrus Root CA # Subject: C=CN, O=iTrusChina Co.,Ltd., CN=vTrus Root CA # SHA256 Fingerprint: 8A:71:DE:65:59:33:6F:42:6C:26:E5:38:80:D0:0D:88:A1:8D:A4:C6:A9:1F:0D:CB:61:94:E2:06:C5:C9:63:87 -----BEGIN CERTIFICATE----- MIIFVjCCAz6gAwIBAgIUQ+NxE9izWRRdt86M/TX9b7wFjUUwDQYJKoZIhvcNAQEL BQAwQzELMAkGA1UEBhMCQ04xHDAaBgNVBAoTE2lUcnVzQ2hpbmEgQ28uLEx0ZC4x FjAUBgNVBAMTDXZUcnVzIFJvb3QgQ0EwHhcNMTgwNzMxMDcyNDA1WhcNNDMwNzMx MDcyNDA1WjBDMQswCQYDVQQGEwJDTjEcMBoGA1UEChMTaVRydXNDaGluYSBDby4s THRkLjEWMBQGA1UEAxMNdlRydXMgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEBBQAD ggIPADCCAgoCggIBAL1VfGHTuB0EYgWgrmy3cLRB6ksDXhA/kFocizuwZotsSKYc IrrVQJLuM7IjWcmOvFjai57QGfIvWcaMY1q6n6MLsLOaXLoRuBLpDLvPbmyAhykU AyyNJJrIZIO1aqwTLDPxn9wsYTwaP3BVm60AUn/PBLn+NvqcwBauYv6WTEN+VRS+ GrPSbcKvdmaVayqwlHeFXgQPYh1jdfdr58tbmnDsPmcF8P4HCIDPKNsFxhQnL4Z9 8Cfe/+Z+M0jnCx5Y0ScrUw5XSmXX+6KAYPxMvDVTAWqXcoKv8R1w6Jz1717CbMdH flqUhSZNO7rrTOiwCcJlwp2dCZtOtZcFrPUGoPc2BX70kLJrxLT5ZOrpGgrIDajt J8nU57O5q4IikCc9Kuh8kO+8T/3iCiSn3mUkpF3qwHYw03dQ+A0Em5Q2AXPKBlim 0zvc+gRGE1WKyURHuFE5Gi7oNOJ5y1lKCn+8pu8fA2dqWSslYpPZUxlmPCdiKYZN pGvu/9ROutW04o5IWgAZCfEF2c6Rsffr6TlP9m8EQ5pV9T4FFL2/s1m02I4zhKOQ UqqzApVg+QxMaPnu1RcN+HFXtSXkKe5lXa/R7jwXC1pDxaWG6iSe4gUH3DRCEpHW OXSuTEGC2/KmSNGzm/MzqvOmwMVO9fSddmPmAsYiS8GVP1BkLFTltvA8Kc9XAgMB AAGjQjBAMB0GA1UdDgQWBBRUYnBj8XWEQ1iO0RYgscasGrz2iTAPBgNVHRMBAf8E BTADAQH/MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAKbqSSaet 8PFww+SX8J+pJdVrnjT+5hpk9jprUrIQeBqfTNqK2uwcN1LgQkv7bHbKJAs5EhWd nxEt/Hlk3ODg9d3gV8mlsnZwUKT+twpw1aA08XXXTUm6EdGz2OyC/+sOxL9kLX1j bhd47F18iMjrjld22VkE+rxSH0Ws8HqA7Oxvdq6R2xCOBNyS36D25q5J08FsEhvM Kar5CKXiNxTKsbhm7xqC5PD48acWabfbqWE8n/Uxy+QARsIvdLGx14HuqCaVvIiv TDUHKgLKeBRtRytAVunLKmChZwOgzoy8sHJnxDHO2zTlJQNgJXtxmOTAGytfdELS S8VZCAeHvsXDf+eW2eHcKJfWjwXj9ZtOyh1QRwVTsMo554WgicEFOwE30z9J4nfr I8iIZjs9OXYhRvHsXyO466JmdXTBQPfYaJqT4i2pLr0cox7IdMakLXogqzu4sEb9 b91fUlV1YvCXoHzXOP0l382gmxDPi7g4Xl7FtKYCNqEeXxzP4padKar9mK5S4fNB UvupLnKWnyfjqnN9+BojZns7q2WwMgFLFT49ok8MKzWixtlnEjUwzXYuFrOZnk1P Ti07NEPhmg4NpGaXutIcSkwsKouLgU9xGqndXHt7CMUADTdA43x7VF8vhV929ven sBxXVsFy6K2ir40zSbofitzmdHxghm+Hl3s= -----END CERTIFICATE----- # Issuer: C=DE, O=D-Trust GmbH, CN=D-TRUST BR Root CA 1 2020 # Subject: C=DE, O=D-Trust GmbH, CN=D-TRUST BR Root CA 1 2020 # SHA256 Fingerprint: E5:9A:AA:81:60:09:C2:2B:FF:5B:25:BA:D3:7D:F3:06:F0:49:79:7C:1F:81:D8:5A:B0:89:E6:57:BD:8F:00:44 -----BEGIN CERTIFICATE----- MIIC2zCCAmCgAwIBAgIQfMmPK4TX3+oPyWWa00tNljAKBggqhkjOPQQDAzBIMQsw CQYDVQQGEwJERTEVMBMGA1UEChMMRC1UcnVzdCBHbWJIMSIwIAYDVQQDExlELVRS VVNUIEJSIFJvb3QgQ0EgMSAyMDIwMB4XDTIwMDIxMTA5NDUwMFoXDTM1MDIxMTA5 NDQ1OVowSDELMAkGA1UEBhMCREUxFTATBgNVBAoTDEQtVHJ1c3QgR21iSDEiMCAG A1UEAxMZRC1UUlVTVCBCUiBSb290IENBIDEgMjAyMDB2MBAGByqGSM49AgEGBSuB BAAiA2IABMbLxyjR+4T1mu9CFCDhQ2tuda38KwOE1HaTJddZO0Flax7mNCq7dPYS zuht56vkPE4/RAiLzRZxy7+SmfSk1zxQVFKQhYN4lGdnoxwJGT11NIXe7WB9xwy0 QVK5buXuQqOCAQ0wggEJMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFHOREKv/ VbNafAkl1bK6CKBrqx9tMA4GA1UdDwEB/wQEAwIBBjCBxgYDVR0fBIG+MIG7MD6g PKA6hjhodHRwOi8vY3JsLmQtdHJ1c3QubmV0L2NybC9kLXRydXN0X2JyX3Jvb3Rf Y2FfMV8yMDIwLmNybDB5oHegdYZzbGRhcDovL2RpcmVjdG9yeS5kLXRydXN0Lm5l dC9DTj1ELVRSVVNUJTIwQlIlMjBSb290JTIwQ0ElMjAxJTIwMjAyMCxPPUQtVHJ1 c3QlMjBHbWJILEM9REU/Y2VydGlmaWNhdGVyZXZvY2F0aW9ubGlzdDAKBggqhkjO PQQDAwNpADBmAjEAlJAtE/rhY/hhY+ithXhUkZy4kzg+GkHaQBZTQgjKL47xPoFW wKrY7RjEsK70PvomAjEA8yjixtsrmfu3Ubgko6SUeho/5jbiA1czijDLgsfWFBHV dWNbFJWcHwHP2NVypw87 -----END CERTIFICATE----- # Issuer: C=DE, O=D-Trust GmbH, CN=D-TRUST EV Root CA 1 2020 # Subject: C=DE, O=D-Trust GmbH, CN=D-TRUST EV Root CA 1 2020 # SHA256 Fingerprint: 08:17:0D:1A:A3:64:53:90:1A:2F:95:92:45:E3:47:DB:0C:8D:37:AB:AA:BC:56:B8:1A:A1:00:DC:95:89:70:DB -----BEGIN CERTIFICATE----- MIIC2zCCAmCgAwIBAgIQXwJB13qHfEwDo6yWjfv/0DAKBggqhkjOPQQDAzBIMQsw CQYDVQQGEwJERTEVMBMGA1UEChMMRC1UcnVzdCBHbWJIMSIwIAYDVQQDExlELVRS VVNUIEVWIFJvb3QgQ0EgMSAyMDIwMB4XDTIwMDIxMTEwMDAwMFoXDTM1MDIxMTA5 NTk1OVowSDELMAkGA1UEBhMCREUxFTATBgNVBAoTDEQtVHJ1c3QgR21iSDEiMCAG A1UEAxMZRC1UUlVTVCBFViBSb290IENBIDEgMjAyMDB2MBAGByqGSM49AgEGBSuB BAAiA2IABPEL3YZDIBnfl4XoIkqbz52Yv7QFJsnL46bSj8WeeHsxiamJrSc8ZRCC /N/DnU7wMyPE0jL1HLDfMxddxfCxivnvubcUyilKwg+pf3VlSSowZ/Rk99Yad9rD wpdhQntJraOCAQ0wggEJMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFH8QARY3 OqQo5FD4pPfsazK2/umLMA4GA1UdDwEB/wQEAwIBBjCBxgYDVR0fBIG+MIG7MD6g PKA6hjhodHRwOi8vY3JsLmQtdHJ1c3QubmV0L2NybC9kLXRydXN0X2V2X3Jvb3Rf Y2FfMV8yMDIwLmNybDB5oHegdYZzbGRhcDovL2RpcmVjdG9yeS5kLXRydXN0Lm5l dC9DTj1ELVRSVVNUJTIwRVYlMjBSb290JTIwQ0ElMjAxJTIwMjAyMCxPPUQtVHJ1 c3QlMjBHbWJILEM9REU/Y2VydGlmaWNhdGVyZXZvY2F0aW9ubGlzdDAKBggqhkjO PQQDAwNpADBmAjEAyjzGKnXCXnViOTYAYFqLwZOZzNnbQTs7h5kXO9XMT8oi96CA y/m0sRtW9XLS/BnRAjEAkfcwkz8QRitxpNA7RJvAKQIFskF3UfN5Wp6OFKBOQtJb gfM0agPnIjhQW+0ZT0MW -----END CERTIFICATE----- # Issuer: C=DE, O=D-Trust GmbH, CN=D-TRUST Root Class 3 CA 2 2009 # Subject: C=DE, O=D-Trust GmbH, CN=D-TRUST Root Class 3 CA 2 2009 # SHA256 Fingerprint: 49:E7:A4:42:AC:F0:EA:62:87:05:00:54:B5:25:64:B6:50:E4:F4:9E:42:E3:48:D6:AA:38:E0:39:E9:57:B1:C1 -----BEGIN CERTIFICATE----- MIIEMzCCAxugAwIBAgIDCYPzMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNVBAYTAkRF MRUwEwYDVQQKDAxELVRydXN0IEdtYkgxJzAlBgNVBAMMHkQtVFJVU1QgUm9vdCBD bGFzcyAzIENBIDIgMjAwOTAeFw0wOTExMDUwODM1NThaFw0yOTExMDUwODM1NTha ME0xCzAJBgNVBAYTAkRFMRUwEwYDVQQKDAxELVRydXN0IEdtYkgxJzAlBgNVBAMM HkQtVFJVU1QgUm9vdCBDbGFzcyAzIENBIDIgMjAwOTCCASIwDQYJKoZIhvcNAQEB BQADggEPADCCAQoCggEBANOySs96R+91myP6Oi/WUEWJNTrGa9v+2wBoqOADER03 UAifTUpolDWzU9GUY6cgVq/eUXjsKj3zSEhQPgrfRlWLJ23DEE0NkVJD2IfgXU42 tSHKXzlABF9bfsyjxiupQB7ZNoTWSPOSHjRGICTBpFGOShrvUD9pXRl/RcPHAY9R ySPocq60vFYJfxLLHLGvKZAKyVXMD9O0Gu1HNVpK7ZxzBCHQqr0ME7UAyiZsxGsM lFqVlNpQmvH/pStmMaTJOKDfHR+4CS7zp+hnUquVH+BGPtikw8paxTGA6Eian5Rp /hnd2HN8gcqW3o7tszIFZYQ05ub9VxC1X3a/L7AQDcUCAwEAAaOCARowggEWMA8G A1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFP3aFMSfMN4hvR5COfyrYyNJ4PGEMA4G A1UdDwEB/wQEAwIBBjCB0wYDVR0fBIHLMIHIMIGAoH6gfIZ6bGRhcDovL2RpcmVj dG9yeS5kLXRydXN0Lm5ldC9DTj1ELVRSVVNUJTIwUm9vdCUyMENsYXNzJTIwMyUy MENBJTIwMiUyMDIwMDksTz1ELVRydXN0JTIwR21iSCxDPURFP2NlcnRpZmljYXRl cmV2b2NhdGlvbmxpc3QwQ6BBoD+GPWh0dHA6Ly93d3cuZC10cnVzdC5uZXQvY3Js L2QtdHJ1c3Rfcm9vdF9jbGFzc18zX2NhXzJfMjAwOS5jcmwwDQYJKoZIhvcNAQEL BQADggEBAH+X2zDI36ScfSF6gHDOFBJpiBSVYEQBrLLpME+bUMJm2H6NMLVwMeni acfzcNsgFYbQDfC+rAF1hM5+n02/t2A7nPPKHeJeaNijnZflQGDSNiH+0LS4F9p0 o3/U37CYAqxva2ssJSRyoWXuJVrl5jLn8t+rSfrzkGkj2wTZ51xY/GXUl77M/C4K zCUqNQT4YJEVdT1B/yMfGchs64JTBKbkTCJNjYy6zltz7GRUUG3RnFX7acM2w4y8 PIWmawomDeCTmGCufsYkl4phX5GOZpIJhzbNi5stPvZR1FDUWSi9g/LMKHtThm3Y Johw1+qRzT65ysCQblrGXnRl11z+o+I= -----END CERTIFICATE----- # Issuer: C=DE, O=D-Trust GmbH, CN=D-TRUST Root Class 3 CA 2 EV 2009 # Subject: C=DE, O=D-Trust GmbH, CN=D-TRUST Root Class 3 CA 2 EV 2009 # SHA256 Fingerprint: EE:C5:49:6B:98:8C:E9:86:25:B9:34:09:2E:EC:29:08:BE:D0:B0:F3:16:C2:D4:73:0C:84:EA:F1:F3:D3:48:81 -----BEGIN CERTIFICATE----- MIIEQzCCAyugAwIBAgIDCYP0MA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNVBAYTAkRF MRUwEwYDVQQKDAxELVRydXN0IEdtYkgxKjAoBgNVBAMMIUQtVFJVU1QgUm9vdCBD bGFzcyAzIENBIDIgRVYgMjAwOTAeFw0wOTExMDUwODUwNDZaFw0yOTExMDUwODUw NDZaMFAxCzAJBgNVBAYTAkRFMRUwEwYDVQQKDAxELVRydXN0IEdtYkgxKjAoBgNV BAMMIUQtVFJVU1QgUm9vdCBDbGFzcyAzIENBIDIgRVYgMjAwOTCCASIwDQYJKoZI hvcNAQEBBQADggEPADCCAQoCggEBAJnxhDRwui+3MKCOvXwEz75ivJn9gpfSegpn ljgJ9hBOlSJzmY3aFS3nBfwZcyK3jpgAvDw9rKFs+9Z5JUut8Mxk2og+KbgPCdM0 3TP1YtHhzRnp7hhPTFiu4h7WDFsVWtg6uMQYZB7jM7K1iXdODL/ZlGsTl28So/6Z qQTMFexgaDbtCHu39b+T7WYxg4zGcTSHThfqr4uRjRxWQa4iN1438h3Z0S0NL2lR p75mpoo6Kr3HGrHhFPC+Oh25z1uxav60sUYgovseO3Dvk5h9jHOW8sXvhXCtKSb8 HgQ+HKDYD8tSg2J87otTlZCpV6LqYQXY+U3EJ/pure3511H3a6UCAwEAAaOCASQw ggEgMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNOUikxiEyoZLsyvcop9Ntea HNxnMA4GA1UdDwEB/wQEAwIBBjCB3QYDVR0fBIHVMIHSMIGHoIGEoIGBhn9sZGFw Oi8vZGlyZWN0b3J5LmQtdHJ1c3QubmV0L0NOPUQtVFJVU1QlMjBSb290JTIwQ2xh c3MlMjAzJTIwQ0ElMjAyJTIwRVYlMjAyMDA5LE89RC1UcnVzdCUyMEdtYkgsQz1E RT9jZXJ0aWZpY2F0ZXJldm9jYXRpb25saXN0MEagRKBChkBodHRwOi8vd3d3LmQt dHJ1c3QubmV0L2NybC9kLXRydXN0X3Jvb3RfY2xhc3NfM19jYV8yX2V2XzIwMDku Y3JsMA0GCSqGSIb3DQEBCwUAA4IBAQA07XtaPKSUiO8aEXUHL7P+PPoeUSbrh/Yp 3uDx1MYkCenBz1UbtDDZzhr+BlGmFaQt77JLvyAoJUnRpjZ3NOhk31KxEcdzes05 nsKtjHEh8lprr988TlWvsoRlFIm5d8sqMb7Po23Pb0iUMkZv53GMoKaEGTcH8gNF CSuGdXzfX2lXANtu2KZyIktQ1HWYVt+3GP9DQ1CuekR78HlR10M9p9OB0/DJT7na xpeG0ILD5EJt/rDiZE4OJudANCa1CInXCGNjOCd1HjPqbqjdn5lPdE2BiYBL3ZqX KVwvvoFBuYz/6n1gBp7N1z3TLqMVvKjmJuVvw9y4AyHqnxbxLFS1 -----END CERTIFICATE----- # Issuer: C=DE, O=T-Systems Enterprise Services GmbH, OU=T-Systems Trust Center, CN=T-TeleSec GlobalRoot Class 2 # Subject: C=DE, O=T-Systems Enterprise Services GmbH, OU=T-Systems Trust Center, CN=T-TeleSec GlobalRoot Class 2 # SHA256 Fingerprint: 91:E2:F5:78:8D:58:10:EB:A7:BA:58:73:7D:E1:54:8A:8E:CA:CD:01:45:98:BC:0B:14:3E:04:1B:17:05:25:52 -----BEGIN CERTIFICATE----- MIIDwzCCAqugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMCREUx KzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnByaXNlIFNlcnZpY2VzIEdtYkgxHzAd BgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50ZXIxJTAjBgNVBAMMHFQtVGVsZVNl YyBHbG9iYWxSb290IENsYXNzIDIwHhcNMDgxMDAxMTA0MDE0WhcNMzMxMDAxMjM1 OTU5WjCBgjELMAkGA1UEBhMCREUxKzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnBy aXNlIFNlcnZpY2VzIEdtYkgxHzAdBgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50 ZXIxJTAjBgNVBAMMHFQtVGVsZVNlYyBHbG9iYWxSb290IENsYXNzIDIwggEiMA0G CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCqX9obX+hzkeXaXPSi5kfl82hVYAUd AqSzm1nzHoqvNK38DcLZSBnuaY/JIPwhqgcZ7bBcrGXHX+0CfHt8LRvWurmAwhiC FoT6ZrAIxlQjgeTNuUk/9k9uN0goOA/FvudocP05l03Sx5iRUKrERLMjfTlH6VJi 1hKTXrcxlkIF+3anHqP1wvzpesVsqXFP6st4vGCvx9702cu+fjOlbpSD8DT6Iavq jnKgP6TeMFvvhk1qlVtDRKgQFRzlAVfFmPHmBiiRqiDFt1MmUUOyCxGVWOHAD3bZ wI18gfNycJ5v/hqO2V81xrJvNHy+SE/iWjnX2J14np+GPgNeGYtEotXHAgMBAAGj QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBS/ WSA2AHmgoCJrjNXyYdK4LMuCSjANBgkqhkiG9w0BAQsFAAOCAQEAMQOiYQsfdOhy NsZt+U2e+iKo4YFWz827n+qrkRk4r6p8FU3ztqONpfSO9kSpp+ghla0+AGIWiPAC uvxhI+YzmzB6azZie60EI4RYZeLbK4rnJVM3YlNfvNoBYimipidx5joifsFvHZVw IEoHNN/q/xWA5brXethbdXwFeilHfkCoMRN3zUA7tFFHei4R40cR3p1m0IvVVGb6 g1XqfMIpiRvpb7PO4gWEyS8+eIVibslfwXhjdFjASBgMmTnrpMwatXlajRWc2BQN 9noHV8cigwUtPJslJj0Ys6lDfMjIq2SPDqO/nBudMNva0Bkuqjzx+zOAduTNrRlP BSeOE6Fuwg== -----END CERTIFICATE----- # Issuer: C=DE, O=T-Systems Enterprise Services GmbH, OU=T-Systems Trust Center, CN=T-TeleSec GlobalRoot Class 3 # Subject: C=DE, O=T-Systems Enterprise Services GmbH, OU=T-Systems Trust Center, CN=T-TeleSec GlobalRoot Class 3 # SHA256 Fingerprint: FD:73:DA:D3:1C:64:4F:F1:B4:3B:EF:0C:CD:DA:96:71:0B:9C:D9:87:5E:CA:7E:31:70:7A:F3:E9:6D:52:2B:BD -----BEGIN CERTIFICATE----- MIIDwzCCAqugAwIBAgIBATANBgkqhkiG9w0BAQsFADCBgjELMAkGA1UEBhMCREUx KzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnByaXNlIFNlcnZpY2VzIEdtYkgxHzAd BgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50ZXIxJTAjBgNVBAMMHFQtVGVsZVNl YyBHbG9iYWxSb290IENsYXNzIDMwHhcNMDgxMDAxMTAyOTU2WhcNMzMxMDAxMjM1 OTU5WjCBgjELMAkGA1UEBhMCREUxKzApBgNVBAoMIlQtU3lzdGVtcyBFbnRlcnBy aXNlIFNlcnZpY2VzIEdtYkgxHzAdBgNVBAsMFlQtU3lzdGVtcyBUcnVzdCBDZW50 ZXIxJTAjBgNVBAMMHFQtVGVsZVNlYyBHbG9iYWxSb290IENsYXNzIDMwggEiMA0G CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9dZPwYiJvJK7genasfb3ZJNW4t/zN 8ELg63iIVl6bmlQdTQyK9tPPcPRStdiTBONGhnFBSivwKixVA9ZIw+A5OO3yXDw/ RLyTPWGrTs0NvvAgJ1gORH8EGoel15YUNpDQSXuhdfsaa3Ox+M6pCSzyU9XDFES4 hqX2iys52qMzVNn6chr3IhUciJFrf2blw2qAsCTz34ZFiP0Zf3WHHx+xGwpzJFu5 ZeAsVMhg02YXP+HMVDNzkQI6pn97djmiH5a2OK61yJN0HZ65tOVgnS9W0eDrXltM EnAMbEQgqxHY9Bn20pxSN+f6tsIxO0rUFJmtxxr1XV/6B7h8DR/Wgx6zAgMBAAGj QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBS1 A/d2O2GCahKqGFPrAyGUv/7OyjANBgkqhkiG9w0BAQsFAAOCAQEAVj3vlNW92nOy WL6ukK2YJ5f+AbGwUgC4TeQbIXQbfsDuXmkqJa9c1h3a0nnJ85cp4IaH3gRZD/FZ 1GSFS5mvJQQeyUapl96Cshtwn5z2r3Ex3XsFpSzTucpH9sry9uetuUg/vBa3wW30 6gmv7PO15wWeph6KU1HWk4HMdJP2udqmJQV0eVp+QD6CSyYRMG7hP0HHRwA11fXT 91Q+gT3aSWqas+8QPebrb9HIIkfLzM8BMZLZGOMivgkeGj5asuRrDFR6fUNOuIml e9eiPZaGzPImNC1qkp2aGtAw4l1OBLBfiyB+d8E9lYLRRpo7PHi4b6HQDWSieB4p TpPDpFQUWw== -----END CERTIFICATE----- # Issuer: C=ES, CN=Autoridad de Certificacion Firmaprofesional CIF A62634068 # Subject: C=ES, CN=Autoridad de Certificacion Firmaprofesional CIF A62634068 # SHA256 Fingerprint: 57:DE:05:83:EF:D2:B2:6E:03:61:DA:99:DA:9D:F4:64:8D:EF:7E:E8:44:1C:3B:72:8A:FA:9B:CD:E0:F9:B2:6A -----BEGIN CERTIFICATE----- MIIGFDCCA/ygAwIBAgIIG3Dp0v+ubHEwDQYJKoZIhvcNAQELBQAwUTELMAkGA1UE BhMCRVMxQjBABgNVBAMMOUF1dG9yaWRhZCBkZSBDZXJ0aWZpY2FjaW9uIEZpcm1h cHJvZmVzaW9uYWwgQ0lGIEE2MjYzNDA2ODAeFw0xNDA5MjMxNTIyMDdaFw0zNjA1 MDUxNTIyMDdaMFExCzAJBgNVBAYTAkVTMUIwQAYDVQQDDDlBdXRvcmlkYWQgZGUg Q2VydGlmaWNhY2lvbiBGaXJtYXByb2Zlc2lvbmFsIENJRiBBNjI2MzQwNjgwggIi MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKlmuO6vj78aI14H9M2uDDUtd9 thDIAl6zQyrET2qyyhxdKJp4ERppWVevtSBC5IsP5t9bpgOSL/UR5GLXMnE42QQM cas9UX4PB99jBVzpv5RvwSmCwLTaUbDBPLutN0pcyvFLNg4kq7/DhHf9qFD0sefG L9ItWY16Ck6WaVICqjaY7Pz6FIMMNx/Jkjd/14Et5cS54D40/mf0PmbR0/RAz15i NA9wBj4gGFrO93IbJWyTdBSTo3OxDqqHECNZXyAFGUftaI6SEspd/NYrspI8IM/h X68gvqB2f3bl7BqGYTM+53u0P6APjqK5am+5hyZvQWyIplD9amML9ZMWGxmPsu2b m8mQ9QEM3xk9Dz44I8kvjwzRAv4bVdZO0I08r0+k8/6vKtMFnXkIoctXMbScyJCy Z/QYFpM6/EfY0XiWMR+6KwxfXZmtY4laJCB22N/9q06mIqqdXuYnin1oKaPnirja EbsXLZmdEyRG98Xi2J+Of8ePdG1asuhy9azuJBCtLxTa/y2aRnFHvkLfuwHb9H/T KI8xWVvTyQKmtFLKbpf7Q8UIJm+K9Lv9nyiqDdVF8xM6HdjAeI9BZzwelGSuewvF 6NkBiDkal4ZkQdU7hwxu+g/GvUgUvzlN1J5Bto+WHWOWk9mVBngxaJ43BjuAiUVh OSPHG0SjFeUc+JIwuwIDAQABo4HvMIHsMB0GA1UdDgQWBBRlzeurNR4APn7VdMAc tHNHDhpkLzASBgNVHRMBAf8ECDAGAQH/AgEBMIGmBgNVHSAEgZ4wgZswgZgGBFUd IAAwgY8wLwYIKwYBBQUHAgEWI2h0dHA6Ly93d3cuZmlybWFwcm9mZXNpb25hbC5j b20vY3BzMFwGCCsGAQUFBwICMFAeTgBQAGEAcwBlAG8AIABkAGUAIABsAGEAIABC AG8AbgBhAG4AbwB2AGEAIAA0ADcAIABCAGEAcgBjAGUAbABvAG4AYQAgADAAOAAw ADEANzAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQELBQADggIBAHSHKAIrdx9m iWTtj3QuRhy7qPj4Cx2Dtjqn6EWKB7fgPiDL4QjbEwj4KKE1soCzC1HA01aajTNF Sa9J8OA9B3pFE1r/yJfY0xgsfZb43aJlQ3CTkBW6kN/oGbDbLIpgD7dvlAceHabJ hfa9NPhAeGIQcDq+fUs5gakQ1JZBu/hfHAsdCPKxsIl68veg4MSPi3i1O1ilI45P Vf42O+AMt8oqMEEgtIDNrvx2ZnOorm7hfNoD6JQg5iKj0B+QXSBTFCZX2lSX3xZE EAEeiGaPcjiT3SC3NL7X8e5jjkd5KAb881lFJWAiMxujX6i6KtoaPc1A6ozuBRWV 1aUsIC+nmCjuRfzxuIgALI9C2lHVnOUTaHFFQ4ueCyE8S1wF3BqfmI7avSKecs2t CsvMo2ebKHTEm9caPARYpoKdrcd7b/+Alun4jWq9GJAd/0kakFI3ky88Al2CdgtR 5xbHV/g4+afNmyJU72OwFW1TZQNKXkqgsqeOSQBZONXH9IBk9W6VULgRfhVwOEqw f9DEMnDAGf/JOC0ULGb0QkTmVXYbgBVX/8Cnp6o5qtjTcNAuuuuUavpfNIbnYrX9 ivAwhZTJryQCL2/W3Wf+47BVTwSYT6RBVuKT0Gro1vP7ZeDOdcQxWQzugsgMYDNK GbqEZycPvEJdvSRUDewdcAZfpLz6IHxV -----END CERTIFICATE----- # Issuer: C=ES, O=FNMT-RCM, OU=AC RAIZ FNMT-RCM # Subject: C=ES, O=FNMT-RCM, OU=AC RAIZ FNMT-RCM # SHA256 Fingerprint: EB:C5:57:0C:29:01:8C:4D:67:B1:AA:12:7B:AF:12:F7:03:B4:61:1E:BC:17:B7:DA:B5:57:38:94:17:9B:93:FA -----BEGIN CERTIFICATE----- MIIFgzCCA2ugAwIBAgIPXZONMGc2yAYdGsdUhGkHMA0GCSqGSIb3DQEBCwUAMDsx CzAJBgNVBAYTAkVTMREwDwYDVQQKDAhGTk1ULVJDTTEZMBcGA1UECwwQQUMgUkFJ WiBGTk1ULVJDTTAeFw0wODEwMjkxNTU5NTZaFw0zMDAxMDEwMDAwMDBaMDsxCzAJ BgNVBAYTAkVTMREwDwYDVQQKDAhGTk1ULVJDTTEZMBcGA1UECwwQQUMgUkFJWiBG Tk1ULVJDTTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALpxgHpMhm5/ yBNtwMZ9HACXjywMI7sQmkCpGreHiPibVmr75nuOi5KOpyVdWRHbNi63URcfqQgf BBckWKo3Shjf5TnUV/3XwSyRAZHiItQDwFj8d0fsjz50Q7qsNI1NOHZnjrDIbzAz WHFctPVrbtQBULgTfmxKo0nRIBnuvMApGGWn3v7v3QqQIecaZ5JCEJhfTzC8PhxF tBDXaEAUwED653cXeuYLj2VbPNmaUtu1vZ5Gzz3rkQUCwJaydkxNEJY7kvqcfw+Z 374jNUUeAlz+taibmSXaXvMiwzn15Cou08YfxGyqxRxqAQVKL9LFwag0Jl1mpdIC IfkYtwb1TplvqKtMUejPUBjFd8g5CSxJkjKZqLsXF3mwWsXmo8RZZUc1g16p6DUL mbvkzSDGm0oGObVo/CK67lWMK07q87Hj/LaZmtVC+nFNCM+HHmpxffnTtOmlcYF7 wk5HlqX2doWjKI/pgG6BU6VtX7hI+cL5NqYuSf+4lsKMB7ObiFj86xsc3i1w4peS MKGJ47xVqCfWS+2QrYv6YyVZLag13cqXM7zlzced0ezvXg5KkAYmY6252TUtB7p2 ZSysV4999AeU14ECll2jB0nVetBX+RvnU0Z1qrB5QstocQjpYL05ac70r8NWQMet UqIJ5G+GR4of6ygnXYMgrwTJbFaai0b1AgMBAAGjgYMwgYAwDwYDVR0TAQH/BAUw AwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFPd9xf3E6Jobd2Sn9R2gzL+H YJptMD4GA1UdIAQ3MDUwMwYEVR0gADArMCkGCCsGAQUFBwIBFh1odHRwOi8vd3d3 LmNlcnQuZm5tdC5lcy9kcGNzLzANBgkqhkiG9w0BAQsFAAOCAgEAB5BK3/MjTvDD nFFlm5wioooMhfNzKWtN/gHiqQxjAb8EZ6WdmF/9ARP67Jpi6Yb+tmLSbkyU+8B1 RXxlDPiyN8+sD8+Nb/kZ94/sHvJwnvDKuO+3/3Y3dlv2bojzr2IyIpMNOmqOFGYM LVN0V2Ue1bLdI4E7pWYjJ2cJj+F3qkPNZVEI7VFY/uY5+ctHhKQV8Xa7pO6kO8Rf 77IzlhEYt8llvhjho6Tc+hj507wTmzl6NLrTQfv6MooqtyuGC2mDOL7Nii4LcK2N JpLuHvUBKwrZ1pebbuCoGRw6IYsMHkCtA+fdZn71uSANA+iW+YJF1DngoABd15jm fZ5nc8OaKveri6E6FO80vFIOiZiaBECEHX5FaZNXzuvO+FB8TxxuBEOb+dY7Ixjp 6o7RTUaN8Tvkasq6+yO3m/qZASlaWFot4/nUbQ4mrcFuNLwy+AwF+mWj2zs3gyLp 1txyM/1d8iC9djwj2ij3+RvrWWTV3F9yfiD8zYm1kGdNYno/Tq0dwzn+evQoFt9B 9kiABdcPUXmsEKvU7ANm5mqwujGSQkBqvjrTcuFqN1W8rB2Vt2lh8kORdOag0wok RqEIr9baRRmW1FMdW4R58MD3R++Lj8UGrp1MYp3/RgT408m2ECVAdf4WqslKYIYv uu8wd+RU4riEmViAqhOLUTpPSPaLtrM= -----END CERTIFICATE----- # Issuer: C=ES, O=FNMT-RCM, OU=Ceres, organizationIdentifier=VATES-Q2826004J, CN=AC RAIZ FNMT-RCM SERVIDORES SEGUROS # Subject: C=ES, O=FNMT-RCM, OU=Ceres, organizationIdentifier=VATES-Q2826004J, CN=AC RAIZ FNMT-RCM SERVIDORES SEGUROS # SHA256 Fingerprint: 55:41:53:B1:3D:2C:F9:DD:B7:53:BF:BE:1A:4E:0A:E0:8D:0A:A4:18:70:58:FE:60:A2:B8:62:B2:E4:B8:7B:CB -----BEGIN CERTIFICATE----- MIICbjCCAfOgAwIBAgIQYvYybOXE42hcG2LdnC6dlTAKBggqhkjOPQQDAzB4MQsw CQYDVQQGEwJFUzERMA8GA1UECgwIRk5NVC1SQ00xDjAMBgNVBAsMBUNlcmVzMRgw FgYDVQRhDA9WQVRFUy1RMjgyNjAwNEoxLDAqBgNVBAMMI0FDIFJBSVogRk5NVC1S Q00gU0VSVklET1JFUyBTRUdVUk9TMB4XDTE4MTIyMDA5MzczM1oXDTQzMTIyMDA5 MzczM1oweDELMAkGA1UEBhMCRVMxETAPBgNVBAoMCEZOTVQtUkNNMQ4wDAYDVQQL DAVDZXJlczEYMBYGA1UEYQwPVkFURVMtUTI4MjYwMDRKMSwwKgYDVQQDDCNBQyBS QUlaIEZOTVQtUkNNIFNFUlZJRE9SRVMgU0VHVVJPUzB2MBAGByqGSM49AgEGBSuB BAAiA2IABPa6V1PIyqvfNkpSIeSX0oNnnvBlUdBeh8dHsVnyV0ebAAKTRBdp20LH sbI6GA60XYyzZl2hNPk2LEnb80b8s0RpRBNm/dfF/a82Tc4DTQdxz69qBdKiQ1oK Um8BA06Oi6NCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD VR0OBBYEFAG5L++/EYZg8k/QQW6rcx/n0m5JMAoGCCqGSM49BAMDA2kAMGYCMQCu SuMrQMN0EfKVrRYj3k4MGuZdpSRea0R7/DjiT8ucRRcRTBQnJlU5dUoDzBOQn5IC MQD6SmxgiHPz7riYYqnOK8LZiqZwMR2vsJRM60/G49HzYqc8/5MuB1xJAWdpEgJy v+c= -----END CERTIFICATE----- # Issuer: C=ES, O=IZENPE S.A., CN=Izenpe.com # Subject: C=ES, O=IZENPE S.A., CN=Izenpe.com # SHA256 Fingerprint: 25:30:CC:8E:98:32:15:02:BA:D9:6F:9B:1F:BA:1B:09:9E:2D:29:9E:0F:45:48:BB:91:4F:36:3B:C0:D4:53:1F -----BEGIN CERTIFICATE----- MIIF8TCCA9mgAwIBAgIQALC3WhZIX7/hy/WL1xnmfTANBgkqhkiG9w0BAQsFADA4 MQswCQYDVQQGEwJFUzEUMBIGA1UECgwLSVpFTlBFIFMuQS4xEzARBgNVBAMMCkl6 ZW5wZS5jb20wHhcNMDcxMjEzMTMwODI4WhcNMzcxMjEzMDgyNzI1WjA4MQswCQYD VQQGEwJFUzEUMBIGA1UECgwLSVpFTlBFIFMuQS4xEzARBgNVBAMMCkl6ZW5wZS5j b20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ03rKDx6sp4boFmVq scIbRTJxldn+EFvMr+eleQGPicPK8lVx93e+d5TzcqQsRNiekpsUOqHnJJAKClaO xdgmlOHZSOEtPtoKct2jmRXagaKH9HtuJneJWK3W6wyyQXpzbm3benhB6QiIEn6H LmYRY2xU+zydcsC8Lv/Ct90NduM61/e0aL6i9eOBbsFGb12N4E3GVFWJGjMxCrFX uaOKmMPsOzTFlUFpfnXCPCDFYbpRR6AgkJOhkEvzTnyFRVSa0QUmQbC1TR0zvsQD yCV8wXDbO/QJLVQnSKwv4cSsPsjLkkxTOTcj7NMB+eAJRE1NZMDhDVqHIrytG6P+ JrUV86f8hBnp7KGItERphIPzidF0BqnMC9bC3ieFUCbKF7jJeodWLBoBHmy+E60Q rLUk9TiRodZL2vG70t5HtfG8gfZZa88ZU+mNFctKy6lvROUbQc/hhqfK0GqfvEyN BjNaooXlkDWgYlwWTvDjovoDGrQscbNYLN57C9saD+veIR8GdwYDsMnvmfzAuU8L hij+0rnq49qlw0dpEuDb8PYZi+17cNcC1u2HGCgsBCRMd+RIihrGO5rUD8r6ddIB QFqNeb+Lz0vPqhbBleStTIo+F5HUsWLlguWABKQDfo2/2n+iD5dPDNMN+9fR5XJ+ HMh3/1uaD7euBUbl8agW7EekFwIDAQABo4H2MIHzMIGwBgNVHREEgagwgaWBD2lu Zm9AaXplbnBlLmNvbaSBkTCBjjFHMEUGA1UECgw+SVpFTlBFIFMuQS4gLSBDSUYg QTAxMzM3MjYwLVJNZXJjLlZpdG9yaWEtR2FzdGVpeiBUMTA1NSBGNjIgUzgxQzBB BgNVBAkMOkF2ZGEgZGVsIE1lZGl0ZXJyYW5lbyBFdG9yYmlkZWEgMTQgLSAwMTAx MCBWaXRvcmlhLUdhc3RlaXowDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC AQYwHQYDVR0OBBYEFB0cZQ6o8iV7tJHP5LGx5r1VdGwFMA0GCSqGSIb3DQEBCwUA A4ICAQB4pgwWSp9MiDrAyw6lFn2fuUhfGI8NYjb2zRlrrKvV9pF9rnHzP7MOeIWb laQnIUdCSnxIOvVFfLMMjlF4rJUT3sb9fbgakEyrkgPH7UIBzg/YsfqikuFgba56 awmqxinuaElnMIAkejEWOVt+8Rwu3WwJrfIxwYJOubv5vr8qhT/AQKM6WfxZSzwo JNu0FXWuDYi6LnPAvViH5ULy617uHjAimcs30cQhbIHsvm0m5hzkQiCeR7Csg1lw LDXWrzY0tM07+DKo7+N4ifuNRSzanLh+QBxh5z6ikixL8s36mLYp//Pye6kfLqCT VyvehQP5aTfLnnhqBbTFMXiJ7HqnheG5ezzevh55hM6fcA5ZwjUukCox2eRFekGk LhObNA5me0mrZJfQRsN5nXJQY6aYWwa9SG3YOYNw6DXwBdGqvOPbyALqfP2C2sJb UjWumDqtujWTI6cfSN01RpiyEGjkpTHCClguGYEQyVB1/OpaFs4R1+7vUIgtYf8/ QnMFlEPVjjxOAToZpR9GTnfQXeWBIiGH/pR9hNiTrdZoQ0iy2+tzJOeRf1SktoA+ naM8THLCV8Sg1Mw4J87VBp6iSNnpn86CcDaTmjvfliHjWbcM2pE38P1ZWrOZyGls QyYBNWNgVYkDOnXYukrZVP/u3oDYLdE41V4tC5h9Pmzb/CaIxw== -----END CERTIFICATE----- # Issuer: C=FI, O=Telia Finland Oyj, CN=Telia Root CA v2 # Subject: C=FI, O=Telia Finland Oyj, CN=Telia Root CA v2 # SHA256 Fingerprint: 24:2B:69:74:2F:CB:1E:5B:2A:BF:98:89:8B:94:57:21:87:54:4E:5B:4D:99:11:78:65:73:62:1F:6A:74:B8:2C -----BEGIN CERTIFICATE----- MIIFdDCCA1ygAwIBAgIPAWdfJ9b+euPkrL4JWwWeMA0GCSqGSIb3DQEBCwUAMEQx CzAJBgNVBAYTAkZJMRowGAYDVQQKDBFUZWxpYSBGaW5sYW5kIE95ajEZMBcGA1UE AwwQVGVsaWEgUm9vdCBDQSB2MjAeFw0xODExMjkxMTU1NTRaFw00MzExMjkxMTU1 NTRaMEQxCzAJBgNVBAYTAkZJMRowGAYDVQQKDBFUZWxpYSBGaW5sYW5kIE95ajEZ MBcGA1UEAwwQVGVsaWEgUm9vdCBDQSB2MjCCAiIwDQYJKoZIhvcNAQEBBQADggIP ADCCAgoCggIBALLQPwe84nvQa5n44ndp586dpAO8gm2h/oFlH0wnrI4AuhZ76zBq AMCzdGh+sq/H1WKzej9Qyow2RCRj0jbpDIX2Q3bVTKFgcmfiKDOlyzG4OiIjNLh9 vVYiQJ3q9HsDrWj8soFPmNB06o3lfc1jw6P23pLCWBnglrvFxKk9pXSW/q/5iaq9 lRdU2HhE8Qx3FZLgmEKnpNaqIJLNwaCzlrI6hEKNfdWV5Nbb6WLEWLN5xYzTNTOD n3WhUidhOPFZPY5Q4L15POdslv5e2QJltI5c0BE0312/UqeBAMN/mUWZFdUXyApT 7GPzmX3MaRKGwhfwAZ6/hLzRUssbkmbOpFPlob/E2wnW5olWK8jjfN7j/4nlNW4o 6GwLI1GpJQXrSPjdscr6bAhR77cYbETKJuFzxokGgeWKrLDiKca5JLNrRBH0pUPC TEPlcDaMtjNXepUugqD0XBCzYYP2AgWGLnwtbNwDRm41k9V6lS/eINhbfpSQBGq6 WT0EBXWdN6IOLj3rwaRSg/7Qa9RmjtzG6RJOHSpXqhC8fF6CfaamyfItufUXJ63R DolUK5X6wK0dmBR4M0KGCqlztft0DbcbMBnEWg4cJ7faGND/isgFuvGqHKI3t+ZI pEYslOqodmJHixBTB0hXbOKSTbauBcvcwUpej6w9GU7C7WB1K9vBykLVAgMBAAGj YzBhMB8GA1UdIwQYMBaAFHKs5DN5qkWH9v2sHZ7Wxy+G2CQ5MB0GA1UdDgQWBBRy rOQzeapFh/b9rB2e1scvhtgkOTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUw AwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAoDtZpwmUPjaE0n4vOaWWl/oRrfxn83EJ 8rKJhGdEr7nv7ZbsnGTbMjBvZ5qsfl+yqwE2foH65IRe0qw24GtixX1LDoJt0nZi 0f6X+J8wfBj5tFJ3gh1229MdqfDBmgC9bXXYfef6xzijnHDoRnkDry5023X4blMM A8iZGok1GTzTyVR8qPAs5m4HeW9q4ebqkYJpCh3DflminmtGFZhb069GHWLIzoBS SRE/yQQSwxN8PzuKlts8oB4KtItUsiRnDe+Cy748fdHif64W1lZYudogsYMVoe+K TTJvQS8TUoKU1xrBeKJR3Stwbbca+few4GeXVtt8YVMJAygCQMez2P2ccGrGKMOF 6eLtGpOg3kuYooQ+BXcBlj37tCAPnHICehIv1aO6UXivKitEZU61/Qrowc15h2Er 3oBXRb9n8ZuRXqWk7FlIEA04x7D6w0RtBPV4UBySllva9bguulvP5fBqnUsvWHMt Ty3EHD70sz+rFQ47GUGKpMFXEmZxTPpT41frYpUJnlTd0cI8Vzy9OK2YZLe4A5pT VmBds9hCG1xLEooc6+t9xnppxyd/pPiL8uSUZodL6ZQHCRJ5irLrdATczvREWeAW ysUsWNc8e89ihmpQfTU2Zqf7N+cox9jQraVplI/owd8k+BsHMYeB2F326CjYSlKA rBPuUBQemMc= -----END CERTIFICATE----- # Issuer: C=FR, O=Dhimyotis, CN=Certigna # Subject: C=FR, O=Dhimyotis, CN=Certigna # SHA256 Fingerprint: E3:B6:A2:DB:2E:D7:CE:48:84:2F:7A:C5:32:41:C7:B7:1D:54:14:4B:FB:40:C1:1F:3F:1D:0B:42:F5:EE:A1:2D -----BEGIN CERTIFICATE----- MIIDqDCCApCgAwIBAgIJAP7c4wEPyUj/MA0GCSqGSIb3DQEBBQUAMDQxCzAJBgNV BAYTAkZSMRIwEAYDVQQKDAlEaGlteW90aXMxETAPBgNVBAMMCENlcnRpZ25hMB4X DTA3MDYyOTE1MTMwNVoXDTI3MDYyOTE1MTMwNVowNDELMAkGA1UEBhMCRlIxEjAQ BgNVBAoMCURoaW15b3RpczERMA8GA1UEAwwIQ2VydGlnbmEwggEiMA0GCSqGSIb3 DQEBAQUAA4IBDwAwggEKAoIBAQDIaPHJ1tazNHUmgh7stL7qXOEm7RFHYeGifBZ4 QCHkYJ5ayGPhxLGWkv8YbWkj4Sti993iNi+RB7lIzw7sebYs5zRLcAglozyHGxny gQcPOJAZ0xH+hrTy0V4eHpbNgGzOOzGTtvKg0KmVEn2lmsxryIRWijOp5yIVUxbw zBfsV1/pogqYCd7jX5xv3EjjhQsVWqa6n6xI4wmy9/Qy3l40vhx4XUJbzg4ij02Q 130yGLMLLGq/jj8UEYkgDncUtT2UCIf3JR7VsmAA7G8qKCVuKj4YYxclPz5EIBb2 JsglrgVKtOdjLPOMFlN+XPsRGgjBRmKfIrjxwo1p3Po6WAbfAgMBAAGjgbwwgbkw DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUGu3+QTmQtCRZvgHyUtVF9lo53BEw ZAYDVR0jBF0wW4AUGu3+QTmQtCRZvgHyUtVF9lo53BGhOKQ2MDQxCzAJBgNVBAYT AkZSMRIwEAYDVQQKDAlEaGlteW90aXMxETAPBgNVBAMMCENlcnRpZ25hggkA/tzj AQ/JSP8wDgYDVR0PAQH/BAQDAgEGMBEGCWCGSAGG+EIBAQQEAwIABzANBgkqhkiG 9w0BAQUFAAOCAQEAhQMeknH2Qq/ho2Ge6/PAD/Kl1NqV5ta+aDY9fm4fTIrv0Q8h bV6lUmPOEvjvKtpv6zf+EwLHyzs+ImvaYS5/1HI93TDhHkxAGYwP15zRgzB7mFnc fca5DClMoTOi62c6ZYTTluLtdkVwj7Ur3vkj1kluPBS1xp81HlDQwY9qcEQCYsuu HWhBp6pX6FOqB9IG9tUUBguRA3UsbHK1YZWaDYu5Def131TN3ubY1gkIl2PlwS6w t0QmwCbAr1UwnjvVNioZBPRcHv/PLLf/0P2HQBHVESO7SMAhqaQoLf0V+LBOK/Qw WyH8EZE0vkHve52Xdf+XlcCWWC/qu0bXu+TZLg== -----END CERTIFICATE----- # Issuer: C=FR, O=Dhimyotis, OU=0002 48146308100036, CN=Certigna Root CA # Subject: C=FR, O=Dhimyotis, OU=0002 48146308100036, CN=Certigna Root CA # SHA256 Fingerprint: D4:8D:3D:23:EE:DB:50:A4:59:E5:51:97:60:1C:27:77:4B:9D:7B:18:C9:4D:5A:05:95:11:A1:02:50:B9:31:68 -----BEGIN CERTIFICATE----- MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of 1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L 6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw 3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= -----END CERTIFICATE----- # Issuer: C=GB, ST=Greater Manchester, L=Salford, O=COMODO CA Limited, CN=COMODO Certification Authority # Subject: C=GB, ST=Greater Manchester, L=Salford, O=COMODO CA Limited, CN=COMODO Certification Authority # SHA256 Fingerprint: 1A:0D:20:44:5D:E5:BA:18:62:D1:9E:F8:80:85:8C:BC:E5:01:02:B3:6E:8F:0A:04:0C:3C:69:E7:45:22:FE:6E -----BEGIN CERTIFICATE----- MIID0DCCArigAwIBAgIQIKTEf93f4cdTYwcTiHdgEjANBgkqhkiG9w0BAQUFADCB gTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxJzAlBgNV BAMTHkNPTU9ETyBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0xMTAxMDEwMDAw MDBaFw0zMDEyMzEyMzU5NTlaMIGBMQswCQYDVQQGEwJHQjEbMBkGA1UECBMSR3Jl YXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHEwdTYWxmb3JkMRowGAYDVQQKExFDT01P RE8gQ0EgTGltaXRlZDEnMCUGA1UEAxMeQ09NT0RPIENlcnRpZmljYXRpb24gQXV0 aG9yaXR5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0ECLi3LjkRv3 UcEbVASY06m/weaKXTuH+7uIzg3jLz8GlvCiKVCZrts7oVewdFFxze1CkU1B/qnI 2GqGd0S7WWaXUF601CxwRM/aN5VCaTwwxHGzUvAhTaHYujl8HJ6jJJ3ygxaYqhZ8 Q5sVW7euNJH+1GImGEaaP+vB+fGQV+useg2L23IwambV4EajcNxo2f8ESIl33rXp +2dtQem8Ob0y2WIC8bGoPW43nOIv4tOiJovGuFVDiOEjPqXSJDlqR6sA1KGzqSX+ DT+nHbrTUcELpNqsOO9VUCQFZUaTNE8tja3G1CEZ0o7KBWFxB3NH5YoZEr0ETc5O nKVIrLsm9wIDAQABo0IwQDAdBgNVHQ4EFgQUC1jli8ZMFTekQKkwqSG+RzZaVv8w DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD ggEBAC/JxBwHO89hAgCx2SFRdXIDMLDEFh9sAIsQrK/xR9SuEDwMGvjUk2ysEDd8 t6aDZK3N3w6HM503sMZ7OHKx8xoOo/lVem0DZgMXlUrxsXrfViEGQo+x06iF3u6X HWLrp+cxEmbDD6ZLLkGC9/3JG6gbr+48zuOcrigHoSybJMIPIyaDMouGDx8rEkYl Fo92kANr3ryqImhrjKGsKxE5pttwwn1y6TPn/CbxdFqR5p2ErPioBhlG5qfpqjQi pKGfeq23sqSaM4hxAjwu1nqyH6LKwN0vEJT9s4yEIHlG1QXUEOTS22RPuFvuG8Ug R1uUq27UlTMdphVx8fiUylQ5PsE= -----END CERTIFICATE----- # Issuer: C=GB, ST=Greater Manchester, L=Salford, O=COMODO CA Limited, CN=COMODO ECC Certification Authority # Subject: C=GB, ST=Greater Manchester, L=Salford, O=COMODO CA Limited, CN=COMODO ECC Certification Authority # SHA256 Fingerprint: 17:93:92:7A:06:14:54:97:89:AD:CE:2F:8F:34:F7:F0:B6:6D:0F:3A:E3:A3:B8:4D:21:EC:15:DB:BA:4F:AD:C7 -----BEGIN CERTIFICATE----- MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTEL MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE BxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMT IkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwMzA2MDAw MDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdy ZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09N T0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlv biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSR FtSrYpn1PlILBs5BAH+X4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0J cfRK9ChQtP6IHG4/bC8vCVlbpVsLM5niwz2J+Wos77LTBumjQjBAMB0GA1UdDgQW BBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/ BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VGFAkK+qDm fQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdv GDeAU/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY= -----END CERTIFICATE----- # Issuer: C=GB, ST=Greater Manchester, L=Salford, O=COMODO CA Limited, CN=COMODO RSA Certification Authority # Subject: C=GB, ST=Greater Manchester, L=Salford, O=COMODO CA Limited, CN=COMODO RSA Certification Authority # SHA256 Fingerprint: 52:F0:E1:C4:E5:8E:C6:29:29:1B:60:31:7F:07:46:71:B8:5D:7E:A8:0D:5B:07:27:34:63:53:4B:32:B4:02:34 -----BEGIN CERTIFICATE----- MIIF2DCCA8CgAwIBAgIQTKr5yttjb+Af907YWwOGnTANBgkqhkiG9w0BAQwFADCB hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMTE5 MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR Q09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNh dGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCR 6FSS0gpWsawNJN3Fz0RndJkrN6N9I3AAcbxT38T6KhKPS38QVr2fcHK3YX/JSw8X pz3jsARh7v8Rl8f0hj4K+j5c+ZPmNHrZFGvnnLOFoIJ6dq9xkNfs/Q36nGz637CC 9BR++b7Epi9Pf5l/tfxnQ3K9DADWietrLNPtj5gcFKt+5eNu/Nio5JIk2kNrYrhV /erBvGy2i/MOjZrkm2xpmfh4SDBF1a3hDTxFYPwyllEnvGfDyi62a+pGx8cgoLEf Zd5ICLqkTqnyg0Y3hOvozIFIQ2dOciqbXL1MGyiKXCJ7tKuY2e7gUYPDCUZObT6Z +pUX2nwzV0E8jVHtC7ZcryxjGt9XyD+86V3Em69FmeKjWiS0uqlWPc9vqv9JWL7w qP/0uK3pN/u6uPQLOvnoQ0IeidiEyxPx2bvhiWC4jChWrBQdnArncevPDt09qZah SL0896+1DSJMwBGB7FY79tOi4lu3sgQiUpWAk2nojkxl8ZEDLXB0AuqLZxUpaVIC u9ffUGpVRr+goyhhf3DQw6KqLCGqR84onAZFdr+CGCe01a60y1Dma/RMhnEw6abf Fobg2P9A3fvQQoh/ozM6LlweQRGBY84YcWsr7KaKtzFcOmpH4MN5WdYgGq/yapiq crxXStJLnbsQ/LBMQeXtHT1eKJ2czL+zUdqnR+WEUwIDAQABo0IwQDAdBgNVHQ4E FgQUu69+Aj36pvE8hI6t7jiY7NkyMtQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB /wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAArx1UaEt65Ru2yyTUEUAJNMnMvl wFTPoCWOAvn9sKIN9SCYPBMtrFaisNZ+EZLpLrqeLppysb0ZRGxhNaKatBYSaVqM 4dc+pBroLwP0rmEdEBsqpIt6xf4FpuHA1sj+nq6PK7o9mfjYcwlYRm6mnPTXJ9OV 2jeDchzTc+CiR5kDOF3VSXkAKRzH7JsgHAckaVd4sjn8OoSgtZx8jb8uk2Intzna FxiuvTwJaP+EmzzV1gsD41eeFPfR60/IvYcjt7ZJQ3mFXLrrkguhxuhoqEwWsRqZ CuhTLJK7oQkYdQxlqHvLI7cawiiFwxv/0Cti76R7CZGYZ4wUAc1oBmpjIXUDgIiK boHGhfKppC3n9KUkEEeDys30jXlYsQab5xoq2Z0B15R97QNKyvDb6KkBPvVWmcke jkk9u+UJueBPSZI9FoJAzMxZxuY67RIuaTxslbH9qh17f4a+Hg4yRvv7E491f0yL S0Zj/gA0QHDBw7mh3aZw4gSzQbzpgJHqZJx64SIDqZxubw5lT2yHh17zbqD5daWb QOhTsiedSrnAdyGN/4fy3ryM7xfft0kL0fJuMAsaDk527RH89elWsn2/x20Kk4yl 0MC2Hb46TpSi125sC8KKfPog88Tk5c0NqMuRkrF8hey1FGlmDoLnzc7ILaZRfyHB NVOFBkpdn627G190 -----END CERTIFICATE----- # Issuer: C=GB, ST=Greater Manchester, L=Salford, O=Comodo CA Limited, CN=AAA Certificate Services # Subject: C=GB, ST=Greater Manchester, L=Salford, O=Comodo CA Limited, CN=AAA Certificate Services # SHA256 Fingerprint: D7:A7:A0:FB:5D:7E:27:31:D7:71:E9:48:4E:BC:DE:F7:1D:5F:0C:3E:0A:29:48:78:2B:C8:3E:E0:EA:69:9E:F4 -----BEGIN CERTIFICATE----- MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe 3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== -----END CERTIFICATE----- # Issuer: C=GR, L=Athens, O=Hellenic Academic and Research Institutions Cert. Authority, CN=Hellenic Academic and Research Institutions ECC RootCA 2015 # Subject: C=GR, L=Athens, O=Hellenic Academic and Research Institutions Cert. Authority, CN=Hellenic Academic and Research Institutions ECC RootCA 2015 # SHA256 Fingerprint: 44:B5:45:AA:8A:25:E6:5A:73:CA:15:DC:27:FC:36:D2:4C:1C:B9:95:3A:06:65:39:B1:15:82:DC:48:7B:48:33 -----BEGIN CERTIFICATE----- MIICwzCCAkqgAwIBAgIBADAKBggqhkjOPQQDAjCBqjELMAkGA1UEBhMCR1IxDzAN BgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJl c2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkxRDBCBgNVBAMTO0hl bGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgRUNDIFJv b3RDQSAyMDE1MB4XDTE1MDcwNzEwMzcxMloXDTQwMDYzMDEwMzcxMlowgaoxCzAJ BgNVBAYTAkdSMQ8wDQYDVQQHEwZBdGhlbnMxRDBCBgNVBAoTO0hlbGxlbmljIEFj YWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgQ2VydC4gQXV0aG9yaXR5 MUQwQgYDVQQDEztIZWxsZW5pYyBBY2FkZW1pYyBhbmQgUmVzZWFyY2ggSW5zdGl0 dXRpb25zIEVDQyBSb290Q0EgMjAxNTB2MBAGByqGSM49AgEGBSuBBAAiA2IABJKg QehLgoRc4vgxEZmGZE4JJS+dQS8KrjVPdJWyUWRrjWvmP3CV8AVER6ZyOFB2lQJa jq4onvktTpnvLEhvTCUp6NFxW98dwXU3tNf6e3pCnGoKVlp8aQuqgAkkbH7BRqNC MEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFLQi C4KZJAEOnLvkDv2/+5cgk5kqMAoGCCqGSM49BAMCA2cAMGQCMGfOFmI4oqxiRaep lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR -----END CERTIFICATE----- # Issuer: C=GR, L=Athens, O=Hellenic Academic and Research Institutions Cert. Authority, CN=Hellenic Academic and Research Institutions RootCA 2015 # Subject: C=GR, L=Athens, O=Hellenic Academic and Research Institutions Cert. Authority, CN=Hellenic Academic and Research Institutions RootCA 2015 # SHA256 Fingerprint: A0:40:92:9A:02:CE:53:B4:AC:F4:F2:FF:C6:98:1C:E4:49:6F:75:5E:6D:45:FE:0B:2A:69:2B:CD:52:52:3F:36 -----BEGIN CERTIFICATE----- MIIGCzCCA/OgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBpjELMAkGA1UEBhMCR1Ix DzANBgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5k IFJlc2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkxQDA+BgNVBAMT N0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgUm9v dENBIDIwMTUwHhcNMTUwNzA3MTAxMTIxWhcNNDAwNjMwMTAxMTIxWjCBpjELMAkG A1UEBhMCR1IxDzANBgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNh ZGVtaWMgYW5kIFJlc2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkx QDA+BgNVBAMTN0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1 dGlvbnMgUm9vdENBIDIwMTUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoIC AQDC+Kk/G4n8PDwEXT2QNrCROnk8ZlrvbTkBSRq0t89/TSNTt5AA4xMqKKYx8ZEA 4yjsriFBzh/a/X0SWwGDD7mwX5nh8hKDgE0GPt+sr+ehiGsxr/CL0BgzuNtFajT0 AoAkKAoCFZVedioNmToUW/bLy1O8E00BiDeUJRtCvCLYjqOWXjrZMts+6PAQZe10 4S+nfK8nNLspfZu2zwnI5dMK/IhlZXQK3HMcXM1AsRzUtoSMTFDPaI6oWa7CJ06C ojXdFPQf/7J31Ycvqm59JCfnxssm5uX+Zwdj2EUN3TpZZTlYepKZcj2chF6IIbjV 9Cz82XBST3i4vTwri5WY9bPRaM8gFH5MXF/ni+X1NYEZN9cRCLdmvtNKzoNXADrD gfgXy5I2XdGj2HUb4Ysn6npIQf1FGQatJ5lOwXBH3bWfgVMS5bGMSF0xQxfjjMZ6 Y5ZLKTBOhE5iGV48zpeQpX8B653g+IuJ3SWYPZK2fu/Z8VFRfS0myGlZYeCsargq NhEEelC9MoS+L9xy1dcdFkfkR2YgP/SWxa+OAXqlD3pk9Q0Yh9muiNX6hME6wGko LfINaFGq46V3xqSQDqE3izEjR8EJCOtu93ib14L8hCCZSRm2Ekax+0VVFqmjZayc Bw/qa9wfLgZy7IaIEuQt218FL+TwA9MmM+eAws1CoRc0CwIDAQABo0IwQDAPBgNV HRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUcRVnyMjJvXVd ctA4GGqd83EkVAswDQYJKoZIhvcNAQELBQADggIBAHW7bVRLqhBYRjTyYtcWNl0I XtVsyIe9tC5G8jH4fOpCtZMWVdyhDBKg2mF+D1hYc2Ryx+hFjtyp8iY/xnmMsVMI M4GwVhO+5lFc2JsKT0ucVlMC6U/2DWDqTUJV6HwbISHTGzrMd/K4kPFox/la/vot 9L/J9UUbzjgQKjeKeaO04wlshYaT/4mWJ3iBj2fjRnRUjtkNaeJK9E10A/+yd+2V Z5fkscWrv2oj6NSU4kQoYsRL4vDY4ilrGnB+JGGTe08DMiUNRSQrlrRGar9KC/ea j8GsGsVn82800vpzY4zvFrCopEYq+OsS7HK07/grfoxSwIuEVPkvPuNVqNxmsdnh X9izjFk0WaSrT2y7HxjbdavYy5LNlDhhDgcGH0tGEPEVvo2FXDtKK4F5D7Rpn0lQ l033DlZdwJVqwjbDG2jJ9SrcR5q+ss7FJej6A7na+RZukYT1HCjI/CbM1xyQVqdf bzoEvM14iQuODy+jqk+iGxI9FghAD/FGTNeqewjBCvVtJ94Cj8rDtSvK6evIIVM4 pcw72Hc3MKJP2W/R8kCtQXoXxdZKNYm3QdV8hn9VTYNKpXMgwDqvkPGaJI7ZjnHK e7iG2rKPmT4dEw0SEe7Uq/DpFXYC5ODfqiAeW2GFZECpkJcNrVPSWh2HagCXZWK0 vm9qp/UsQu0yrbYhnr68 -----END CERTIFICATE----- # Issuer: C=GR, O=Hellenic Academic and Research Institutions CA, CN=HARICA TLS ECC Root CA 2021 # Subject: C=GR, O=Hellenic Academic and Research Institutions CA, CN=HARICA TLS ECC Root CA 2021 # SHA256 Fingerprint: 3F:99:CC:47:4A:CF:CE:4D:FE:D5:87:94:66:5E:47:8D:15:47:73:9F:2E:78:0F:1B:B4:CA:9B:13:30:97:D4:01 -----BEGIN CERTIFICATE----- MIICVDCCAdugAwIBAgIQZ3SdjXfYO2rbIvT/WeK/zjAKBggqhkjOPQQDAzBsMQsw CQYDVQQGEwJHUjE3MDUGA1UECgwuSGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJlc2Vh cmNoIEluc3RpdHV0aW9ucyBDQTEkMCIGA1UEAwwbSEFSSUNBIFRMUyBFQ0MgUm9v dCBDQSAyMDIxMB4XDTIxMDIxOTExMDExMFoXDTQ1MDIxMzExMDEwOVowbDELMAkG A1UEBhMCR1IxNzA1BgNVBAoMLkhlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJj aCBJbnN0aXR1dGlvbnMgQ0ExJDAiBgNVBAMMG0hBUklDQSBUTFMgRUNDIFJvb3Qg Q0EgMjAyMTB2MBAGByqGSM49AgEGBSuBBAAiA2IABDgI/rGgltJ6rK9JOtDA4MM7 KKrxcm1lAEeIhPyaJmuqS7psBAqIXhfyVYf8MLA04jRYVxqEU+kw2anylnTDUR9Y STHMmE5gEYd103KUkE+bECUqqHgtvpBBWJAVcqeht6NCMEAwDwYDVR0TAQH/BAUw AwEB/zAdBgNVHQ4EFgQUyRtTgRL+BNUW0aq8mm+3oJUZbsowDgYDVR0PAQH/BAQD AgGGMAoGCCqGSM49BAMDA2cAMGQCMBHervjcToiwqfAircJRQO9gcS3ujwLEXQNw SaSS6sUUiHCm0w2wqsosQJz76YJumgIwK0eaB8bRwoF8yguWGEEbo/QwCZ61IygN nxS2PFOiTAZpffpskcYqSUXm7LcT4Tps -----END CERTIFICATE----- # Issuer: C=GR, O=Hellenic Academic and Research Institutions CA, CN=HARICA TLS RSA Root CA 2021 # Subject: C=GR, O=Hellenic Academic and Research Institutions CA, CN=HARICA TLS RSA Root CA 2021 # SHA256 Fingerprint: D9:5D:0E:8E:DA:79:52:5B:F9:BE:B1:1B:14:D2:10:0D:32:94:98:5F:0C:62:D9:FA:BD:9C:D9:99:EC:CB:7B:1D -----BEGIN CERTIFICATE----- MIIFpDCCA4ygAwIBAgIQOcqTHO9D88aOk8f0ZIk4fjANBgkqhkiG9w0BAQsFADBs MQswCQYDVQQGEwJHUjE3MDUGA1UECgwuSGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJl c2VhcmNoIEluc3RpdHV0aW9ucyBDQTEkMCIGA1UEAwwbSEFSSUNBIFRMUyBSU0Eg Um9vdCBDQSAyMDIxMB4XDTIxMDIxOTEwNTUzOFoXDTQ1MDIxMzEwNTUzN1owbDEL MAkGA1UEBhMCR1IxNzA1BgNVBAoMLkhlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNl YXJjaCBJbnN0aXR1dGlvbnMgQ0ExJDAiBgNVBAMMG0hBUklDQSBUTFMgUlNBIFJv b3QgQ0EgMjAyMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAIvC569l mwVnlskNJLnQDmT8zuIkGCyEf3dRywQRNrhe7Wlxp57kJQmXZ8FHws+RFjZiPTgE 4VGC/6zStGndLuwRo0Xua2s7TL+MjaQenRG56Tj5eg4MmOIjHdFOY9TnuEFE+2uv a9of08WRiFukiZLRgeaMOVig1mlDqa2YUlhu2wr7a89o+uOkXjpFc5gH6l8Cct4M pbOfrqkdtx2z/IpZ525yZa31MJQjB/OCFks1mJxTuy/K5FrZx40d/JiZ+yykgmvw Kh+OC19xXFyuQnspiYHLA6OZyoieC0AJQTPb5lh6/a6ZcMBaD9YThnEvdmn8kN3b LW7R8pv1GmuebxWMevBLKKAiOIAkbDakO/IwkfN4E8/BPzWr8R0RI7VDIp4BkrcY AuUR0YLbFQDMYTfBKnya4dC6s1BG7oKsnTH4+yPiAwBIcKMJJnkVU2DzOFytOOqB AGMUuTNe3QvboEUHGjMJ+E20pwKmafTCWQWIZYVWrkvL4N48fS0ayOn7H6NhStYq E613TBoYm5EPWNgGVMWX+Ko/IIqmhaZ39qb8HOLubpQzKoNQhArlT4b4UEV4AIHr W2jjJo3Me1xR9BQsQL4aYB16cmEdH2MtiKrOokWQCPxrvrNQKlr9qEgYRtaQQJKQ CoReaDH46+0N0x3GfZkYVVYnZS6NRcUk7M7jAgMBAAGjQjBAMA8GA1UdEwEB/wQF MAMBAf8wHQYDVR0OBBYEFApII6ZgpJIKM+qTW8VX6iVNvRLuMA4GA1UdDwEB/wQE AwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAPpBIqm5iFSVmewzVjIuJndftTgfvnNAU X15QvWiWkKQUEapobQk1OUAJ2vQJLDSle1mESSmXdMgHHkdt8s4cUCbjnj1AUz/3 f5Z2EMVGpdAgS1D0NTsY9FVqQRtHBmg8uwkIYtlfVUKqrFOFrJVWNlar5AWMxaja H6NpvVMPxP/cyuN+8kyIhkdGGvMA9YCRotxDQpSbIPDRzbLrLFPCU3hKTwSUQZqP JzLB5UkZv/HywouoCjkxKLR9YjYsTewfM7Z+d21+UPCfDtcRj88YxeMn/ibvBZ3P zzfF0HvaO7AWhAw6k9a+F9sPPg4ZeAnHqQJyIkv3N3a6dcSFA1pj1bF1BcK5vZSt jBWZp5N99sXzqnTPBIWUmAD04vnKJGW/4GKvyMX6ssmeVkjaef2WdhW+o45WxLM0 /L5H9MG0qPzVMIho7suuyWPEdr6sOBjhXlzPrjoiUevRi7PzKzMHVIf6tLITe7pT BGIBnfHAT+7hOtSLIBD6Alfm78ELt5BGnBkpjNxvoEppaZS3JGWg/6w/zgH7IS79 aPib8qXPMThcFarmlwDB31qlpzmq6YR/PFGoOtmUW4y/Twhx5duoXNTSpv4Ao8YW xw/ogM4cKGR0GQjTQuPOAF1/sdwTsOEFy9EgqoZ0njnnkf3/W9b3raYvAwtt41dU 63ZTGI0RmLo= -----END CERTIFICATE----- # Issuer: C=HK, ST=Hong Kong, L=Hong Kong, O=Hongkong Post, CN=Hongkong Post Root CA 3 # Subject: C=HK, ST=Hong Kong, L=Hong Kong, O=Hongkong Post, CN=Hongkong Post Root CA 3 # SHA256 Fingerprint: 5A:2F:C0:3F:0C:83:B0:90:BB:FA:40:60:4B:09:88:44:6C:76:36:18:3D:F9:84:6E:17:10:1A:44:7F:B8:EF:D6 -----BEGIN CERTIFICATE----- MIIFzzCCA7egAwIBAgIUCBZfikyl7ADJk0DfxMauI7gcWqQwDQYJKoZIhvcNAQEL BQAwbzELMAkGA1UEBhMCSEsxEjAQBgNVBAgTCUhvbmcgS29uZzESMBAGA1UEBxMJ SG9uZyBLb25nMRYwFAYDVQQKEw1Ib25na29uZyBQb3N0MSAwHgYDVQQDExdIb25n a29uZyBQb3N0IFJvb3QgQ0EgMzAeFw0xNzA2MDMwMjI5NDZaFw00MjA2MDMwMjI5 NDZaMG8xCzAJBgNVBAYTAkhLMRIwEAYDVQQIEwlIb25nIEtvbmcxEjAQBgNVBAcT CUhvbmcgS29uZzEWMBQGA1UEChMNSG9uZ2tvbmcgUG9zdDEgMB4GA1UEAxMXSG9u Z2tvbmcgUG9zdCBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK AoICAQCziNfqzg8gTr7m1gNt7ln8wlffKWihgw4+aMdoWJwcYEuJQwy51BWy7sFO dem1p+/l6TWZ5Mwc50tfjTMwIDNT2aa71T4Tjukfh0mtUC1Qyhi+AViiE3CWu4mI VoBc+L0sPOFMV4i707mV78vH9toxdCim5lSJ9UExyuUmGs2C4HDaOym71QP1mbpV 9WTRYA6ziUm4ii8F0oRFKHyPaFASePwLtVPLwpgchKOesL4jpNrcyCse2m5FHomY 2vkALgbpDDtw1VAliJnLzXNg99X/NWfFobxeq81KuEXryGgeDQ0URhLj0mRiikKY vLTGCAj4/ahMZJx2Ab0vqWwzD9g/KLg8aQFChn5pwckGyuV6RmXpwtZQQS4/t+Tt bNe/JgERohYpSms0BpDsE9K2+2p20jzt8NYt3eEV7KObLyzJPivkaTv/ciWxNoZb x39ri1UbSsUgYT2uy1DhCDq+sI9jQVMwCFk8mB13umOResoQUGC/8Ne8lYePl8X+ l2oBlKN8W4UdKjk60FSh0Tlxnf0h+bV78OLgAo9uliQlLKAeLKjEiafv7ZkGL7YK TE/bosw3Gq9HhS2KX8Q0NEwA/RiTZxPRN+ZItIsGxVd7GYYKecsAyVKvQv83j+Gj Hno9UKtjBucVtT+2RTeUN7F+8kjDf8V1/peNRY8apxpyKBpADwIDAQABo2MwYTAP BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBQXnc0e i9Y5K3DTXNSguB+wAPzFYTAdBgNVHQ4EFgQUF53NHovWOStw01zUoLgfsAD8xWEw DQYJKoZIhvcNAQELBQADggIBAFbVe27mIgHSQpsY1Q7XZiNc4/6gx5LS6ZStS6LG 7BJ8dNVI0lkUmcDrudHr9EgwW62nV3OZqdPlt9EuWSRY3GguLmLYauRwCy0gUCCk MpXRAJi70/33MvJJrsZ64Ee+bs7Lo3I6LWldy8joRTnU+kLBEUx3XZL7av9YROXr gZ6voJmtvqkBZss4HTzfQx/0TW60uhdG/H39h4F5ag0zD/ov+BS5gLNdTaqX4fnk GMX41TiMJjz98iji7lpJiCzfeT2OnpA8vUFKOt1b9pq0zj8lMH8yfaIDlNDceqFS 3m6TjRgm/VWsvY+b0s+v54Ysyx8Jb6NvqYTUc79NoXQbTiNg8swOqn+knEwlqLJm Ozj/2ZQw9nKEvmhVEA/GcywWaZMH/rFF7buiVWqw2rVKAiUnhde3t4ZEFolsgCs+ l6mc1X5VTMbeRRAc6uk7nwNT7u56AQIWeNTowr5GdogTPyK7SBIdUgC0An4hGh6c JfTzPV4e0hz5sy229zdcxsshTrD3mUcYhcErulWuBurQB7Lcq9CClnXO0lD+mefP L5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB60PZ2Pierc+xYw5F9KBa LJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fqdBb9HxEG mpv0 -----END CERTIFICATE----- # Issuer: C=HU, L=Budapest, O=Microsec Ltd., CN=Microsec e-Szigno Root CA 2009, emailAddress=info@e-szigno.hu # Subject: C=HU, L=Budapest, O=Microsec Ltd., CN=Microsec e-Szigno Root CA 2009, emailAddress=info@e-szigno.hu # SHA256 Fingerprint: 3C:5F:81:FE:A5:FA:B8:2C:64:BF:A2:EA:EC:AF:CD:E8:E0:77:FC:86:20:A7:CA:E5:37:16:3D:F3:6E:DB:F3:78 -----BEGIN CERTIFICATE----- MIIECjCCAvKgAwIBAgIJAMJ+QwRORz8ZMA0GCSqGSIb3DQEBCwUAMIGCMQswCQYD VQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFjAUBgNVBAoMDU1pY3Jvc2VjIEx0 ZC4xJzAlBgNVBAMMHk1pY3Jvc2VjIGUtU3ppZ25vIFJvb3QgQ0EgMjAwOTEfMB0G CSqGSIb3DQEJARYQaW5mb0BlLXN6aWduby5odTAeFw0wOTA2MTYxMTMwMThaFw0y OTEyMzAxMTMwMThaMIGCMQswCQYDVQQGEwJIVTERMA8GA1UEBwwIQnVkYXBlc3Qx FjAUBgNVBAoMDU1pY3Jvc2VjIEx0ZC4xJzAlBgNVBAMMHk1pY3Jvc2VjIGUtU3pp Z25vIFJvb3QgQ0EgMjAwOTEfMB0GCSqGSIb3DQEJARYQaW5mb0BlLXN6aWduby5o dTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOn4j/NjrdqG2KfgQvvP kd6mJviZpWNwrZuuyjNAfW2WbqEORO7hE52UQlKavXWFdCyoDh2Tthi3jCyoz/tc cbna7P7ofo/kLx2yqHWH2Leh5TvPmUpG0IMZfcChEhyVbUr02MelTTMuhTlAdX4U fIASmFDHQWe4oIBhVKZsTh/gnQ4H6cm6M+f+wFUoLAKApxn1ntxVUwOXewdI/5n7 N4okxFnMUBBjjqqpGrCEGob5X7uxUG6k0QrM1XF+H6cbfPVTbiJfyyvm1HxdrtbC xkzlBQHZ7Vf8wSN5/PrIJIOV87VqUQHQd9bpEqH5GoP7ghu5sJf0dgYzQ0mg/wu1 +rUCAwEAAaOBgDB+MA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0G A1UdDgQWBBTLD8bfQkPMPcu1SCOhGnqmKrs0aDAfBgNVHSMEGDAWgBTLD8bfQkPM Pcu1SCOhGnqmKrs0aDAbBgNVHREEFDASgRBpbmZvQGUtc3ppZ25vLmh1MA0GCSqG SIb3DQEBCwUAA4IBAQDJ0Q5eLtXMs3w+y/w9/w0olZMEyL/azXm4Q5DwpL7v8u8h mLzU1F0G9u5C7DBsoKqpyvGvivo/C3NqPuouQH4frlRheesuCDfXI/OMn74dseGk ddug4lQUsbocKaQY9hK6ohQU4zE1yED/t+AFdlfBHFny+L/k7SViXITwfn4fs775 tyERzAMBVnCnEJIeGzSBHq2cGsMEPO0CYdYeBvNfOofyK/FFh+U9rNHHV4S9a67c 2Pm2G2JwCz02yULyMtd6YebS2z3PyKnJm9zbWETXbzivf3jTo60adbocwTZ8jx5t HMN1Rq41Bab2XD0h7lbwyYIiLXpUq3DDfSJlgnCW -----END CERTIFICATE----- # Issuer: C=HU, L=Budapest, O=Microsec Ltd., organizationIdentifier=VATHU-23584497, CN=e-Szigno Root CA 2017 # Subject: C=HU, L=Budapest, O=Microsec Ltd., organizationIdentifier=VATHU-23584497, CN=e-Szigno Root CA 2017 # SHA256 Fingerprint: BE:B0:0B:30:83:9B:9B:C3:2C:32:E4:44:79:05:95:06:41:F2:64:21:B1:5E:D0:89:19:8B:51:8A:E2:EA:1B:99 -----BEGIN CERTIFICATE----- MIICQDCCAeWgAwIBAgIMAVRI7yH9l1kN9QQKMAoGCCqGSM49BAMCMHExCzAJBgNV BAYTAkhVMREwDwYDVQQHDAhCdWRhcGVzdDEWMBQGA1UECgwNTWljcm9zZWMgTHRk LjEXMBUGA1UEYQwOVkFUSFUtMjM1ODQ0OTcxHjAcBgNVBAMMFWUtU3ppZ25vIFJv b3QgQ0EgMjAxNzAeFw0xNzA4MjIxMjA3MDZaFw00MjA4MjIxMjA3MDZaMHExCzAJ BgNVBAYTAkhVMREwDwYDVQQHDAhCdWRhcGVzdDEWMBQGA1UECgwNTWljcm9zZWMg THRkLjEXMBUGA1UEYQwOVkFUSFUtMjM1ODQ0OTcxHjAcBgNVBAMMFWUtU3ppZ25v IFJvb3QgQ0EgMjAxNzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABJbcPYrYsHtv xie+RJCxs1YVe45DJH0ahFnuY2iyxl6H0BVIHqiQrb1TotreOpCmYF9oMrWGQd+H Wyx7xf58etqjYzBhMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0G A1UdDgQWBBSHERUI0arBeAyxr87GyZDvvzAEwDAfBgNVHSMEGDAWgBSHERUI0arB eAyxr87GyZDvvzAEwDAKBggqhkjOPQQDAgNJADBGAiEAtVfd14pVCzbhhkT61Nlo jbjcI4qKDdQvfepz7L9NbKgCIQDLpbQS+ue16M9+k/zzNY9vTlp8tLxOsvxyqltZ +efcMQ== -----END CERTIFICATE----- # Issuer: C=HU, L=Budapest, O=NetLock Kft., OU=Tan\xC3\xBAs\xC3\xADtv\xC3\xA1nykiad\xC3\xB3k (Certification Services), CN=NetLock Arany (Class Gold) F\xC5\x91tan\xC3\xBAs\xC3\xADtv\xC3\xA1ny # Subject: C=HU, L=Budapest, O=NetLock Kft., OU=Tan\xC3\xBAs\xC3\xADtv\xC3\xA1nykiad\xC3\xB3k (Certification Services), CN=NetLock Arany (Class Gold) F\xC5\x91tan\xC3\xBAs\xC3\xADtv\xC3\xA1ny # SHA256 Fingerprint: 6C:61:DA:C3:A2:DE:F0:31:50:6B:E0:36:D2:A6:FE:40:19:94:FB:D1:3D:F9:C8:D4:66:59:92:74:C4:46:EC:98 -----BEGIN CERTIFICATE----- MIIEFTCCAv2gAwIBAgIGSUEs5AAQMA0GCSqGSIb3DQEBCwUAMIGnMQswCQYDVQQG EwJIVTERMA8GA1UEBwwIQnVkYXBlc3QxFTATBgNVBAoMDE5ldExvY2sgS2Z0LjE3 MDUGA1UECwwuVGFuw7pzw610dsOhbnlraWFkw7NrIChDZXJ0aWZpY2F0aW9uIFNl cnZpY2VzKTE1MDMGA1UEAwwsTmV0TG9jayBBcmFueSAoQ2xhc3MgR29sZCkgRsWR dGFuw7pzw610dsOhbnkwHhcNMDgxMjExMTUwODIxWhcNMjgxMjA2MTUwODIxWjCB pzELMAkGA1UEBhMCSFUxETAPBgNVBAcMCEJ1ZGFwZXN0MRUwEwYDVQQKDAxOZXRM b2NrIEtmdC4xNzA1BgNVBAsMLlRhbsO6c8OtdHbDoW55a2lhZMOzayAoQ2VydGlm aWNhdGlvbiBTZXJ2aWNlcykxNTAzBgNVBAMMLE5ldExvY2sgQXJhbnkgKENsYXNz IEdvbGQpIEbFkXRhbsO6c8OtdHbDoW55MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A MIIBCgKCAQEAxCRec75LbRTDofTjl5Bu0jBFHjzuZ9lk4BqKf8owyoPjIMHj9DrT lF8afFttvzBPhCf2nx9JvMaZCpDyD/V/Q4Q3Y1GLeqVw/HpYzY6b7cNGbIRwXdrz AZAj/E4wqX7hJ2Pn7WQ8oLjJM2P+FpD/sLj916jAwJRDC7bVWaaeVtAkH3B5r9s5 VA1lddkVQZQBr17s9o3x/61k/iCa11zr/qYfCGSji3ZVrR47KGAuhyXoqq8fxmRG ILdwfzzeSNuWU7c5d+Qa4scWhHaXWy+7GRWF+GmF9ZmnqfI0p6m2pgP8b4Y9VHx2 BJtr+UBdADTHLpl1neWIA6pN+APSQnbAGwIDAKiLo0UwQzASBgNVHRMBAf8ECDAG AQH/AgEEMA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUzPpnk/C2uNClwB7zU/2M U9+D15YwDQYJKoZIhvcNAQELBQADggEBAKt/7hwWqZw8UQCgwBEIBaeZ5m8BiFRh bvG5GK1Krf6BQCOUL/t1fC8oS2IkgYIL9WHxHG64YTjrgfpioTtaYtOUZcTh5m2C +C8lcLIhJsFyUR+MLMOEkMNaj7rP9KdlpeuY0fsFskZ1FSNqb4VjMIDw1Z4fKRzC bLBQWV2QWzuoDTDPv31/zvGdg73JRm4gpvlhUbohL3u+pRVjodSVh/GeufOJ8z2F uLjbvrW5KfnaNwUASZQDhETnv0Mxz3WLJdH0pmT1kvarBes96aULNmLazAZfNou2 XjG4Kvte9nHfRCaexOYNkbQudZWAUWpLMKawYqGT8ZvYzsRjdT9ZR7E= -----END CERTIFICATE----- # Issuer: C=IE, O=Baltimore, OU=CyberTrust, CN=Baltimore CyberTrust Root # Subject: C=IE, O=Baltimore, OU=CyberTrust, CN=Baltimore CyberTrust Root # SHA256 Fingerprint: 16:AF:57:A9:F6:76:B0:AB:12:60:95:AA:5E:BA:DE:F2:2A:B3:11:19:D6:44:AC:95:CD:4B:93:DB:F3:F2:6A:EB -----BEGIN CERTIFICATE----- MIIDdzCCAl+gAwIBAgIEAgAAuTANBgkqhkiG9w0BAQUFADBaMQswCQYDVQQGEwJJ RTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJlclRydXN0MSIwIAYD VQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTAwMDUxMjE4NDYwMFoX DTI1MDUxMjIzNTkwMFowWjELMAkGA1UEBhMCSUUxEjAQBgNVBAoTCUJhbHRpbW9y ZTETMBEGA1UECxMKQ3liZXJUcnVzdDEiMCAGA1UEAxMZQmFsdGltb3JlIEN5YmVy VHJ1c3QgUm9vdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKMEuyKr mD1X6CZymrV51Cni4eiVgLGw41uOKymaZN+hXe2wCQVt2yguzmKiYv60iNoS6zjr IZ3AQSsBUnuId9Mcj8e6uYi1agnnc+gRQKfRzMpijS3ljwumUNKoUMMo6vWrJYeK mpYcqWe4PwzV9/lSEy/CG9VwcPCPwBLKBsua4dnKM3p31vjsufFoREJIE9LAwqSu XmD+tqYF/LTdB1kC1FkYmGP1pWPgkAx9XbIGevOF6uvUA65ehD5f/xXtabz5OTZy dc93Uk3zyZAsuT3lySNTPx8kmCFcB5kpvcY67Oduhjprl3RjM71oGDHweI12v/ye jl0qhqdNkNwnGjkCAwEAAaNFMEMwHQYDVR0OBBYEFOWdWTCCR1jMrPoIVDaGezq1 BE3wMBIGA1UdEwEB/wQIMAYBAf8CAQMwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3 DQEBBQUAA4IBAQCFDF2O5G9RaEIFoN27TyclhAO992T9Ldcw46QQF+vaKSm2eT92 9hkTI7gQCvlYpNRhcL0EYWoSihfVCr3FvDB81ukMJY2GQE/szKN+OMY3EU/t3Wgx jkzSswF07r51XgdIGn9w/xZchMB5hbgF/X++ZRGjD8ACtPhSNzkE1akxehi/oCr0 Epn3o0WC4zxe9Z2etciefC7IpJ5OCBRLbf1wbWsaY71k5h+3zvDyny67G7fyUIhz ksLi4xaNmjICq44Y3ekQEe5+NauQrz4wlHrQMz2nZQ/1/I6eYs9HRCwBXbsdtTLS R9I4LtD+gdwyah617jzV/OeBHRnDJELqYzmp -----END CERTIFICATE----- # Issuer: C=IN, OU=emSign PKI, O=eMudhra Technologies Limited, CN=emSign ECC Root CA - G3 # Subject: C=IN, OU=emSign PKI, O=eMudhra Technologies Limited, CN=emSign ECC Root CA - G3 # SHA256 Fingerprint: 86:A1:EC:BA:08:9C:4A:8D:3B:BE:27:34:C6:12:BA:34:1D:81:3E:04:3C:F9:E8:A8:62:CD:5C:57:A3:6B:BE:6B -----BEGIN CERTIFICATE----- MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD +JbNR6iC8hZVdyR+EhCVBCyj -----END CERTIFICATE----- # Issuer: C=IN, OU=emSign PKI, O=eMudhra Technologies Limited, CN=emSign Root CA - G1 # Subject: C=IN, OU=emSign PKI, O=eMudhra Technologies Limited, CN=emSign Root CA - G1 # SHA256 Fingerprint: 40:F6:AF:03:46:A9:9A:A1:CD:1D:55:5A:4E:9C:CE:62:C7:F9:63:46:03:EE:40:66:15:83:3D:C8:C8:D0:03:67 -----BEGIN CERTIFICATE----- MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO 8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH 6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx iN66zB+Afko= -----END CERTIFICATE----- # Issuer: C=IT, L=Milan, O=Actalis S.p.A./03358520967, CN=Actalis Authentication Root CA # Subject: C=IT, L=Milan, O=Actalis S.p.A./03358520967, CN=Actalis Authentication Root CA # SHA256 Fingerprint: 55:92:60:84:EC:96:3A:64:B9:6E:2A:BE:01:CE:0B:A8:6A:64:FB:FE:BC:C7:AA:B5:AF:C1:55:B3:7F:D7:60:66 -----BEGIN CERTIFICATE----- MIIFuzCCA6OgAwIBAgIIVwoRl0LE48wwDQYJKoZIhvcNAQELBQAwazELMAkGA1UE BhMCSVQxDjAMBgNVBAcMBU1pbGFuMSMwIQYDVQQKDBpBY3RhbGlzIFMucC5BLi8w MzM1ODUyMDk2NzEnMCUGA1UEAwweQWN0YWxpcyBBdXRoZW50aWNhdGlvbiBSb290 IENBMB4XDTExMDkyMjExMjIwMloXDTMwMDkyMjExMjIwMlowazELMAkGA1UEBhMC SVQxDjAMBgNVBAcMBU1pbGFuMSMwIQYDVQQKDBpBY3RhbGlzIFMucC5BLi8wMzM1 ODUyMDk2NzEnMCUGA1UEAwweQWN0YWxpcyBBdXRoZW50aWNhdGlvbiBSb290IENB MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAp8bEpSmkLO/lGMWwUKNv UTufClrJwkg4CsIcoBh/kbWHuUA/3R1oHwiD1S0eiKD4j1aPbZkCkpAW1V8IbInX 4ay8IMKx4INRimlNAJZaby/ARH6jDuSRzVju3PvHHkVH3Se5CAGfpiEd9UEtL0z9 KK3giq0itFZljoZUj5NDKd45RnijMCO6zfB9E1fAXdKDa0hMxKufgFpbOr3JpyI/ gCczWw63igxdBzcIy2zSekciRDXFzMwujt0q7bd9Zg1fYVEiVRvjRuPjPdA1Yprb rxTIW6HMiRvhMCb8oJsfgadHHwTrozmSBp+Z07/T6k9QnBn+locePGX2oxgkg4YQ 51Q+qDp2JE+BIcXjDwL4k5RHILv+1A7TaLndxHqEguNTVHnd25zS8gebLra8Pu2F be8lEfKXGkJh90qX6IuxEAf6ZYGyojnP9zz/GPvG8VqLWeICrHuS0E4UT1lF9gxe KF+w6D9Fz8+vm2/7hNN3WpVvrJSEnu68wEqPSpP4RCHiMUVhUE4Q2OM1fEwZtN4F v6MGn8i1zeQf1xcGDXqVdFUNaBr8EBtiZJ1t4JWgw5QHVw0U5r0F+7if5t+L4sbn fpb2U8WANFAoWPASUHEXMLrmeGO89LKtmyuy/uE5jF66CyCU3nuDuP/jVo23Eek7 jPKxwV2dpAtMK9myGPW1n0sCAwEAAaNjMGEwHQYDVR0OBBYEFFLYiDrIn3hm7Ynz ezhwlMkCAjbQMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUUtiIOsifeGbt ifN7OHCUyQICNtAwDgYDVR0PAQH/BAQDAgEGMA0GCSqGSIb3DQEBCwUAA4ICAQAL e3KHwGCmSUyIWOYdiPcUZEim2FgKDk8TNd81HdTtBjHIgT5q1d07GjLukD0R0i70 jsNjLiNmsGe+b7bAEzlgqqI0JZN1Ut6nna0Oh4lScWoWPBkdg/iaKWW+9D+a2fDz WochcYBNy+A4mz+7+uAwTc+G02UQGRjRlwKxK3JCaKygvU5a2hi/a5iB0P2avl4V SM0RFbnAKVy06Ij3Pjaut2L9HmLecHgQHEhb2rykOLpn7VU+Xlff1ANATIGk0k9j pwlCCRT8AKnCgHNPLsBA2RF7SOp6AsDT6ygBJlh0wcBzIm2Tlf05fbsq4/aC4yyX X04fkZT6/iyj2HYauE2yOE+b+h1IYHkm4vP9qdCa6HCPSXrW5b0KDtst842/6+Ok fcvHlXHo2qN8xcL4dJIEG4aspCJTQLas/kx2z/uUMsA1n3Y/buWQbqCmJqK4LL7R K4X9p2jIugErsWx0Hbhzlefut8cl8ABMALJ+tguLHPPAUJ4lueAI3jZm/zel0btU ZCzJJ7VLkn5l/9Mt4blOvH+kQSGQQXemOR/qnuOf0GZvBeyqdn6/axag67XH/JJU LysRJyU3eExRarDzzFhdFPFqSBX/wge2sY0PjlxQRrM9vwGYT7JZVEc+NHt4bVaT LnPqZih4zR0Uv6CPLy64Lo7yFIrM6bV8+2ydDKXhlg== -----END CERTIFICATE----- # Issuer: C=JP, O=Japan Certification Services, Inc., CN=SecureSign RootCA11 # Subject: C=JP, O=Japan Certification Services, Inc., CN=SecureSign RootCA11 # SHA256 Fingerprint: BF:0F:EE:FB:9E:3A:58:1A:D5:F9:E9:DB:75:89:98:57:43:D2:61:08:5C:4D:31:4F:6F:5D:72:59:AA:42:16:12 -----BEGIN CERTIFICATE----- MIIDbTCCAlWgAwIBAgIBATANBgkqhkiG9w0BAQUFADBYMQswCQYDVQQGEwJKUDEr MCkGA1UEChMiSmFwYW4gQ2VydGlmaWNhdGlvbiBTZXJ2aWNlcywgSW5jLjEcMBoG A1UEAxMTU2VjdXJlU2lnbiBSb290Q0ExMTAeFw0wOTA0MDgwNDU2NDdaFw0yOTA0 MDgwNDU2NDdaMFgxCzAJBgNVBAYTAkpQMSswKQYDVQQKEyJKYXBhbiBDZXJ0aWZp Y2F0aW9uIFNlcnZpY2VzLCBJbmMuMRwwGgYDVQQDExNTZWN1cmVTaWduIFJvb3RD QTExMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA/XeqpRyQBTvLTJsz i1oURaTnkBbR31fSIRCkF/3frNYfp+TbfPfs37gD2pRY/V1yfIw/XwFndBWW4wI8 h9uuywGOwvNmxoVF9ALGOrVisq/6nL+k5tSAMJjzDbaTj6nU2DbysPyKyiyhFTOV MdrAG/LuYpmGYz+/3ZMqg6h2uRMft85OQoWPIucuGvKVCbIFtUROd6EgvanyTgp9 UK31BQ1FT0Zx/Sg+U/sE2C3XZR1KG/rPO7AxmjVuyIsG0wCR8pQIZUyxNAYAeoni 8McDWc/V1uinMrPmmECGxc0nEovMe863ETxiYAcjPitAbpSACW22s293bzUIUPsC h8U+iQIDAQABo0IwQDAdBgNVHQ4EFgQUW/hNT7KlhtQ60vFjmqC+CfZXt94wDgYD VR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB AKChOBZmLqdWHyGcBvod7bkixTgm2E5P7KN/ed5GIaGHd48HCJqypMWvDzKYC3xm KbabfSVSSUOrTC4rbnpwrxYO4wJs+0LmGJ1F2FXI6Dvd5+H0LgscNFxsWEr7jIhQ X5Ucv+2rIrVls4W6ng+4reV6G4pQOh29Dbx7VFALuUKvVaAYga1lme++5Jy/xIWr QbJUb9wlze144o4MjQlJ3WN7WmmWAiGovVJZ6X01y8hSyn+B/tlr0/cR7SXf+Of5 pPpyl4RTDaXQMhhRdlkUbA/r7F+AjHVDg8OFmP9Mni0N5HeDk061lgeLKBObjBmN QSdJQO7e5iNEOdyhIta6A/I= -----END CERTIFICATE----- # Issuer: C=JP, O=SECOM Trust Systems CO.,LTD., CN=Security Communication ECC RootCA1 # Subject: C=JP, O=SECOM Trust Systems CO.,LTD., CN=Security Communication ECC RootCA1 # SHA256 Fingerprint: E7:4F:BD:A5:5B:D5:64:C4:73:A3:6B:44:1A:A7:99:C8:A6:8E:07:74:40:E8:28:8B:9F:A1:E5:0E:4B:BA:CA:11 -----BEGIN CERTIFICATE----- MIICODCCAb6gAwIBAgIJANZdm7N4gS7rMAoGCCqGSM49BAMDMGExCzAJBgNVBAYT AkpQMSUwIwYDVQQKExxTRUNPTSBUcnVzdCBTeXN0ZW1zIENPLixMVEQuMSswKQYD VQQDEyJTZWN1cml0eSBDb21tdW5pY2F0aW9uIEVDQyBSb290Q0ExMB4XDTE2MDYx NjA1MTUyOFoXDTM4MDExODA1MTUyOFowYTELMAkGA1UEBhMCSlAxJTAjBgNVBAoT HFNFQ09NIFRydXN0IFN5c3RlbXMgQ08uLExURC4xKzApBgNVBAMTIlNlY3VyaXR5 IENvbW11bmljYXRpb24gRUNDIFJvb3RDQTEwdjAQBgcqhkjOPQIBBgUrgQQAIgNi AASkpW9gAwPDvTH00xecK4R1rOX9PVdu12O/5gSJko6BnOPpR27KkBLIE+Cnnfdl dB9sELLo5OnvbYUymUSxXv3MdhDYW72ixvnWQuRXdtyQwjWpS4g8EkdtXP9JTxpK ULGjQjBAMB0GA1UdDgQWBBSGHOf+LaVKiwj+KBH6vqNm+GBZLzAOBgNVHQ8BAf8E BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjAVXUI9/Lbu 9zuxNuie9sRGKEkz0FhDKmMpzE2xtHqiuQ04pV1IKv3LsnNdo4gIxwwCMQDAqy0O be0YottT6SXbVQjgUMzfRGEWgqtJsLKB7HOHeLRMsmIbEvoWTSVLY70eN9k= -----END CERTIFICATE----- # Issuer: C=JP, O=SECOM Trust Systems CO.,LTD., OU=Security Communication RootCA2 # Subject: C=JP, O=SECOM Trust Systems CO.,LTD., OU=Security Communication RootCA2 # SHA256 Fingerprint: 51:3B:2C:EC:B8:10:D4:CD:E5:DD:85:39:1A:DF:C6:C2:DD:60:D8:7B:B7:36:D2:B5:21:48:4A:A4:7A:0E:BE:F6 -----BEGIN CERTIFICATE----- MIIDdzCCAl+gAwIBAgIBADANBgkqhkiG9w0BAQsFADBdMQswCQYDVQQGEwJKUDEl MCMGA1UEChMcU0VDT00gVHJ1c3QgU3lzdGVtcyBDTy4sTFRELjEnMCUGA1UECxMe U2VjdXJpdHkgQ29tbXVuaWNhdGlvbiBSb290Q0EyMB4XDTA5MDUyOTA1MDAzOVoX DTI5MDUyOTA1MDAzOVowXTELMAkGA1UEBhMCSlAxJTAjBgNVBAoTHFNFQ09NIFRy dXN0IFN5c3RlbXMgQ08uLExURC4xJzAlBgNVBAsTHlNlY3VyaXR5IENvbW11bmlj YXRpb24gUm9vdENBMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANAV OVKxUrO6xVmCxF1SrjpDZYBLx/KWvNs2l9amZIyoXvDjChz335c9S672XewhtUGr zbl+dp+++T42NKA7wfYxEUV0kz1XgMX5iZnK5atq1LXaQZAQwdbWQonCv/Q4EpVM VAX3NuRFg3sUZdbcDE3R3n4MqzvEFb46VqZab3ZpUql6ucjrappdUtAtCms1FgkQ hNBqyjoGADdH5H5XTz+L62e4iKrFvlNVspHEfbmwhRkGeC7bYRr6hfVKkaHnFtWO ojnflLhwHyg/i/xAXmODPIMqGplrz95Zajv8bxbXH/1KEOtOghY6rCcMU/Gt1SSw awNQwS08Ft1ENCcadfsCAwEAAaNCMEAwHQYDVR0OBBYEFAqFqXdlBZh8QIH4D5cs OPEK7DzPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 DQEBCwUAA4IBAQBMOqNErLlFsceTfsgLCkLfZOoc7llsCLqJX2rKSpWeeo8HxdpF coJxDjrSzG+ntKEju/Ykn8sX/oymzsLS28yN/HH8AynBbF0zX2S2ZTuJbxh2ePXc okgfGT+Ok+vx+hfuzU7jBBJV1uXk3fs+BXziHV7Gp7yXT2g69ekuCkO2r1dcYmh8 t/2jioSgrGK+KwmHNPBqAbubKVY8/gA3zyNs8U6qtnRGEmyR7jTV7JqR50S+kDFy 1UkC9gLl9B/rfNmWVan/7Ir5mUf/NVoCqgTLiluHcSmRvaS0eg29mvVXIwAHIRc/ SjnRBUkLp7Y3gaVdjKozXoEofKd9J+sAro03 -----END CERTIFICATE----- # Issuer: C=KR, O=NAVER BUSINESS PLATFORM Corp., CN=NAVER Global Root Certification Authority # Subject: C=KR, O=NAVER BUSINESS PLATFORM Corp., CN=NAVER Global Root Certification Authority # SHA256 Fingerprint: 88:F4:38:DC:F8:FF:D1:FA:8F:42:91:15:FF:E5:F8:2A:E1:E0:6E:0C:70:C3:75:FA:AD:71:7B:34:A4:9E:72:65 -----BEGIN CERTIFICATE----- MIIFojCCA4qgAwIBAgIUAZQwHqIL3fXFMyqxQ0Rx+NZQTQ0wDQYJKoZIhvcNAQEM BQAwaTELMAkGA1UEBhMCS1IxJjAkBgNVBAoMHU5BVkVSIEJVU0lORVNTIFBMQVRG T1JNIENvcnAuMTIwMAYDVQQDDClOQVZFUiBHbG9iYWwgUm9vdCBDZXJ0aWZpY2F0 aW9uIEF1dGhvcml0eTAeFw0xNzA4MTgwODU4NDJaFw0zNzA4MTgyMzU5NTlaMGkx CzAJBgNVBAYTAktSMSYwJAYDVQQKDB1OQVZFUiBCVVNJTkVTUyBQTEFURk9STSBD b3JwLjEyMDAGA1UEAwwpTkFWRVIgR2xvYmFsIFJvb3QgQ2VydGlmaWNhdGlvbiBB dXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC21PGTXLVA iQqrDZBbUGOukJR0F0Vy1ntlWilLp1agS7gvQnXp2XskWjFlqxcX0TM62RHcQDaH 38dq6SZeWYp34+hInDEW+j6RscrJo+KfziFTowI2MMtSAuXaMl3Dxeb57hHHi8lE HoSTGEq0n+USZGnQJoViAbbJAh2+g1G7XNr4rRVqmfeSVPc0W+m/6imBEtRTkZaz kVrd/pBzKPswRrXKCAfHcXLJZtM0l/aM9BhK4dA9WkW2aacp+yPOiNgSnABIqKYP szuSjXEOdMWLyEz59JuOuDxp7W87UC9Y7cSw0BwbagzivESq2M0UXZR4Yb8Obtoq vC8MC3GmsxY/nOb5zJ9TNeIDoKAYv7vxvvTWjIcNQvcGufFt7QSUqP620wbGQGHf nZ3zVHbOUzoBppJB7ASjjw2i1QnK1sua8e9DXcCrpUHPXFNwcMmIpi3Ua2FzUCaG YQ5fG8Ir4ozVu53BA0K6lNpfqbDKzE0K70dpAy8i+/Eozr9dUGWokG2zdLAIx6yo 0es+nPxdGoMuK8u180SdOqcXYZaicdNwlhVNt0xz7hlcxVs+Qf6sdWA7G2POAN3a CJBitOUt7kinaxeZVL6HSuOpXgRM6xBtVNbv8ejyYhbLgGvtPe31HzClrkvJE+2K AQHJuFFYwGY6sWZLxNUxAmLpdIQM201GLQIDAQABo0IwQDAdBgNVHQ4EFgQU0p+I 36HNLL3s9TsBAZMzJ7LrYEswDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB Af8wDQYJKoZIhvcNAQEMBQADggIBADLKgLOdPVQG3dLSLvCkASELZ0jKbY7gyKoN qo0hV4/GPnrK21HUUrPUloSlWGB/5QuOH/XcChWB5Tu2tyIvCZwTFrFsDDUIbatj cu3cvuzHV+YwIHHW1xDBE1UBjCpD5EHxzzp6U5LOogMFDTjfArsQLtk70pt6wKGm +LUx5vR1yblTmXVHIloUFcd4G7ad6Qz4G3bxhYTeodoS76TiEJd6eN4MUZeoIUCL hr0N8F5OSza7OyAfikJW4Qsav3vQIkMsRIz75Sq0bBwcupTgE34h5prCy8VCZLQe lHsIJchxzIdFV4XTnyliIoNRlwAYl3dqmJLJfGBs32x9SuRwTMKeuB330DTHD8z7 p/8Dvq1wkNoL3chtl1+afwkyQf3NosxabUzyqkn+Zvjp2DXrDige7kgvOtB5CTh8 piKCk5XQA76+AqAF3SAi428diDRgxuYKuQl1C/AH6GmWNcf7I4GOODm4RStDeKLR LBT/DShycpWbXgnbiUSYqqFJu3FS8r/2/yehNq+4tneI3TqkbZs0kNwUXTC/t+sX 5Ie3cdCh13cV1ELX8vMxmV2b3RZtP+oGI/hGoiLtk/bdmuYqh7GYVPEi92tF4+KO dh2ajcQGjTa3FPOdVGm3jjzVpG2Tgbet9r1ke8LJaDmgkpzNNIaRkPpkUZ3+/uul 9XXeifdy -----END CERTIFICATE----- # Issuer: C=NO, O=Buypass AS-983163327, CN=Buypass Class 2 Root CA # Subject: C=NO, O=Buypass AS-983163327, CN=Buypass Class 2 Root CA # SHA256 Fingerprint: 9A:11:40:25:19:7C:5B:B9:5D:94:E6:3D:55:CD:43:79:08:47:B6:46:B2:3C:DF:11:AD:A4:A0:0E:FF:15:FB:48 -----BEGIN CERTIFICATE----- MIIFWTCCA0GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQGEwJOTzEd MBsGA1UECgwUQnV5cGFzcyBBUy05ODMxNjMzMjcxIDAeBgNVBAMMF0J1eXBhc3Mg Q2xhc3MgMiBSb290IENBMB4XDTEwMTAyNjA4MzgwM1oXDTQwMTAyNjA4MzgwM1ow TjELMAkGA1UEBhMCTk8xHTAbBgNVBAoMFEJ1eXBhc3MgQVMtOTgzMTYzMzI3MSAw HgYDVQQDDBdCdXlwYXNzIENsYXNzIDIgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEB BQADggIPADCCAgoCggIBANfHXvfBB9R3+0Mh9PT1aeTuMgHbo4Yf5FkNuud1g1Lr 6hxhFUi7HQfKjK6w3Jad6sNgkoaCKHOcVgb/S2TwDCo3SbXlzwx87vFKu3MwZfPV L4O2fuPn9Z6rYPnT8Z2SdIrkHJasW4DptfQxh6NR/Md+oW+OU3fUl8FVM5I+GC91 1K2GScuVr1QGbNgGE41b/+EmGVnAJLqBcXmQRFBoJJRfuLMR8SlBYaNByyM21cHx MlAQTn/0hpPshNOOvEu/XAFOBz3cFIqUCqTqc/sLUegTBxj6DvEr0VQVfTzh97QZ QmdiXnfgolXsttlpF9U6r0TtSsWe5HonfOV116rLJeffawrbD02TTqigzXsu8lkB arcNuAeBfos4GzjmCleZPe4h6KP1DBbdi+w0jpwqHAAVF41og9JwnxgIzRFo1clr Us3ERo/ctfPYV3Me6ZQ5BL/T3jjetFPsaRyifsSP5BtwrfKi+fv3FmRmaZ9JUaLi FRhnBkp/1Wy1TbMz4GHrXb7pmA8y1x1LPC5aAVKRCfLf6o3YBkBjqhHk/sM3nhRS P/TizPJhk9H9Z2vXUq6/aKtAQ6BXNVN48FP4YUIHZMbXb5tMOA1jrGKvNouicwoN 9SG9dKpN6nIDSdvHXx1iY8f93ZHsM+71bbRuMGjeyNYmsHVee7QHIJihdjK4TWxP AgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMmAd+BikoL1Rpzz uvdMw964o605MA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAU18h 9bqwOlI5LJKwbADJ784g7wbylp7ppHR/ehb8t/W2+xUbP6umwHJdELFx7rxP462s A20ucS6vxOOto70MEae0/0qyexAQH6dXQbLArvQsWdZHEIjzIVEpMMpghq9Gqx3t OluwlN5E40EIosHsHdb9T7bWR9AUC8rmyrV7d35BH16Dx7aMOZawP5aBQW9gkOLo +fsicdl9sz1Gv7SEr5AcD48Saq/v7h56rgJKihcrdv6sVIkkLE8/trKnToyokZf7 KcZ7XC25y2a2t6hbElGFtQl+Ynhw/qlqYLYdDnkM/crqJIByw5c/8nerQyIKx+u2 DISCLIBrQYoIwOula9+ZEsuK1V6ADJHgJgg2SMX6OBE1/yWDLfJ6v9r9jv6ly0Us H8SIU653DtmadsWOLB2jutXsMq7Aqqz30XpN69QH4kj3Io6wpJ9qzo6ysmD0oyLQ I+uUWnpp3Q+/QFesa1lQ2aOZ4W7+jQF5JyMV3pKdewlNWudLSDBaGOYKbeaP4NK7 5t98biGCwWg5TbSYWGZizEqQXsP6JwSxeRV0mcy+rSDeJmAc61ZRpqPq5KM/p/9h 3PFaTWwyI0PurKju7koSCTxdccK+efrCh2gdC/1cacwG0Jp9VJkqyTkaGa9LKkPz Y11aWOIv4x3kqdbQCtCev9eBCfHJxyYNrJgWVqA= -----END CERTIFICATE----- # Issuer: C=NO, O=Buypass AS-983163327, CN=Buypass Class 3 Root CA # Subject: C=NO, O=Buypass AS-983163327, CN=Buypass Class 3 Root CA # SHA256 Fingerprint: ED:F7:EB:BC:A2:7A:2A:38:4D:38:7B:7D:40:10:C6:66:E2:ED:B4:84:3E:4C:29:B4:AE:1D:5B:93:32:E6:B2:4D -----BEGIN CERTIFICATE----- MIIFWTCCA0GgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQGEwJOTzEd MBsGA1UECgwUQnV5cGFzcyBBUy05ODMxNjMzMjcxIDAeBgNVBAMMF0J1eXBhc3Mg Q2xhc3MgMyBSb290IENBMB4XDTEwMTAyNjA4Mjg1OFoXDTQwMTAyNjA4Mjg1OFow TjELMAkGA1UEBhMCTk8xHTAbBgNVBAoMFEJ1eXBhc3MgQVMtOTgzMTYzMzI3MSAw HgYDVQQDDBdCdXlwYXNzIENsYXNzIDMgUm9vdCBDQTCCAiIwDQYJKoZIhvcNAQEB BQADggIPADCCAgoCggIBAKXaCpUWUOOV8l6ddjEGMnqb8RB2uACatVI2zSRHsJ8Y ZLya9vrVediQYkwiL944PdbgqOkcLNt4EemOaFEVcsfzM4fkoF0LXOBXByow9c3E N3coTRiR5r/VUv1xLXA+58bEiuPwKAv0dpihi4dVsjoT/Lc+JzeOIuOoTyrvYLs9 tznDDgFHmV0ST9tD+leh7fmdvhFHJlsTmKtdFoqwNxxXnUX/iJY2v7vKB3tvh2PX 0DJq1l1sDPGzbjniazEuOQAnFN44wOwZZoYS6J1yFhNkUsepNxz9gjDthBgd9K5c /3ATAOux9TN6S9ZV+AWNS2mw9bMoNlwUxFFzTWsL8TQH2xc519woe2v1n/MuwU8X KhDzzMro6/1rqy6any2CbgTUUgGTLT2G/H783+9CHaZr77kgxve9oKeV/afmiSTY zIw0bOIjL9kSGiG5VZFvC5F5GQytQIgLcOJ60g7YaEi7ghM5EFjp2CoHxhLbWNvS O1UQRwUVZ2J+GGOmRj8JDlQyXr8NYnon74Do29lLBlo3WiXQCBJ31G8JUJc9yB3D 34xFMFbG02SrZvPAXpacw8Tvw3xrizp5f7NJzz3iiZ+gMEuFuZyUJHmPfWupRWgP K9Dx2hzLabjKSWJtyNBjYt1gD1iqj6G8BaVmos8bdrKEZLFMOVLAMLrwjEsCsLa3 AgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFEe4zf/lb+74suwv Tg75JbCOPGvDMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAACAj QTUEkMJAYmDv4jVM1z+s4jSQuKFvdvoWFqRINyzpkMLyPPgKn9iB5btb2iUspKdV cSQy9sgL8rxq+JOssgfCX5/bzMiKqr5qb+FJEMwx14C7u8jYog5kV+qi9cKpMRXS IGrs/CIBKM+GuIAeqcwRpTzyFrNHnfzSgCHEy9BHcEGhyoMZCCxt8l13nIoUE9Q2 HJLw5QY33KbmkJs4j1xrG0aGQ0JfPgEHU1RdZX33inOhmlRaHylDFCfChQ+1iHsa O5S3HWCntZznKWlXWpuTekMwGwPXYshApqr8ZORK15FTAaggiG6cX0S5y2CBNOxv 033aSF/rtJC8LakcC6wc1aJoIIAE1vyxjy+7SjENSoYc6+I2KSb12tjE8nVhz36u dmNKekBlk4f4HoCMhuWG1o8O/FMsYOgWYRqiPkN7zTlgVGr18okmAWiDSKIz6MkE kbIRNBE+6tBDGR8Dk5AM/1E9V/RBbuHLoL7ryWPNbczk+DaqaJ3tvV2XcEQNtg41 3OEMXbugUZTLfhbrES+jkkXITHHZvMmZUldGL1DPvTVp9D0VzgalLA8+9oG6lLvD u79leNKGef9JOxqDDPDeeOzI8k1MGt6CKfjBWtrt7uYnXuhF0J0cUahoq0Tj0Itq 4/g7u9xN12TyUb7mqqta6THuBrxzvxNiCp/HuZc= -----END CERTIFICATE----- # Issuer: C=PL, O=Asseco Data Systems S.A., OU=Certum Certification Authority, CN=Certum EC-384 CA # Subject: C=PL, O=Asseco Data Systems S.A., OU=Certum Certification Authority, CN=Certum EC-384 CA # SHA256 Fingerprint: 6B:32:80:85:62:53:18:AA:50:D1:73:C9:8D:8B:DA:09:D5:7E:27:41:3D:11:4C:F7:87:A0:F5:D0:6C:03:0C:F6 -----BEGIN CERTIFICATE----- MIICZTCCAeugAwIBAgIQeI8nXIESUiClBNAt3bpz9DAKBggqhkjOPQQDAzB0MQsw CQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBTLkEuMScw JQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxGTAXBgNVBAMT EENlcnR1bSBFQy0zODQgQ0EwHhcNMTgwMzI2MDcyNDU0WhcNNDMwMzI2MDcyNDU0 WjB0MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBT LkEuMScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxGTAX BgNVBAMTEENlcnR1bSBFQy0zODQgQ0EwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAATE KI6rGFtqvm5kN2PkzeyrOvfMobgOgknXhimfoZTy42B4mIF4Bk3y7JoOV2CDn7Tm Fy8as10CW4kjPMIRBSqniBMY81CE1700LCeJVf/OTOffph8oxPBUw7l8t1Ot68Kj QjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI0GZnQkdjrzife81r1HfS+8 EF9LMA4GA1UdDwEB/wQEAwIBBjAKBggqhkjOPQQDAwNoADBlAjADVS2m5hjEfO/J UG7BJw+ch69u1RsIGL2SKcHvlJF40jocVYli5RsJHrpka/F2tNQCMQC0QoSZ/6vn nvuRlydd3LBbMHHOXjgaatkl5+r3YZJW+OraNsKHZZYuciUvf9/DE8k= -----END CERTIFICATE----- # Issuer: C=PL, O=Asseco Data Systems S.A., OU=Certum Certification Authority, CN=Certum Trusted Root CA # Subject: C=PL, O=Asseco Data Systems S.A., OU=Certum Certification Authority, CN=Certum Trusted Root CA # SHA256 Fingerprint: FE:76:96:57:38:55:77:3E:37:A9:5E:7A:D4:D9:CC:96:C3:01:57:C1:5D:31:76:5B:A9:B1:57:04:E1:AE:78:FD -----BEGIN CERTIFICATE----- MIIFwDCCA6igAwIBAgIQHr9ZULjJgDdMBvfrVU+17TANBgkqhkiG9w0BAQ0FADB6 MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEgU3lzdGVtcyBTLkEu MScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxHzAdBgNV BAMTFkNlcnR1bSBUcnVzdGVkIFJvb3QgQ0EwHhcNMTgwMzE2MTIxMDEzWhcNNDMw MzE2MTIxMDEzWjB6MQswCQYDVQQGEwJQTDEhMB8GA1UEChMYQXNzZWNvIERhdGEg U3lzdGVtcyBTLkEuMScwJQYDVQQLEx5DZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRo b3JpdHkxHzAdBgNVBAMTFkNlcnR1bSBUcnVzdGVkIFJvb3QgQ0EwggIiMA0GCSqG SIb3DQEBAQUAA4ICDwAwggIKAoICAQDRLY67tzbqbTeRn06TpwXkKQMlzhyC93yZ n0EGze2jusDbCSzBfN8pfktlL5On1AFrAygYo9idBcEq2EXxkd7fO9CAAozPOA/q p1x4EaTByIVcJdPTsuclzxFUl6s1wB52HO8AU5853BSlLCIls3Jy/I2z5T4IHhQq NwuIPMqw9MjCoa68wb4pZ1Xi/K1ZXP69VyywkI3C7Te2fJmItdUDmj0VDT06qKhF 8JVOJVkdzZhpu9PMMsmN74H+rX2Ju7pgE8pllWeg8xn2A1bUatMn4qGtg/BKEiJ3 HAVz4hlxQsDsdUaakFjgao4rpUYwBI4Zshfjvqm6f1bxJAPXsiEodg42MEx51UGa mqi4NboMOvJEGyCI98Ul1z3G4z5D3Yf+xOr1Uz5MZf87Sst4WmsXXw3Hw09Omiqi 7VdNIuJGmj8PkTQkfVXjjJU30xrwCSss0smNtA0Aq2cpKNgB9RkEth2+dv5yXMSF ytKAQd8FqKPVhJBPC/PgP5sZ0jeJP/J7UhyM9uH3PAeXjA6iWYEMspA90+NZRu0P qafegGtaqge2Gcu8V/OXIXoMsSt0Puvap2ctTMSYnjYJdmZm/Bo/6khUHL4wvYBQ v3y1zgD2DGHZ5yQD4OMBgQ692IU0iL2yNqh7XAjlRICMb/gv1SHKHRzQ+8S1h9E6 Tsd2tTVItQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSM+xx1 vALTn04uSNn5YFSqxLNP+jAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQENBQAD ggIBAEii1QALLtA/vBzVtVRJHlpr9OTy4EA34MwUe7nJ+jW1dReTagVphZzNTxl4 WxmB82M+w85bj/UvXgF2Ez8sALnNllI5SW0ETsXpD4YN4fqzX4IS8TrOZgYkNCvo zMrnadyHncI013nR03e4qllY/p0m+jiGPp2Kh2RX5Rc64vmNueMzeMGQ2Ljdt4NR 5MTMI9UGfOZR0800McD2RrsLrfw9EAUqO0qRJe6M1ISHgCq8CYyqOhNf6DR5UMEQ GfnTKB7U0VEwKbOukGfWHwpjscWpxkIxYxeU72nLL/qMFH3EQxiJ2fAyQOaA4kZf 5ePBAFmo+eggvIksDkc0C+pXwlM2/KfUrzHN/gLldfq5Jwn58/U7yn2fqSLLiMmq 0Uc9NneoWWRrJ8/vJ8HjJLWG965+Mk2weWjROeiQWMODvA8s1pfrzgzhIMfatz7D P78v3DSk+yshzWePS/Tj6tQ/50+6uaWTRRxmHyH6ZF5v4HaUMst19W7l9o/HuKTM qJZ9ZPskWkoDbGs4xugDQ5r3V7mzKWmTOPQD8rv7gmsHINFSH5pkAnuYZttcTVoP 0ISVoDwUQwbKytu4QTbaakRnh6+v40URFWkIsr4WOZckbxJF0WddCajJFdr60qZf E2Efv4WstK2tBZQIgx51F9NxO5NQI1mg7TyRVJ12AMXDuDjb -----END CERTIFICATE----- # Issuer: C=PL, O=Krajowa Izba Rozliczeniowa S.A., CN=SZAFIR ROOT CA2 # Subject: C=PL, O=Krajowa Izba Rozliczeniowa S.A., CN=SZAFIR ROOT CA2 # SHA256 Fingerprint: A1:33:9D:33:28:1A:0B:56:E5:57:D3:D3:2B:1C:E7:F9:36:7E:B0:94:BD:5F:A7:2A:7E:50:04:C8:DE:D7:CA:FE -----BEGIN CERTIFICATE----- MIIDcjCCAlqgAwIBAgIUPopdB+xV0jLVt+O2XwHrLdzk1uQwDQYJKoZIhvcNAQEL BQAwUTELMAkGA1UEBhMCUEwxKDAmBgNVBAoMH0tyYWpvd2EgSXpiYSBSb3psaWN6 ZW5pb3dhIFMuQS4xGDAWBgNVBAMMD1NaQUZJUiBST09UIENBMjAeFw0xNTEwMTkw NzQzMzBaFw0zNTEwMTkwNzQzMzBaMFExCzAJBgNVBAYTAlBMMSgwJgYDVQQKDB9L cmFqb3dhIEl6YmEgUm96bGljemVuaW93YSBTLkEuMRgwFgYDVQQDDA9TWkFGSVIg Uk9PVCBDQTIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3vD5QqEvN QLXOYeeWyrSh2gwisPq1e3YAd4wLz32ohswmUeQgPYUM1ljj5/QqGJ3a0a4m7utT 3PSQ1hNKDJA8w/Ta0o4NkjrcsbH/ON7Dui1fgLkCvUqdGw+0w8LBZwPd3BucPbOw 3gAeqDRHu5rr/gsUvTaE2g0gv/pby6kWIK05YO4vdbbnl5z5Pv1+TW9NL++IDWr6 3fE9biCloBK0TXC5ztdyO4mTp4CEHCdJckm1/zuVnsHMyAHs6A6KCpbns6aH5db5 BSsNl0BwPLqsdVqc1U2dAgrSS5tmS0YHF2Wtn2yIANwiieDhZNRnvDF5YTy7ykHN XGoAyDw4jlivAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD AgEGMB0GA1UdDgQWBBQuFqlKGLXLzPVvUPMjX/hd56zwyDANBgkqhkiG9w0BAQsF AAOCAQEAtXP4A9xZWx126aMqe5Aosk3AM0+qmrHUuOQn/6mWmc5G4G18TKI4pAZw 8PRBEew/R40/cof5O/2kbytTAOD/OblqBw7rHRz2onKQy4I9EYKL0rufKq8h5mOG nXkZ7/e7DDWQw4rtTw/1zBLZpD67oPwglV9PJi8RI4NOdQcPv5vRtB3pEAT+ymCP oky4rc/hkA/NrgrHXXu3UNLUYfrVFdvXn4dRVOul4+vJhaAlIDf7js4MNIThPIGy d05DpYhfhmehPea0XGG2Ptv+tyjFogeutcrKjSoS75ftwjCkySp6+/NNIxuZMzSg LvWpCz/UXeHPhJ/iGcJfitYgHuNztw== -----END CERTIFICATE----- # Issuer: C=PL, O=Unizeto Technologies S.A., OU=Certum Certification Authority, CN=Certum Trusted Network CA 2 # Subject: C=PL, O=Unizeto Technologies S.A., OU=Certum Certification Authority, CN=Certum Trusted Network CA 2 # SHA256 Fingerprint: B6:76:F2:ED:DA:E8:77:5C:D3:6C:B0:F6:3C:D1:D4:60:39:61:F4:9E:62:65:BA:01:3A:2F:03:07:B6:D0:B8:04 -----BEGIN CERTIFICATE----- MIIF0jCCA7qgAwIBAgIQIdbQSk8lD8kyN/yqXhKN6TANBgkqhkiG9w0BAQ0FADCB gDELMAkGA1UEBhMCUEwxIjAgBgNVBAoTGVVuaXpldG8gVGVjaG5vbG9naWVzIFMu QS4xJzAlBgNVBAsTHkNlcnR1bSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTEkMCIG A1UEAxMbQ2VydHVtIFRydXN0ZWQgTmV0d29yayBDQSAyMCIYDzIwMTExMDA2MDgz OTU2WhgPMjA0NjEwMDYwODM5NTZaMIGAMQswCQYDVQQGEwJQTDEiMCAGA1UEChMZ VW5pemV0byBUZWNobm9sb2dpZXMgUy5BLjEnMCUGA1UECxMeQ2VydHVtIENlcnRp ZmljYXRpb24gQXV0aG9yaXR5MSQwIgYDVQQDExtDZXJ0dW0gVHJ1c3RlZCBOZXR3 b3JrIENBIDIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC9+Xj45tWA DGSdhhuWZGc/IjoedQF97/tcZ4zJzFxrqZHmuULlIEub2pt7uZld2ZuAS9eEQCsn 0+i6MLs+CRqnSZXvK0AkwpfHp+6bJe+oCgCXhVqqndwpyeI1B+twTUrWwbNWuKFB OJvR+zF/j+Bf4bE/D44WSWDXBo0Y+aomEKsq09DRZ40bRr5HMNUuctHFY9rnY3lE fktjJImGLjQ/KUxSiyqnwOKRKIm5wFv5HdnnJ63/mgKXwcZQkpsCLL2puTRZCr+E Sv/f/rOf69me4Jgj7KZrdxYq28ytOxykh9xGc14ZYmhFV+SQgkK7QtbwYeDBoz1m o130GO6IyY0XRSmZMnUCMe4pJshrAua1YkV/NxVaI2iJ1D7eTiew8EAMvE0Xy02i sx7QBlrd9pPPV3WZ9fqGGmd4s7+W/jTcvedSVuWz5XV710GRBdxdaeOVDUO5/IOW OZV7bIBaTxNyxtd9KXpEulKkKtVBRgkg/iKgtlswjbyJDNXXcPiHUv3a76xRLgez Tv7QCdpw75j6VuZt27VXS9zlLCUVyJ4ueE742pyehizKV/Ma5ciSixqClnrDvFAS adgOWkaLOusm+iPJtrCBvkIApPjW/jAux9JG9uWOdf3yzLnQh1vMBhBgu4M1t15n 3kfsmUjxpKEV/q2MYo45VU85FrmxY53/twIDAQABo0IwQDAPBgNVHRMBAf8EBTAD AQH/MB0GA1UdDgQWBBS2oVQ5AsOgP46KvPrU+Bym0ToO/TAOBgNVHQ8BAf8EBAMC AQYwDQYJKoZIhvcNAQENBQADggIBAHGlDs7k6b8/ONWJWsQCYftMxRQXLYtPU2sQ F/xlhMcQSZDe28cmk4gmb3DWAl45oPePq5a1pRNcgRRtDoGCERuKTsZPpd1iHkTf CVn0W3cLN+mLIMb4Ck4uWBzrM9DPhmDJ2vuAL55MYIR4PSFk1vtBHxgP58l1cb29 XN40hz5BsA72udY/CROWFC/emh1auVbONTqwX3BNXuMp8SMoclm2q8KMZiYcdywm djWLKKdpoPk79SPdhRB0yZADVpHnr7pH1BKXESLjokmUbOe3lEu6LaTaM4tMpkT/ WjzGHWTYtTHkpjx6qFcL2+1hGsvxznN3Y6SHb0xRONbkX8eftoEq5IVIeVheO/jb AoJnwTnbw3RLPTYe+SmTiGhbqEQZIfCn6IENLOiTNrQ3ssqwGyZ6miUfmpqAnksq P/ujmv5zMnHCnsZy4YpoJ/HkD7TETKVhk/iXEAcqMCWpuchxuO9ozC1+9eB+D4Ko b7a6bINDd82Kkhehnlt4Fj1F4jNy3eFmypnTycUm/Q1oBEauttmbjL4ZvrHG8hnj XALKLNhvSgfZyTXaQHXyxKcZb55CEJh15pWLYLztxRLXis7VmFxWlgPF7ncGNf/P 5O4/E2Hu29othfDNrp2yGAlFw5Khchf8R7agCyzxxN5DaAhqXzvwdmP7zAYspsbi DrW5viSP -----END CERTIFICATE----- # Issuer: C=PL, O=Unizeto Technologies S.A., OU=Certum Certification Authority, CN=Certum Trusted Network CA # Subject: C=PL, O=Unizeto Technologies S.A., OU=Certum Certification Authority, CN=Certum Trusted Network CA # SHA256 Fingerprint: 5C:58:46:8D:55:F5:8E:49:7E:74:39:82:D2:B5:00:10:B6:D1:65:37:4A:CF:83:A7:D4:A3:2D:B7:68:C4:40:8E -----BEGIN CERTIFICATE----- MIIDuzCCAqOgAwIBAgIDBETAMA0GCSqGSIb3DQEBBQUAMH4xCzAJBgNVBAYTAlBM MSIwIAYDVQQKExlVbml6ZXRvIFRlY2hub2xvZ2llcyBTLkEuMScwJQYDVQQLEx5D ZXJ0dW0gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxIjAgBgNVBAMTGUNlcnR1bSBU cnVzdGVkIE5ldHdvcmsgQ0EwHhcNMDgxMDIyMTIwNzM3WhcNMjkxMjMxMTIwNzM3 WjB+MQswCQYDVQQGEwJQTDEiMCAGA1UEChMZVW5pemV0byBUZWNobm9sb2dpZXMg Uy5BLjEnMCUGA1UECxMeQ2VydHVtIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MSIw IAYDVQQDExlDZXJ0dW0gVHJ1c3RlZCBOZXR3b3JrIENBMIIBIjANBgkqhkiG9w0B AQEFAAOCAQ8AMIIBCgKCAQEA4/t9o3K6wvDJFIf1awFO4W5AB7ptJ11/91sts1rH UV+rpDKmYYe2bg+G0jACl/jXaVehGDldamR5xgFZrDwxSjh80gTSSyjoIF87B6LM TXPb865Px1bVWqeWifrzq2jUI4ZZJ88JJ7ysbnKDHDBy3+Ci6dLhdHUZvSqeexVU BBvXQzmtVSjF4hq79MDkrjhJM8x2hZ85RdKknvISjFH4fOQtf/WsX+sWn7Et0brM kUJ3TCXJkDhv2/DM+44el1k+1WBO5gUo7Ul5E0u6SNsv+XLTOcr+H9g0cvW0QM8x AcPs3hEtF10fuFDRXhmnad4HMyjKUJX5p1TLVIZQRan5SQIDAQABo0IwQDAPBgNV HRMBAf8EBTADAQH/MB0GA1UdDgQWBBQIds3LB/8k9sXN7buQvOKEN0Z19zAOBgNV HQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEFBQADggEBAKaorSLOAT2mo/9i0Eidi15y sHhE49wcrwn9I0j6vSrEuVUEtRCjjSfeC4Jj0O7eDDd5QVsisrCaQVymcODU0HfL I9MA4GxWL+FpDQ3Zqr8hgVDZBqWo/5U30Kr+4rP1mS1FhIrlQgnXdAIv94nYmem8 J9RHjboNRhx3zxSkHLmkMcScKHQDNP8zGSal6Q10tz6XxnboJ5ajZt3hrvJBW8qY VoNzcOSGGtIxQbovvi0TWnZvTuhOgQ4/WwMioBK+ZlgRSssDxLQqKi2WF+A5VLxI 03YnnZotBqbJ7DnSq9ufmgsnAjUpsUCV5/nonFWIGUbWtzT1fs45mtk48VH3Tyw= -----END CERTIFICATE----- # Issuer: C=RO, O=CERTSIGN SA, OU=certSIGN ROOT CA G2 # Subject: C=RO, O=CERTSIGN SA, OU=certSIGN ROOT CA G2 # SHA256 Fingerprint: 65:7C:FE:2F:A7:3F:AA:38:46:25:71:F3:32:A2:36:3A:46:FC:E7:02:09:51:71:07:02:CD:FB:B6:EE:DA:33:05 -----BEGIN CERTIFICATE----- MIIFRzCCAy+gAwIBAgIJEQA0tk7GNi02MA0GCSqGSIb3DQEBCwUAMEExCzAJBgNV BAYTAlJPMRQwEgYDVQQKEwtDRVJUU0lHTiBTQTEcMBoGA1UECxMTY2VydFNJR04g Uk9PVCBDQSBHMjAeFw0xNzAyMDYwOTI3MzVaFw00MjAyMDYwOTI3MzVaMEExCzAJ BgNVBAYTAlJPMRQwEgYDVQQKEwtDRVJUU0lHTiBTQTEcMBoGA1UECxMTY2VydFNJ R04gUk9PVCBDQSBHMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMDF dRmRfUR0dIf+DjuW3NgBFszuY5HnC2/OOwppGnzC46+CjobXXo9X69MhWf05N0Iw vlDqtg+piNguLWkh59E3GE59kdUWX2tbAMI5Qw02hVK5U2UPHULlj88F0+7cDBrZ uIt4ImfkabBoxTzkbFpG583H+u/E7Eu9aqSs/cwoUe+StCmrqzWaTOTECMYmzPhp n+Sc8CnTXPnGFiWeI8MgwT0PPzhAsP6CRDiqWhqKa2NYOLQV07YRaXseVO6MGiKs cpc/I1mbySKEwQdPzH/iV8oScLumZfNpdWO9lfsbl83kqK/20U6o2YpxJM02PbyW xPFsqa7lzw1uKA2wDrXKUXt4FMMgL3/7FFXhEZn91QqhngLjYl/rNUssuHLoPj1P rCy7Lobio3aP5ZMqz6WryFyNSwb/EkaseMsUBzXgqd+L6a8VTxaJW732jcZZroiF DsGJ6x9nxUWO/203Nit4ZoORUSs9/1F3dmKh7Gc+PoGD4FapUB8fepmrY7+EF3fx DTvf95xhszWYijqy7DwaNz9+j5LP2RIUZNoQAhVB/0/E6xyjyfqZ90bp4RjZsbgy LcsUDFDYg2WD7rlcz8sFWkz6GZdr1l0T08JcVLwyc6B49fFtHsufpaafItzRUZ6C eWRgKRM+o/1Pcmqr4tTluCRVLERLiohEnMqE0yo7AgMBAAGjQjBAMA8GA1UdEwEB /wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSCIS1mxteg4BXrzkwJ d8RgnlRuAzANBgkqhkiG9w0BAQsFAAOCAgEAYN4auOfyYILVAzOBywaK8SJJ6ejq kX/GM15oGQOGO0MBzwdw5AgeZYWR5hEit/UCI46uuR59H35s5r0l1ZUa8gWmr4UC b6741jH/JclKyMeKqdmfS0mbEVeZkkMR3rYzpMzXjWR91M08KCy0mpbqTfXERMQl qiCA2ClV9+BB/AYm/7k29UMUA2Z44RGx2iBfRgB4ACGlHgAoYXhvqAEBj500mv/0 OJD7uNGzcgbJceaBxXntC6Z58hMLnPddDnskk7RI24Zf3lCGeOdA5jGokHZwYa+c NywRtYK3qq4kNFtyDGkNzVmf9nGvnAvRCjj5BiKDUyUM/FHE5r7iOZULJK2v0ZXk ltd0ZGtxTgI8qoXzIKNDOXZbbFD+mpwUHmUUihW9o4JFWklWatKcsWMy5WHgUyIO pwpJ6st+H6jiYoD2EEVSmAYY3qXNL3+q1Ok+CHLsIwMCPKaq2LxndD0UF/tUSxfj 03k9bWtJySgOLnRQvwzZRjoQhsmnP+mg7H/rpXdYaXHmgwo38oZJar55CJD2AhZk PuXaTH4MNMn5X7azKFGnpyuqSfqNZSlO42sTp5SjLVFteAxEy9/eCG/Oo2Sr05WE 1LlSVHJ7liXMvGnjSG4N0MedJ5qq+BOS3R7fY581qRY27Iy4g/Q9iY/NtBde17MX QRBdJ3NghVdJIgc= -----END CERTIFICATE----- # Issuer: C=RO, O=certSIGN, OU=certSIGN ROOT CA # Subject: C=RO, O=certSIGN, OU=certSIGN ROOT CA # SHA256 Fingerprint: EA:A9:62:C4:FA:4A:6B:AF:EB:E4:15:19:6D:35:1C:CD:88:8D:4F:53:F3:FA:8A:E6:D7:C4:66:A9:4E:60:42:BB -----BEGIN CERTIFICATE----- MIIDODCCAiCgAwIBAgIGIAYFFnACMA0GCSqGSIb3DQEBBQUAMDsxCzAJBgNVBAYT AlJPMREwDwYDVQQKEwhjZXJ0U0lHTjEZMBcGA1UECxMQY2VydFNJR04gUk9PVCBD QTAeFw0wNjA3MDQxNzIwMDRaFw0zMTA3MDQxNzIwMDRaMDsxCzAJBgNVBAYTAlJP MREwDwYDVQQKEwhjZXJ0U0lHTjEZMBcGA1UECxMQY2VydFNJR04gUk9PVCBDQTCC ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALczuX7IJUqOtdu0KBuqV5Do 0SLTZLrTk+jUrIZhQGpgV2hUhE28alQCBf/fm5oqrl0Hj0rDKH/v+yv6efHHrfAQ UySQi2bJqIirr1qjAOm+ukbuW3N7LBeCgV5iLKECZbO9xSsAfsT8AzNXDe3i+s5d RdY4zTW2ssHQnIFKquSyAVwdj1+ZxLGt24gh65AIgoDzMKND5pCCrlUoSe1b16kQ OA7+j0xbm0bqQfWwCHTD0IgztnzXdN/chNFDDnU5oSVAKOp4yw4sLjmdjItuFhwv JoIQ4uNllAoEwF73XVv4EOLQunpL+943AAAaWyjj0pxzPjKHmKHJUS/X3qwzs08C AwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAcYwHQYDVR0O BBYEFOCMm9slSbPxfIbWskKHC9BroNnkMA0GCSqGSIb3DQEBBQUAA4IBAQA+0hyJ LjX8+HXd5n9liPRyTMks1zJO890ZeUe9jjtbkw9QSSQTaxQGcu8J06Gh40CEyecY MnQ8SG4Pn0vU9x7Tk4ZkVJdjclDVVc/6IJMCopvDI5NOFlV2oHB5bc0hH88vLbwZ 44gx+FkagQnIl6Z0x2DEW8xXjrJ1/RsCCdtZb3KTafcxQdaIOL+Hsr0Wefmq5L6I Jd1hJyMctTEHBDa0GpC9oHRxUIltvBTjD4au8as+x6AJzKNI0eDbZOeStc+vckNw i/nDhDwTqn6Sm1dTk/pwwpEOMfmbZ13pljheX7NzTogVZ96edhBiIL5VaZVDADlN 9u6wWk5JRFRYX0KD -----END CERTIFICATE----- # Issuer: C=SK, L=Bratislava, O=Disig a.s., CN=CA Disig Root R2 # Subject: C=SK, L=Bratislava, O=Disig a.s., CN=CA Disig Root R2 # SHA256 Fingerprint: E2:3D:4A:03:6D:7B:70:E9:F5:95:B1:42:20:79:D2:B9:1E:DF:BB:1F:B6:51:A0:63:3E:AA:8A:9D:C5:F8:07:03 -----BEGIN CERTIFICATE----- MIIFaTCCA1GgAwIBAgIJAJK4iNuwisFjMA0GCSqGSIb3DQEBCwUAMFIxCzAJBgNV BAYTAlNLMRMwEQYDVQQHEwpCcmF0aXNsYXZhMRMwEQYDVQQKEwpEaXNpZyBhLnMu MRkwFwYDVQQDExBDQSBEaXNpZyBSb290IFIyMB4XDTEyMDcxOTA5MTUzMFoXDTQy MDcxOTA5MTUzMFowUjELMAkGA1UEBhMCU0sxEzARBgNVBAcTCkJyYXRpc2xhdmEx EzARBgNVBAoTCkRpc2lnIGEucy4xGTAXBgNVBAMTEENBIERpc2lnIFJvb3QgUjIw ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCio8QACdaFXS1tFPbCw3Oe NcJxVX6B+6tGUODBfEl45qt5WDza/3wcn9iXAng+a0EE6UG9vgMsRfYvZNSrXaNH PWSb6WiaxswbP7q+sos0Ai6YVRn8jG+qX9pMzk0DIaPY0jSTVpbLTAwAFjxfGs3I x2ymrdMxp7zo5eFm1tL7A7RBZckQrg4FY8aAamkw/dLukO8NJ9+flXP04SXabBbe QTg06ov80egEFGEtQX6sx3dOy1FU+16SGBsEWmjGycT6txOgmLcRK7fWV8x8nhfR yyX+hk4kLlYMeE2eARKmK6cBZW58Yh2EhN/qwGu1pSqVg8NTEQxzHQuyRpDRQjrO QG6Vrf/GlK1ul4SOfW+eioANSW1z4nuSHsPzwfPrLgVv2RvPN3YEyLRa5Beny912 H9AZdugsBbPWnDTYltxhh5EF5EQIM8HauQhl1K6yNg3ruji6DOWbnuuNZt2Zz9aJ QfYEkoopKW1rOhzndX0CcQ7zwOe9yxndnWCywmZgtrEE7snmhrmaZkCo5xHtgUUD i/ZnWejBBhG93c+AAk9lQHhcR1DIm+YfgXvkRKhbhZri3lrVx/k6RGZL5DJUfORs nLMOPReisjQS1n6yqEm70XooQL6iFh/f5DcfEXP7kAplQ6INfPgGAVUzfbANuPT1 rqVCV3w2EYx7XsQDnYx5nQIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1Ud DwEB/wQEAwIBBjAdBgNVHQ4EFgQUtZn4r7CU9eMg1gqtzk5WpC5uQu0wDQYJKoZI hvcNAQELBQADggIBACYGXnDnZTPIgm7ZnBc6G3pmsgH2eDtpXi/q/075KMOYKmFM tCQSin1tERT3nLXK5ryeJ45MGcipvXrA1zYObYVybqjGom32+nNjf7xueQgcnYqf GopTpti72TVVsRHFqQOzVju5hJMiXn7B9hJSi+osZ7z+Nkz1uM/Rs0mSO9MpDpkb lvdhuDvEK7Z4bLQjb/D907JedR+Zlais9trhxTF7+9FGs9K8Z7RiVLoJ92Owk6Ka +elSLotgEqv89WBW7xBci8QaQtyDW2QOy7W81k/BfDxujRNt+3vrMNDcTa/F1bal TFtxyegxvug4BkihGuLq0t4SOVga/4AOgnXmt8kHbA7v/zjxmHHEt38OFdAlab0i nSvtBfZGR6ztwPDUO+Ls7pZbkBNOHlY667DvlruWIxG68kOGdGSVyCh13x01utI3 gzhTODY7z2zp+WsO0PsE6E9312UBeIYMej4hYvF/Y3EMyZ9E26gnonW+boE+18Dr G5gPcFw0sorMwIUY6256s/daoQe/qUKS82Ail+QUoQebTnbAjn39pCXHR+3/H3Os zMOl6W8KjptlwlCFtaOgUxLMVYdh84GuEEZhvUQhuMI9dM9+JDX6HAcOmz0iyu8x L4ysEr3vQCj8KWefshNPZiTEUxnpHikV7+ZtsH8tZ/3zbBt1RqPlShfppNcL -----END CERTIFICATE----- # Issuer: C=TN, O=Agence Nationale de Certification Electronique, CN=TunTrust Root CA # Subject: C=TN, O=Agence Nationale de Certification Electronique, CN=TunTrust Root CA # SHA256 Fingerprint: 2E:44:10:2A:B5:8C:B8:54:19:45:1C:8E:19:D9:AC:F3:66:2C:AF:BC:61:4B:6A:53:96:0A:30:F7:D0:E2:EB:41 -----BEGIN CERTIFICATE----- MIIFszCCA5ugAwIBAgIUEwLV4kBMkkaGFmddtLu7sms+/BMwDQYJKoZIhvcNAQEL BQAwYTELMAkGA1UEBhMCVE4xNzA1BgNVBAoMLkFnZW5jZSBOYXRpb25hbGUgZGUg Q2VydGlmaWNhdGlvbiBFbGVjdHJvbmlxdWUxGTAXBgNVBAMMEFR1blRydXN0IFJv b3QgQ0EwHhcNMTkwNDI2MDg1NzU2WhcNNDQwNDI2MDg1NzU2WjBhMQswCQYDVQQG EwJUTjE3MDUGA1UECgwuQWdlbmNlIE5hdGlvbmFsZSBkZSBDZXJ0aWZpY2F0aW9u IEVsZWN0cm9uaXF1ZTEZMBcGA1UEAwwQVHVuVHJ1c3QgUm9vdCBDQTCCAiIwDQYJ KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMPN0/y9BFPdDCA61YguBUtB9YOCfvdZ n56eY+hz2vYGqU8ftPkLHzmMmiDQfgbU7DTZhrx1W4eI8NLZ1KMKsmwb60ksPqxd 2JQDoOw05TDENX37Jk0bbjBU2PWARZw5rZzJJQRNmpA+TkBuimvNKWfGzC3gdOgF VwpIUPp6Q9p+7FuaDmJ2/uqdHYVy7BG7NegfJ7/Boce7SBbdVtfMTqDhuazb1YMZ GoXRlJfXyqNlC/M4+QKu3fZnz8k/9YosRxqZbwUN/dAdgjH8KcwAWJeRTIAAHDOF li/LQcKLEITDCSSJH7UP2dl3RxiSlGBcx5kDPP73lad9UKGAwqmDrViWVSHbhlnU r8a83YFuB9tgYv7sEG7aaAH0gxupPqJbI9dkxt/con3YS7qC0lH4Zr8GRuR5KiY2 eY8fTpkdso8MDhz/yV3A/ZAQprE38806JG60hZC/gLkMjNWb1sjxVj8agIl6qeIb MlEsPvLfe/ZdeikZjuXIvTZxi11Mwh0/rViizz1wTaZQmCXcI/m4WEEIcb9PuISg jwBUFfyRbVinljvrS5YnzWuioYasDXxU5mZMZl+QviGaAkYt5IPCgLnPSz7ofzwB 7I9ezX/SKEIBlYrilz0QIX32nRzFNKHsLA4KUiwSVXAkPcvCFDVDXSdOvsC9qnyW 5/yeYa1E0wCXAgMBAAGjYzBhMB0GA1UdDgQWBBQGmpsfU33x9aTI04Y+oXNZtPdE ITAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFAaamx9TffH1pMjThj6hc1m0 90QhMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAqgVutt0Vyb+z xiD2BkewhpMl0425yAA/l/VSJ4hxyXT968pk21vvHl26v9Hr7lxpuhbI87mP0zYu QEkHDVneixCwSQXi/5E/S7fdAo74gShczNxtr18UnH1YeA32gAm56Q6XKRm4t+v4 FstVEuTGfbvE7Pi1HE4+Z7/FXxttbUcoqgRYYdZ2vyJ/0Adqp2RT8JeNnYA/u8EH 22Wv5psymsNUk8QcCMNE+3tjEUPRahphanltkE8pjkcFwRJpadbGNjHh/PqAulxP xOu3Mqz4dWEX1xAZufHSCe96Qp1bWgvUxpVOKs7/B9dPfhgGiPEZtdmYu65xxBzn dFlY7wyJz4sfdZMaBBSSSFCp61cpABbjNhzI+L/wM9VBD8TMPN3pM0MBkRArHtG5 Xc0yGYuPjCB31yLEQtyEFpslbei0VXF/sHyz03FJuc9SpAQ/3D2gu68zngowYI7b nV2UqL1g52KAdoGDDIzMMEZJ4gzSqK/rYXHv5yJiqfdcZGyfFoxnNidF9Ql7v/YQ CvGwjVRDjAS6oz/v4jXH+XTgbzRB0L9zZVcg+ZtnemZoJE6AZb0QmQZZ8mWvuMZH u/2QeItBcy6vVR/cO5JyboTT0GFMDcx2V+IthSIVNg3rAZ3r2OvEhJn7wAzMMujj d9qDRIueVSjAi1jTkD5OGwDxFa2DK5o= -----END CERTIFICATE----- # Issuer: C=TR, L=Gebze - Kocaeli, O=Turkiye Bilimsel ve Teknolojik Arastirma Kurumu - TUBITAK, OU=Kamu Sertifikasyon Merkezi - Kamu SM, CN=TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1 # Subject: C=TR, L=Gebze - Kocaeli, O=Turkiye Bilimsel ve Teknolojik Arastirma Kurumu - TUBITAK, OU=Kamu Sertifikasyon Merkezi - Kamu SM, CN=TUBITAK Kamu SM SSL Kok Sertifikasi - Surum 1 # SHA256 Fingerprint: 46:ED:C3:68:90:46:D5:3A:45:3F:B3:10:4A:B8:0D:CA:EC:65:8B:26:60:EA:16:29:DD:7E:86:79:90:64:87:16 -----BEGIN CERTIFICATE----- MIIEYzCCA0ugAwIBAgIBATANBgkqhkiG9w0BAQsFADCB0jELMAkGA1UEBhMCVFIx GDAWBgNVBAcTD0dlYnplIC0gS29jYWVsaTFCMEAGA1UEChM5VHVya2l5ZSBCaWxp bXNlbCB2ZSBUZWtub2xvamlrIEFyYXN0aXJtYSBLdXJ1bXUgLSBUVUJJVEFLMS0w KwYDVQQLEyRLYW11IFNlcnRpZmlrYXN5b24gTWVya2V6aSAtIEthbXUgU00xNjA0 BgNVBAMTLVRVQklUQUsgS2FtdSBTTSBTU0wgS29rIFNlcnRpZmlrYXNpIC0gU3Vy dW0gMTAeFw0xMzExMjUwODI1NTVaFw00MzEwMjUwODI1NTVaMIHSMQswCQYDVQQG EwJUUjEYMBYGA1UEBxMPR2ViemUgLSBLb2NhZWxpMUIwQAYDVQQKEzlUdXJraXll IEJpbGltc2VsIHZlIFRla25vbG9qaWsgQXJhc3Rpcm1hIEt1cnVtdSAtIFRVQklU QUsxLTArBgNVBAsTJEthbXUgU2VydGlmaWthc3lvbiBNZXJrZXppIC0gS2FtdSBT TTE2MDQGA1UEAxMtVFVCSVRBSyBLYW11IFNNIFNTTCBLb2sgU2VydGlmaWthc2kg LSBTdXJ1bSAxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr3UwM6q7 a9OZLBI3hNmNe5eA027n/5tQlT6QlVZC1xl8JoSNkvoBHToP4mQ4t4y86Ij5iySr LqP1N+RAjhgleYN1Hzv/bKjFxlb4tO2KRKOrbEz8HdDc72i9z+SqzvBV96I01INr N3wcwv61A+xXzry0tcXtAA9TNypN9E8Mg/uGz8v+jE69h/mniyFXnHrfA2eJLJ2X YacQuFWQfw4tJzh03+f92k4S400VIgLI4OD8D62K18lUUMw7D8oWgITQUVbDjlZ/ iSIzL+aFCr2lqBs23tPcLG07xxO9WSMs5uWk99gL7eqQQESolbuT1dCANLZGeA4f AJNG4e7p+exPFwIDAQABo0IwQDAdBgNVHQ4EFgQUZT/HiobGPN08VFw1+DrtUgxH V8gwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL BQADggEBACo/4fEyjq7hmFxLXs9rHmoJ0iKpEsdeV31zVmSAhHqT5Am5EM2fKifh AHe+SMg1qIGf5LgsyX8OsNJLN13qudULXjS99HMpw+0mFZx+CFOKWI3QSyjfwbPf IPP54+M638yclNhOT8NrF7f3cuitZjO1JVOr4PhMqZ398g26rrnZqsZr+ZO7rqu4 lzwDGrpDxpa5RXI4s6ehlj2Re37AIVNMh+3yC1SVUZPVIqUNivGTDj5UDrDYyU7c 8jEyVupk+eq1nRZmQnLzf9OxMUP8pI4X8W0jq5Rm+K37DwhuJi1/FwcJsoz7UMCf lo3Ptv0AnVoUmr8CRPXBwp8iXqIPoeM= -----END CERTIFICATE----- # Issuer: C=TW, O=Chunghwa Telecom Co., Ltd., CN=HiPKI Root CA - G1 # Subject: C=TW, O=Chunghwa Telecom Co., Ltd., CN=HiPKI Root CA - G1 # SHA256 Fingerprint: F0:15:CE:3C:C2:39:BF:EF:06:4B:E9:F1:D2:C4:17:E1:A0:26:4A:0A:94:BE:1F:0C:8D:12:18:64:EB:69:49:CC -----BEGIN CERTIFICATE----- MIIFajCCA1KgAwIBAgIQLd2szmKXlKFD6LDNdmpeYDANBgkqhkiG9w0BAQsFADBP MQswCQYDVQQGEwJUVzEjMCEGA1UECgwaQ2h1bmdod2EgVGVsZWNvbSBDby4sIEx0 ZC4xGzAZBgNVBAMMEkhpUEtJIFJvb3QgQ0EgLSBHMTAeFw0xOTAyMjIwOTQ2MDRa Fw0zNzEyMzExNTU5NTlaME8xCzAJBgNVBAYTAlRXMSMwIQYDVQQKDBpDaHVuZ2h3 YSBUZWxlY29tIENvLiwgTHRkLjEbMBkGA1UEAwwSSGlQS0kgUm9vdCBDQSAtIEcx MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA9B5/UnMyDHPkvRN0o9Qw qNCuS9i233VHZvR85zkEHmpwINJaR3JnVfSl6J3VHiGh8Ge6zCFovkRTv4354twv Vcg3Px+kwJyz5HdcoEb+d/oaoDjq7Zpy3iu9lFc6uux55199QmQ5eiY29yTw1S+6 lZgRZq2XNdZ1AYDgr/SEYYwNHl98h5ZeQa/rh+r4XfEuiAU+TCK72h8q3VJGZDnz Qs7ZngyzsHeXZJzA9KMuH5UHsBffMNsAGJZMoYFL3QRtU6M9/Aes1MU3guvklQgZ KILSQjqj2FPseYlgSGDIcpJQ3AOPgz+yQlda22rpEZfdhSi8MEyr48KxRURHH+CK FgeW0iEPU8DtqX7UTuybCeyvQqww1r/REEXgphaypcXTT3OUM3ECoWqj1jOXTyFj HluP2cFeRXF3D4FdXyGarYPM+l7WjSNfGz1BryB1ZlpK9p/7qxj3ccC2HTHsOyDr y+K49a6SsvfhhEvyovKTmiKe0xRvNlS9H15ZFblzqMF8b3ti6RZsR1pl8w4Rm0bZ /W3c1pzAtH2lsN0/Vm+h+fbkEkj9Bn8SV7apI09bA8PgcSojt/ewsTu8mL3WmKgM a/aOEmem8rJY5AIJEzypuxC00jBF8ez3ABHfZfjcK0NVvxaXxA/VLGGEqnKG/uY6 fsI/fe78LxQ+5oXdUG+3Se0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAdBgNV HQ4EFgQU8ncX+l6o/vY9cdVouslGDDjYr7AwDgYDVR0PAQH/BAQDAgGGMA0GCSqG SIb3DQEBCwUAA4ICAQBQUfB13HAE4/+qddRxosuej6ip0691x1TPOhwEmSKsxBHi 7zNKpiMdDg1H2DfHb680f0+BazVP6XKlMeJ45/dOlBhbQH3PayFUhuaVevvGyuqc SE5XCV0vrPSltJczWNWseanMX/mF+lLFjfiRFOs6DRfQUsJ748JzjkZ4Bjgs6Fza ZsT0pPBWGTMpWmWSBUdGSquEwx4noR8RkpkndZMPvDY7l1ePJlsMu5wP1G4wB9Tc XzZoZjmDlicmisjEOf6aIW/Vcobpf2Lll07QJNBAsNB1CI69aO4I1258EHBGG3zg iLKecoaZAeO/n0kZtCW+VmWuF2PlHt/o/0elv+EmBYTksMCv5wiZqAxeJoBF1Pho L5aPruJKHJwWDBNvOIf2u8g0X5IDUXlwpt/L9ZlNec1OvFefQ05rLisY+GpzjLrF Ne85akEez3GoorKGB1s6yeHvP2UEgEcyRHCVTjFnanRbEEV16rCf0OY1/k6fi8wr kkVbbiVghUbN0aqwdmaTd5a+g744tiROJgvM7XpWGuDpWsZkrUx6AEhEL7lAuxM+ vhV4nYWBSipX3tUZQ9rbyltHhoMLP7YNdnhzeSJesYAfz77RP1YQmCuVh6EfnWQU YDksswBVLuT1sw5XxJFBAJw/6KXf6vb/yPCtbVKoF6ubYfwSUTXkJf2vqmqGOQ== -----END CERTIFICATE----- # Issuer: C=TW, O=Chunghwa Telecom Co., Ltd., OU=ePKI Root Certification Authority # Subject: C=TW, O=Chunghwa Telecom Co., Ltd., OU=ePKI Root Certification Authority # SHA256 Fingerprint: C0:A6:F4:DC:63:A2:4B:FD:CF:54:EF:2A:6A:08:2A:0A:72:DE:35:80:3E:2F:F5:FF:52:7A:E5:D8:72:06:DF:D5 -----BEGIN CERTIFICATE----- MIIFsDCCA5igAwIBAgIQFci9ZUdcr7iXAF7kBtK8nTANBgkqhkiG9w0BAQUFADBe MQswCQYDVQQGEwJUVzEjMCEGA1UECgwaQ2h1bmdod2EgVGVsZWNvbSBDby4sIEx0 ZC4xKjAoBgNVBAsMIWVQS0kgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAe Fw0wNDEyMjAwMjMxMjdaFw0zNDEyMjAwMjMxMjdaMF4xCzAJBgNVBAYTAlRXMSMw IQYDVQQKDBpDaHVuZ2h3YSBUZWxlY29tIENvLiwgTHRkLjEqMCgGA1UECwwhZVBL SSBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF AAOCAg8AMIICCgKCAgEA4SUP7o3biDN1Z82tH306Tm2d0y8U82N0ywEhajfqhFAH SyZbCUNsIZ5qyNUD9WBpj8zwIuQf5/dqIjG3LBXy4P4AakP/h2XGtRrBp0xtInAh ijHyl3SJCRImHJ7K2RKilTza6We/CKBk49ZCt0Xvl/T29de1ShUCWH2YWEtgvM3X DZoTM1PRYfl61dd4s5oz9wCGzh1NlDivqOx4UXCKXBCDUSH3ET00hl7lSM2XgYI1 TBnsZfZrxQWh7kcT1rMhJ5QQCtkkO7q+RBNGMD+XPNjX12ruOzjjK9SXDrkb5wdJ fzcq+Xd4z1TtW0ado4AOkUPB1ltfFLqfpo0kR0BZv3I4sjZsN/+Z0V0OWQqraffA sgRFelQArr5T9rXn4fg8ozHSqf4hUmTFpmfwdQcGlBSBVcYn5AGPF8Fqcde+S/uU WH1+ETOxQvdibBjWzwloPn9s9h6PYq2lY9sJpx8iQkEeb5mKPtf5P0B6ebClAZLS nT0IFaUQAS2zMnaolQ2zepr7BxB4EW/hj8e6DyUadCrlHJhBmd8hh+iVBmoKs2pH dmX2Os+PYhcZewoozRrSgx4hxyy/vv9haLdnG7t4TY3OZ+XkwY63I2binZB1NJip NiuKmpS5nezMirH4JYlcWrYvjB9teSSnUmjDhDXiZo1jDiVN1Rmy5nk3pyKdVDEC AwEAAaNqMGgwHQYDVR0OBBYEFB4M97Zn8uGSJglFwFU5Lnc/QkqiMAwGA1UdEwQF MAMBAf8wOQYEZyoHAAQxMC8wLQIBADAJBgUrDgMCGgUAMAcGBWcqAwAABBRFsMLH ClZ87lt4DJX5GFPBphzYEDANBgkqhkiG9w0BAQUFAAOCAgEACbODU1kBPpVJufGB uvl2ICO1J2B01GqZNF5sAFPZn/KmsSQHRGoqxqWOeBLoR9lYGxMqXnmbnwoqZ6Yl PwZpVnPDimZI+ymBV3QGypzqKOg4ZyYr8dW1P2WT+DZdjo2NQCCHGervJ8A9tDkP JXtoUHRVnAxZfVo9QZQlUgjgRywVMRnVvwdVxrsStZf0X4OFunHB2WyBEXYKCrC/ gpf36j36+uwtqSiUO1bd0lEursC9CBWMd1I0ltabrNMdjmEPNXubrjlpC2JgQCA2 j6/7Nu4tCEoduL+bXPjqpRugc6bY+G7gMwRfaKonh+3ZwZCc7b3jajWvY9+rGNm6 5ulK6lCKD2GTHuItGeIwlDWSXQ62B68ZgI9HkFFLLk3dheLSClIKF5r8GrBQAuUB o2M3IUxExJtRmREOc5wGj1QupyheRDmHVi03vYVElOEMSyycw5KFNGHLD7ibSkNS /jQ6fbjpKdx2qcgw+BRxgMYeNkh0IkFch4LoGHGLQYlE535YW6i4jRPpp2zDR+2z Gp1iro2C6pSe3VkQw63d4k3jMdXH7OjysP6SHhYKGvzZ8/gntsm+HbRsZJB/9OTE W9c3rkIO3aQab3yIVMUWbuF6aC74Or8NpDyJO3inTmODBCEIZ43ygknQW/2xzQ+D hNQ+IIX3Sj0rnP0qCglN6oH4EZw= -----END CERTIFICATE----- # Issuer: C=TW, O=TAIWAN-CA, OU=Root CA, CN=TWCA Global Root CA # Subject: C=TW, O=TAIWAN-CA, OU=Root CA, CN=TWCA Global Root CA # SHA256 Fingerprint: 59:76:90:07:F7:68:5D:0F:CD:50:87:2F:9F:95:D5:75:5A:5B:2B:45:7D:81:F3:69:2B:61:0A:98:67:2F:0E:1B -----BEGIN CERTIFICATE----- MIIFQTCCAymgAwIBAgICDL4wDQYJKoZIhvcNAQELBQAwUTELMAkGA1UEBhMCVFcx EjAQBgNVBAoTCVRBSVdBTi1DQTEQMA4GA1UECxMHUm9vdCBDQTEcMBoGA1UEAxMT VFdDQSBHbG9iYWwgUm9vdCBDQTAeFw0xMjA2MjcwNjI4MzNaFw0zMDEyMzExNTU5 NTlaMFExCzAJBgNVBAYTAlRXMRIwEAYDVQQKEwlUQUlXQU4tQ0ExEDAOBgNVBAsT B1Jvb3QgQ0ExHDAaBgNVBAMTE1RXQ0EgR2xvYmFsIFJvb3QgQ0EwggIiMA0GCSqG SIb3DQEBAQUAA4ICDwAwggIKAoICAQCwBdvI64zEbooh745NnHEKH1Jw7W2CnJfF 10xORUnLQEK1EjRsGcJ0pDFfhQKX7EMzClPSnIyOt7h52yvVavKOZsTuKwEHktSz 0ALfUPZVr2YOy+BHYC8rMjk1Ujoog/h7FsYYuGLWRyWRzvAZEk2tY/XTP3VfKfCh MBwqoJimFb3u/Rk28OKRQ4/6ytYQJ0lM793B8YVwm8rqqFpD/G2Gb3PpN0Wp8DbH zIh1HrtsBv+baz4X7GGqcXzGHaL3SekVtTzWoWH1EfcFbx39Eb7QMAfCKbAJTibc 46KokWofwpFFiFzlmLhxpRUZyXx1EcxwdE8tmx2RRP1WKKD+u4ZqyPpcC1jcxkt2 yKsi2XMPpfRaAok/T54igu6idFMqPVMnaR1sjjIsZAAmY2E2TqNGtz99sy2sbZCi laLOz9qC5wc0GZbpuCGqKX6mOL6OKUohZnkfs8O1CWfe1tQHRvMq2uYiN2DLgbYP oA/pyJV/v1WRBXrPPRXAb94JlAGD1zQbzECl8LibZ9WYkTunhHiVJqRaCPgrdLQA BDzfuBSO6N+pjWxnkjMdwLfS7JLIvgm/LCkFbwJrnu+8vyq8W8BQj0FwcYeyTbcE qYSjMq+u7msXi7Kx/mzhkIyIqJdIzshNy/MGz19qCkKxHh53L46g5pIOBvwFItIm 4TFRfTLcDwIDAQABoyMwITAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB /zANBgkqhkiG9w0BAQsFAAOCAgEAXzSBdu+WHdXltdkCY4QWwa6gcFGn90xHNcgL 1yg9iXHZqjNB6hQbbCEAwGxCGX6faVsgQt+i0trEfJdLjbDorMjupWkEmQqSpqsn LhpNgb+E1HAerUf+/UqdM+DyucRFCCEK2mlpc3INvjT+lIutwx4116KD7+U4x6WF H6vPNOw/KP4M8VeGTslV9xzU2KV9Bnpv1d8Q34FOIWWxtuEXeZVFBs5fzNxGiWNo RI2T9GRwoD2dKAXDOXC4Ynsg/eTb6QihuJ49CcdP+yz4k3ZB3lLg4VfSnQO8d57+ nile98FRYB/e2guyLXW3Q0iT5/Z5xoRdgFlglPx4mI88k1HtQJAH32RjJMtOcQWh 15QaiDLxInQirqWm2BJpTGCjAu4r7NRjkgtevi92a6O2JryPA9gK8kxkRr05YuWW 6zRjESjMlfGt7+/cgFhI6Uu46mWs6fyAtbXIRfmswZ/ZuepiiI7E8UuDEq3mi4TW nsLrgxifarsbJGAzcMzs9zLzXNl5fe+epP7JI8Mk7hWSsT2RTyaGvWZzJBPqpK5j wa19hAM8EHiGG3njxPPyBJUgriOCxLM6AGK/5jYk4Ve6xx6QddVfP5VhK8E7zeWz aGHQRiapIVJpLesux+t3zqY6tQMzT3bR51xUAV3LePTJDL/PEo4XLSNolOer/qmy KwbQBM0= -----END CERTIFICATE----- # Issuer: C=TW, O=TAIWAN-CA, OU=Root CA, CN=TWCA Root Certification Authority # Subject: C=TW, O=TAIWAN-CA, OU=Root CA, CN=TWCA Root Certification Authority # SHA256 Fingerprint: BF:D8:8F:E1:10:1C:41:AE:3E:80:1B:F8:BE:56:35:0E:E9:BA:D1:A6:B9:BD:51:5E:DC:5C:6D:5B:87:11:AC:44 -----BEGIN CERTIFICATE----- MIIDezCCAmOgAwIBAgIBATANBgkqhkiG9w0BAQUFADBfMQswCQYDVQQGEwJUVzES MBAGA1UECgwJVEFJV0FOLUNBMRAwDgYDVQQLDAdSb290IENBMSowKAYDVQQDDCFU V0NBIFJvb3QgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwODI4MDcyNDMz WhcNMzAxMjMxMTU1OTU5WjBfMQswCQYDVQQGEwJUVzESMBAGA1UECgwJVEFJV0FO LUNBMRAwDgYDVQQLDAdSb290IENBMSowKAYDVQQDDCFUV0NBIFJvb3QgQ2VydGlm aWNhdGlvbiBBdXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB AQCwfnK4pAOU5qfeCTiRShFAh6d8WWQUe7UREN3+v9XAu1bihSX0NXIP+FPQQeFE AcK0HMMxQhZHhTMidrIKbw/lJVBPhYa+v5guEGcevhEFhgWQxFnQfHgQsIBct+HH K3XLfJ+utdGdIzdjp9xCoi2SBBtQwXu4PhvJVgSLL1KbralW6cH/ralYhzC2gfeX RfwZVzsrb+RH9JlF/h3x+JejiB03HFyP4HYlmlD4oFT/RJB2I9IyxsOrBr/8+7/z rX2SYgJbKdM1o5OaQ2RgXbL6Mv87BK9NQGr5x+PvI/1ry+UPizgN7gr8/g+YnzAx 3WxSZfmLgb4i4RxYA7qRG4kHAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV HRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqOFsmjd6LWvJPelSDGRjjCDWmujANBgkq hkiG9w0BAQUFAAOCAQEAPNV3PdrfibqHDAhUaiBQkr6wQT25JmSDCi/oQMCXKCeC MErJk/9q56YAf4lCmtYR5VPOL8zy2gXE/uJQxDqGfczafhAJO5I1KlOy/usrBdls XebQ79NqZp4VKIV66IIArB6nCWlWQtNoURi+VJq/REG6Sb4gumlc7rh3zc5sH62D lhh9DrUUOYTxKOkto557HnpyWoOzeW/vtPzQCqVYT0bf+215WfKEIlKuD8z7fDvn aspHYcN6+NOSBB+4IIThNlQWx0DeO4pz3N/GCUzf7Nr/1FNCocnyYh0igzyXxfkZ YiesZSLX0zzG5Y6yU8xJzrww/nsOM5D77dIUkR8Hrw== -----END CERTIFICATE----- # Issuer: C=US, O=AffirmTrust, CN=AffirmTrust Commercial # Subject: C=US, O=AffirmTrust, CN=AffirmTrust Commercial # SHA256 Fingerprint: 03:76:AB:1D:54:C5:F9:80:3C:E4:B2:E2:01:A0:EE:7E:EF:7B:57:B6:36:E8:A9:3C:9B:8D:48:60:C9:6F:5F:A7 -----BEGIN CERTIFICATE----- MIIDTDCCAjSgAwIBAgIId3cGJyapsXwwDQYJKoZIhvcNAQELBQAwRDELMAkGA1UE BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZpcm1UcnVz dCBDb21tZXJjaWFsMB4XDTEwMDEyOTE0MDYwNloXDTMwMTIzMTE0MDYwNlowRDEL MAkGA1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZp cm1UcnVzdCBDb21tZXJjaWFsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC AQEA9htPZwcroRX1BiLLHwGy43NFBkRJLLtJJRTWzsO3qyxPxkEylFf6EqdbDuKP Hx6GGaeqtS25Xw2Kwq+FNXkyLbscYjfysVtKPcrNcV/pQr6U6Mje+SJIZMblq8Yr ba0F8PrVC8+a5fBQpIs7R6UjW3p6+DM/uO+Zl+MgwdYoic+U+7lF7eNAFxHUdPAL MeIrJmqbTFeurCA+ukV6BfO9m2kVrn1OIGPENXY6BwLJN/3HR+7o8XYdcxXyl6S1 yHp52UKqK39c/s4mT6NmgTWvRLpUHhwwMmWd5jyTXlBOeuM61G7MGvv50jeuJCqr VwMiKA1JdX+3KNp1v47j3A55MQIDAQABo0IwQDAdBgNVHQ4EFgQUnZPGU4teyq8/ nx4P5ZmVvCT2lI8wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJ KoZIhvcNAQELBQADggEBAFis9AQOzcAN/wr91LoWXym9e2iZWEnStB03TX8nfUYG XUPGhi4+c7ImfU+TqbbEKpqrIZcUsd6M06uJFdhrJNTxFq7YpFzUf1GO7RgBsZNj vbz4YYCanrHOQnDiqX0GJX0nof5v7LMeJNrjS1UaADs1tDvZ110w/YETifLCBivt Z8SOyUOyXGsViQK8YvxO8rUzqrJv0wqiUOP2O+guRMLbZjipM1ZI8W0bM40NjD9g N53Tym1+NH4Nn3J2ixufcv1SNUFFApYvHLKac0khsUlHRUe072o0EclNmsxZt9YC nlpOZbWUrhvfKbAW8b8Angc6F2S1BLUjIZkKlTuXfO8= -----END CERTIFICATE----- # Issuer: C=US, O=AffirmTrust, CN=AffirmTrust Networking # Subject: C=US, O=AffirmTrust, CN=AffirmTrust Networking # SHA256 Fingerprint: 0A:81:EC:5A:92:97:77:F1:45:90:4A:F3:8D:5D:50:9F:66:B5:E2:C5:8F:CD:B5:31:05:8B:0E:17:F3:F0:B4:1B -----BEGIN CERTIFICATE----- MIIDTDCCAjSgAwIBAgIIfE8EORzUmS0wDQYJKoZIhvcNAQEFBQAwRDELMAkGA1UE BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZpcm1UcnVz dCBOZXR3b3JraW5nMB4XDTEwMDEyOTE0MDgyNFoXDTMwMTIzMTE0MDgyNFowRDEL MAkGA1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MR8wHQYDVQQDDBZBZmZp cm1UcnVzdCBOZXR3b3JraW5nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC AQEAtITMMxcua5Rsa2FSoOujz3mUTOWUgJnLVWREZY9nZOIG41w3SfYvm4SEHi3y YJ0wTsyEheIszx6e/jarM3c1RNg1lho9Nuh6DtjVR6FqaYvZ/Ls6rnla1fTWcbua kCNrmreIdIcMHl+5ni36q1Mr3Lt2PpNMCAiMHqIjHNRqrSK6mQEubWXLviRmVSRL QESxG9fhwoXA3hA/Pe24/PHxI1Pcv2WXb9n5QHGNfb2V1M6+oF4nI979ptAmDgAp 6zxG8D1gvz9Q0twmQVGeFDdCBKNwV6gbh+0t+nvujArjqWaJGctB+d1ENmHP4ndG yH329JKBNv3bNPFyfvMMFr20FQIDAQABo0IwQDAdBgNVHQ4EFgQUBx/S55zawm6i QLSwelAQUHTEyL0wDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJ KoZIhvcNAQEFBQADggEBAIlXshZ6qML91tmbmzTCnLQyFE2npN/svqe++EPbkTfO tDIuUFUaNU52Q3Eg75N3ThVwLofDwR1t3Mu1J9QsVtFSUzpE0nPIxBsFZVpikpzu QY0x2+c06lkh1QF612S4ZDnNye2v7UsDSKegmQGA3GWjNq5lWUhPgkvIZfFXHeVZ Lgo/bNjR9eUJtGxUAArgFU2HdW23WJZa3W3SAKD0m0i+wzekujbgfIeFlxoVot4u olu9rxj5kFDNcFn4J2dHy8egBzp90SxdbBk6ZrV9/ZFvgrG+CJPbFEfxojfHRZ48 x3evZKiT3/Zpg4Jg8klCNO1aAFSFHBY2kgxc+qatv9s= -----END CERTIFICATE----- # Issuer: C=US, O=AffirmTrust, CN=AffirmTrust Premium ECC # Subject: C=US, O=AffirmTrust, CN=AffirmTrust Premium ECC # SHA256 Fingerprint: BD:71:FD:F6:DA:97:E4:CF:62:D1:64:7A:DD:25:81:B0:7D:79:AD:F8:39:7E:B4:EC:BA:9C:5E:84:88:82:14:23 -----BEGIN CERTIFICATE----- MIIB/jCCAYWgAwIBAgIIdJclisc/elQwCgYIKoZIzj0EAwMwRTELMAkGA1UEBhMC VVMxFDASBgNVBAoMC0FmZmlybVRydXN0MSAwHgYDVQQDDBdBZmZpcm1UcnVzdCBQ cmVtaXVtIEVDQzAeFw0xMDAxMjkxNDIwMjRaFw00MDEyMzExNDIwMjRaMEUxCzAJ BgNVBAYTAlVTMRQwEgYDVQQKDAtBZmZpcm1UcnVzdDEgMB4GA1UEAwwXQWZmaXJt VHJ1c3QgUHJlbWl1bSBFQ0MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQNMF4bFZ0D 0KF5Nbc6PJJ6yhUczWLznCZcBz3lVPqj1swS6vQUX+iOGasvLkjmrBhDeKzQN8O9 ss0s5kfiGuZjuD0uL3jET9v0D6RoTFVya5UdThhClXjMNzyR4ptlKymjQjBAMB0G A1UdDgQWBBSaryl6wBE1NSZRMADDav5A1a7WPDAPBgNVHRMBAf8EBTADAQH/MA4G A1UdDwEB/wQEAwIBBjAKBggqhkjOPQQDAwNnADBkAjAXCfOHiFBar8jAQr9HX/Vs aobgxCd05DhT1wV/GzTjxi+zygk8N53X57hG8f2h4nECMEJZh0PUUd+60wkyWs6I flc9nF9Ca/UHLbXwgpP5WW+uZPpY5Yse42O+tYHNbwKMeQ== -----END CERTIFICATE----- # Issuer: C=US, O=AffirmTrust, CN=AffirmTrust Premium # Subject: C=US, O=AffirmTrust, CN=AffirmTrust Premium # SHA256 Fingerprint: 70:A7:3F:7F:37:6B:60:07:42:48:90:45:34:B1:14:82:D5:BF:0E:69:8E:CC:49:8D:F5:25:77:EB:F2:E9:3B:9A -----BEGIN CERTIFICATE----- MIIFRjCCAy6gAwIBAgIIbYwURrGmCu4wDQYJKoZIhvcNAQEMBQAwQTELMAkGA1UE BhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MRwwGgYDVQQDDBNBZmZpcm1UcnVz dCBQcmVtaXVtMB4XDTEwMDEyOTE0MTAzNloXDTQwMTIzMTE0MTAzNlowQTELMAkG A1UEBhMCVVMxFDASBgNVBAoMC0FmZmlybVRydXN0MRwwGgYDVQQDDBNBZmZpcm1U cnVzdCBQcmVtaXVtMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxBLf qV/+Qd3d9Z+K4/as4Tx4mrzY8H96oDMq3I0gW64tb+eT2TZwamjPjlGjhVtnBKAQ JG9dKILBl1fYSCkTtuG+kU3fhQxTGJoeJKJPj/CihQvL9Cl/0qRY7iZNyaqoe5rZ +jjeRFcV5fiMyNlI4g0WJx0eyIOFJbe6qlVBzAMiSy2RjYvmia9mx+n/K+k8rNrS s8PhaJyJ+HoAVt70VZVs+7pk3WKL3wt3MutizCaam7uqYoNMtAZ6MMgpv+0GTZe5 HMQxK9VfvFMSF5yZVylmd2EhMQcuJUmdGPLu8ytxjLW6OQdJd/zvLpKQBY0tL3d7 70O/Nbua2Plzpyzy0FfuKE4mX4+QaAkvuPjcBukumj5Rp9EixAqnOEhss/n/fauG V+O61oV4d7pD6kh/9ti+I20ev9E2bFhc8e6kGVQa9QPSdubhjL08s9NIS+LI+H+S qHZGnEJlPqQewQcDWkYtuJfzt9WyVSHvutxMAJf7FJUnM7/oQ0dG0giZFmA7mn7S 5u046uwBHjxIVkkJx0w3AJ6IDsBz4W9m6XJHMD4Q5QsDyZpCAGzFlH5hxIrff4Ia C1nEWTJ3s7xgaVY5/bQGeyzWZDbZvUjthB9+pSKPKrhC9IK31FOQeE4tGv2Bb0TX OwF0lkLgAOIua+rF7nKsu7/+6qqo+Nz2snmKtmcCAwEAAaNCMEAwHQYDVR0OBBYE FJ3AZ6YMItkm9UWrpmVSESfYRaxjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ BAQDAgEGMA0GCSqGSIb3DQEBDAUAA4ICAQCzV00QYk465KzquByvMiPIs0laUZx2 KI15qldGF9X1Uva3ROgIRL8YhNILgM3FEv0AVQVhh0HctSSePMTYyPtwni94loMg Nt58D2kTiKV1NpgIpsbfrM7jWNa3Pt668+s0QNiigfV4Py/VpfzZotReBA4Xrf5B 8OWycvpEgjNC6C1Y91aMYj+6QrCcDFx+LmUmXFNPALJ4fqENmS2NuB2OosSw/WDQ MKSOyARiqcTtNd56l+0OOF6SL5Nwpamcb6d9Ex1+xghIsV5n61EIJenmJWtSKZGc 0jlzCFfemQa0W50QBuHCAKi4HEoCChTQwUHK+4w1IX2COPKpVJEZNZOUbWo6xbLQ u4mGk+ibyQ86p3q4ofB4Rvr8Ny/lioTz3/4E2aFooC8k4gmVBtWVyuEklut89pMF u+1z6S3RdTnX5yTb2E5fQ4+e0BQ5v1VwSJlXMbSc7kqYA5YwH2AG7hsj/oFgIxpH YoWlzBk0gG+zrBrjn/B7SK3VAdlntqlyk+otZrWyuOQ9PLLvTIzq6we/qzWaVYa8 GKa1qF60g2xraUDTn9zxw2lrueFtCfTxqlB2Cnp9ehehVZZCmTEJ3WARjQUwfuaO RtGdFNrHF+QFlozEJLUbzxQHskD4o55BhrwE0GuWyCqANP2/7waj3VjFhT0+j/6e KeC2uAloGRwYQw== -----END CERTIFICATE----- # Issuer: C=US, O=Amazon, CN=Amazon Root CA 1 # Subject: C=US, O=Amazon, CN=Amazon Root CA 1 # SHA256 Fingerprint: 8E:CD:E6:88:4F:3D:87:B1:12:5B:A3:1A:C3:FC:B1:3D:70:16:DE:7F:57:CC:90:4F:E1:CB:97:C6:AE:98:19:6E -----BEGIN CERTIFICATE----- MIIDQTCCAimgAwIBAgITBmyfz5m/jAo54vB4ikPmljZbyjANBgkqhkiG9w0BAQsF ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 b24gUm9vdCBDQSAxMB4XDTE1MDUyNjAwMDAwMFoXDTM4MDExNzAwMDAwMFowOTEL MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv b3QgQ0EgMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALJ4gHHKeNXj ca9HgFB0fW7Y14h29Jlo91ghYPl0hAEvrAIthtOgQ3pOsqTQNroBvo3bSMgHFzZM 9O6II8c+6zf1tRn4SWiw3te5djgdYZ6k/oI2peVKVuRF4fn9tBb6dNqcmzU5L/qw IFAGbHrQgLKm+a/sRxmPUDgH3KKHOVj4utWp+UhnMJbulHheb4mjUcAwhmahRWa6 VOujw5H5SNz/0egwLX0tdHA114gk957EWW67c4cX8jJGKLhD+rcdqsq08p8kDi1L 93FcXmn/6pUCyziKrlA4b9v7LWIbxcceVOF34GfID5yHI9Y/QCB/IIDEgEw+OyQm jgSubJrIqg0CAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC AYYwHQYDVR0OBBYEFIQYzIU07LwMlJQuCFmcx7IQTgoIMA0GCSqGSIb3DQEBCwUA A4IBAQCY8jdaQZChGsV2USggNiMOruYou6r4lK5IpDB/G/wkjUu0yKGX9rbxenDI U5PMCCjjmCXPI6T53iHTfIUJrU6adTrCC2qJeHZERxhlbI1Bjjt/msv0tadQ1wUs N+gDS63pYaACbvXy8MWy7Vu33PqUXHeeE6V/Uq2V8viTO96LXFvKWlJbYK8U90vv o/ufQJVtMVT8QtPHRh8jrdkPSHCa2XV4cdFyQzR1bldZwgJcJmApzyMZFo6IQ6XU 5MsI+yMRQ+hDKXJioaldXgjUkK642M4UwtBV8ob2xJNDd2ZhwLnoQdeXeGADbkpy rqXRfboQnoZsG4q5WTP468SQvvG5 -----END CERTIFICATE----- # Issuer: C=US, O=Amazon, CN=Amazon Root CA 2 # Subject: C=US, O=Amazon, CN=Amazon Root CA 2 # SHA256 Fingerprint: 1B:A5:B2:AA:8C:65:40:1A:82:96:01:18:F8:0B:EC:4F:62:30:4D:83:CE:C4:71:3A:19:C3:9C:01:1E:A4:6D:B4 -----BEGIN CERTIFICATE----- MIIFQTCCAymgAwIBAgITBmyf0pY1hp8KD+WGePhbJruKNzANBgkqhkiG9w0BAQwF ADA5MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6 b24gUm9vdCBDQSAyMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTEL MAkGA1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJv b3QgQ0EgMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK2Wny2cSkxK gXlRmeyKy2tgURO8TW0G/LAIjd0ZEGrHJgw12MBvIITplLGbhQPDW9tK6Mj4kHbZ W0/jTOgGNk3Mmqw9DJArktQGGWCsN0R5hYGCrVo34A3MnaZMUnbqQ523BNFQ9lXg 1dKmSYXpN+nKfq5clU1Imj+uIFptiJXZNLhSGkOQsL9sBbm2eLfq0OQ6PBJTYv9K 8nu+NQWpEjTj82R0Yiw9AElaKP4yRLuH3WUnAnE72kr3H9rN9yFVkE8P7K6C4Z9r 2UXTu/Bfh+08LDmG2j/e7HJV63mjrdvdfLC6HM783k81ds8P+HgfajZRRidhW+me z/CiVX18JYpvL7TFz4QuK/0NURBs+18bvBt+xa47mAExkv8LV/SasrlX6avvDXbR 8O70zoan4G7ptGmh32n2M8ZpLpcTnqWHsFcQgTfJU7O7f/aS0ZzQGPSSbtqDT6Zj mUyl+17vIWR6IF9sZIUVyzfpYgwLKhbcAS4y2j5L9Z469hdAlO+ekQiG+r5jqFoz 7Mt0Q5X5bGlSNscpb/xVA1wf+5+9R+vnSUeVC06JIglJ4PVhHvG/LopyboBZ/1c6 +XUyo05f7O0oYtlNc/LMgRdg7c3r3NunysV+Ar3yVAhU/bQtCSwXVEqY0VThUWcI 0u1ufm8/0i2BWSlmy5A5lREedCf+3euvAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMB Af8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSwDPBMMPQFWAJI/TPlUq9LhONm UjANBgkqhkiG9w0BAQwFAAOCAgEAqqiAjw54o+Ci1M3m9Zh6O+oAA7CXDpO8Wqj2 LIxyh6mx/H9z/WNxeKWHWc8w4Q0QshNabYL1auaAn6AFC2jkR2vHat+2/XcycuUY +gn0oJMsXdKMdYV2ZZAMA3m3MSNjrXiDCYZohMr/+c8mmpJ5581LxedhpxfL86kS k5Nrp+gvU5LEYFiwzAJRGFuFjWJZY7attN6a+yb3ACfAXVU3dJnJUH/jWS5E4ywl 7uxMMne0nxrpS10gxdr9HIcWxkPo1LsmmkVwXqkLN1PiRnsn/eBG8om3zEK2yygm btmlyTrIQRNg91CMFa6ybRoVGld45pIq2WWQgj9sAq+uEjonljYE1x2igGOpm/Hl urR8FLBOybEfdF849lHqm/osohHUqS0nGkWxr7JOcQ3AWEbWaQbLU8uz/mtBzUF+ fUwPfHJ5elnNXkoOrJupmHN5fLT0zLm4BwyydFy4x2+IoZCn9Kr5v2c69BoVYh63 n749sSmvZ6ES8lgQGVMDMBu4Gon2nL2XA46jCfMdiyHxtN/kHNGfZQIG6lzWE7OE 76KlXIx3KadowGuuQNKotOrN8I1LOJwZmhsoVLiJkO/KdYE+HvJkJMcYr07/R54H 9jVlpNMKVv/1F2Rs76giJUmTtt8AF9pYfl3uxRuw0dFfIRDH+fO6AgonB8Xx1sfT 4PsJYGw= -----END CERTIFICATE----- # Issuer: C=US, O=Amazon, CN=Amazon Root CA 3 # Subject: C=US, O=Amazon, CN=Amazon Root CA 3 # SHA256 Fingerprint: 18:CE:6C:FE:7B:F1:4E:60:B2:E3:47:B8:DF:E8:68:CB:31:D0:2E:BB:3A:DA:27:15:69:F5:03:43:B4:6D:B3:A4 -----BEGIN CERTIFICATE----- MIIBtjCCAVugAwIBAgITBmyf1XSXNmY/Owua2eiedgPySjAKBggqhkjOPQQDAjA5 MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g Um9vdCBDQSAzMB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg Q0EgMzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCmXp8ZBf8ANm+gBG1bG8lKl ui2yEujSLtf6ycXYqm0fc4E7O5hrOXwzpcVOho6AF2hiRVd9RFgdszflZwjrZt6j QjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQWBBSr ttvXBp43rDCGB5Fwx5zEGbF4wDAKBggqhkjOPQQDAgNJADBGAiEA4IWSoxe3jfkr BqWTrBqYaGFy+uGh0PsceGCmQ5nFuMQCIQCcAu/xlJyzlvnrxir4tiz+OpAUFteM YyRIHN8wfdVoOw== -----END CERTIFICATE----- # Issuer: C=US, O=Amazon, CN=Amazon Root CA 4 # Subject: C=US, O=Amazon, CN=Amazon Root CA 4 # SHA256 Fingerprint: E3:5D:28:41:9E:D0:20:25:CF:A6:90:38:CD:62:39:62:45:8D:A5:C6:95:FB:DE:A3:C2:2B:0B:FB:25:89:70:92 -----BEGIN CERTIFICATE----- MIIB8jCCAXigAwIBAgITBmyf18G7EEwpQ+Vxe3ssyBrBDjAKBggqhkjOPQQDAzA5 MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRkwFwYDVQQDExBBbWF6b24g Um9vdCBDQSA0MB4XDTE1MDUyNjAwMDAwMFoXDTQwMDUyNjAwMDAwMFowOTELMAkG A1UEBhMCVVMxDzANBgNVBAoTBkFtYXpvbjEZMBcGA1UEAxMQQW1hem9uIFJvb3Qg Q0EgNDB2MBAGByqGSM49AgEGBSuBBAAiA2IABNKrijdPo1MN/sGKe0uoe0ZLY7Bi 9i0b2whxIdIA6GO9mif78DluXeo9pcmBqqNbIJhFXRbb/egQbeOc4OO9X4Ri83Bk M6DLJC9wuoihKqB1+IGuYgbEgds5bimwHvouXKNCMEAwDwYDVR0TAQH/BAUwAwEB /zAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0OBBYEFNPsxzplbszh2naaVvuc84ZtV+WB MAoGCCqGSM49BAMDA2gAMGUCMDqLIfG9fhGt0O9Yli/W651+kI0rz2ZVwyzjKKlw CkcO8DdZEv8tmZQoTipPNU0zWgIxAOp1AE47xDqUEpHJWEadIRNyp4iciuRMStuW 1KyLa2tJElMzrdfkviT8tQp21KW8EA== -----END CERTIFICATE----- # Issuer: C=US, O=Certainly, CN=Certainly Root E1 # Subject: C=US, O=Certainly, CN=Certainly Root E1 # SHA256 Fingerprint: B4:58:5F:22:E4:AC:75:6A:4E:86:12:A1:36:1C:5D:9D:03:1A:93:FD:84:FE:BB:77:8F:A3:06:8B:0F:C4:2D:C2 -----BEGIN CERTIFICATE----- MIIB9zCCAX2gAwIBAgIQBiUzsUcDMydc+Y2aub/M+DAKBggqhkjOPQQDAzA9MQsw CQYDVQQGEwJVUzESMBAGA1UEChMJQ2VydGFpbmx5MRowGAYDVQQDExFDZXJ0YWlu bHkgUm9vdCBFMTAeFw0yMTA0MDEwMDAwMDBaFw00NjA0MDEwMDAwMDBaMD0xCzAJ BgNVBAYTAlVTMRIwEAYDVQQKEwlDZXJ0YWlubHkxGjAYBgNVBAMTEUNlcnRhaW5s eSBSb290IEUxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE3m/4fxzf7flHh4axpMCK +IKXgOqPyEpeKn2IaKcBYhSRJHpcnqMXfYqGITQYUBsQ3tA3SybHGWCA6TS9YBk2 QNYphwk8kXr2vBMj3VlOBF7PyAIcGFPBMdjaIOlEjeR2o0IwQDAOBgNVHQ8BAf8E BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU8ygYy2R17ikq6+2uI1g4 hevIIgcwCgYIKoZIzj0EAwMDaAAwZQIxALGOWiDDshliTd6wT99u0nCK8Z9+aozm ut6Dacpps6kFtZaSF4fC0urQe87YQVt8rgIwRt7qy12a7DLCZRawTDBcMPPaTnOG BtjOiQRINzf43TNRnXCve1XYAS59BWQOhriR -----END CERTIFICATE----- # Issuer: C=US, O=Certainly, CN=Certainly Root R1 # Subject: C=US, O=Certainly, CN=Certainly Root R1 # SHA256 Fingerprint: 77:B8:2C:D8:64:4C:43:05:F7:AC:C5:CB:15:6B:45:67:50:04:03:3D:51:C6:0C:62:02:A8:E0:C3:34:67:D3:A0 -----BEGIN CERTIFICATE----- MIIFRzCCAy+gAwIBAgIRAI4P+UuQcWhlM1T01EQ5t+AwDQYJKoZIhvcNAQELBQAw PTELMAkGA1UEBhMCVVMxEjAQBgNVBAoTCUNlcnRhaW5seTEaMBgGA1UEAxMRQ2Vy dGFpbmx5IFJvb3QgUjEwHhcNMjEwNDAxMDAwMDAwWhcNNDYwNDAxMDAwMDAwWjA9 MQswCQYDVQQGEwJVUzESMBAGA1UEChMJQ2VydGFpbmx5MRowGAYDVQQDExFDZXJ0 YWlubHkgUm9vdCBSMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANA2 1B/q3avk0bbm+yLA3RMNansiExyXPGhjZjKcA7WNpIGD2ngwEc/csiu+kr+O5MQT vqRoTNoCaBZ0vrLdBORrKt03H2As2/X3oXyVtwxwhi7xOu9S98zTm/mLvg7fMbed aFySpvXl8wo0tf97ouSHocavFwDvA5HtqRxOcT3Si2yJ9HiG5mpJoM610rCrm/b0 1C7jcvk2xusVtyWMOvwlDbMicyF0yEqWYZL1LwsYpfSt4u5BvQF5+paMjRcCMLT5 r3gajLQ2EBAHBXDQ9DGQilHFhiZ5shGIXsXwClTNSaa/ApzSRKft43jvRl5tcdF5 cBxGX1HpyTfcX35pe0HfNEXgO4T0oYoKNp43zGJS4YkNKPl6I7ENPT2a/Z2B7yyQ wHtETrtJ4A5KVpK8y7XdeReJkd5hiXSSqOMyhb5OhaRLWcsrxXiOcVTQAjeZjOVJ 6uBUcqQRBi8LjMFbvrWhsFNunLhgkR9Za/kt9JQKl7XsxXYDVBtlUrpMklZRNaBA 2CnbrlJ2Oy0wQJuK0EJWtLeIAaSHO1OWzaMWj/Nmqhexx2DgwUMFDO6bW2BvBlyH Wyf5QBGenDPBt+U1VwV/J84XIIwc/PH72jEpSe31C4SnT8H2TsIonPru4K8H+zMR eiFPCyEQtkA6qyI6BJyLm4SGcprSp6XEtHWRqSsjAgMBAAGjQjBAMA4GA1UdDwEB /wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTgqj8ljZ9EXME66C6u d0yEPmcM9DANBgkqhkiG9w0BAQsFAAOCAgEAuVevuBLaV4OPaAszHQNTVfSVcOQr PbA56/qJYv331hgELyE03fFo8NWWWt7CgKPBjcZq91l3rhVkz1t5BXdm6ozTaw3d 8VkswTOlMIAVRQdFGjEitpIAq5lNOo93r6kiyi9jyhXWx8bwPWz8HA2YEGGeEaIi 1wrykXprOQ4vMMM2SZ/g6Q8CRFA3lFV96p/2O7qUpUzpvD5RtOjKkjZUbVwlKNrd rRT90+7iIgXr0PK3aBLXWopBGsaSpVo7Y0VPv+E6dyIvXL9G+VoDhRNCX8reU9di taY1BMJH/5n9hN9czulegChB8n3nHpDYT3Y+gjwN/KUD+nsa2UUeYNrEjvn8K8l7 lcUq/6qJ34IxD3L/DCfXCh5WAFAeDJDBlrXYFIW7pw0WwfgHJBu6haEaBQmAupVj yTrsJZ9/nbqkRxWbRHDxakvWOF5D8xh+UG7pWijmZeZ3Gzr9Hb4DJqPb1OG7fpYn Kx3upPvaJVQTA945xsMfTZDsjxtK0hzthZU4UHlG1sGQUDGpXJpuHfUzVounmdLy yCwzk5Iwx06MZTMQZBf9JBeW0Y3COmor6xOLRPIh80oat3df1+2IpHLlOR+Vnb5n wXARPbv0+Em34yaXOp/SX3z7wJl8OSngex2/DaeP0ik0biQVy96QXr8axGbqwua6 OV+KmalBWQewLK8= -----END CERTIFICATE----- # Issuer: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Assured ID Root CA # Subject: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Assured ID Root CA # SHA256 Fingerprint: 3E:90:99:B5:01:5E:8F:48:6C:00:BC:EA:9D:11:1E:E7:21:FA:BA:35:5A:89:BC:F1:DF:69:56:1E:3D:C6:32:5C -----BEGIN CERTIFICATE----- MIIDtzCCAp+gAwIBAgIQDOfg5RfYRv6P5WD8G/AwOTANBgkqhkiG9w0BAQUFADBl MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv b3QgQ0EwHhcNMDYxMTEwMDAwMDAwWhcNMzExMTEwMDAwMDAwWjBlMQswCQYDVQQG EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgQ0EwggEi MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCtDhXO5EOAXLGH87dg+XESpa7c JpSIqvTO9SA5KFhgDPiA2qkVlTJhPLWxKISKityfCgyDF3qPkKyK53lTXDGEKvYP mDI2dsze3Tyoou9q+yHyUmHfnyDXH+Kx2f4YZNISW1/5WBg1vEfNoTb5a3/UsDg+ wRvDjDPZ2C8Y/igPs6eD1sNuRMBhNZYW/lmci3Zt1/GiSw0r/wty2p5g0I6QNcZ4 VYcgoc/lbQrISXwxmDNsIumH0DJaoroTghHtORedmTpyoeb6pNnVFzF1roV9Iq4/ AUaG9ih5yLHa5FcXxH4cDrC0kqZWs72yl+2qp/C3xag/lRbQ/6GW6whfGHdPAgMB AAGjYzBhMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW BBRF66Kv9JLLgjEtUYunpyGd823IDzAfBgNVHSMEGDAWgBRF66Kv9JLLgjEtUYun pyGd823IDzANBgkqhkiG9w0BAQUFAAOCAQEAog683+Lt8ONyc3pklL/3cmbYMuRC dWKuh+vy1dneVrOfzM4UKLkNl2BcEkxY5NM9g0lFWJc1aRqoR+pWxnmrEthngYTf fwk8lOa4JiwgvT2zKIn3X/8i4peEH+ll74fg38FnSbNd67IJKusm7Xi+fT8r87cm NW1fiQG2SVufAQWbqz0lwcy2f8Lxb4bG+mRo64EtlOtCt/qMHt1i8b5QZ7dsvfPx H2sMNgcWfzd8qVttevESRmCD1ycEvkvOl77DZypoEd+A5wwzZr8TDRRu838fYxAe +o0bJW1sj6W3YQGx0qMmoRBxna3iw/nDmVG3KwcIzi7mULKn+gpFL6Lw8g== -----END CERTIFICATE----- # Issuer: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Assured ID Root G2 # Subject: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Assured ID Root G2 # SHA256 Fingerprint: 7D:05:EB:B6:82:33:9F:8C:94:51:EE:09:4E:EB:FE:FA:79:53:A1:14:ED:B2:F4:49:49:45:2F:AB:7D:2F:C1:85 -----BEGIN CERTIFICATE----- MIIDljCCAn6gAwIBAgIQC5McOtY5Z+pnI7/Dr5r0SzANBgkqhkiG9w0BAQsFADBl MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 d3cuZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJv b3QgRzIwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQG EwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNl cnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzIwggEi MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZ5ygvUj82ckmIkzTz+GoeMVSA n61UQbVH35ao1K+ALbkKz3X9iaV9JPrjIgwrvJUXCzO/GU1BBpAAvQxNEP4Htecc biJVMWWXvdMX0h5i89vqbFCMP4QMls+3ywPgym2hFEwbid3tALBSfK+RbLE4E9Hp EgjAALAcKxHad3A2m67OeYfcgnDmCXRwVWmvo2ifv922ebPynXApVfSr/5Vh88lA bx3RvpO704gqu52/clpWcTs/1PPRCv4o76Pu2ZmvA9OPYLfykqGxvYmJHzDNw6Yu YjOuFgJ3RFrngQo8p0Quebg/BLxcoIfhG69Rjs3sLPr4/m3wOnyqi+RnlTGNAgMB AAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMB0GA1UdDgQW BBTOw0q5mVXyuNtgv6l+vVa1lzan1jANBgkqhkiG9w0BAQsFAAOCAQEAyqVVjOPI QW5pJ6d1Ee88hjZv0p3GeDgdaZaikmkuOGybfQTUiaWxMTeKySHMq2zNixya1r9I 0jJmwYrA8y8678Dj1JGG0VDjA9tzd29KOVPt3ibHtX2vK0LRdWLjSisCx1BL4Gni lmwORGYQRI+tBev4eaymG+g3NJ1TyWGqolKvSnAWhsI6yLETcDbYz+70CjTVW0z9 B5yiutkBclzzTcHdDrEcDcRjvq30FPuJ7KJBDkzMyFdA0G4Dqs0MjomZmWzwPDCv ON9vvKO+KSAnq3T/EyJ43pdSVR6DtVQgA+6uwE9W3jfMw3+qBCe703e4YtsXfJwo IhNzbM8m9Yop5w== -----END CERTIFICATE----- # Issuer: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Assured ID Root G3 # Subject: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Assured ID Root G3 # SHA256 Fingerprint: 7E:37:CB:8B:4C:47:09:0C:AB:36:55:1B:A6:F4:5D:B8:40:68:0F:BA:16:6A:95:2D:B1:00:71:7F:43:05:3F:C2 -----BEGIN CERTIFICATE----- MIICRjCCAc2gAwIBAgIQC6Fa+h3foLVJRK/NJKBs7DAKBggqhkjOPQQDAzBlMQsw CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu ZGlnaWNlcnQuY29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3Qg RzMwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBlMQswCQYDVQQGEwJV UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu Y29tMSQwIgYDVQQDExtEaWdpQ2VydCBBc3N1cmVkIElEIFJvb3QgRzMwdjAQBgcq hkjOPQIBBgUrgQQAIgNiAAQZ57ysRGXtzbg/WPuNsVepRC0FFfLvC/8QdJ+1YlJf Zn4f5dwbRXkLzMZTCp2NXQLZqVneAlr2lSoOjThKiknGvMYDOAdfVdp+CW7if17Q RSAPWXYQ1qAk8C3eNvJsKTmjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/ BAQDAgGGMB0GA1UdDgQWBBTL0L2p4ZgFUaFNN6KDec6NHSrkhDAKBggqhkjOPQQD AwNnADBkAjAlpIFFAmsSS3V0T8gj43DydXLefInwz5FyYZ5eEJJZVrmDxxDnOOlY JjZ91eQ0hjkCMHw2U/Aw5WJjOpnitqM7mzT6HtoQknFekROn3aRukswy1vUhZscv 6pZjamVFkpUBtA== -----END CERTIFICATE----- # Issuer: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Global Root CA # Subject: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Global Root CA # SHA256 Fingerprint: 43:48:A0:E9:44:4C:78:CB:26:5E:05:8D:5E:89:44:B4:D8:4F:96:62:BD:26:DB:25:7F:89:34:A4:43:C7:01:61 -----BEGIN CERTIFICATE----- MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD QTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG 9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsB CSDMAZOnTjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97 nh6Vfe63SKMI2tavegw5BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt 43C/dxC//AH2hdmoRBBYMql1GNXRor5H4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7P T19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y7vrTC0LUq7dBMtoM1O/4 gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQABo2MwYTAO BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbR TLtm8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUw DQYJKoZIhvcNAQEFBQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/Esr hMAtudXH/vTBH1jLuG2cenTnmCmrEbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg 06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIttep3Sp+dWOIrWcBAI+0tKIJF PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4= -----END CERTIFICATE----- # Issuer: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Global Root G2 # Subject: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Global Root G2 # SHA256 Fingerprint: CB:3C:CB:B7:60:31:E5:E0:13:8F:8D:D3:9A:23:F9:DE:47:FF:C3:5E:43:C1:14:4C:EA:27:D4:6A:5A:B1:CB:5F -----BEGIN CERTIFICATE----- MIIDjjCCAnagAwIBAgIQAzrx5qcRqaC7KGSxHQn65TANBgkqhkiG9w0BAQsFADBh MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBH MjAeFw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVT MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEcyMIIBIjANBgkqhkiG 9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzfNNNx7a8myaJCtSnX/RrohCgiN9RlUyfuI 2/Ou8jqJkTx65qsGGmvPrC3oXgkkRLpimn7Wo6h+4FR1IAWsULecYxpsMNzaHxmx 1x7e/dfgy5SDN67sH0NO3Xss0r0upS/kqbitOtSZpLYl6ZtrAGCSYP9PIUkY92eQ q2EGnI/yuum06ZIya7XzV+hdG82MHauVBJVJ8zUtluNJbd134/tJS7SsVQepj5Wz tCO7TG1F8PapspUwtP1MVYwnSlcUfIKdzXOS0xZKBgyMUNGPHgm+F6HmIcr9g+UQ vIOlCsRnKPZzFBQ9RnbDhxSJITRNrw9FDKZJobq7nMWxM4MphQIDAQABo0IwQDAP BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAdBgNVHQ4EFgQUTiJUIBiV 5uNu5g/6+rkS7QYXjzkwDQYJKoZIhvcNAQELBQADggEBAGBnKJRvDkhj6zHd6mcY 1Yl9PMWLSn/pvtsrF9+wX3N3KjITOYFnQoQj8kVnNeyIv/iPsGEMNKSuIEyExtv4 NeF22d+mQrvHRAiGfzZ0JFrabA0UWTW98kndth/Jsw1HKj2ZL7tcu7XUIOGZX1NG Fdtom/DzMNU+MeKNhJ7jitralj41E6Vf8PlwUHBHQRFXGU7Aj64GxJUTFy8bJZ91 8rGOmaFvE7FBcf6IKshPECBV1/MUReXgRPTqh5Uykw7+U0b6LJ3/iyK5S9kJRaTe pLiaWN0bfVKfjllDiIGknibVb63dDcY3fe0Dkhvld1927jyNxF1WW6LZZm6zNTfl MrY= -----END CERTIFICATE----- # Issuer: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Global Root G3 # Subject: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Global Root G3 # SHA256 Fingerprint: 31:AD:66:48:F8:10:41:38:C7:38:F3:9E:A4:32:01:33:39:3E:3A:18:CC:02:29:6E:F9:7C:2A:C9:EF:67:31:D0 -----BEGIN CERTIFICATE----- MIICPzCCAcWgAwIBAgIQBVVWvPJepDU1w6QP1atFcjAKBggqhkjOPQQDAzBhMQsw CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cu ZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBHMzAe Fw0xMzA4MDExMjAwMDBaFw0zODAxMTUxMjAwMDBaMGExCzAJBgNVBAYTAlVTMRUw EwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5jb20x IDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IEczMHYwEAYHKoZIzj0CAQYF K4EEACIDYgAE3afZu4q4C/sLfyHS8L6+c/MzXRq8NOrexpu80JX28MzQC7phW1FG fp4tn+6OYwwX7Adw9c+ELkCDnOg/QW07rdOkFFk2eJ0DQ+4QE2xy3q6Ip6FrtUPO Z9wj/wMco+I+o0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBhjAd BgNVHQ4EFgQUs9tIpPmhxdiuNkHMEWNpYim8S8YwCgYIKoZIzj0EAwMDaAAwZQIx AK288mw/EkrRLTnDCgmXc/SINoyIJ7vmiI1Qhadj+Z4y3maTD/HMsQmP3Wyr+mt/ oAIwOWZbwmSNuJ5Q3KjVSaLtx9zRSX8XAbjIho9OjIgrqJqpisXRAL34VOKa5Vt8 sycX -----END CERTIFICATE----- # Issuer: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert High Assurance EV Root CA # Subject: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert High Assurance EV Root CA # SHA256 Fingerprint: 74:31:E5:F4:C3:C1:CE:46:90:77:4F:0B:61:E0:54:40:88:3B:A9:A0:1E:D0:0B:A6:AB:D7:80:6E:D3:B1:18:CF -----BEGIN CERTIFICATE----- MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm +9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3 hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2 Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep +OkuE6N36B9K -----END CERTIFICATE----- # Issuer: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Trusted Root G4 # Subject: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert Trusted Root G4 # SHA256 Fingerprint: 55:2F:7B:DC:F1:A7:AF:9E:6C:E6:72:01:7F:4F:12:AB:F7:72:40:C7:8E:76:1A:C2:03:D1:D9:D2:0A:C8:99:88 -----BEGIN CERTIFICATE----- MIIFkDCCA3igAwIBAgIQBZsbV56OITLiOQe9p3d1XDANBgkqhkiG9w0BAQwFADBi MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 d3cuZGlnaWNlcnQuY29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3Qg RzQwHhcNMTMwODAxMTIwMDAwWhcNMzgwMTE1MTIwMDAwWjBiMQswCQYDVQQGEwJV UzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQu Y29tMSEwHwYDVQQDExhEaWdpQ2VydCBUcnVzdGVkIFJvb3QgRzQwggIiMA0GCSqG SIb3DQEBAQUAA4ICDwAwggIKAoICAQC/5pBzaN675F1KPDAiMGkz7MKnJS7JIT3y ithZwuEppz1Yq3aaza57G4QNxDAf8xukOBbrVsaXbR2rsnnyyhHS5F/WBTxSD1If xp4VpX6+n6lXFllVcq9ok3DCsrp1mWpzMpTREEQQLt+C8weE5nQ7bXHiLQwb7iDV ySAdYyktzuxeTsiT+CFhmzTrBcZe7FsavOvJz82sNEBfsXpm7nfISKhmV1efVFiO DCu3T6cw2Vbuyntd463JT17lNecxy9qTXtyOj4DatpGYQJB5w3jHtrHEtWoYOAMQ jdjUN6QuBX2I9YI+EJFwq1WCQTLX2wRzKm6RAXwhTNS8rhsDdV14Ztk6MUSaM0C/ CNdaSaTC5qmgZ92kJ7yhTzm1EVgX9yRcRo9k98FpiHaYdj1ZXUJ2h4mXaXpI8OCi EhtmmnTK3kse5w5jrubU75KSOp493ADkRSWJtppEGSt+wJS00mFt6zPZxd9LBADM fRyVw4/3IbKyEbe7f/LVjHAsQWCqsWMYRJUadmJ+9oCw++hkpjPRiQfhvbfmQ6QY uKZ3AeEPlAwhHbJUKSWJbOUOUlFHdL4mrLZBdd56rF+NP8m800ERElvlEFDrMcXK chYiCd98THU/Y+whX8QgUWtvsauGi0/C1kVfnSD8oR7FwI+isX4KJpn15GkvmB0t 9dmpsh3lGwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIB hjAdBgNVHQ4EFgQU7NfjgtJxXWRM3y5nP+e6mK4cD08wDQYJKoZIhvcNAQEMBQAD ggIBALth2X2pbL4XxJEbw6GiAI3jZGgPVs93rnD5/ZpKmbnJeFwMDF/k5hQpVgs2 SV1EY+CtnJYYZhsjDT156W1r1lT40jzBQ0CuHVD1UvyQO7uYmWlrx8GnqGikJ9yd +SeuMIW59mdNOj6PWTkiU0TryF0Dyu1Qen1iIQqAyHNm0aAFYF/opbSnr6j3bTWc fFqK1qI4mfN4i/RN0iAL3gTujJtHgXINwBQy7zBZLq7gcfJW5GqXb5JQbZaNaHqa sjYUegbyJLkJEVDXCLG4iXqEI2FCKeWjzaIgQdfRnGTZ6iahixTXTBmyUEFxPT9N cCOGDErcgdLMMpSEDQgJlxxPwO5rIHQw0uA5NBCFIRUBCOhVMt5xSdkoF1BN5r5N 0XWs0Mr7QbhDparTwwVETyw2m+L64kW4I1NsBm9nVX9GtUw/bihaeSbSpKhil9Ie 4u1Ki7wb/UdKDd9nZn6yW0HQO+T0O/QEY+nvwlQAUaCKKsnOeMzV6ocEGLPOr0mI r/OSmbaz5mEP0oUA51Aa5BuVnRmhuZyxm7EAHu/QD09CbMkKvO5D+jpxpchNJqU1 /YldvIViHTLSoCtU7ZpXwdv6EM8Zt4tKG48BtieVU+i2iW1bvGjUI+iLUaJW+fCm gKDWHrO8Dw9TdSmq6hN35N6MgSGtBxBHEa2HPQfRdbzP82Z+ -----END CERTIFICATE----- # Issuer: C=US, O=DigiCert, Inc., CN=DigiCert TLS ECC P384 Root G5 # Subject: C=US, O=DigiCert, Inc., CN=DigiCert TLS ECC P384 Root G5 # SHA256 Fingerprint: 01:8E:13:F0:77:25:32:CF:80:9B:D1:B1:72:81:86:72:83:FC:48:C6:E1:3B:E9:C6:98:12:85:4A:49:0C:1B:05 -----BEGIN CERTIFICATE----- MIICGTCCAZ+gAwIBAgIQCeCTZaz32ci5PhwLBCou8zAKBggqhkjOPQQDAzBOMQsw CQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJjAkBgNVBAMTHURp Z2lDZXJ0IFRMUyBFQ0MgUDM4NCBSb290IEc1MB4XDTIxMDExNTAwMDAwMFoXDTQ2 MDExNDIzNTk1OVowTjELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDkRpZ2lDZXJ0LCBJ bmMuMSYwJAYDVQQDEx1EaWdpQ2VydCBUTFMgRUNDIFAzODQgUm9vdCBHNTB2MBAG ByqGSM49AgEGBSuBBAAiA2IABMFEoc8Rl1Ca3iOCNQfN0MsYndLxf3c1TzvdlHJS 7cI7+Oz6e2tYIOyZrsn8aLN1udsJ7MgT9U7GCh1mMEy7H0cKPGEQQil8pQgO4CLp 0zVozptjn4S1mU1YoI71VOeVyaNCMEAwHQYDVR0OBBYEFMFRRVBZqz7nLFr6ICIS B4CIfBFqMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49 BAMDA2gAMGUCMQCJao1H5+z8blUD2WdsJk6Dxv3J+ysTvLd6jLRl0mlpYxNjOyZQ LgGheQaRnUi/wr4CMEfDFXuxoJGZSZOoPHzoRgaLLPIxAJSdYsiJvRmEFOml+wG4 DXZDjC5Ty3zfDBeWUA== -----END CERTIFICATE----- # Issuer: C=US, O=DigiCert, Inc., CN=DigiCert TLS RSA4096 Root G5 # Subject: C=US, O=DigiCert, Inc., CN=DigiCert TLS RSA4096 Root G5 # SHA256 Fingerprint: 37:1A:00:DC:05:33:B3:72:1A:7E:EB:40:E8:41:9E:70:79:9D:2B:0A:0F:2C:1D:80:69:31:65:F7:CE:C4:AD:75 -----BEGIN CERTIFICATE----- MIIFZjCCA06gAwIBAgIQCPm0eKj6ftpqMzeJ3nzPijANBgkqhkiG9w0BAQwFADBN MQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQsIEluYy4xJTAjBgNVBAMT HERpZ2lDZXJ0IFRMUyBSU0E0MDk2IFJvb3QgRzUwHhcNMjEwMTE1MDAwMDAwWhcN NDYwMTE0MjM1OTU5WjBNMQswCQYDVQQGEwJVUzEXMBUGA1UEChMORGlnaUNlcnQs IEluYy4xJTAjBgNVBAMTHERpZ2lDZXJ0IFRMUyBSU0E0MDk2IFJvb3QgRzUwggIi MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCz0PTJeRGd/fxmgefM1eS87IE+ ajWOLrfn3q/5B03PMJ3qCQuZvWxX2hhKuHisOjmopkisLnLlvevxGs3npAOpPxG0 2C+JFvuUAT27L/gTBaF4HI4o4EXgg/RZG5Wzrn4DReW+wkL+7vI8toUTmDKdFqgp wgscONyfMXdcvyej/Cestyu9dJsXLfKB2l2w4SMXPohKEiPQ6s+d3gMXsUJKoBZM pG2T6T867jp8nVid9E6P/DsjyG244gXazOvswzH016cpVIDPRFtMbzCe88zdH5RD nU1/cHAN1DrRN/BsnZvAFJNY781BOHW8EwOVfH/jXOnVDdXifBBiqmvwPXbzP6Po sMH976pXTayGpxi0KcEsDr9kvimM2AItzVwv8n/vFfQMFawKsPHTDU9qTXeXAaDx Zre3zu/O7Oyldcqs4+Fj97ihBMi8ez9dLRYiVu1ISf6nL3kwJZu6ay0/nTvEF+cd Lvvyz6b84xQslpghjLSR6Rlgg/IwKwZzUNWYOwbpx4oMYIwo+FKbbuH2TbsGJJvX KyY//SovcfXWJL5/MZ4PbeiPT02jP/816t9JXkGPhvnxd3lLG7SjXi/7RgLQZhNe XoVPzthwiHvOAbWWl9fNff2C+MIkwcoBOU+NosEUQB+cZtUMCUbW8tDRSHZWOkPL tgoRObqME2wGtZ7P6wIDAQABo0IwQDAdBgNVHQ4EFgQUUTMc7TZArxfTJc1paPKv TiM+s0EwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcN AQEMBQADggIBAGCmr1tfV9qJ20tQqcQjNSH/0GEwhJG3PxDPJY7Jv0Y02cEhJhxw GXIeo8mH/qlDZJY6yFMECrZBu8RHANmfGBg7sg7zNOok992vIGCukihfNudd5N7H PNtQOa27PShNlnx2xlv0wdsUpasZYgcYQF+Xkdycx6u1UQ3maVNVzDl92sURVXLF O4uJ+DQtpBflF+aZfTCIITfNMBc9uPK8qHWgQ9w+iUuQrm0D4ByjoJYJu32jtyoQ REtGBzRj7TG5BO6jm5qu5jF49OokYTurWGT/u4cnYiWB39yhL/btp/96j1EuMPik AdKFOV8BmZZvWltwGUb+hmA+rYAQCd05JS9Yf7vSdPD3Rh9GOUrYU9DzLjtxpdRv /PNn5AeP3SYZ4Y1b+qOTEZvpyDrDVWiakuFSdjjo4bq9+0/V77PnSIMx8IIh47a+ p6tv75/fTM8BuGJqIz3nCU2AG3swpMPdB380vqQmsvZB6Akd4yCYqjdP//fx4ilw MUc/dNAUFvohigLVigmUdy7yWSiLfFCSCmZ4OIN1xLVaqBHG5cGdZlXPU8Sv13WF qUITVuwhd4GTWgzqltlJyqEI8pc7bZsEGCREjnwB8twl2F6GmrE52/WRMmrRpnCK ovfepEWFJqgejF0pW8hL2JpqA15w8oVPbEtoL8pU9ozaMv7Da4M/OMZ+ -----END CERTIFICATE----- # Issuer: C=US, O=Entrust, Inc., OU=See www.entrust.net/legal-terms, OU=(c) 2009 Entrust, Inc. - for authorized use only, CN=Entrust Root Certification Authority - G2 # Subject: C=US, O=Entrust, Inc., OU=See www.entrust.net/legal-terms, OU=(c) 2009 Entrust, Inc. - for authorized use only, CN=Entrust Root Certification Authority - G2 # SHA256 Fingerprint: 43:DF:57:74:B0:3E:7F:EF:5F:E4:0D:93:1A:7B:ED:F1:BB:2E:6B:42:73:8C:4E:6D:38:41:10:3D:3A:A7:F3:39 -----BEGIN CERTIFICATE----- MIIEPjCCAyagAwIBAgIESlOMKDANBgkqhkiG9w0BAQsFADCBvjELMAkGA1UEBhMC VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50 cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3Qs IEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVz dCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRzIwHhcNMDkwNzA3MTcy NTU0WhcNMzAxMjA3MTc1NTU0WjCBvjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUVu dHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3d3cuZW50cnVzdC5uZXQvbGVnYWwt dGVybXMxOTA3BgNVBAsTMChjKSAyMDA5IEVudHJ1c3QsIEluYy4gLSBmb3IgYXV0 aG9yaXplZCB1c2Ugb25seTEyMDAGA1UEAxMpRW50cnVzdCBSb290IENlcnRpZmlj YXRpb24gQXV0aG9yaXR5IC0gRzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK AoIBAQC6hLZy254Ma+KZ6TABp3bqMriVQRrJ2mFOWHLP/vaCeb9zYQYKpSfYs1/T RU4cctZOMvJyig/3gxnQaoCAAEUesMfnmr8SVycco2gvCoe9amsOXmXzHHfV1IWN cCG0szLni6LVhjkCsbjSR87kyUnEO6fe+1R9V77w6G7CebI6C1XiUJgWMhNcL3hW wcKUs/Ja5CeanyTXxuzQmyWC48zCxEXFjJd6BmsqEZ+pCm5IO2/b1BEZQvePB7/1 U1+cPvQXLOZprE4yTGJ36rfo5bs0vBmLrpxR57d+tVOxMyLlbc9wPBr64ptntoP0 jaWvYkxN4FisZDQSA/i2jZRjJKRxAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRqciZ60B7vfec7aVHUbI2fkBJmqzAN BgkqhkiG9w0BAQsFAAOCAQEAeZ8dlsa2eT8ijYfThwMEYGprmi5ZiXMRrEPR9RP/ jTkrwPK9T3CMqS/qF8QLVJ7UG5aYMzyorWKiAHarWWluBh1+xLlEjZivEtRh2woZ Rkfz6/djwUAFQKXSt/S1mja/qYh2iARVBCuch38aNzx+LaUa2NSJXsq9rD1s2G2v 1fN2D807iDginWyTmsQ9v4IbZT+mD12q/OWyFcq1rca8PdCE6OoGcrBNOTJ4vz4R nAuknZoh8/CbCzB428Hch0P+vGOaysXCHMnHjf87ElgI5rY97HosTvuDls4MPGmH VHOkc8KT/1EQrBVUAdj8BbGJoX90g5pJ19xOe4pIb4tF9g== -----END CERTIFICATE----- # Issuer: C=US, O=Entrust, Inc., OU=See www.entrust.net/legal-terms, OU=(c) 2012 Entrust, Inc. - for authorized use only, CN=Entrust Root Certification Authority - EC1 # Subject: C=US, O=Entrust, Inc., OU=See www.entrust.net/legal-terms, OU=(c) 2012 Entrust, Inc. - for authorized use only, CN=Entrust Root Certification Authority - EC1 # SHA256 Fingerprint: 02:ED:0E:B2:8C:14:DA:45:16:5C:56:67:91:70:0D:64:51:D7:FB:56:F0:B2:AB:1D:3B:8E:B0:70:E5:6E:DF:F5 -----BEGIN CERTIFICATE----- MIIC+TCCAoCgAwIBAgINAKaLeSkAAAAAUNCR+TAKBggqhkjOPQQDAzCBvzELMAkG A1UEBhMCVVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xKDAmBgNVBAsTH1NlZSB3 d3cuZW50cnVzdC5uZXQvbGVnYWwtdGVybXMxOTA3BgNVBAsTMChjKSAyMDEyIEVu dHJ1c3QsIEluYy4gLSBmb3IgYXV0aG9yaXplZCB1c2Ugb25seTEzMDEGA1UEAxMq RW50cnVzdCBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IC0gRUMxMB4XDTEy MTIxODE1MjUzNloXDTM3MTIxODE1NTUzNlowgb8xCzAJBgNVBAYTAlVTMRYwFAYD VQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1c3QubmV0 L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxMiBFbnRydXN0LCBJbmMuIC0g Zm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxMzAxBgNVBAMTKkVudHJ1c3QgUm9vdCBD ZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEVDMTB2MBAGByqGSM49AgEGBSuBBAAi A2IABIQTydC6bUF74mzQ61VfZgIaJPRbiWlH47jCffHyAsWfoPZb1YsGGYZPUxBt ByQnoaD41UcZYUx9ypMn6nQM72+WCf5j7HBdNq1nd67JnXxVRDqiY1Ef9eNi1KlH Bz7MIKNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0O BBYEFLdj5xrdjekIplWDpOBqUEFlEUJJMAoGCCqGSM49BAMDA2cAMGQCMGF52OVC R98crlOZF7ZvHH3hvxGU0QOIdeSNiaSKd0bebWHvAvX7td/M/k7//qnmpwIwW5nX hTcGtXsI/esni0qU+eH6p44mCOh8kmhtc9hvJqwhAriZtyZBWyVgrtBIGu4G -----END CERTIFICATE----- # Issuer: C=US, O=Entrust, Inc., OU=See www.entrust.net/legal-terms, OU=(c) 2015 Entrust, Inc. - for authorized use only, CN=Entrust Root Certification Authority - G4 # Subject: C=US, O=Entrust, Inc., OU=See www.entrust.net/legal-terms, OU=(c) 2015 Entrust, Inc. - for authorized use only, CN=Entrust Root Certification Authority - G4 # SHA256 Fingerprint: DB:35:17:D1:F6:73:2A:2D:5A:B9:7C:53:3E:C7:07:79:EE:32:70:A6:2F:B4:AC:42:38:37:24:60:E6:F0:1E:88 -----BEGIN CERTIFICATE----- MIIGSzCCBDOgAwIBAgIRANm1Q3+vqTkPAAAAAFVlrVgwDQYJKoZIhvcNAQELBQAw gb4xCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQL Ex9TZWUgd3d3LmVudHJ1c3QubmV0L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykg MjAxNSBFbnRydXN0LCBJbmMuIC0gZm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxMjAw BgNVBAMTKUVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEc0 MB4XDTE1MDUyNzExMTExNloXDTM3MTIyNzExNDExNlowgb4xCzAJBgNVBAYTAlVT MRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1 c3QubmV0L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxNSBFbnRydXN0LCBJ bmMuIC0gZm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxMjAwBgNVBAMTKUVudHJ1c3Qg Um9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEc0MIICIjANBgkqhkiG9w0B AQEFAAOCAg8AMIICCgKCAgEAsewsQu7i0TD/pZJH4i3DumSXbcr3DbVZwbPLqGgZ 2K+EbTBwXX7zLtJTmeH+H17ZSK9dE43b/2MzTdMAArzE+NEGCJR5WIoV3imz/f3E T+iq4qA7ec2/a0My3dl0ELn39GjUu9CH1apLiipvKgS1sqbHoHrmSKvS0VnM1n4j 5pds8ELl3FFLFUHtSUrJ3hCX1nbB76W1NhSXNdh4IjVS70O92yfbYVaCNNzLiGAM C1rlLAHGVK/XqsEQe9IFWrhAnoanw5CGAlZSCXqc0ieCU0plUmr1POeo8pyvi73T DtTUXm6Hnmo9RR3RXRv06QqsYJn7ibT/mCzPfB3pAqoEmh643IhuJbNsZvc8kPNX wbMv9W3y+8qh+CmdRouzavbmZwe+LGcKKh9asj5XxNMhIWNlUpEbsZmOeX7m640A 2Vqq6nPopIICR5b+W45UYaPrL0swsIsjdXJ8ITzI9vF01Bx7owVV7rtNOzK+mndm nqxpkCIHH2E6lr7lmk/MBTwoWdPBDFSoWWG9yHJM6Nyfh3+9nEg2XpWjDrk4JFX8 dWbrAuMINClKxuMrLzOg2qOGpRKX/YAr2hRC45K9PvJdXmd0LhyIRyk0X+IyqJwl N4y6mACXi0mWHv0liqzc2thddG5msP9E36EYxr5ILzeUePiVSj9/E15dWf10hkNj c0kCAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD VR0OBBYEFJ84xFYjwznooHFs6FRM5Og6sb9nMA0GCSqGSIb3DQEBCwUAA4ICAQAS 5UKme4sPDORGpbZgQIeMJX6tuGguW8ZAdjwD+MlZ9POrYs4QjbRaZIxowLByQzTS Gwv2LFPSypBLhmb8qoMi9IsabyZIrHZ3CL/FmFz0Jomee8O5ZDIBf9PD3Vht7LGr hFV0d4QEJ1JrhkzO3bll/9bGXp+aEJlLdWr+aumXIOTkdnrG0CSqkM0gkLpHZPt/ B7NTeLUKYvJzQ85BK4FqLoUWlFPUa19yIqtRLULVAJyZv967lDtX/Zr1hstWO1uI AeV8KEsD+UmDfLJ/fOPtjqF/YFOOVZ1QNBIPt5d7bIdKROf1beyAN/BYGW5KaHbw H5Lk6rWS02FREAutp9lfx1/cH6NcjKF+m7ee01ZvZl4HliDtC3T7Zk6LERXpgUl+ b7DUUH8i119lAg2m9IUe2K4GS0qn0jFmwvjO5QimpAKWRGhXxNUzzxkvFMSUHHuk 2fCfDrGA4tGeEWSpiBE6doLlYsKA2KSD7ZPvfC+QsDJMlhVoSFLUmQjAJOgc47Ol IQ6SwJAfzyBfyjs4x7dtOvPmRLgOMWuIjnDrnBdSqEGULoe256YSxXXfW8AKbnuk 5F6G+TaU33fD6Q3AOfF5u0aOq0NZJ7cguyPpVkAh7DE9ZapD8j3fcEThuk0mEDuY n/PIjhs4ViFqUZPTkcpG2om3PVODLAgfi49T3f+sHw== -----END CERTIFICATE----- # Issuer: C=US, O=Entrust, Inc., OU=www.entrust.net/CPS is incorporated by reference, OU=(c) 2006 Entrust, Inc., CN=Entrust Root Certification Authority # Subject: C=US, O=Entrust, Inc., OU=www.entrust.net/CPS is incorporated by reference, OU=(c) 2006 Entrust, Inc., CN=Entrust Root Certification Authority # SHA256 Fingerprint: 73:C1:76:43:4F:1B:C6:D5:AD:F4:5B:0E:76:E7:27:28:7C:8D:E5:76:16:C1:E6:E6:14:1A:2B:2C:BC:7D:8E:4C -----BEGIN CERTIFICATE----- MIIEkTCCA3mgAwIBAgIERWtQVDANBgkqhkiG9w0BAQUFADCBsDELMAkGA1UEBhMC VVMxFjAUBgNVBAoTDUVudHJ1c3QsIEluYy4xOTA3BgNVBAsTMHd3dy5lbnRydXN0 Lm5ldC9DUFMgaXMgaW5jb3Jwb3JhdGVkIGJ5IHJlZmVyZW5jZTEfMB0GA1UECxMW KGMpIDIwMDYgRW50cnVzdCwgSW5jLjEtMCsGA1UEAxMkRW50cnVzdCBSb290IENl cnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA2MTEyNzIwMjM0MloXDTI2MTEyNzIw NTM0MlowgbAxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMTkw NwYDVQQLEzB3d3cuZW50cnVzdC5uZXQvQ1BTIGlzIGluY29ycG9yYXRlZCBieSBy ZWZlcmVuY2UxHzAdBgNVBAsTFihjKSAyMDA2IEVudHJ1c3QsIEluYy4xLTArBgNV BAMTJEVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASIwDQYJ KoZIhvcNAQEBBQADggEPADCCAQoCggEBALaVtkNC+sZtKm9I35RMOVcF7sN5EUFo Nu3s/poBj6E4KPz3EEZmLk0eGrEaTsbRwJWIsMn/MYszA9u3g3s+IIRe7bJWKKf4 4LlAcTfFy0cOlypowCKVYhXbR9n10Cv/gkvJrT7eTNuQgFA/CYqEAOwwCj0Yzfv9 KlmaI5UXLEWeH25DeW0MXJj+SKfFI0dcXv1u5x609mhF0YaDW6KKjbHjKYD+JXGI rb68j6xSlkuqUY3kEzEZ6E5Nn9uss2rVvDlUccp6en+Q3X0dgNmBu1kmwhH+5pPi 94DkZfs0Nw4pgHBNrziGLp5/V6+eF67rHMsoIV+2HNjnogQi+dPa2MsCAwEAAaOB sDCBrTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zArBgNVHRAEJDAi gA8yMDA2MTEyNzIwMjM0MlqBDzIwMjYxMTI3MjA1MzQyWjAfBgNVHSMEGDAWgBRo kORnpKZTgMeGZqTx90tD+4S9bTAdBgNVHQ4EFgQUaJDkZ6SmU4DHhmak8fdLQ/uE vW0wHQYJKoZIhvZ9B0EABBAwDhsIVjcuMTo0LjADAgSQMA0GCSqGSIb3DQEBBQUA A4IBAQCT1DCw1wMgKtD5Y+iRDAUgqV8ZyntyTtSx29CW+1RaGSwMCPeyvIWonX9t O1KzKtvn1ISMY/YPyyYBkVBs9F8U4pN0wBOeMDpQ47RgxRzwIkSNcUesyBrJ6Zua AGAT/3B+XxFNSRuzFVJ7yVTav52Vr2ua2J7p8eRDjeIRRDq/r72DQnNSi6q7pynP 9WQcCk3RvKqsnyrQ/39/2n3qse0wJcGE2jTSW3iDVuycNsMm4hH2Z0kdkquM++v/ eu6FSqdQgPCnXEqULl8FmTxSQeDNtGPPAUO6nIPcj2A781q0tHuu2guQOHXvgR1m 0vdXcDazv/wor3ElhVsT/h5/WrQ8 -----END CERTIFICATE----- # Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R1 # Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R1 # SHA256 Fingerprint: D9:47:43:2A:BD:E7:B7:FA:90:FC:2E:6B:59:10:1B:12:80:E0:E1:C7:E4:E4:0F:A3:C6:88:7F:FF:57:A7:F4:CF -----BEGIN CERTIFICATE----- MIIFVzCCAz+gAwIBAgINAgPlk28xsBNJiGuiFzANBgkqhkiG9w0BAQwFADBHMQsw CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU MBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEBAQUA A4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaMf/vo 27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vXmX7w Cl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7zUjw TcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0Pfybl qAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtcvfaH szVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4Zor8 Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUspzBmk MiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOORc92 wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYWk70p aDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+DVrN VjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgFlQID AQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E FgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBAJ+qQibb C5u+/x6Wki4+omVKapi6Ist9wTrYggoGxval3sBOh2Z5ofmmWJyq+bXmYOfg6LEe QkEzCzc9zolwFcq1JKjPa7XSQCGYzyI0zzvFIoTgxQ6KfF2I5DUkzps+GlQebtuy h6f88/qBVRRiClmpIgUxPoLW7ttXNLwzldMXG+gnoot7TiYaelpkttGsN/H9oPM4 7HLwEXWdyzRSjeZ2axfG34arJ45JK3VmgRAhpuo+9K4l/3wV3s6MJT/KYnAK9y8J ZgfIPxz88NtFMN9iiMG1D53Dn0reWVlHxYciNuaCp+0KueIHoI17eko8cdLiA6Ef MgfdG+RCzgwARWGAtQsgWSl4vflVy2PFPEz0tv/bal8xa5meLMFrUKTX5hgUvYU/ Z6tGn6D/Qqc6f1zLXbBwHSs09dR2CQzreExZBfMzQsNhFRAbd03OIozUhfJFfbdT 6u9AWpQKXCBfTkBdYiJ23//OYb2MI3jSNwLgjt7RETeJ9r/tSQdirpLsQBqvFAnZ 0E6yove+7u7Y/9waLd64NnHi/Hm3lCXRSHNboTXns5lndcEZOitHTtNCjv0xyBZm 2tIMPNuzjsmhDYAPexZ3FL//2wmUspO8IFgV6dtxQ/PeEMMA3KgqlbbC1j+Qa3bb bP6MvPJwNQzcmRk13NfIRmPVNnGuV/u3gm3c -----END CERTIFICATE----- # Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R2 # Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R2 # SHA256 Fingerprint: 8D:25:CD:97:22:9D:BF:70:35:6B:DA:4E:B3:CC:73:40:31:E2:4C:F0:0F:AF:CF:D3:2D:C7:6E:B5:84:1C:7E:A8 -----BEGIN CERTIFICATE----- MIIFVzCCAz+gAwIBAgINAgPlrsWNBCUaqxElqjANBgkqhkiG9w0BAQwFADBHMQsw CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU MBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEBAQUA A4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3LvCvpt nfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3KgGjSY 6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9BuXvAu MC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOdre7k RXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXuPuWg f9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1mKPV +3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K8Yzo dDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqjx5RW Ir9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsRnTKa G73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0kzCq gc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9OktwID AQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E FgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBAB/Kzt3H vqGf2SdMC9wXmBFqiN495nFWcrKeGk6c1SuYJF2ba3uwM4IJvd8lRuqYnrYb/oM8 0mJhwQTtzuDFycgTE1XnqGOtjHsB/ncw4c5omwX4Eu55MaBBRTUoCnGkJE+M3DyC B19m3H0Q/gxhswWV7uGugQ+o+MePTagjAiZrHYNSVc61LwDKgEDg4XSsYPWHgJ2u NmSRXbBoGOqKYcl3qJfEycel/FVL8/B/uWU9J2jQzGv6U53hkRrJXRqWbTKH7QMg yALOWr7Z6v2yTcQvG99fevX4i8buMTolUVVnjWQye+mew4K6Ki3pHrTgSAai/Gev HyICc/sgCq+dVEuhzf9gR7A/Xe8bVr2XIZYtCtFenTgCR2y59PYjJbigapordwj6 xLEokCZYCDzifqrXPW+6MYgKBesntaFJ7qBFVHvmJ2WZICGoo7z7GJa7Um8M7YNR TOlZ4iBgxcJlkoKM8xAfDoqXvneCbT+PHV28SSe9zE8P4c52hgQjxcCMElv924Sg JPFI/2R80L5cFtHvma3AH/vLrrw4IgYmZNralw4/KBVEqE8AyvCazM90arQ+POuV 7LXTWtiBmelDGDfrs7vRWGJB82bSj6p4lVQgw1oudCvV0b4YacCs1aTPObpRhANl 6WLAYv7YTVWW4tAR+kg0Eeye7QUd5MjWHYbL -----END CERTIFICATE----- # Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R3 # Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R3 # SHA256 Fingerprint: 34:D8:A7:3E:E2:08:D9:BC:DB:0D:95:65:20:93:4B:4E:40:E6:94:82:59:6E:8B:6F:73:C8:42:6B:01:0A:6F:48 -----BEGIN CERTIFICATE----- MIICCTCCAY6gAwIBAgINAgPluILrIPglJ209ZjAKBggqhkjOPQQDAzBHMQswCQYD VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG A1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQAIgNi AAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout736G jOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2ADDL2 4CejQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW BBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEA9uEglRR7 VKOQFhG/hMjqb2sXnh5GmCCbn9MN2azTL818+FsuVbu/3ZL3pAzcMeGiAjEA/Jdm ZuVDFhOD3cffL74UOO0BzrEXGhF16b0DjyZ+hOXJYKaV11RZt+cRLInUue4X -----END CERTIFICATE----- # Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R4 # Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R4 # SHA256 Fingerprint: 34:9D:FA:40:58:C5:E2:63:12:3B:39:8A:E7:95:57:3C:4E:13:13:C8:3F:E6:8F:93:55:6C:D5:E8:03:1B:3C:7D -----BEGIN CERTIFICATE----- MIICCTCCAY6gAwIBAgINAgPlwGjvYxqccpBQUjAKBggqhkjOPQQDAzBHMQswCQYD VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG A1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQAIgNi AATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzuhXyi QHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/lxKvR HYqjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW BBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNpADBmAjEA6ED/g94D 9J+uHXqnLrmvT/aDHQ4thQEd0dlq7A/Cr8deVl5c1RxYIigL9zC2L7F8AjEA8GE8 p/SgguMh1YQdc4acLa/KNJvxn7kjNuK8YAOdgLOaVsjh4rsUecrNIdSUtUlD -----END CERTIFICATE----- # Issuer: C=US, O=IdenTrust, CN=IdenTrust Commercial Root CA 1 # Subject: C=US, O=IdenTrust, CN=IdenTrust Commercial Root CA 1 # SHA256 Fingerprint: 5D:56:49:9B:E4:D2:E0:8B:CF:CA:D0:8A:3E:38:72:3D:50:50:3B:DE:70:69:48:E4:2F:55:60:30:19:E5:28:AE -----BEGIN CERTIFICATE----- MIIFYDCCA0igAwIBAgIQCgFCgAAAAUUjyES1AAAAAjANBgkqhkiG9w0BAQsFADBK MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScwJQYDVQQDEx5JZGVu VHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwHhcNMTQwMTE2MTgxMjIzWhcNMzQw MTE2MTgxMjIzWjBKMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScw JQYDVQQDEx5JZGVuVHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwggIiMA0GCSqG SIb3DQEBAQUAA4ICDwAwggIKAoICAQCnUBneP5k91DNG8W9RYYKyqU+PZ4ldhNlT 3Qwo2dfw/66VQ3KZ+bVdfIrBQuExUHTRgQ18zZshq0PirK1ehm7zCYofWjK9ouuU +ehcCuz/mNKvcbO0U59Oh++SvL3sTzIwiEsXXlfEU8L2ApeN2WIrvyQfYo3fw7gp S0l4PJNgiCL8mdo2yMKi1CxUAGc1bnO/AljwpN3lsKImesrgNqUZFvX9t++uP0D1 bVoE/c40yiTcdCMbXTMTEl3EASX2MN0CXZ/g1Ue9tOsbobtJSdifWwLziuQkkORi T0/Br4sOdBeo0XKIanoBScy0RnnGF7HamB4HWfp1IYVl3ZBWzvurpWCdxJ35UrCL vYf5jysjCiN2O/cz4ckA82n5S6LgTrx+kzmEB/dEcH7+B1rlsazRGMzyNeVJSQjK Vsk9+w8YfYs7wRPCTY/JTw436R+hDmrfYi7LNQZReSzIJTj0+kuniVyc0uMNOYZK dHzVWYfCP04MXFL0PfdSgvHqo6z9STQaKPNBiDoT7uje/5kdX7rL6B7yuVBgwDHT c+XvvqDtMwt0viAgxGds8AgDelWAf0ZOlqf0Hj7h9tgJ4TNkK2PXMl6f+cB7D3hv l7yTmvmcEpB4eoCHFddydJxVdHixuuFucAS6T6C6aMN7/zHwcz09lCqxC0EOoP5N iGVreTO01wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB /zAdBgNVHQ4EFgQU7UQZwNPwBovupHu+QucmVMiONnYwDQYJKoZIhvcNAQELBQAD ggIBAA2ukDL2pkt8RHYZYR4nKM1eVO8lvOMIkPkp165oCOGUAFjvLi5+U1KMtlwH 6oi6mYtQlNeCgN9hCQCTrQ0U5s7B8jeUeLBfnLOic7iPBZM4zY0+sLj7wM+x8uwt LRvM7Kqas6pgghstO8OEPVeKlh6cdbjTMM1gCIOQ045U8U1mwF10A0Cj7oV+wh93 nAbowacYXVKV7cndJZ5t+qntozo00Fl72u1Q8zW/7esUTTHHYPTa8Yec4kjixsU3 +wYQ+nVZZjFHKdp2mhzpgq7vmrlR94gjmmmVYjzlVYA211QC//G5Xc7UI2/YRYRK W2XviQzdFKcgyxilJbQN+QHwotL0AMh0jqEqSI5l2xPE4iUXfeu+h1sXIFRRk0pT AwvsXcoz7WL9RccvW9xYoIA55vrX/hMUpu09lEpCdNTDd1lzzY9GvlU47/rokTLq l1gEIt44w8y8bckzOmoKaT+gyOpyj4xjhiO9bTyWnpXgSUyqorkqG5w2gXjtw+hG 4iZZRHUe2XWJUc0QhJ1hYMtd+ZciTY6Y5uN/9lu7rs3KSoFrXgvzUeF0K+l+J6fZ mUlO+KWA2yUPHGNiiskzZ2s8EIPGrd6ozRaOjfAHN3Gf8qv8QfXBi+wAN10J5U6A 7/qxXDgGpRtK4dw4LTzcqx+QGtVKnO7RcGzM7vRX+Bi6hG6H -----END CERTIFICATE----- # Issuer: C=US, O=IdenTrust, CN=IdenTrust Public Sector Root CA 1 # Subject: C=US, O=IdenTrust, CN=IdenTrust Public Sector Root CA 1 # SHA256 Fingerprint: 30:D0:89:5A:9A:44:8A:26:20:91:63:55:22:D1:F5:20:10:B5:86:7A:CA:E1:2C:78:EF:95:8F:D4:F4:38:9F:2F -----BEGIN CERTIFICATE----- MIIFZjCCA06gAwIBAgIQCgFCgAAAAUUjz0Z8AAAAAjANBgkqhkiG9w0BAQsFADBN MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MSowKAYDVQQDEyFJZGVu VHJ1c3QgUHVibGljIFNlY3RvciBSb290IENBIDEwHhcNMTQwMTE2MTc1MzMyWhcN MzQwMTE2MTc1MzMyWjBNMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0 MSowKAYDVQQDEyFJZGVuVHJ1c3QgUHVibGljIFNlY3RvciBSb290IENBIDEwggIi MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC2IpT8pEiv6EdrCvsnduTyP4o7 ekosMSqMjbCpwzFrqHd2hCa2rIFCDQjrVVi7evi8ZX3yoG2LqEfpYnYeEe4IFNGy RBb06tD6Hi9e28tzQa68ALBKK0CyrOE7S8ItneShm+waOh7wCLPQ5CQ1B5+ctMlS bdsHyo+1W/CD80/HLaXIrcuVIKQxKFdYWuSNG5qrng0M8gozOSI5Cpcu81N3uURF /YTLNiCBWS2ab21ISGHKTN9T0a9SvESfqy9rg3LvdYDaBjMbXcjaY8ZNzaxmMc3R 3j6HEDbhuaR672BQssvKplbgN6+rNBM5Jeg5ZuSYeqoSmJxZZoY+rfGwyj4GD3vw EUs3oERte8uojHH01bWRNszwFcYr3lEXsZdMUD2xlVl8BX0tIdUAvwFnol57plzy 9yLxkA2T26pEUWbMfXYD62qoKjgZl3YNa4ph+bz27nb9cCvdKTz4Ch5bQhyLVi9V GxyhLrXHFub4qjySjmm2AcG1hp2JDws4lFTo6tyePSW8Uybt1as5qsVATFSrsrTZ 2fjXctscvG29ZV/viDUqZi/u9rNl8DONfJhBaUYPQxxp+pu10GFqzcpL2UyQRqsV WaFHVCkugyhfHMKiq3IXAAaOReyL4jM9f9oZRORicsPfIsbyVtTdX5Vy7W1f90gD W/3FKqD2cyOEEBsB5wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/ BAUwAwEB/zAdBgNVHQ4EFgQU43HgntinQtnbcZFrlJPrw6PRFKMwDQYJKoZIhvcN AQELBQADggIBAEf63QqwEZE4rU1d9+UOl1QZgkiHVIyqZJnYWv6IAcVYpZmxI1Qj t2odIFflAWJBF9MJ23XLblSQdf4an4EKwt3X9wnQW3IV5B4Jaj0z8yGa5hV+rVHV DRDtfULAj+7AmgjVQdZcDiFpboBhDhXAuM/FSRJSzL46zNQuOAXeNf0fb7iAaJg9 TaDKQGXSc3z1i9kKlT/YPyNtGtEqJBnZhbMX73huqVjRI9PHE+1yJX9dsXNw0H8G lwmEKYBhHfpe/3OsoOOJuBxxFcbeMX8S3OFtm6/n6J91eEyrRjuazr8FGF1NFTwW mhlQBJqymm9li1JfPFgEKCXAZmExfrngdbkaqIHWchezxQMxNRF4eKLg6TCMf4Df WN88uieW4oA0beOY02QnrEh+KHdcxiVhJfiFDGX6xDIvpZgF5PgLZxYWxoK4Mhn5 +bl53B/N66+rDt0b20XkeucC4pVd/GnwU2lhlXV5C15V5jgclKlZM57IcXR5f1GJ tshquDDIajjDbp7hNxbqBWJMWxJH7ae0s1hWx0nzfxJoCTFx8G34Tkf71oXuxVhA GaQdp/lLQzfcaFpPz+vCZHTetBXZ9FRUGi8c15dxVJCO2SCdUyt/q4/i6jC8UDfv 8Ue1fXwsBOxonbRJRBD0ckscZOf85muQ3Wl9af0AVqW3rLatt8o+Ae+c -----END CERTIFICATE----- # Issuer: C=US, O=Internet Security Research Group, CN=ISRG Root X1 # Subject: C=US, O=Internet Security Research Group, CN=ISRG Root X1 # SHA256 Fingerprint: 96:BC:EC:06:26:49:76:F3:74:60:77:9A:CF:28:C5:A7:CF:E8:A3:C0:AA:E1:1A:8F:FC:EE:05:C0:BD:DF:08:C6 -----BEGIN CERTIFICATE----- MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4 WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+ 0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ 3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5 ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq 4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= -----END CERTIFICATE----- # Issuer: C=US, O=Internet Security Research Group, CN=ISRG Root X2 # Subject: C=US, O=Internet Security Research Group, CN=ISRG Root X2 # SHA256 Fingerprint: 69:72:9B:8E:15:A8:6E:FC:17:7A:57:AF:B7:17:1D:FC:64:AD:D2:8C:2F:CA:8C:F1:50:7E:34:45:3C:CB:14:70 -----BEGIN CERTIFICATE----- MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00 MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW +1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9 ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1 /q4AaOeMSQ+2b1tbFfLn -----END CERTIFICATE----- # Issuer: C=US, O=Microsoft Corporation, CN=Microsoft ECC Root Certificate Authority 2017 # Subject: C=US, O=Microsoft Corporation, CN=Microsoft ECC Root Certificate Authority 2017 # SHA256 Fingerprint: 35:8D:F3:9D:76:4A:F9:E1:B7:66:E9:C9:72:DF:35:2E:E1:5C:FA:C2:27:AF:6A:D1:D7:0E:8E:4A:6E:DC:BA:02 -----BEGIN CERTIFICATE----- MIICWTCCAd+gAwIBAgIQZvI9r4fei7FK6gxXMQHC7DAKBggqhkjOPQQDAzBlMQsw CQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYD VQQDEy1NaWNyb3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIw MTcwHhcNMTkxMjE4MjMwNjQ1WhcNNDIwNzE4MjMxNjA0WjBlMQswCQYDVQQGEwJV UzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1NaWNy b3NvZnQgRUNDIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwdjAQBgcq hkjOPQIBBgUrgQQAIgNiAATUvD0CQnVBEyPNgASGAlEvaqiBYgtlzPbKnR5vSmZR ogPZnZH6thaxjG7efM3beaYvzrvOcS/lpaso7GMEZpn4+vKTEAXhgShC48Zo9OYb hGBKia/teQ87zvH2RPUBeMCjVDBSMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8E BTADAQH/MB0GA1UdDgQWBBTIy5lycFIM+Oa+sgRXKSrPQhDtNTAQBgkrBgEEAYI3 FQEEAwIBADAKBggqhkjOPQQDAwNoADBlAjBY8k3qDPlfXu5gKcs68tvWMoQZP3zV L8KxzJOuULsJMsbG7X7JNpQS5GiFBqIb0C8CMQCZ6Ra0DvpWSNSkMBaReNtUjGUB iudQZsIxtzm6uBoiB078a1QWIP8rtedMDE2mT3M= -----END CERTIFICATE----- # Issuer: C=US, O=Microsoft Corporation, CN=Microsoft RSA Root Certificate Authority 2017 # Subject: C=US, O=Microsoft Corporation, CN=Microsoft RSA Root Certificate Authority 2017 # SHA256 Fingerprint: C7:41:F7:0F:4B:2A:8D:88:BF:2E:71:C1:41:22:EF:53:EF:10:EB:A0:CF:A5:E6:4C:FA:20:F4:18:85:30:73:E0 -----BEGIN CERTIFICATE----- MIIFqDCCA5CgAwIBAgIQHtOXCV/YtLNHcB6qvn9FszANBgkqhkiG9w0BAQwFADBl MQswCQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYw NAYDVQQDEy1NaWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5 IDIwMTcwHhcNMTkxMjE4MjI1MTIyWhcNNDIwNzE4MjMwMDIzWjBlMQswCQYDVQQG EwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1N aWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwggIi MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKW76UM4wplZEWCpW9R2LBifOZ Nt9GkMml7Xhqb0eRaPgnZ1AzHaGm++DlQ6OEAlcBXZxIQIJTELy/xztokLaCLeX0 ZdDMbRnMlfl7rEqUrQ7eS0MdhweSE5CAg2Q1OQT85elss7YfUJQ4ZVBcF0a5toW1 HLUX6NZFndiyJrDKxHBKrmCk3bPZ7Pw71VdyvD/IybLeS2v4I2wDwAW9lcfNcztm gGTjGqwu+UcF8ga2m3P1eDNbx6H7JyqhtJqRjJHTOoI+dkC0zVJhUXAoP8XFWvLJ jEm7FFtNyP9nTUwSlq31/niol4fX/V4ggNyhSyL71Imtus5Hl0dVe49FyGcohJUc aDDv70ngNXtk55iwlNpNhTs+VcQor1fznhPbRiefHqJeRIOkpcrVE7NLP8TjwuaG YaRSMLl6IE9vDzhTyzMMEyuP1pq9KsgtsRx9S1HKR9FIJ3Jdh+vVReZIZZ2vUpC6 W6IYZVcSn2i51BVrlMRpIpj0M+Dt+VGOQVDJNE92kKz8OMHY4Xu54+OU4UZpyw4K UGsTuqwPN1q3ErWQgR5WrlcihtnJ0tHXUeOrO8ZV/R4O03QK0dqq6mm4lyiPSMQH +FJDOvTKVTUssKZqwJz58oHhEmrARdlns87/I6KJClTUFLkqqNfs+avNJVgyeY+Q W5g5xAgGwax/Dj0ApQIDAQABo1QwUjAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/ BAUwAwEB/zAdBgNVHQ4EFgQUCctZf4aycI8awznjwNnpv7tNsiMwEAYJKwYBBAGC NxUBBAMCAQAwDQYJKoZIhvcNAQEMBQADggIBAKyvPl3CEZaJjqPnktaXFbgToqZC LgLNFgVZJ8og6Lq46BrsTaiXVq5lQ7GPAJtSzVXNUzltYkyLDVt8LkS/gxCP81OC gMNPOsduET/m4xaRhPtthH80dK2Jp86519efhGSSvpWhrQlTM93uCupKUY5vVau6 tZRGrox/2KJQJWVggEbbMwSubLWYdFQl3JPk+ONVFT24bcMKpBLBaYVu32TxU5nh SnUgnZUP5NbcA/FZGOhHibJXWpS2qdgXKxdJ5XbLwVaZOjex/2kskZGT4d9Mozd2 TaGf+G0eHdP67Pv0RR0Tbc/3WeUiJ3IrhvNXuzDtJE3cfVa7o7P4NHmJweDyAmH3 pvwPuxwXC65B2Xy9J6P9LjrRk5Sxcx0ki69bIImtt2dmefU6xqaWM/5TkshGsRGR xpl/j8nWZjEgQRCHLQzWwa80mMpkg/sTV9HB8Dx6jKXB/ZUhoHHBk2dxEuqPiApp GWSZI1b7rCoucL5mxAyE7+WL85MB+GqQk2dLsmijtWKP6T+MejteD+eMuMZ87zf9 dOLITzNy4ZQ5bb0Sr74MTnB8G2+NszKTc0QWbej09+CVgI+WXTik9KveCjCHk9hN AHFiRSdLOkKEW39lt2c0Ui2cFmuqqNh7o0JMcccMyj6D5KbvtwEwXlGjefVwaaZB RA+GsCyRxj3qrg+E -----END CERTIFICATE----- # Issuer: C=US, O=SecureTrust Corporation, CN=Secure Global CA # Subject: C=US, O=SecureTrust Corporation, CN=Secure Global CA # SHA256 Fingerprint: 42:00:F5:04:3A:C8:59:0E:BB:52:7D:20:9E:D1:50:30:29:FB:CB:D4:1C:A1:B5:06:EC:27:F1:5A:DE:7D:AC:69 -----BEGIN CERTIFICATE----- MIIDvDCCAqSgAwIBAgIQB1YipOjUiolN9BPI8PjqpTANBgkqhkiG9w0BAQUFADBK MQswCQYDVQQGEwJVUzEgMB4GA1UEChMXU2VjdXJlVHJ1c3QgQ29ycG9yYXRpb24x GTAXBgNVBAMTEFNlY3VyZSBHbG9iYWwgQ0EwHhcNMDYxMTA3MTk0MjI4WhcNMjkx MjMxMTk1MjA2WjBKMQswCQYDVQQGEwJVUzEgMB4GA1UEChMXU2VjdXJlVHJ1c3Qg Q29ycG9yYXRpb24xGTAXBgNVBAMTEFNlY3VyZSBHbG9iYWwgQ0EwggEiMA0GCSqG SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvNS7YrGxVaQZx5RNoJLNP2MwhR/jxYDiJ iQPpvepeRlMJ3Fz1Wuj3RSoC6zFh1ykzTM7HfAo3fg+6MpjhHZevj8fcyTiW89sa /FHtaMbQbqR8JNGuQsiWUGMu4P51/pinX0kuleM5M2SOHqRfkNJnPLLZ/kG5VacJ jnIFHovdRIWCQtBJwB1g8NEXLJXr9qXBkqPFwqcIYA1gBBCWeZ4WNOaptvolRTnI HmX5k/Wq8VLcmZg9pYYaDDUz+kulBAYVHDGA76oYa8J719rO+TMg1fW9ajMtgQT7 sFzUnKPiXB3jqUJ1XnvUd+85VLrJChgbEplJL4hL/VBi0XPnj3pDAgMBAAGjgZ0w gZowEwYJKwYBBAGCNxQCBAYeBABDAEEwCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQF MAMBAf8wHQYDVR0OBBYEFK9EBMJBfkiD2045AuzshHrmzsmkMDQGA1UdHwQtMCsw KaAnoCWGI2h0dHA6Ly9jcmwuc2VjdXJldHJ1c3QuY29tL1NHQ0EuY3JsMBAGCSsG AQQBgjcVAQQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQBjGghAfaReUw132HquHw0L URYD7xh8yOOvaliTFGCRsoTciE6+OYo68+aCiV0BN7OrJKQVDpI1WkpEXk5X+nXO H0jOZvQ8QCaSmGwb7iRGDBezUqXbpZGRzzfTb+cnCDpOGR86p1hcF895P4vkp9Mm I50mD1hp/Ed+stCNi5O/KU9DaXR2Z0vPB4zmAve14bRDtUstFJ/53CYNv6ZHdAbY iNE6KTCEztI5gGIbqMdXSbxqVVFnFUq+NQfk1XWYN3kwFNspnWzFacxHVaIw98xc f8LDmBxrThaA63p4ZUWiABqvDA1VZDRIuJK58bRQKfJPIx/abKwfROHdI3hRW8cW -----END CERTIFICATE----- # Issuer: C=US, O=SecureTrust Corporation, CN=SecureTrust CA # Subject: C=US, O=SecureTrust Corporation, CN=SecureTrust CA # SHA256 Fingerprint: F1:C1:B5:0A:E5:A2:0D:D8:03:0E:C9:F6:BC:24:82:3D:D3:67:B5:25:57:59:B4:E7:1B:61:FC:E9:F7:37:5D:73 -----BEGIN CERTIFICATE----- MIIDuDCCAqCgAwIBAgIQDPCOXAgWpa1Cf/DrJxhZ0DANBgkqhkiG9w0BAQUFADBI MQswCQYDVQQGEwJVUzEgMB4GA1UEChMXU2VjdXJlVHJ1c3QgQ29ycG9yYXRpb24x FzAVBgNVBAMTDlNlY3VyZVRydXN0IENBMB4XDTA2MTEwNzE5MzExOFoXDTI5MTIz MTE5NDA1NVowSDELMAkGA1UEBhMCVVMxIDAeBgNVBAoTF1NlY3VyZVRydXN0IENv cnBvcmF0aW9uMRcwFQYDVQQDEw5TZWN1cmVUcnVzdCBDQTCCASIwDQYJKoZIhvcN AQEBBQADggEPADCCAQoCggEBAKukgeWVzfX2FI7CT8rU4niVWJxB4Q2ZQCQXOZEz Zum+4YOvYlyJ0fwkW2Gz4BERQRwdbvC4u/jep4G6pkjGnx29vo6pQT64lO0pGtSO 0gMdA+9tDWccV9cGrcrI9f4Or2YlSASWC12juhbDCE/RRvgUXPLIXgGZbf2IzIao wW8xQmxSPmjL8xk037uHGFaAJsTQ3MBv396gwpEWoGQRS0S8Hvbn+mPeZqx2pHGj 7DaUaHp3pLHnDi+BeuK1cobvomuL8A/b01k/unK8RCSc43Oz969XL0Imnal0ugBS 8kvNU3xHCzaFDmapCJcWNFfBZveA4+1wVMeT4C4oFVmHursCAwEAAaOBnTCBmjAT BgkrBgEEAYI3FAIEBh4EAEMAQTALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB /zAdBgNVHQ4EFgQUQjK2FvoE/f5dS3rD/fdMQB1aQ68wNAYDVR0fBC0wKzApoCeg JYYjaHR0cDovL2NybC5zZWN1cmV0cnVzdC5jb20vU1RDQS5jcmwwEAYJKwYBBAGC NxUBBAMCAQAwDQYJKoZIhvcNAQEFBQADggEBADDtT0rhWDpSclu1pqNlGKa7UTt3 6Z3q059c4EVlew3KW+JwULKUBRSuSceNQQcSc5R+DCMh/bwQf2AQWnL1mA6s7Ll/ 3XpvXdMc9P+IBWlCqQVxyLesJugutIxq/3HcuLHfmbx8IVQr5Fiiu1cprp6poxkm D5kuCLDv/WnPmRoJjeOnnyvJNjR7JLN4TJUXpAYmHrZkUjZfYGfZnMUFdAvnZyPS CPyI6a6Lf+Ew9Dd+/cYy2i2eRDAwbO4H3tI0/NL/QPZL9GZGBlSm8jIKYyYwa5vR 3ItHuuG51WLQoqD0ZwV4KWMabwTW+MZMo5qxN7SN5ShLHZ4swrhovO0C7jE= -----END CERTIFICATE----- # Issuer: C=US, O=Starfield Technologies, Inc., OU=Starfield Class 2 Certification Authority # Subject: C=US, O=Starfield Technologies, Inc., OU=Starfield Class 2 Certification Authority # SHA256 Fingerprint: 14:65:FA:20:53:97:B8:76:FA:A6:F0:A9:95:8E:55:90:E4:0F:CC:7F:AA:4F:B7:C2:C8:67:75:21:FB:5F:B6:58 -----BEGIN CERTIFICATE----- MIIEDzCCAvegAwIBAgIBADANBgkqhkiG9w0BAQUFADBoMQswCQYDVQQGEwJVUzEl MCMGA1UEChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMp U3RhcmZpZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQw NjI5MTczOTE2WhcNMzQwNjI5MTczOTE2WjBoMQswCQYDVQQGEwJVUzElMCMGA1UE ChMcU3RhcmZpZWxkIFRlY2hub2xvZ2llcywgSW5jLjEyMDAGA1UECxMpU3RhcmZp ZWxkIENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwggEgMA0GCSqGSIb3 DQEBAQUAA4IBDQAwggEIAoIBAQC3Msj+6XGmBIWtDBFk385N78gDGIc/oav7PKaf 8MOh2tTYbitTkPskpD6E8J7oX+zlJ0T1KKY/e97gKvDIr1MvnsoFAZMej2YcOadN +lq2cwQlZut3f+dZxkqZJRRU6ybH838Z1TBwj6+wRir/resp7defqgSHo9T5iaU0 X9tDkYI22WY8sbi5gv2cOj4QyDvvBmVmepsZGD3/cVE8MC5fvj13c7JdBmzDI1aa K4UmkhynArPkPw2vCHmCuDY96pzTNbO8acr1zJ3o/WSNF4Azbl5KXZnJHoe0nRrA 1W4TNSNe35tfPe/W93bC6j67eA0cQmdrBNj41tpvi/JEoAGrAgEDo4HFMIHCMB0G A1UdDgQWBBS/X7fRzt0fhvRbVazc1xDCDqmI5zCBkgYDVR0jBIGKMIGHgBS/X7fR zt0fhvRbVazc1xDCDqmI56FspGowaDELMAkGA1UEBhMCVVMxJTAjBgNVBAoTHFN0 YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAsTKVN0YXJmaWVsZCBD bGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8w DQYJKoZIhvcNAQEFBQADggEBAAWdP4id0ckaVaGsafPzWdqbAYcaT1epoXkJKtv3 L7IezMdeatiDh6GX70k1PncGQVhiv45YuApnP+yz3SFmH8lU+nLMPUxA2IGvd56D eruix/U0F47ZEUD0/CwqTRV/p2JdLiXTAAsgGh1o+Re49L2L7ShZ3U0WixeDyLJl xy16paq8U4Zt3VekyvggQQto8PT7dL5WXXp59fkdheMtlb71cZBDzI0fmgAKhynp VSJYACPq4xJDKVtHCN2MQWplBqjlIapBtJUhlbl90TSrE9atvNziPTnNvT51cKEY WQPJIrSPnNVeKtelttQKbfi3QBFGmh95DmK/D5fs4C8fF5Q= -----END CERTIFICATE----- # Issuer: C=US, O=The Go Daddy Group, Inc., OU=Go Daddy Class 2 Certification Authority # Subject: C=US, O=The Go Daddy Group, Inc., OU=Go Daddy Class 2 Certification Authority # SHA256 Fingerprint: C3:84:6B:F2:4B:9E:93:CA:64:27:4C:0E:C6:7C:1E:CC:5E:02:4F:FC:AC:D2:D7:40:19:35:0E:81:FE:54:6A:E4 -----BEGIN CERTIFICATE----- MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEh MB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBE YWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3 MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRo ZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3Mg MiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggEN ADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCA PVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6w wdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXi EqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMY avx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+ YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLE sNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h /t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5 IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmlj YXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQAD ggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNy OO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7P TMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQ HmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mER dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf ReYNnyicsbkqWletNw+vHX/bvZ8= -----END CERTIFICATE----- # Issuer: C=US, OU=emSign PKI, O=eMudhra Inc, CN=emSign ECC Root CA - C3 # Subject: C=US, OU=emSign PKI, O=eMudhra Inc, CN=emSign ECC Root CA - C3 # SHA256 Fingerprint: BC:4D:80:9B:15:18:9D:78:DB:3E:1D:8C:F4:F9:72:6A:79:5D:A1:64:3C:A5:F1:35:8E:1D:DB:0E:DC:0D:7E:B3 -----BEGIN CERTIFICATE----- MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c 3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J 0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ== -----END CERTIFICATE----- # Issuer: C=US, OU=emSign PKI, O=eMudhra Inc, CN=emSign Root CA - C1 # Subject: C=US, OU=emSign PKI, O=eMudhra Inc, CN=emSign Root CA - C1 # SHA256 Fingerprint: 12:56:09:AA:30:1D:A0:A2:49:B9:7A:82:39:CB:6A:34:21:6F:44:DC:AC:9F:39:54:B1:42:92:F2:E8:C8:60:8F -----BEGIN CERTIFICATE----- MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH 3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1 aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87 /kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4 kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT +xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo WXzhriKi4gp6D/piq1JM4fHfyr6DDUI= -----END CERTIFICATE----- # Issuer: C=US, OU=www.xrampsecurity.com, O=XRamp Security Services Inc, CN=XRamp Global Certification Authority # Subject: C=US, OU=www.xrampsecurity.com, O=XRamp Security Services Inc, CN=XRamp Global Certification Authority # SHA256 Fingerprint: CE:CD:DC:90:50:99:D8:DA:DF:C5:B1:D2:09:B7:37:CB:E2:C1:8C:FB:2C:10:C0:FF:0B:CF:0D:32:86:FC:1A:A2 -----BEGIN CERTIFICATE----- MIIEMDCCAxigAwIBAgIQUJRs7Bjq1ZxN1ZfvdY+grTANBgkqhkiG9w0BAQUFADCB gjELMAkGA1UEBhMCVVMxHjAcBgNVBAsTFXd3dy54cmFtcHNlY3VyaXR5LmNvbTEk MCIGA1UEChMbWFJhbXAgU2VjdXJpdHkgU2VydmljZXMgSW5jMS0wKwYDVQQDEyRY UmFtcCBHbG9iYWwgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDQxMTAxMTcx NDA0WhcNMzUwMTAxMDUzNzE5WjCBgjELMAkGA1UEBhMCVVMxHjAcBgNVBAsTFXd3 dy54cmFtcHNlY3VyaXR5LmNvbTEkMCIGA1UEChMbWFJhbXAgU2VjdXJpdHkgU2Vy dmljZXMgSW5jMS0wKwYDVQQDEyRYUmFtcCBHbG9iYWwgQ2VydGlmaWNhdGlvbiBB dXRob3JpdHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYJB69FbS6 38eMpSe2OAtp87ZOqCwuIR1cRN8hXX4jdP5efrRKt6atH67gBhbim1vZZ3RrXYCP KZ2GG9mcDZhtdhAoWORlsH9KmHmf4MMxfoArtYzAQDsRhtDLooY2YKTVMIJt2W7Q DxIEM5dfT2Fa8OT5kavnHTu86M/0ay00fOJIYRyO82FEzG+gSqmUsE3a56k0enI4 qEHMPJQRfevIpoy3hsvKMzvZPTeL+3o+hiznc9cKV6xkmxnr9A8ECIqsAxcZZPRa JSKNNCyy9mgdEm3Tih4U2sSPpuIjhdV6Db1q4Ons7Be7QhtnqiXtRYMh/MHJfNVi PvryxS3T/dRlAgMBAAGjgZ8wgZwwEwYJKwYBBAGCNxQCBAYeBABDAEEwCwYDVR0P BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFMZPoj0GY4QJnM5i5ASs jVy16bYbMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9jcmwueHJhbXBzZWN1cml0 eS5jb20vWEdDQS5jcmwwEAYJKwYBBAGCNxUBBAMCAQEwDQYJKoZIhvcNAQEFBQAD ggEBAJEVOQMBG2f7Shz5CmBbodpNl2L5JFMn14JkTpAuw0kbK5rc/Kh4ZzXxHfAR vbdI4xD2Dd8/0sm2qlWkSLoC295ZLhVbO50WfUfXN+pfTXYSNrsf16GBBEYgoyxt qZ4Bfj8pzgCT3/3JknOJiWSe5yvkHJEs0rnOfc5vMZnT5r7SHpDwCRR5XCOrTdLa IR9NmXmd4c8nnxCbHIgNsIpkQTG4DmyQJKSbXHGPurt+HBvbaoAPIbzp26a3QPSy i6mx5O+aGtA9aZnuqCij4Tyz8LIRnM98QObd50N9otg6tamN8jSZxNQQ4Qb9CYQQ O+7ETPTsJ3xCwnR8gooJybQDJbw= -----END CERTIFICATE----- # Issuer: C=US, ST=Arizona, L=Scottsdale, O=GoDaddy.com, Inc., CN=Go Daddy Root Certificate Authority - G2 # Subject: C=US, ST=Arizona, L=Scottsdale, O=GoDaddy.com, Inc., CN=Go Daddy Root Certificate Authority - G2 # SHA256 Fingerprint: 45:14:0B:32:47:EB:9C:C8:C5:B4:F0:D7:B5:30:91:F7:32:92:08:9E:6E:5A:63:E2:74:9D:D3:AC:A9:19:8E:DA -----BEGIN CERTIFICATE----- MIIDxTCCAq2gAwIBAgIBADANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMx EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoT EUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRp ZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAwMFoXDTM3MTIzMTIz NTk1OVowgYMxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQH EwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjExMC8GA1UE AxMoR28gRGFkZHkgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIw DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAL9xYgjx+lk09xvJGKP3gElY6SKD E6bFIEMBO4Tx5oVJnyfq9oQbTqC023CYxzIBsQU+B07u9PpPL1kwIuerGVZr4oAH /PMWdYA5UXvl+TW2dE6pjYIT5LY/qQOD+qK+ihVqf94Lw7YZFAXK6sOoBJQ7Rnwy DfMAZiLIjWltNowRGLfTshxgtDj6AozO091GB94KPutdfMh8+7ArU6SSYmlRJQVh GkSBjCypQ5Yj36w6gZoOKcUcqeldHraenjAKOc7xiID7S13MMuyFYkMlNAJWJwGR tDtwKj9useiciAF9n9T521NtYJ2/LOdYq7hfRvzOxBsDPAnrSTFcaUaz4EcCAwEA AaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE FDqahQcQZyi27/a9BUFuIMGU2g/eMA0GCSqGSIb3DQEBCwUAA4IBAQCZ21151fmX WWcDYfF+OwYxdS2hII5PZYe096acvNjpL9DbWu7PdIxztDhC2gV7+AJ1uP2lsdeu 9tfeE8tTEH6KRtGX+rcuKxGrkLAngPnon1rpN5+r5N9ss4UXnT3ZJE95kTXWXwTr gIOrmgIttRD02JDHBHNA7XIloKmf7J6raBKZV8aPEjoJpL1E/QYVN8Gb5DKj7Tjo 2GTzLH4U/ALqn83/B2gX2yKQOC16jdFU8WnjXzPKej17CuPKf1855eJ1usV2GDPO LPAvTK33sefOT6jEm0pUBsV/fdUID+Ic/n4XuKxe9tQWskMJDE32p2u0mYRlynqI 4uJEvlz36hz1 -----END CERTIFICATE----- # Issuer: C=US, ST=Arizona, L=Scottsdale, O=Starfield Technologies, Inc., CN=Starfield Root Certificate Authority - G2 # Subject: C=US, ST=Arizona, L=Scottsdale, O=Starfield Technologies, Inc., CN=Starfield Root Certificate Authority - G2 # SHA256 Fingerprint: 2C:E1:CB:0B:F9:D2:F9:E1:02:99:3F:BE:21:51:52:C3:B2:DD:0C:AB:DE:1C:68:E5:31:9B:83:91:54:DB:B7:F5 -----BEGIN CERTIFICATE----- MIID3TCCAsWgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBjzELMAkGA1UEBhMCVVMx EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xMjAwBgNVBAMTKVN0YXJmaWVs ZCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5MDkwMTAwMDAw MFoXDTM3MTIzMTIzNTk1OVowgY8xCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6 b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFyZmllbGQgVGVj aG5vbG9naWVzLCBJbmMuMTIwMAYDVQQDEylTdGFyZmllbGQgUm9vdCBDZXJ0aWZp Y2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC ggEBAL3twQP89o/8ArFvW59I2Z154qK3A2FWGMNHttfKPTUuiUP3oWmb3ooa/RMg nLRJdzIpVv257IzdIvpy3Cdhl+72WoTsbhm5iSzchFvVdPtrX8WJpRBSiUZV9Lh1 HOZ/5FSuS/hVclcCGfgXcVnrHigHdMWdSL5stPSksPNkN3mSwOxGXn/hbVNMYq/N Hwtjuzqd+/x5AJhhdM8mgkBj87JyahkNmcrUDnXMN/uLicFZ8WJ/X7NfZTD4p7dN dloedl40wOiWVpmKs/B/pM293DIxfJHP4F8R+GuqSVzRmZTRouNjWwl2tVZi4Ut0 HZbUJtQIBFnQmA4O5t78w+wfkPECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAO BgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFHwMMh+n2TB/xH1oo2Kooc6rB1snMA0G CSqGSIb3DQEBCwUAA4IBAQARWfolTwNvlJk7mh+ChTnUdgWUXuEok21iXQnCoKjU sHU48TRqneSfioYmUeYs0cYtbpUgSpIB7LiKZ3sx4mcujJUDJi5DnUox9g61DLu3 4jd/IroAow57UvtruzvE03lRTs2Q9GcHGcg8RnoNAX3FWOdt5oUwF5okxBDgBPfg 8n/Uqgr/Qh037ZTlZFkSIHc40zI+OIF1lnP6aI+xy84fxez6nH7PfrHxBy22/L/K pL/QlwVKvOoYKAKQvVR4CSFx09F9HdkWsKlhPdAKACL8x3vLCWRFCztAgfd9fDL1 mMpYjn0q7pBZc2T5NnReJaH1ZgUufzkVqSr7UIuOhWn0 -----END CERTIFICATE----- # Issuer: C=US, ST=Arizona, L=Scottsdale, O=Starfield Technologies, Inc., CN=Starfield Services Root Certificate Authority - G2 # Subject: C=US, ST=Arizona, L=Scottsdale, O=Starfield Technologies, Inc., CN=Starfield Services Root Certificate Authority - G2 # SHA256 Fingerprint: 56:8D:69:05:A2:C8:87:08:A4:B3:02:51:90:ED:CF:ED:B1:97:4A:60:6A:13:C6:E5:29:0F:CB:2A:E6:3E:DA:B5 -----BEGIN CERTIFICATE----- MIID7zCCAtegAwIBAgIBADANBgkqhkiG9w0BAQsFADCBmDELMAkGA1UEBhMCVVMx EDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxJTAjBgNVBAoT HFN0YXJmaWVsZCBUZWNobm9sb2dpZXMsIEluYy4xOzA5BgNVBAMTMlN0YXJmaWVs ZCBTZXJ2aWNlcyBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTA5 MDkwMTAwMDAwMFoXDTM3MTIzMTIzNTk1OVowgZgxCzAJBgNVBAYTAlVTMRAwDgYD VQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMSUwIwYDVQQKExxTdGFy ZmllbGQgVGVjaG5vbG9naWVzLCBJbmMuMTswOQYDVQQDEzJTdGFyZmllbGQgU2Vy dmljZXMgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgLSBHMjCCASIwDQYJKoZI hvcNAQEBBQADggEPADCCAQoCggEBANUMOsQq+U7i9b4Zl1+OiFOxHz/Lz58gE20p OsgPfTz3a3Y4Y9k2YKibXlwAgLIvWX/2h/klQ4bnaRtSmpDhcePYLQ1Ob/bISdm2 8xpWriu2dBTrz/sm4xq6HZYuajtYlIlHVv8loJNwU4PahHQUw2eeBGg6345AWh1K Ts9DkTvnVtYAcMtS7nt9rjrnvDH5RfbCYM8TWQIrgMw0R9+53pBlbQLPLJGmpufe hRhJfGZOozptqbXuNC66DQO4M99H67FrjSXZm86B0UVGMpZwh94CDklDhbZsc7tk 6mFBrMnUVN+HL8cisibMn1lUaJ/8viovxFUcdUBgF4UCVTmLfwUCAwEAAaNCMEAw DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFJxfAN+q AdcwKziIorhtSpzyEZGDMA0GCSqGSIb3DQEBCwUAA4IBAQBLNqaEd2ndOxmfZyMI bw5hyf2E3F/YNoHN2BtBLZ9g3ccaaNnRbobhiCPPE95Dz+I0swSdHynVv/heyNXB ve6SbzJ08pGCL72CQnqtKrcgfU28elUSwhXqvfdqlS5sdJ/PHLTyxQGjhdByPq1z qwubdQxtRbeOlKyWN7Wg0I8VRw7j6IPdj/3vQQF3zCepYoUz8jcI73HPdwbeyBkd iEDPfUYd/x7H4c7/I9vG+o1VTqkC50cRRj70/b17KSa7qWFiNyi2LSr2EIZkyXCn 0q23KXB56jzaYyWf/Wi3MOxw+3WKt21gZ7IeyLnp2KhvAotnDU0mV3HaIPzBSlCN sSi6 -----END CERTIFICATE----- # Issuer: C=US, ST=Illinois, L=Chicago, O=Trustwave Holdings, Inc., CN=Trustwave Global Certification Authority # Subject: C=US, ST=Illinois, L=Chicago, O=Trustwave Holdings, Inc., CN=Trustwave Global Certification Authority # SHA256 Fingerprint: 97:55:20:15:F5:DD:FC:3C:87:88:C0:06:94:45:55:40:88:94:45:00:84:F1:00:86:70:86:BC:1A:2B:B5:8D:C8 -----BEGIN CERTIFICATE----- MIIF2jCCA8KgAwIBAgIMBfcOhtpJ80Y1LrqyMA0GCSqGSIb3DQEBCwUAMIGIMQsw CQYDVQQGEwJVUzERMA8GA1UECAwISWxsaW5vaXMxEDAOBgNVBAcMB0NoaWNhZ28x ITAfBgNVBAoMGFRydXN0d2F2ZSBIb2xkaW5ncywgSW5jLjExMC8GA1UEAwwoVHJ1 c3R3YXZlIEdsb2JhbCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0xNzA4MjMx OTM0MTJaFw00MjA4MjMxOTM0MTJaMIGIMQswCQYDVQQGEwJVUzERMA8GA1UECAwI SWxsaW5vaXMxEDAOBgNVBAcMB0NoaWNhZ28xITAfBgNVBAoMGFRydXN0d2F2ZSBI b2xkaW5ncywgSW5jLjExMC8GA1UEAwwoVHJ1c3R3YXZlIEdsb2JhbCBDZXJ0aWZp Y2F0aW9uIEF1dGhvcml0eTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB ALldUShLPDeS0YLOvR29zd24q88KPuFd5dyqCblXAj7mY2Hf8g+CY66j96xz0Xzn swuvCAAJWX/NKSqIk4cXGIDtiLK0thAfLdZfVaITXdHG6wZWiYj+rDKd/VzDBcdu 7oaJuogDnXIhhpCujwOl3J+IKMujkkkP7NAP4m1ET4BqstTnoApTAbqOl5F2brz8 1Ws25kCI1nsvXwXoLG0R8+eyvpJETNKXpP7ScoFDB5zpET71ixpZfR9oWN0EACyW 80OzfpgZdNmcc9kYvkHHNHnZ9GLCQ7mzJ7Aiy/k9UscwR7PJPrhq4ufogXBeQotP JqX+OsIgbrv4Fo7NDKm0G2x2EOFYeUY+VM6AqFcJNykbmROPDMjWLBz7BegIlT1l RtzuzWniTY+HKE40Cz7PFNm73bZQmq131BnW2hqIyE4bJ3XYsgjxroMwuREOzYfw hI0Vcnyh78zyiGG69Gm7DIwLdVcEuE4qFC49DxweMqZiNu5m4iK4BUBjECLzMx10 coos9TkpoNPnG4CELcU9402x/RpvumUHO1jsQkUm+9jaJXLE9gCxInm943xZYkqc BW89zubWR2OZxiRvchLIrH+QtAuRcOi35hYQcRfO3gZPSEF9NUqjifLJS3tBEW1n twiYTOURGa5CgNz7kAXU+FDKvuStx8KU1xad5hePrzb7AgMBAAGjQjBAMA8GA1Ud EwEB/wQFMAMBAf8wHQYDVR0OBBYEFJngGWcNYtt2s9o9uFvo/ULSMQ6HMA4GA1Ud DwEB/wQEAwIBBjANBgkqhkiG9w0BAQsFAAOCAgEAmHNw4rDT7TnsTGDZqRKGFx6W 0OhUKDtkLSGm+J1WE2pIPU/HPinbbViDVD2HfSMF1OQc3Og4ZYbFdada2zUFvXfe uyk3QAUHw5RSn8pk3fEbK9xGChACMf1KaA0HZJDmHvUqoai7PF35owgLEQzxPy0Q lG/+4jSHg9bP5Rs1bdID4bANqKCqRieCNqcVtgimQlRXtpla4gt5kNdXElE1GYhB aCXUNxeEFfsBctyV3lImIJgm4nb1J2/6ADtKYdkNy1GTKv0WBpanI5ojSP5RvbbE sLFUzt5sQa0WZ37b/TjNuThOssFgy50X31ieemKyJo90lZvkWx3SD92YHJtZuSPT MaCm/zjdzyBP6VhWOmfD0faZmZ26NraAL4hHT4a/RDqA5Dccprrql5gR0IRiR2Qe qu5AvzSxnI9O4fKSTx+O856X3vOmeWqJcU9LJxdI/uz0UA9PSX3MReO9ekDFQdxh VicGaeVyQYHTtgGJoC86cnn+OjC/QezHYj6RS8fZMXZC+fc8Y+wmjHMMfRod6qh8 h6jCJ3zhM0EPz8/8AKAigJ5Kp28AsEFFtyLKaEjFQqKu3R3y4G5OBVixwJAWKqQ9 EEC+j2Jjg6mcgn0tAumDMHzLJ8n9HmYAsC7TIS+OMxZsmO0QqAfWzJPP29FpHOTK yeC2nOnOcXHebD8WpHk= -----END CERTIFICATE----- # Issuer: C=US, ST=Illinois, L=Chicago, O=Trustwave Holdings, Inc., CN=Trustwave Global ECC P256 Certification Authority # Subject: C=US, ST=Illinois, L=Chicago, O=Trustwave Holdings, Inc., CN=Trustwave Global ECC P256 Certification Authority # SHA256 Fingerprint: 94:5B:BC:82:5E:A5:54:F4:89:D1:FD:51:A7:3D:DF:2E:A6:24:AC:70:19:A0:52:05:22:5C:22:A7:8C:CF:A8:B4 -----BEGIN CERTIFICATE----- MIICYDCCAgegAwIBAgIMDWpfCD8oXD5Rld9dMAoGCCqGSM49BAMCMIGRMQswCQYD VQQGEwJVUzERMA8GA1UECBMISWxsaW5vaXMxEDAOBgNVBAcTB0NoaWNhZ28xITAf BgNVBAoTGFRydXN0d2F2ZSBIb2xkaW5ncywgSW5jLjE6MDgGA1UEAxMxVHJ1c3R3 YXZlIEdsb2JhbCBFQ0MgUDI1NiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0x NzA4MjMxOTM1MTBaFw00MjA4MjMxOTM1MTBaMIGRMQswCQYDVQQGEwJVUzERMA8G A1UECBMISWxsaW5vaXMxEDAOBgNVBAcTB0NoaWNhZ28xITAfBgNVBAoTGFRydXN0 d2F2ZSBIb2xkaW5ncywgSW5jLjE6MDgGA1UEAxMxVHJ1c3R3YXZlIEdsb2JhbCBF Q0MgUDI1NiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTBZMBMGByqGSM49AgEGCCqG SM49AwEHA0IABH77bOYj43MyCMpg5lOcunSNGLB4kFKA3TjASh3RqMyTpJcGOMoN FWLGjgEqZZ2q3zSRLoHB5DOSMcT9CTqmP62jQzBBMA8GA1UdEwEB/wQFMAMBAf8w DwYDVR0PAQH/BAUDAwcGADAdBgNVHQ4EFgQUo0EGrJBt0UrrdaVKEJmzsaGLSvcw CgYIKoZIzj0EAwIDRwAwRAIgB+ZU2g6gWrKuEZ+Hxbb/ad4lvvigtwjzRM4q3wgh DDcCIC0mA6AFvWvR9lz4ZcyGbbOcNEhjhAnFjXca4syc4XR7 -----END CERTIFICATE----- # Issuer: C=US, ST=Illinois, L=Chicago, O=Trustwave Holdings, Inc., CN=Trustwave Global ECC P384 Certification Authority # Subject: C=US, ST=Illinois, L=Chicago, O=Trustwave Holdings, Inc., CN=Trustwave Global ECC P384 Certification Authority # SHA256 Fingerprint: 55:90:38:59:C8:C0:C3:EB:B8:75:9E:CE:4E:25:57:22:5F:F5:75:8B:BD:38:EB:D4:82:76:60:1E:1B:D5:80:97 -----BEGIN CERTIFICATE----- MIICnTCCAiSgAwIBAgIMCL2Fl2yZJ6SAaEc7MAoGCCqGSM49BAMDMIGRMQswCQYD VQQGEwJVUzERMA8GA1UECBMISWxsaW5vaXMxEDAOBgNVBAcTB0NoaWNhZ28xITAf BgNVBAoTGFRydXN0d2F2ZSBIb2xkaW5ncywgSW5jLjE6MDgGA1UEAxMxVHJ1c3R3 YXZlIEdsb2JhbCBFQ0MgUDM4NCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0x NzA4MjMxOTM2NDNaFw00MjA4MjMxOTM2NDNaMIGRMQswCQYDVQQGEwJVUzERMA8G A1UECBMISWxsaW5vaXMxEDAOBgNVBAcTB0NoaWNhZ28xITAfBgNVBAoTGFRydXN0 d2F2ZSBIb2xkaW5ncywgSW5jLjE6MDgGA1UEAxMxVHJ1c3R3YXZlIEdsb2JhbCBF Q0MgUDM4NCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTB2MBAGByqGSM49AgEGBSuB BAAiA2IABGvaDXU1CDFHBa5FmVXxERMuSvgQMSOjfoPTfygIOiYaOs+Xgh+AtycJ j9GOMMQKmw6sWASr9zZ9lCOkmwqKi6vr/TklZvFe/oyujUF5nQlgziip04pt89ZF 1PKYhDhloKNDMEEwDwYDVR0TAQH/BAUwAwEB/zAPBgNVHQ8BAf8EBQMDBwYAMB0G A1UdDgQWBBRVqYSJ0sEyvRjLbKYHTsjnnb6CkDAKBggqhkjOPQQDAwNnADBkAjA3 AZKXRRJ+oPM+rRk6ct30UJMDEr5E0k9BpIycnR+j9sKS50gU/k6bpZFXrsY3crsC MGclCrEMXu6pY5Jv5ZAL/mYiykf9ijH3g/56vxC+GCsej/YpHpRZ744hN8tRmKVu Sw== -----END CERTIFICATE----- # Issuer: C=US, ST=New Jersey, L=Jersey City, O=The USERTRUST Network, CN=USERTrust ECC Certification Authority # Subject: C=US, ST=New Jersey, L=Jersey City, O=The USERTRUST Network, CN=USERTrust ECC Certification Authority # SHA256 Fingerprint: 4F:F4:60:D5:4B:9C:86:DA:BF:BC:FC:57:12:E0:40:0D:2B:ED:3F:BC:4D:4F:BD:AA:86:E0:6A:DC:D2:A9:AD:7A -----BEGIN CERTIFICATE----- MIICjzCCAhWgAwIBAgIQXIuZxVqUxdJxVt7NiYDMJjAKBggqhkjOPQQDAzCBiDEL MAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNl eSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMT JVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAwMjAx MDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNVBAgT Ck5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVUaGUg VVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBFQ0MgQ2VydGlm aWNhdGlvbiBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQarFRaqflo I+d61SRvU8Za2EurxtW20eZzca7dnNYMYf3boIkDuAUU7FfO7l0/4iGzzvfUinng o4N+LZfQYcTxmdwlkWOrfzCjtHDix6EznPO/LlxTsV+zfTJ/ijTjeXmjQjBAMB0G A1UdDgQWBBQ64QmG1M8ZwpZ2dEl23OA1xmNjmjAOBgNVHQ8BAf8EBAMCAQYwDwYD VR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjA2Z6EWCNzklwBBHU6+4WMB zzuqQhFkoJ2UOQIReVx7Hfpkue4WQrO/isIJxOzksU0CMQDpKmFHjFJKS04YcPbW RNZu9YO6bVi9JNlWSOrvxKJGgYhqOkbRqZtNyWHa0V1Xahg= -----END CERTIFICATE----- # Issuer: C=US, ST=New Jersey, L=Jersey City, O=The USERTRUST Network, CN=USERTrust RSA Certification Authority # Subject: C=US, ST=New Jersey, L=Jersey City, O=The USERTRUST Network, CN=USERTrust RSA Certification Authority # SHA256 Fingerprint: E7:93:C9:B0:2F:D8:AA:13:E2:1C:31:22:8A:CC:B0:81:19:64:3B:74:9C:89:89:64:B1:74:6D:46:C3:D4:CB:D2 -----BEGIN CERTIFICATE----- MIIF3jCCA8agAwIBAgIQAf1tMPyjylGoG7xkDjUDLTANBgkqhkiG9w0BAQwFADCB iDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0pl cnNleSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNV BAMTJVVTRVJUcnVzdCBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTAw MjAxMDAwMDAwWhcNMzgwMTE4MjM1OTU5WjCBiDELMAkGA1UEBhMCVVMxEzARBgNV BAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNleSBDaXR5MR4wHAYDVQQKExVU aGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMTJVVTRVJUcnVzdCBSU0EgQ2Vy dGlmaWNhdGlvbiBBdXRob3JpdHkwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK AoICAQCAEmUXNg7D2wiz0KxXDXbtzSfTTK1Qg2HiqiBNCS1kCdzOiZ/MPans9s/B 3PHTsdZ7NygRK0faOca8Ohm0X6a9fZ2jY0K2dvKpOyuR+OJv0OwWIJAJPuLodMkY tJHUYmTbf6MG8YgYapAiPLz+E/CHFHv25B+O1ORRxhFnRghRy4YUVD+8M/5+bJz/ Fp0YvVGONaanZshyZ9shZrHUm3gDwFA66Mzw3LyeTP6vBZY1H1dat//O+T23LLb2 VN3I5xI6Ta5MirdcmrS3ID3KfyI0rn47aGYBROcBTkZTmzNg95S+UzeQc0PzMsNT 79uq/nROacdrjGCT3sTHDN/hMq7MkztReJVni+49Vv4M0GkPGw/zJSZrM233bkf6 c0Plfg6lZrEpfDKEY1WJxA3Bk1QwGROs0303p+tdOmw1XNtB1xLaqUkL39iAigmT Yo61Zs8liM2EuLE/pDkP2QKe6xJMlXzzawWpXhaDzLhn4ugTncxbgtNMs+1b/97l c6wjOy0AvzVVdAlJ2ElYGn+SNuZRkg7zJn0cTRe8yexDJtC/QV9AqURE9JnnV4ee UB9XVKg+/XRjL7FQZQnmWEIuQxpMtPAlR1n6BB6T1CZGSlCBst6+eLf8ZxXhyVeE Hg9j1uliutZfVS7qXMYoCAQlObgOK6nyTJccBz8NUvXt7y+CDwIDAQABo0IwQDAd BgNVHQ4EFgQUU3m/WqorSs9UgOHYm8Cd8rIDZsswDgYDVR0PAQH/BAQDAgEGMA8G A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAFzUfA3P9wF9QZllDHPF Up/L+M+ZBn8b2kMVn54CVVeWFPFSPCeHlCjtHzoBN6J2/FNQwISbxmtOuowhT6KO VWKR82kV2LyI48SqC/3vqOlLVSoGIG1VeCkZ7l8wXEskEVX/JJpuXior7gtNn3/3 ATiUFJVDBwn7YKnuHKsSjKCaXqeYalltiz8I+8jRRa8YFWSQEg9zKC7F4iRO/Fjs 8PRF/iKz6y+O0tlFYQXBl2+odnKPi4w2r78NBc5xjeambx9spnFixdjQg3IM8WcR iQycE0xyNN+81XHfqnHd4blsjDwSXWXavVcStkNr/+XeTWYRUc+ZruwXtuhxkYze Sf7dNXGiFSeUHM9h4ya7b6NnJSFd5t0dCy5oGzuCr+yDZ4XUmFF0sbmZgIn/f3gZ XHlKYC6SQK5MNyosycdiyA5d9zZbyuAlJQG03RoHnHcAP9Dc1ew91Pq7P8yF1m9/ qS3fuQL39ZeatTXaw2ewh0qpKJ4jjv9cJ2vhsE/zB+4ALtRZh8tSQZXq9EfX7mRB VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG jjxDah2nGN59PRbxYvnKkKj9 -----END CERTIFICATE----- # Issuer: C=US, ST=Texas, L=Houston, O=SSL Corporation, CN=SSL.com EV Root Certification Authority ECC # Subject: C=US, ST=Texas, L=Houston, O=SSL Corporation, CN=SSL.com EV Root Certification Authority ECC # SHA256 Fingerprint: 22:A2:C1:F7:BD:ED:70:4C:C1:E7:01:B5:F4:08:C3:10:88:0F:E9:56:B5:DE:2A:4A:44:F9:9C:87:3A:25:A7:C8 -----BEGIN CERTIFICATE----- MIIClDCCAhqgAwIBAgIILCmcWxbtBZUwCgYIKoZIzj0EAwIwfzELMAkGA1UEBhMC VVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQKDA9T U0wgQ29ycG9yYXRpb24xNDAyBgNVBAMMK1NTTC5jb20gRVYgUm9vdCBDZXJ0aWZp Y2F0aW9uIEF1dGhvcml0eSBFQ0MwHhcNMTYwMjEyMTgxNTIzWhcNNDEwMjEyMTgx NTIzWjB/MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hv dXN0b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjE0MDIGA1UEAwwrU1NMLmNv bSBFViBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IEVDQzB2MBAGByqGSM49 AgEGBSuBBAAiA2IABKoSR5CYG/vvw0AHgyBO8TCCogbR8pKGYfL2IWjKAMTH6kMA VIbc/R/fALhBYlzccBYy3h+Z1MzFB8gIH2EWB1E9fVwHU+M1OIzfzZ/ZLg1Kthku WnBaBu2+8KGwytAJKaNjMGEwHQYDVR0OBBYEFFvKXuXe0oGqzagtZFG22XKbl+ZP MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX 5k8wDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMCA2gAMGUCMQCK5kCJN+vp1RPZ ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== -----END CERTIFICATE----- # Issuer: C=US, ST=Texas, L=Houston, O=SSL Corporation, CN=SSL.com EV Root Certification Authority RSA R2 # Subject: C=US, ST=Texas, L=Houston, O=SSL Corporation, CN=SSL.com EV Root Certification Authority RSA R2 # SHA256 Fingerprint: 2E:7B:F1:6C:C2:24:85:A7:BB:E2:AA:86:96:75:07:61:B0:AE:39:BE:3B:2F:E9:D0:CC:6D:4E:F7:34:91:42:5C -----BEGIN CERTIFICATE----- MIIF6zCCA9OgAwIBAgIIVrYpzTS8ePYwDQYJKoZIhvcNAQELBQAwgYIxCzAJBgNV BAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEQMA4GA1UEBwwHSG91c3RvbjEYMBYGA1UE CgwPU1NMIENvcnBvcmF0aW9uMTcwNQYDVQQDDC5TU0wuY29tIEVWIFJvb3QgQ2Vy dGlmaWNhdGlvbiBBdXRob3JpdHkgUlNBIFIyMB4XDTE3MDUzMTE4MTQzN1oXDTQy MDUzMDE4MTQzN1owgYIxCzAJBgNVBAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEQMA4G A1UEBwwHSG91c3RvbjEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMTcwNQYDVQQD DC5TU0wuY29tIEVWIFJvb3QgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgUlNBIFIy MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAjzZlQOHWTcDXtOlG2mvq M0fNTPl9fb69LT3w23jhhqXZuglXaO1XPqDQCEGD5yhBJB/jchXQARr7XnAjssuf OePPxU7Gkm0mxnu7s9onnQqG6YE3Bf7wcXHswxzpY6IXFJ3vG2fThVUCAtZJycxa 4bH3bzKfydQ7iEGonL3Lq9ttewkfokxykNorCPzPPFTOZw+oz12WGQvE43LrrdF9 HSfvkusQv1vrO6/PgN3B0pYEW3p+pKk8OHakYo6gOV7qd89dAFmPZiw+B6KjBSYR aZfqhbcPlgtLyEDhULouisv3D5oi53+aNxPN8k0TayHRwMwi8qFG9kRpnMphNQcA b9ZhCBHqurj26bNg5U257J8UZslXWNvNh2n4ioYSA0e/ZhN2rHd9NCSFg83XqpyQ Gp8hLH94t2S42Oim9HizVcuE0jLEeK6jj2HdzghTreyI/BXkmg3mnxp3zkyPuBQV PWKchjgGAGYS5Fl2WlPAApiiECtoRHuOec4zSnaqW4EWG7WK2NAAe15itAnWhmMO pgWVSbooi4iTsjQc2KRVbrcc0N6ZVTsj9CLg+SlmJuwgUHfbSguPvuUCYHBBXtSu UDkiFCbLsjtzdFVHB3mBOagwE0TlBIqulhMlQg+5U8Sb/M3kHN48+qvWBkofZ6aY MBzdLNvcGJVXZsb/XItW9XcCAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAfBgNV HSMEGDAWgBT5YLvU49U09rj1BoAlp3PbRmmonjAdBgNVHQ4EFgQU+WC71OPVNPa4 9QaAJadz20ZpqJ4wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQBW s47LCp1Jjr+kxJG7ZhcFUZh1++VQLHqe8RT6q9OKPv+RKY9ji9i0qVQBDb6Thi/5 Sm3HXvVX+cpVHBK+Rw82xd9qt9t1wkclf7nxY/hoLVUE0fKNsKTPvDxeH3jnpaAg cLAExbf3cqfeIg29MyVGjGSSJuM+LmOW2puMPfgYCdcDzH2GguDKBAdRUNf/ktUM 79qGn5nX67evaOI5JpS6aLe/g9Pqemc9YmeuJeVy6OLk7K4S9ksrPJ/psEDzOFSz /bdoyNrGj1E8svuR3Bznm53htw1yj+KkxKl4+esUrMZDBcJlOSgYAsOCsp0FvmXt ll9ldDz7CTUue5wT/RsPXcdtgTpWD8w74a8CLyKsRspGPKAcTNZEtF4uXBVmCeEm Kf7GUmG6sXP/wwyc5WxqlD8UykAWlYTzWamsX0xhk23RO8yilQwipmdnRC652dKK QbNmC1r7fSOl8hqw/96bg5Qu0T/fkreRrwU7ZcegbLHNYhLDkBvjJc40vG93drEQ w/cFGsDWr3RiSBd3kmmQYRzelYB0VI8YHMPzA9C/pEN1hlMYegouCRw2n5H9gooi S9EOUCXdywMMF8mDAAhONU2Ki+3wApRmLER/y5UnlhetCTCstnEXbosX9hwJ1C07 mKVx01QT2WDz9UtmT/rx7iASjbSsV7FFY6GsdqnC+w== -----END CERTIFICATE----- # Issuer: C=US, ST=Texas, L=Houston, O=SSL Corporation, CN=SSL.com Root Certification Authority ECC # Subject: C=US, ST=Texas, L=Houston, O=SSL Corporation, CN=SSL.com Root Certification Authority ECC # SHA256 Fingerprint: 34:17:BB:06:CC:60:07:DA:1B:96:1C:92:0B:8A:B4:CE:3F:AD:82:0E:4A:A3:0B:9A:CB:C4:A7:4E:BD:CE:BC:65 -----BEGIN CERTIFICATE----- MIICjTCCAhSgAwIBAgIIdebfy8FoW6gwCgYIKoZIzj0EAwIwfDELMAkGA1UEBhMC VVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQKDA9T U0wgQ29ycG9yYXRpb24xMTAvBgNVBAMMKFNTTC5jb20gUm9vdCBDZXJ0aWZpY2F0 aW9uIEF1dGhvcml0eSBFQ0MwHhcNMTYwMjEyMTgxNDAzWhcNNDEwMjEyMTgxNDAz WjB8MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hvdXN0 b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjExMC8GA1UEAwwoU1NMLmNvbSBS b290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IEVDQzB2MBAGByqGSM49AgEGBSuB BAAiA2IABEVuqVDEpiM2nl8ojRfLliJkP9x6jh3MCLOicSS6jkm5BBtHllirLZXI 7Z4INcgn64mMU1jrYor+8FsPazFSY0E7ic3s7LaNGdM0B9y7xgZ/wkWV7Mt/qCPg CemB+vNH06NjMGEwHQYDVR0OBBYEFILRhXMw5zUE044CkvvlpNHEIejNMA8GA1Ud EwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUgtGFczDnNQTTjgKS++Wk0cQh6M0wDgYD VR0PAQH/BAQDAgGGMAoGCCqGSM49BAMCA2cAMGQCMG/n61kRpGDPYbCWe+0F+S8T kdzt5fxQaxFGRrMcIQBiu77D5+jNB5n5DQtdcj7EqgIwH7y6C+IwJPt8bYBVCpk+ gA0z5Wajs6O7pdWLjwkspl1+4vAHCGht0nxpbl/f5Wpl -----END CERTIFICATE----- # Issuer: C=US, ST=Texas, L=Houston, O=SSL Corporation, CN=SSL.com Root Certification Authority RSA # Subject: C=US, ST=Texas, L=Houston, O=SSL Corporation, CN=SSL.com Root Certification Authority RSA # SHA256 Fingerprint: 85:66:6A:56:2E:E0:BE:5C:E9:25:C1:D8:89:0A:6F:76:A8:7E:C1:6D:4D:7D:5F:29:EA:74:19:CF:20:12:3B:69 -----BEGIN CERTIFICATE----- MIIF3TCCA8WgAwIBAgIIeyyb0xaAMpkwDQYJKoZIhvcNAQELBQAwfDELMAkGA1UE BhMCVVMxDjAMBgNVBAgMBVRleGFzMRAwDgYDVQQHDAdIb3VzdG9uMRgwFgYDVQQK DA9TU0wgQ29ycG9yYXRpb24xMTAvBgNVBAMMKFNTTC5jb20gUm9vdCBDZXJ0aWZp Y2F0aW9uIEF1dGhvcml0eSBSU0EwHhcNMTYwMjEyMTczOTM5WhcNNDEwMjEyMTcz OTM5WjB8MQswCQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxEDAOBgNVBAcMB0hv dXN0b24xGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjExMC8GA1UEAwwoU1NMLmNv bSBSb290IENlcnRpZmljYXRpb24gQXV0aG9yaXR5IFJTQTCCAiIwDQYJKoZIhvcN AQEBBQADggIPADCCAgoCggIBAPkP3aMrfcvQKv7sZ4Wm5y4bunfh4/WvpOz6Sl2R xFdHaxh3a3by/ZPkPQ/CFp4LZsNWlJ4Xg4XOVu/yFv0AYvUiCVToZRdOQbngT0aX qhvIuG5iXmmxX9sqAn78bMrzQdjt0Oj8P2FI7bADFB0QDksZ4LtO7IZl/zbzXmcC C52GVWH9ejjt/uIZALdvoVBidXQ8oPrIJZK0bnoix/geoeOy3ZExqysdBP+lSgQ3 6YWkMyv94tZVNHwZpEpox7Ko07fKoZOI68GXvIz5HdkihCR0xwQ9aqkpk8zruFvh /l8lqjRYyMEjVJ0bmBHDOJx+PYZspQ9AhnwC9FwCTyjLrnGfDzrIM/4RJTXq/LrF YD3ZfBjVsqnTdXgDciLKOsMf7yzlLqn6niy2UUb9rwPW6mBo6oUWNmuF6R7As93E JNyAKoFBbZQ+yODJgUEAnl6/f8UImKIYLEJAs/lvOCdLToD0PYFH4Ih86hzOtXVc US4cK38acijnALXRdMbX5J+tB5O2UzU1/Dfkw/ZdFr4hc96SCvigY2q8lpJqPvi8 ZVWb3vUNiSYE/CUapiVpy8JtynziWV+XrOvvLsi81xtZPCvM8hnIk2snYxnP/Okm +Mpxm3+T/jRnhE6Z6/yzeAkzcLpmpnbtG3PrGqUNxCITIJRWCk4sbE6x/c+cCbqi M+2HAgMBAAGjYzBhMB0GA1UdDgQWBBTdBAkHovV6fVJTEpKV7jiAJQ2mWTAPBgNV HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFN0ECQei9Xp9UlMSkpXuOIAlDaZZMA4G A1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAIBgRlCn7Jp0cHh5wYfGV cpNxJK1ok1iOMq8bs3AD/CUrdIWQPXhq9LmLpZc7tRiRux6n+UBbkflVma8eEdBc Hadm47GUBwwyOabqG7B52B2ccETjit3E+ZUfijhDPwGFpUenPUayvOUiaPd7nNgs PgohyC0zrL/FgZkxdMF1ccW+sfAjRfSda/wZY52jvATGGAslu1OJD7OAUN5F7kR/ q5R4ZJjT9ijdh9hwZXT7DrkT66cPYakylszeu+1jTBi7qUD3oFRuIIhxdRjqerQ0 cuAjJ3dctpDqhiVAq+8zD8ufgr6iIPv2tS0a5sKFsXQP+8hlAqRSAUfdSSLBv9jr a6x+3uxjMxW3IwiPxg+NQVrdjsW5j+VFP3jbutIbQLH+cU0/4IGiul607BXgk90I H37hVZkLId6Tngr75qNJvTYw/ud3sqB1l7UtgYgXZSD32pAAn8lSzDLKNXz1PQ/Y K9f1JmzJBjSWFupwWRoyeXkLtoh/D1JIPb9s2KJELtFOt3JY04kTlf5Eq/jXixtu nLwsoFvVagCvXzfh1foQC5ichucmj87w7G6KVwuA406ywKBjYZC6VWg3dGq2ktuf oYYitmUnDuy2n0Jg5GfCtdpBC8TTi2EbvPofkSvXRAdeuims2cXp71NIWuuA8ShY Ic2wBlX7Jz9TkHCpBB5XJ7k= -----END CERTIFICATE----- # Issuer: CN=ACCVRAIZ1, OU=PKIACCV, O=ACCV, C=ES # Subject: CN=ACCVRAIZ1, OU=PKIACCV, O=ACCV, C=ES # SHA256 Fingerprint: 9A:6E:C0:12:E1:A7:DA:9D:BE:34:19:4D:47:8A:D7:C0:DB:18:22:FB:07:1D:F1:29:81:49:6E:D1:04:38:41:13 -----BEGIN CERTIFICATE----- MIIH0zCCBbugAwIBAgIIXsO3pkN/pOAwDQYJKoZIhvcNAQEFBQAwQjESMBAGA1UE AwwJQUNDVlJBSVoxMRAwDgYDVQQLDAdQS0lBQ0NWMQ0wCwYDVQQKDARBQ0NWMQsw CQYDVQQGEwJFUzAeFw0xMTA1MDUwOTM3MzdaFw0zMDEyMzEwOTM3MzdaMEIxEjAQ BgNVBAMMCUFDQ1ZSQUlaMTEQMA4GA1UECwwHUEtJQUNDVjENMAsGA1UECgwEQUND VjELMAkGA1UEBhMCRVMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCb qau/YUqXry+XZpp0X9DZlv3P4uRm7x8fRzPCRKPfmt4ftVTdFXxpNRFvu8gMjmoY HtiP2Ra8EEg2XPBjs5BaXCQ316PWywlxufEBcoSwfdtNgM3802/J+Nq2DoLSRYWo G2ioPej0RGy9ocLLA76MPhMAhN9KSMDjIgro6TenGEyxCQ0jVn8ETdkXhBilyNpA lHPrzg5XPAOBOp0KoVdDaaxXbXmQeOW1tDvYvEyNKKGno6e6Ak4l0Squ7a4DIrhr IA8wKFSVf+DuzgpmndFALW4ir50awQUZ0m/A8p/4e7MCQvtQqR0tkw8jq8bBD5L/ 0KIV9VMJcRz/RROE5iZe+OCIHAr8Fraocwa48GOEAqDGWuzndN9wrqODJerWx5eH k6fGioozl2A3ED6XPm4pFdahD9GILBKfb6qkxkLrQaLjlUPTAYVtjrs78yM2x/47 4KElB0iryYl0/wiPgL/AlmXz7uxLaL2diMMxs0Dx6M/2OLuc5NF/1OVYm3z61PMO m3WR5LpSLhl+0fXNWhn8ugb2+1KoS5kE3fj5tItQo05iifCHJPqDQsGH+tUtKSpa cXpkatcnYGMN285J9Y0fkIkyF/hzQ7jSWpOGYdbhdQrqeWZ2iE9x6wQl1gpaepPl uUsXQA+xtrn13k/c4LOsOxFwYIRKQ26ZIMApcQrAZQIDAQABo4ICyzCCAscwfQYI KwYBBQUHAQEEcTBvMEwGCCsGAQUFBzAChkBodHRwOi8vd3d3LmFjY3YuZXMvZmls ZWFkbWluL0FyY2hpdm9zL2NlcnRpZmljYWRvcy9yYWl6YWNjdjEuY3J0MB8GCCsG AQUFBzABhhNodHRwOi8vb2NzcC5hY2N2LmVzMB0GA1UdDgQWBBTSh7Tj3zcnk1X2 VuqB5TbMjB4/vTAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFNKHtOPfNyeT VfZW6oHlNsyMHj+9MIIBcwYDVR0gBIIBajCCAWYwggFiBgRVHSAAMIIBWDCCASIG CCsGAQUFBwICMIIBFB6CARAAQQB1AHQAbwByAGkAZABhAGQAIABkAGUAIABDAGUA cgB0AGkAZgBpAGMAYQBjAGkA8wBuACAAUgBhAO0AegAgAGQAZQAgAGwAYQAgAEEA QwBDAFYAIAAoAEEAZwBlAG4AYwBpAGEAIABkAGUAIABUAGUAYwBuAG8AbABvAGcA 7QBhACAAeQAgAEMAZQByAHQAaQBmAGkAYwBhAGMAaQDzAG4AIABFAGwAZQBjAHQA cgDzAG4AaQBjAGEALAAgAEMASQBGACAAUQA0ADYAMAAxADEANQA2AEUAKQAuACAA QwBQAFMAIABlAG4AIABoAHQAdABwADoALwAvAHcAdwB3AC4AYQBjAGMAdgAuAGUA czAwBggrBgEFBQcCARYkaHR0cDovL3d3dy5hY2N2LmVzL2xlZ2lzbGFjaW9uX2Mu aHRtMFUGA1UdHwROMEwwSqBIoEaGRGh0dHA6Ly93d3cuYWNjdi5lcy9maWxlYWRt aW4vQXJjaGl2b3MvY2VydGlmaWNhZG9zL3JhaXphY2N2MV9kZXIuY3JsMA4GA1Ud DwEB/wQEAwIBBjAXBgNVHREEEDAOgQxhY2N2QGFjY3YuZXMwDQYJKoZIhvcNAQEF BQADggIBAJcxAp/n/UNnSEQU5CmH7UwoZtCPNdpNYbdKl02125DgBS4OxnnQ8pdp D70ER9m+27Up2pvZrqmZ1dM8MJP1jaGo/AaNRPTKFpV8M9xii6g3+CfYCS0b78gU JyCpZET/LtZ1qmxNYEAZSUNUY9rizLpm5U9EelvZaoErQNV/+QEnWCzI7UiRfD+m AM/EKXMRNt6GGT6d7hmKG9Ww7Y49nCrADdg9ZuM8Db3VlFzi4qc1GwQA9j9ajepD vV+JHanBsMyZ4k0ACtrJJ1vnE5Bc5PUzolVt3OAJTS+xJlsndQAJxGJ3KQhfnlms tn6tn1QwIgPBHnFk/vk4CpYY3QIUrCPLBhwepH2NDd4nQeit2hW3sCPdK6jT2iWH 7ehVRE2I9DZ+hJp4rPcOVkkO1jMl1oRQQmwgEh0q1b688nCBpHBgvgW1m54ERL5h I6zppSSMEYCUWqKiuUnSwdzRp+0xESyeGabu4VXhwOrPDYTkF7eifKXeVSUG7szA h1xA2syVP1XgNce4hL60Xc16gwFy7ofmXx2utYXGJt/mwZrpHgJHnyqobalbz+xF d3+YJ5oyXSrjhO7FmGYvliAd3djDJ9ew+f7Zfc3Qn48LFFhRny+Lwzgt3uiP1o2H pPVWQxaZLPSkVrQ0uGE3ycJYgBugl6H8WY3pEfbRD0tVNEYqi4Y7 -----END CERTIFICATE----- # Issuer: CN=Atos TrustedRoot 2011, O=Atos, C=DE # Subject: CN=Atos TrustedRoot 2011, O=Atos, C=DE # SHA256 Fingerprint: F3:56:BE:A2:44:B7:A9:1E:B3:5D:53:CA:9A:D7:86:4A:CE:01:8E:2D:35:D5:F8:F9:6D:DF:68:A6:F4:1A:A4:74 -----BEGIN CERTIFICATE----- MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ 4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed -----END CERTIFICATE----- # Issuer: CN=Atos TrustedRoot Root CA ECC TLS 2021, O=Atos, C=DE # Subject: CN=Atos TrustedRoot Root CA ECC TLS 2021, O=Atos, C=DE # SHA256 Fingerprint: B2:FA:E5:3E:14:CC:D7:AB:92:12:06:47:01:AE:27:9C:1D:89:88:FA:CB:77:5F:A8:A0:08:91:4E:66:39:88:A8 -----BEGIN CERTIFICATE----- MIICFTCCAZugAwIBAgIQPZg7pmY9kGP3fiZXOATvADAKBggqhkjOPQQDAzBMMS4w LAYDVQQDDCVBdG9zIFRydXN0ZWRSb290IFJvb3QgQ0EgRUNDIFRMUyAyMDIxMQ0w CwYDVQQKDARBdG9zMQswCQYDVQQGEwJERTAeFw0yMTA0MjIwOTI2MjNaFw00MTA0 MTcwOTI2MjJaMEwxLjAsBgNVBAMMJUF0b3MgVHJ1c3RlZFJvb3QgUm9vdCBDQSBF Q0MgVExTIDIwMjExDTALBgNVBAoMBEF0b3MxCzAJBgNVBAYTAkRFMHYwEAYHKoZI zj0CAQYFK4EEACIDYgAEloZYKDcKZ9Cg3iQZGeHkBQcfl+3oZIK59sRxUM6KDP/X tXa7oWyTbIOiaG6l2b4siJVBzV3dscqDY4PMwL502eCdpO5KTlbgmClBk1IQ1SQ4 AjJn8ZQSb+/Xxd4u/RmAo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR2 KCXWfeBmmnoJsmo7jjPXNtNPojAOBgNVHQ8BAf8EBAMCAYYwCgYIKoZIzj0EAwMD aAAwZQIwW5kp85wxtolrbNa9d+F851F+uDrNozZffPc8dz7kUK2o59JZDCaOMDtu CCrCp1rIAjEAmeMM56PDr9NJLkaCI2ZdyQAUEv049OGYa3cpetskz2VAv9LcjBHo 9H1/IISpQuQo -----END CERTIFICATE----- # Issuer: CN=Atos TrustedRoot Root CA RSA TLS 2021, O=Atos, C=DE # Subject: CN=Atos TrustedRoot Root CA RSA TLS 2021, O=Atos, C=DE # SHA256 Fingerprint: 81:A9:08:8E:A5:9F:B3:64:C5:48:A6:F8:55:59:09:9B:6F:04:05:EF:BF:18:E5:32:4E:C9:F4:57:BA:00:11:2F -----BEGIN CERTIFICATE----- MIIFZDCCA0ygAwIBAgIQU9XP5hmTC/srBRLYwiqipDANBgkqhkiG9w0BAQwFADBM MS4wLAYDVQQDDCVBdG9zIFRydXN0ZWRSb290IFJvb3QgQ0EgUlNBIFRMUyAyMDIx MQ0wCwYDVQQKDARBdG9zMQswCQYDVQQGEwJERTAeFw0yMTA0MjIwOTIxMTBaFw00 MTA0MTcwOTIxMDlaMEwxLjAsBgNVBAMMJUF0b3MgVHJ1c3RlZFJvb3QgUm9vdCBD QSBSU0EgVExTIDIwMjExDTALBgNVBAoMBEF0b3MxCzAJBgNVBAYTAkRFMIICIjAN BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtoAOxHm9BYx9sKOdTSJNy/BBl01Z 4NH+VoyX8te9j2y3I49f1cTYQcvyAh5x5en2XssIKl4w8i1mx4QbZFc4nXUtVsYv Ye+W/CBGvevUez8/fEc4BKkbqlLfEzfTFRVOvV98r61jx3ncCHvVoOX3W3WsgFWZ kmGbzSoXfduP9LVq6hdKZChmFSlsAvFr1bqjM9xaZ6cF4r9lthawEO3NUDPJcFDs GY6wx/J0W2tExn2WuZgIWWbeKQGb9Cpt0xU6kGpn8bRrZtkh68rZYnxGEFzedUln nkL5/nWpo63/dgpnQOPF943HhZpZnmKaau1Fh5hnstVKPNe0OwANwI8f4UDErmwh 3El+fsqyjW22v5MvoVw+j8rtgI5Y4dtXz4U2OLJxpAmMkokIiEjxQGMYsluMWuPD 0xeqqxmjLBvk1cbiZnrXghmmOxYsL3GHX0WelXOTwkKBIROW1527k2gV+p2kHYzy geBYBr3JtuP2iV2J+axEoctr+hbxx1A9JNr3w+SH1VbxT5Aw+kUJWdo0zuATHAR8 ANSbhqRAvNncTFd+rrcztl524WWLZt+NyteYr842mIycg5kDcPOvdO3GDjbnvezB c6eUWsuSZIKmAMFwoW4sKeFYV+xafJlrJaSQOoD0IJ2azsct+bJLKZWD6TWNp0lI pw9MGZHQ9b8Q4HECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU dEmZ0f+0emhFdcN+tNzMzjkz2ggwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB DAUAA4ICAQAjQ1MkYlxt/T7Cz1UAbMVWiLkO3TriJQ2VSpfKgInuKs1l+NsW4AmS 4BjHeJi78+xCUvuppILXTdiK/ORO/auQxDh1MoSf/7OwKwIzNsAQkG8dnK/haZPs o0UvFJ/1TCplQ3IM98P4lYsU84UgYt1UU90s3BiVaU+DR3BAM1h3Egyi61IxHkzJ qM7F78PRreBrAwA0JrRUITWXAdxfG/F851X6LWh3e9NpzNMOa7pNdkTWwhWaJuyw xfW70Xp0wmzNxbVe9kzmWy2B27O3Opee7c9GslA9hGCZcbUztVdF5kJHdWoOsAgM rr3e97sPWD2PAzHoPYJQyi9eDF20l74gNAf0xBLh7tew2VktafcxBPTy+av5EzH4 AXcOPUIjJsyacmdRIXrMPIWo6iFqO9taPKU0nprALN+AnCng33eU0aKAQv9qTFsR 0PXNor6uzFFcw9VUewyu1rkGd4Di7wcaaMxZUa1+XGdrudviB0JbuAEFWDlN5LuY o7Ey7Nmj1m+UI/87tyll5gfp77YZ6ufCOB0yiJA8EytuzO+rdwY0d4RPcuSBhPm5 dDTedk+SKlOxJTnbPP/lPqYO5Wue/9vsL3SD3460s6neFE3/MaNFcyT6lSnMEpcE oji2jbDwN/zIIX8/syQbPYtuzE2wFg2WHYMfRsCbvUOZ58SWLs5fyQ== -----END CERTIFICATE----- # Issuer: O=Entrust.net, OU=www.entrust.net/CPS_2048 incorp. by ref. (limits liab.), OU=(c) 1999 Entrust.net Limited, CN=Entrust.net Certification Authority (2048) # Subject: O=Entrust.net, OU=www.entrust.net/CPS_2048 incorp. by ref. (limits liab.), OU=(c) 1999 Entrust.net Limited, CN=Entrust.net Certification Authority (2048) # SHA256 Fingerprint: 6D:C4:71:72:E0:1C:BC:B0:BF:62:58:0D:89:5F:E2:B8:AC:9A:D4:F8:73:80:1E:0C:10:B9:C8:37:D2:1E:B1:77 -----BEGIN CERTIFICATE----- MIIEKjCCAxKgAwIBAgIEOGPe+DANBgkqhkiG9w0BAQUFADCBtDEUMBIGA1UEChML RW50cnVzdC5uZXQxQDA+BgNVBAsUN3d3dy5lbnRydXN0Lm5ldC9DUFNfMjA0OCBp bmNvcnAuIGJ5IHJlZi4gKGxpbWl0cyBsaWFiLikxJTAjBgNVBAsTHChjKSAxOTk5 IEVudHJ1c3QubmV0IExpbWl0ZWQxMzAxBgNVBAMTKkVudHJ1c3QubmV0IENlcnRp ZmljYXRpb24gQXV0aG9yaXR5ICgyMDQ4KTAeFw05OTEyMjQxNzUwNTFaFw0yOTA3 MjQxNDE1MTJaMIG0MRQwEgYDVQQKEwtFbnRydXN0Lm5ldDFAMD4GA1UECxQ3d3d3 LmVudHJ1c3QubmV0L0NQU18yMDQ4IGluY29ycC4gYnkgcmVmLiAobGltaXRzIGxp YWIuKTElMCMGA1UECxMcKGMpIDE5OTkgRW50cnVzdC5uZXQgTGltaXRlZDEzMDEG A1UEAxMqRW50cnVzdC5uZXQgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkgKDIwNDgp MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArU1LqRKGsuqjIAcVFmQq K0vRvwtKTY7tgHalZ7d4QMBzQshowNtTK91euHaYNZOLGp18EzoOH1u3Hs/lJBQe sYGpjX24zGtLA/ECDNyrpUAkAH90lKGdCCmziAv1h3edVc3kw37XamSrhRSGlVuX MlBvPci6Zgzj/L24ScF2iUkZ/cCovYmjZy/Gn7xxGWC4LeksyZB2ZnuU4q941mVT XTzWnLLPKQP5L6RQstRIzgUyVYr9smRMDuSYB3Xbf9+5CFVghTAp+XtIpGmG4zU/ HoZdenoVve8AjhUiVBcAkCaTvA5JaJG/+EfTnZVCwQ5N328mz8MYIWJmQ3DW1cAH 4QIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV HQ4EFgQUVeSB0RGAvtiJuQijMfmhJAkWuXAwDQYJKoZIhvcNAQEFBQADggEBADub j1abMOdTmXx6eadNl9cZlZD7Bh/KM3xGY4+WZiT6QBshJ8rmcnPyT/4xmf3IDExo U8aAghOY+rat2l098c5u9hURlIIM7j+VrxGrD9cv3h8Dj1csHsm7mhpElesYT6Yf zX1XEC+bBAlahLVu2B064dae0Wx5XnkcFMXj0EyTO2U87d89vqbllRrDtRnDvV5b u/8j72gZyxKTJ1wDLW8w0B62GqzeWvfRqqgnpv55gcR5mTNXuhKwqeBCbJPKVt7+ bYQLCIt+jerXmCHG8+c8eS9enNFMFY3h7CI3zJpDC5fcgJCNs2ebb0gIFVbPv/Er fF6adulZkMV8gzURZVE= -----END CERTIFICATE----- # Issuer: O=TeliaSonera, CN=TeliaSonera Root CA v1 # Subject: O=TeliaSonera, CN=TeliaSonera Root CA v1 # SHA256 Fingerprint: DD:69:36:FE:21:F8:F0:77:C1:23:A1:A5:21:C1:22:24:F7:22:55:B7:3E:03:A7:26:06:93:E8:A2:4B:0F:A3:89 -----BEGIN CERTIFICATE----- MIIFODCCAyCgAwIBAgIRAJW+FqD3LkbxezmCcvqLzZYwDQYJKoZIhvcNAQEFBQAw NzEUMBIGA1UECgwLVGVsaWFTb25lcmExHzAdBgNVBAMMFlRlbGlhU29uZXJhIFJv b3QgQ0EgdjEwHhcNMDcxMDE4MTIwMDUwWhcNMzIxMDE4MTIwMDUwWjA3MRQwEgYD VQQKDAtUZWxpYVNvbmVyYTEfMB0GA1UEAwwWVGVsaWFTb25lcmEgUm9vdCBDQSB2 MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMK+6yfwIaPzaSZVfp3F VRaRXP3vIb9TgHot0pGMYzHw7CTww6XScnwQbfQ3t+XmfHnqjLWCi65ItqwA3GV1 7CpNX8GH9SBlK4GoRz6JI5UwFpB/6FcHSOcZrr9FZ7E3GwYq/t75rH2D+1665I+X Z75Ljo1kB1c4VWk0Nj0TSO9P4tNmHqTPGrdeNjPUtAa9GAH9d4RQAEX1jF3oI7x+ /jXh7VB7qTCNGdMJjmhnXb88lxhTuylixcpecsHHltTbLaC0H2kD7OriUPEMPPCs 81Mt8Bz17Ww5OXOAFshSsCPN4D7c3TxHoLs1iuKYaIu+5b9y7tL6pe0S7fyYGKkm dtwoSxAgHNN/Fnct7W+A90m7UwW7XWjH1Mh1Fj+JWov3F0fUTPHSiXk+TT2YqGHe Oh7S+F4D4MHJHIzTjU3TlTazN19jY5szFPAtJmtTfImMMsJu7D0hADnJoWjiUIMu sDor8zagrC/kb2HCUQk5PotTubtn2txTuXZZNp1D5SDgPTJghSJRt8czu90VL6R4 pgd7gUY2BIbdeTXHlSw7sKMXNeVzH7RcWe/a6hBle3rQf5+ztCo3O3CLm1u5K7fs slESl1MpWtTwEhDcTwK7EpIvYtQ/aUN8Ddb8WHUBiJ1YFkveupD/RwGJBmr2X7KQ arMCpgKIv7NHfirZ1fpoeDVNAgMBAAGjPzA9MA8GA1UdEwEB/wQFMAMBAf8wCwYD VR0PBAQDAgEGMB0GA1UdDgQWBBTwj1k4ALP1j5qWDNXr+nuqF+gTEjANBgkqhkiG 9w0BAQUFAAOCAgEAvuRcYk4k9AwI//DTDGjkk0kiP0Qnb7tt3oNmzqjMDfz1mgbl dxSR651Be5kqhOX//CHBXfDkH1e3damhXwIm/9fH907eT/j3HEbAek9ALCI18Bmx 0GtnLLCo4MBANzX2hFxc469CeP6nyQ1Q6g2EdvZR74NTxnr/DlZJLo961gzmJ1Tj TQpgcmLNkQfWpb/ImWvtxBnmq0wROMVvMeJuScg/doAmAyYp4Db29iBT4xdwNBed Y2gea+zDTYa4EzAvXUYNR0PVG6pZDrlcjQZIrXSHX8f8MVRBE+LHIQ6e4B4N4cB7 Q4WQxYpYxmUKeFfyxiMPAdkgS94P+5KFdSpcc41teyWRyu5FrgZLAMzTsVlQ2jqI OylDRl6XK1TOU2+NSueW+r9xDkKLfP0ooNBIytrEgUy7onOTJsjrDNYmiLbAJM+7 vVvrdX3pCI6GMyx5dwlppYn8s3CQh3aP0yK7Qs69cwsgJirQmz1wHiRszYd2qReW t88NkvuOGKmYSdGe/mBEciG5Ge3C9THxOUiIkCR1VBatzvT4aRRkOfujuLpwQMcn HL/EVlP6Y2XQ8xwOFvVrhlhNGNTkDY6lnVuR3HYkUD/GKvvZt5y11ubQ2egZixVx SK236thZiNSQvxaz2emsWWFUyBy6ysHK4bkgTI86k4mloMy/0/Z1pHWWbVY= -----END CERTIFICATE----- # Issuer: OU=GlobalSign ECC Root CA - R4, O=GlobalSign, CN=GlobalSign # Subject: OU=GlobalSign ECC Root CA - R4, O=GlobalSign, CN=GlobalSign # SHA256 Fingerprint: B0:85:D7:0B:96:4F:19:1A:73:E4:AF:0D:54:AE:7A:0E:07:AA:FD:AF:9B:71:DD:08:62:13:8A:B7:32:5A:24:A2 -----BEGIN CERTIFICATE----- MIIB3DCCAYOgAwIBAgINAgPlfvU/k/2lCSGypjAKBggqhkjOPQQDAjBQMSQwIgYD VQQLExtHbG9iYWxTaWduIEVDQyBSb290IENBIC0gUjQxEzARBgNVBAoTCkdsb2Jh bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTIxMTEzMDAwMDAwWhcNMzgw MTE5MDMxNDA3WjBQMSQwIgYDVQQLExtHbG9iYWxTaWduIEVDQyBSb290IENBIC0g UjQxEzARBgNVBAoTCkdsb2JhbFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wWTAT BgcqhkjOPQIBBggqhkjOPQMBBwNCAAS4xnnTj2wlDp8uORkcA6SumuU5BwkWymOx uYb4ilfBV85C+nOh92VC/x7BALJucw7/xyHlGKSq2XE/qNS5zowdo0IwQDAOBgNV HQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUVLB7rUW44kB/ +wpu+74zyTyjhNUwCgYIKoZIzj0EAwIDRwAwRAIgIk90crlgr/HmnKAWBVBfw147 bmF0774BxL4YSFlhgjICICadVGNA3jdgUM/I2O2dgq43mLyjj0xMqTQrbO/7lZsm -----END CERTIFICATE----- # Issuer: OU=GlobalSign ECC Root CA - R5, O=GlobalSign, CN=GlobalSign # Subject: OU=GlobalSign ECC Root CA - R5, O=GlobalSign, CN=GlobalSign # SHA256 Fingerprint: 17:9F:BC:14:8A:3D:D0:0F:D2:4E:A1:34:58:CC:43:BF:A7:F5:9C:81:82:D7:83:A5:13:F6:EB:EC:10:0C:89:24 -----BEGIN CERTIFICATE----- MIICHjCCAaSgAwIBAgIRYFlJ4CYuu1X5CneKcflK2GwwCgYIKoZIzj0EAwMwUDEk MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI1MRMwEQYDVQQKEwpH bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD QSAtIFI1MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAER0UOlvt9Xb/pOdEh+J8LttV7HpI6SFkc 8GIxLcB6KP4ap1yztsyX50XUWPrRd21DosCHZTQKH3rd6zwzocWdTaRvQZU4f8ke hOvRnkmSh5SHDDqFSmafnVmTTZdhBoZKo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYD VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUPeYpSJvqB8ohREom3m7e0oPQn1kwCgYI KoZIzj0EAwMDaAAwZQIxAOVpEslu28YxuglB4Zf4+/2a4n0Sye18ZNPLBSWLVtmg 515dTguDnFt2KaAJJiFqYgIwcdK1j1zqO+F4CYWodZI7yFz9SO8NdCKoCOJuxUnO xwy8p2Fp8fc74SrL+SvzZpA3 -----END CERTIFICATE----- # Issuer: OU=GlobalSign Root CA - R3, O=GlobalSign, CN=GlobalSign # Subject: OU=GlobalSign Root CA - R3, O=GlobalSign, CN=GlobalSign # SHA256 Fingerprint: CB:B5:22:D7:B7:F1:27:AD:6A:01:13:86:5B:DF:1C:D4:10:2E:7D:07:59:AF:63:5A:7C:F4:72:0D:C9:63:C5:3B -----BEGIN CERTIFICATE----- MIIDXzCCAkegAwIBAgILBAAAAAABIVhTCKIwDQYJKoZIhvcNAQELBQAwTDEgMB4G A1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjMxEzARBgNVBAoTCkdsb2JhbFNp Z24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMDkwMzE4MTAwMDAwWhcNMjkwMzE4 MTAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSMzETMBEG A1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCASIwDQYJKoZI hvcNAQEBBQADggEPADCCAQoCggEBAMwldpB5BngiFvXAg7aEyiie/QV2EcWtiHL8 RgJDx7KKnQRfJMsuS+FggkbhUqsMgUdwbN1k0ev1LKMPgj0MK66X17YUhhB5uzsT gHeMCOFJ0mpiLx9e+pZo34knlTifBtc+ycsmWQ1z3rDI6SYOgxXG71uL0gRgykmm KPZpO/bLyCiR5Z2KYVc3rHQU3HTgOu5yLy6c+9C7v/U9AOEGM+iCK65TpjoWc4zd QQ4gOsC0p6Hpsk+QLjJg6VfLuQSSaGjlOCZgdbKfd/+RFO+uIEn8rUAVSNECMWEZ XriX7613t2Saer9fwRPvm2L7DWzgVGkWqQPabumDk3F2xmmFghcCAwEAAaNCMEAw DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFI/wS3+o LkUkrk1Q+mOai97i3Ru8MA0GCSqGSIb3DQEBCwUAA4IBAQBLQNvAUKr+yAzv95ZU RUm7lgAJQayzE4aGKAczymvmdLm6AC2upArT9fHxD4q/c2dKg8dEe3jgr25sbwMp jjM5RcOO5LlXbKr8EpbsU8Yt5CRsuZRj+9xTaGdWPoO4zzUhw8lo/s7awlOqzJCK 6fBdRoyV3XpYKBovHd7NADdBj+1EbddTKJd+82cEHhXXipa0095MJ6RMG3NzdvQX mcIfeg7jLQitChws/zyrVQ4PkX4268NXSb7hLi18YIvDQVETI53O9zJrlAGomecs Mx86OyXShkDOOyyGeMlhLxS67ttVb9+E7gUJTb0o2HLO02JQZR7rkpeDMdmztcpH WD9f -----END CERTIFICATE----- # Issuer: OU=GlobalSign Root CA - R6, O=GlobalSign, CN=GlobalSign # Subject: OU=GlobalSign Root CA - R6, O=GlobalSign, CN=GlobalSign # SHA256 Fingerprint: 2C:AB:EA:FE:37:D0:6C:A2:2A:BA:73:91:C0:03:3D:25:98:29:52:C4:53:64:73:49:76:3A:3A:B5:AD:6C:CF:69 -----BEGIN CERTIFICATE----- MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw 1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R 8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= -----END CERTIFICATE----- # Issuer: serialNumber=G63287510, C=ES, O=ANF Autoridad de Certificacion, OU=ANF CA Raiz, CN=ANF Secure Server Root CA # Subject: serialNumber=G63287510, C=ES, O=ANF Autoridad de Certificacion, OU=ANF CA Raiz, CN=ANF Secure Server Root CA # SHA256 Fingerprint: FB:8F:EC:75:91:69:B9:10:6B:1E:51:16:44:C6:18:C5:13:04:37:3F:6C:06:43:08:8D:8B:EF:FD:1B:99:75:99 -----BEGIN CERTIFICATE----- MIIF7zCCA9egAwIBAgIIDdPjvGz5a7EwDQYJKoZIhvcNAQELBQAwgYQxEjAQBgNV BAUTCUc2MzI4NzUxMDELMAkGA1UEBhMCRVMxJzAlBgNVBAoTHkFORiBBdXRvcmlk YWQgZGUgQ2VydGlmaWNhY2lvbjEUMBIGA1UECxMLQU5GIENBIFJhaXoxIjAgBgNV BAMTGUFORiBTZWN1cmUgU2VydmVyIFJvb3QgQ0EwHhcNMTkwOTA0MTAwMDM4WhcN MzkwODMwMTAwMDM4WjCBhDESMBAGA1UEBRMJRzYzMjg3NTEwMQswCQYDVQQGEwJF UzEnMCUGA1UEChMeQU5GIEF1dG9yaWRhZCBkZSBDZXJ0aWZpY2FjaW9uMRQwEgYD VQQLEwtBTkYgQ0EgUmFpejEiMCAGA1UEAxMZQU5GIFNlY3VyZSBTZXJ2ZXIgUm9v dCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANvrayvmZFSVgpCj cqQZAZ2cC4Ffc0m6p6zzBE57lgvsEeBbphzOG9INgxwruJ4dfkUyYA8H6XdYfp9q yGFOtibBTI3/TO80sh9l2Ll49a2pcbnvT1gdpd50IJeh7WhM3pIXS7yr/2WanvtH 2Vdy8wmhrnZEE26cLUQ5vPnHO6RYPUG9tMJJo8gN0pcvB2VSAKduyK9o7PQUlrZX H1bDOZ8rbeTzPvY1ZNoMHKGESy9LS+IsJJ1tk0DrtSOOMspvRdOoiXsezx76W0OL zc2oD2rKDF65nkeP8Nm2CgtYZRczuSPkdxl9y0oukntPLxB3sY0vaJxizOBQ+OyR p1RMVwnVdmPF6GUe7m1qzwmd+nxPrWAI/VaZDxUse6mAq4xhj0oHdkLePfTdsiQz W7i1o0TJrH93PB0j7IKppuLIBkwC/qxcmZkLLxCKpvR/1Yd0DVlJRfbwcVw5Kda/ SiOL9V8BY9KHcyi1Swr1+KuCLH5zJTIdC2MKF4EA/7Z2Xue0sUDKIbvVgFHlSFJn LNJhiQcND85Cd8BEc5xEUKDbEAotlRyBr+Qc5RQe8TZBAQIvfXOn3kLMTOmJDVb3 n5HUA8ZsyY/b2BzgQJhdZpmYgG4t/wHFzstGH6wCxkPmrqKEPMVOHj1tyRRM4y5B u8o5vzY8KhmqQYdOpc5LMnndkEl/AgMBAAGjYzBhMB8GA1UdIwQYMBaAFJxf0Gxj o1+TypOYCK2Mh6UsXME3MB0GA1UdDgQWBBScX9BsY6Nfk8qTmAitjIelLFzBNzAO BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOC AgEATh65isagmD9uw2nAalxJUqzLK114OMHVVISfk/CHGT0sZonrDUL8zPB1hT+L 9IBdeeUXZ701guLyPI59WzbLWoAAKfLOKyzxj6ptBZNscsdW699QIyjlRRA96Gej rw5VD5AJYu9LWaL2U/HANeQvwSS9eS9OICI7/RogsKQOLHDtdD+4E5UGUcjohybK pFtqFiGS3XNgnhAY3jyB6ugYw3yJ8otQPr0R4hUDqDZ9MwFsSBXXiJCZBMXM5gf0 vPSQ7RPi6ovDj6MzD8EpTBNO2hVWcXNyglD2mjN8orGoGjR0ZVzO0eurU+AagNjq OknkJjCb5RyKqKkVMoaZkgoQI1YS4PbOTOK7vtuNknMBZi9iPrJyJ0U27U1W45eZ /zo1PqVUSlJZS2Db7v54EX9K3BR5YLZrZAPbFYPhor72I5dQ8AkzNqdxliXzuUJ9 2zg/LFis6ELhDtjTO0wugumDLmsx2d1Hhk9tl5EuT+IocTUW0fJz/iUrB0ckYyfI +PbZa/wSMVYIwFNCr5zQM378BvAxRAMU8Vjq8moNqRGyg77FGr8H6lnco4g175x2 MjxNBiLOFeXdntiP2t7SxDnlF4HPOEfrf4htWRvfn0IUrn7PqLBmZdo3r5+qPeoo tt7VMVgWglvquxl1AnMaykgaIZOQCo6ThKd9OyMYkomgjaw= -----END CERTIFICATE----- metrics_reporter.py 0000644 00000011164 15025013616 0010510 0 ustar 00 # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Script for reporting metrics.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import pickle import random import string import sys _LOG_FILE_BASENAME = 'metrics.log' # pylint:disable=g-import-not-at-top try: from gslib.utils.boto_util import ConfigureCertsFile from gslib.utils.boto_util import GetGsutilStateDir from gslib.utils.boto_util import GetNewHttp from gslib.utils.system_util import CreateDirIfNeeded _LOG_FILE_PARENT_DIR = GetGsutilStateDir() except: # pylint: disable=bare-except # Some environments import their own version of standard Python libraries # which might cause the import of some of our gslib.utils modules to fail # (e.g. if httplib2 is missing, etc). Try these alternative definitions in # such cases. _LOG_FILE_PARENT_DIR = os.path.expanduser(os.path.join('~', '.gsutil')) # Some environments don't have the httplib2 functionality we need. try: import errno # Fall back to httplib (no proxy) if we can't import libraries normally. from six.moves import http_client def GetNewHttp(): """Returns an httplib-based metrics reporter.""" class HttplibReporter(object): def __init__(self): pass # pylint: disable=invalid-name def request(self, endpoint, method=None, body=None, headers=None): # Strip 'https://' https_con = http_client.HTTPSConnection(endpoint[8:].split('/')[0]) https_con.request(method, endpoint, body=body, headers=headers) response = https_con.getresponse() # Return status like an httplib2 response. return ({'status': response.status},) # pylint: enable=invalid-name return HttplibReporter() def ConfigureCertsFile(): pass def CreateDirIfNeeded(dir_path, mode=0o777): """See the same-named method in gslib.utils.system_util.""" if not os.path.exists(dir_path): try: os.makedirs(dir_path, mode) except OSError as e: if e.errno != errno.EEXIST: raise except: # pylint: disable=bare-except sys.exit(0) LOG_FILE_PATH = os.path.join(_LOG_FILE_PARENT_DIR, _LOG_FILE_BASENAME) def ReportMetrics(metrics_file_path, log_level, log_file_path=None): """Sends the specified anonymous usage event to the given analytics endpoint. Args: metrics_file_path: str, File with pickled metrics (list of tuples). log_level: int, The logging level of gsutil's root logger. log_file_path: str, The file that this module should write its logs to. This parameter is intended for use by tests that need to evaluate the contents of the file at this path. """ logger = logging.getLogger() if log_file_path is not None: # Use a separate logger so that we don't add another handler to the default # module-level logger. This is intended to prevent multiple calls from tests # running in parallel from writing output to the same file. new_name = '%s.%s' % (logger.name, ''.join( random.choice(string.ascii_lowercase) for _ in range(8))) logger = logging.getLogger(new_name) log_file_path = log_file_path or LOG_FILE_PATH log_file_parent_dir = os.path.dirname(log_file_path) CreateDirIfNeeded(log_file_parent_dir) handler = logging.FileHandler(log_file_path, mode='w') logger.addHandler(handler) logger.setLevel(log_level) with open(metrics_file_path, 'rb') as metrics_file: metrics = pickle.load(metrics_file) os.remove(metrics_file_path) ConfigureCertsFile() http = GetNewHttp() for metric in metrics: try: headers = {'User-Agent': metric.user_agent} response = http.request(metric.endpoint, method=metric.method, body=metric.body, headers=headers) logger.debug(metric) logger.debug('RESPONSE: %s', response[0]['status']) except Exception as e: # pylint: disable=broad-except logger.debug(e) gcs_json_media.py 0000644 00000070774 15025013616 0010100 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Media helper functions and classes for Google Cloud Storage JSON API.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import copy import logging import re import socket import types import six from six.moves import http_client from six.moves import urllib from six.moves import cStringIO from apitools.base.py import exceptions as apitools_exceptions from gslib.cloud_api import BadRequestException from gslib.lazy_wrapper import LazyWrapper from gslib.progress_callback import ProgressCallbackWithTimeout from gslib.utils.constants import DEBUGLEVEL_DUMP_REQUESTS from gslib.utils.constants import SSL_TIMEOUT_SEC from gslib.utils.constants import TRANSFER_BUFFER_SIZE from gslib.utils.constants import UTF8 from gslib.utils import text_util import httplib2 from httplib2 import parse_uri if six.PY3: long = int # A regex for matching any series of decimal digits. DECIMAL_REGEX = LazyWrapper(lambda: (re.compile(r'\d+'))) class BytesTransferredContainer(object): """Container class for passing number of bytes transferred to lower layers. For resumed transfers or connection rebuilds in the middle of a transfer, we need to rebuild the connection class with how much we've transferred so far. For uploads, we don't know the total number of bytes uploaded until we've queried the server, but we need to create the connection class to pass to httplib2 before we can query the server. This container object allows us to pass a reference into Upload/DownloadCallbackConnection. """ def __init__(self): self.__bytes_transferred = 0 @property def bytes_transferred(self): return self.__bytes_transferred @bytes_transferred.setter def bytes_transferred(self, value): self.__bytes_transferred = value class UploadCallbackConnectionClassFactory(object): """Creates a class that can override an httplib2 connection. This is used to provide progress callbacks and disable dumping the upload payload during debug statements. It can later be used to provide on-the-fly hash digestion during upload. """ def __init__(self, bytes_uploaded_container, buffer_size=TRANSFER_BUFFER_SIZE, total_size=0, progress_callback=None, logger=None, debug=0): self.bytes_uploaded_container = bytes_uploaded_container self.buffer_size = buffer_size self.total_size = total_size self.progress_callback = progress_callback self.logger = logger self.debug = debug def GetConnectionClass(self): """Returns a connection class that overrides send.""" outer_bytes_uploaded_container = self.bytes_uploaded_container outer_buffer_size = self.buffer_size outer_total_size = self.total_size outer_progress_callback = self.progress_callback outer_logger = self.logger outer_debug = self.debug class UploadCallbackConnection(httplib2.HTTPSConnectionWithTimeout): """Connection class override for uploads.""" bytes_uploaded_container = outer_bytes_uploaded_container # After we instantiate this class, apitools will check with the server # to find out how many bytes remain for a resumable upload. This allows # us to update our progress once based on that number. processed_initial_bytes = False GCS_JSON_BUFFER_SIZE = outer_buffer_size callback_processor = None size = outer_total_size header_encoding = '' header_length = None header_range = None size_modifier = 1.0 def __init__(self, *args, **kwargs): kwargs['timeout'] = SSL_TIMEOUT_SEC httplib2.HTTPSConnectionWithTimeout.__init__(self, *args, **kwargs) # Override httplib.HTTPConnection._send_output for debug logging. # Because the distinction between headers and message body occurs # only in this httplib function, we can only differentiate them here. def _send_output(self, message_body=None, encode_chunked=False): r"""Send the currently buffered request and clear the buffer. Appends an extra \r\n to the buffer. Args: message_body: if specified, this is appended to the request. """ # TODO: Presently, apitools will set http2lib2.debuglevel to 0 # (no prints) or 4 (dump upload payload, httplib prints to stdout). # Refactor to allow our media-handling functions to handle # debuglevel == 4 and print messages to stderr. self._buffer.extend((b'', b'')) if six.PY2: items = self._buffer else: items = [] for item in self._buffer: if isinstance(item, bytes): items.append(item) else: items.append(item.encode(UTF8)) msg = b'\r\n'.join(items) num_metadata_bytes = len(msg) if outer_debug == DEBUGLEVEL_DUMP_REQUESTS and outer_logger: outer_logger.debug('send: %s' % msg) del self._buffer[:] # If msg and message_body are sent in a single send() call, # it will avoid performance problems caused by the interaction # between delayed ack and the Nagle algorithm. if isinstance(message_body, str): msg += message_body message_body = None self.send(msg, num_metadata_bytes=num_metadata_bytes) if message_body is not None: # message_body was not a string (i.e. it is a file) and # we must run the risk of Nagle self.send(message_body) def putheader(self, header, *values): """Overrides HTTPConnection.putheader. Send a request header line to the server. For example: h.putheader('Accept', 'text/html'). This override records the content encoding, length, and range of the payload. For uploads where the content-range difference does not match the content-length, progress printing will under-report progress. These headers are used to calculate a multiplier to correct the progress. For example: the content-length for gzip transport encoded data represents the compressed size of the data while the content-range difference represents the uncompressed size. Dividing the content-range difference by the content-length gives the ratio to multiply the progress by to correctly report the relative progress. Args: header: The header. *values: A set of values for the header. """ if header == 'content-encoding': value = ''.join([str(v) for v in values]) self.header_encoding = value if outer_debug == DEBUGLEVEL_DUMP_REQUESTS and outer_logger: outer_logger.debug( 'send: Using gzip transport encoding for the request.') elif header == 'content-length': try: value = int(''.join([str(v) for v in values])) self.header_length = value except ValueError: pass elif header == 'content-range': try: # There are 3 valid header formats: # '*/%d', '%d-%d/*', and '%d-%d/%d' value = ''.join([str(v) for v in values]) ranges = DECIMAL_REGEX().findall(value) # If there are 2 or more range values, they will always # correspond to the start and end ranges in the header. if len(ranges) > 1: # Subtract the end position from the start position. self.header_range = (int(ranges[1]) - int(ranges[0])) + 1 except ValueError: pass # If the content header is gzip, and a range and length are set, # update the modifier. if (self.header_encoding == 'gzip' and self.header_length and self.header_range): # Update the modifier self.size_modifier = self.header_range / float(self.header_length) # Reset the headers self.header_encoding = '' self.header_length = None self.header_range = None # Log debug information to catch in tests. if outer_debug == DEBUGLEVEL_DUMP_REQUESTS and outer_logger: outer_logger.debug('send: Setting progress modifier to %s.' % (self.size_modifier)) # Propagate header values. http_client.HTTPSConnection.putheader(self, header, *values) def send(self, data, num_metadata_bytes=0): """Overrides HTTPConnection.send. Args: data: string or file-like object (implements read()) of data to send. num_metadata_bytes: number of bytes that consist of metadata (headers, etc.) not representing the data being uploaded. """ if not self.processed_initial_bytes: self.processed_initial_bytes = True if outer_progress_callback: self.callback_processor = ProgressCallbackWithTimeout( outer_total_size, outer_progress_callback) self.callback_processor.Progress( self.bytes_uploaded_container.bytes_transferred) # httplib.HTTPConnection.send accepts either a string or a file-like # object (anything that implements read()). if isinstance(data, six.text_type): full_buffer = cStringIO(data) elif isinstance(data, six.binary_type): full_buffer = six.BytesIO(data) else: full_buffer = data partial_buffer = full_buffer.read(self.GCS_JSON_BUFFER_SIZE) while partial_buffer: if six.PY2: httplib2.HTTPSConnectionWithTimeout.send(self, partial_buffer) else: if isinstance(partial_buffer, bytes): httplib2.HTTPSConnectionWithTimeout.send(self, partial_buffer) else: httplib2.HTTPSConnectionWithTimeout.send( self, partial_buffer.encode(UTF8)) sent_data_bytes = len(partial_buffer) if num_metadata_bytes: if num_metadata_bytes <= sent_data_bytes: sent_data_bytes -= num_metadata_bytes num_metadata_bytes = 0 else: num_metadata_bytes -= sent_data_bytes sent_data_bytes = 0 if self.callback_processor: # Modify the sent data bytes by the size modifier. These are # stored as floats, so the result should be floored. sent_data_bytes = int(sent_data_bytes * self.size_modifier) # TODO: We can't differentiate the multipart upload # metadata in the request body from the actual upload bytes, so we # will actually report slightly more bytes than desired to the # callback handler. Get the number of multipart upload metadata # bytes from apitools and subtract from sent_data_bytes. self.callback_processor.Progress(sent_data_bytes) partial_buffer = full_buffer.read(self.GCS_JSON_BUFFER_SIZE) return UploadCallbackConnection def WrapUploadHttpRequest(upload_http): """Wraps upload_http so we only use our custom connection_type on PUTs. POSTs are used to refresh oauth tokens, and we don't want to process the data sent in those requests. Args: upload_http: httplib2.Http instance to wrap """ request_orig = upload_http.request def NewRequest(uri, method='GET', body=None, headers=None, redirections=httplib2.DEFAULT_MAX_REDIRECTS, connection_type=None): if method == 'PUT' or method == 'POST': override_connection_type = connection_type else: override_connection_type = None return request_orig(uri, method=method, body=body, headers=headers, redirections=redirections, connection_type=override_connection_type) # Replace the request method with our own closure. upload_http.request = NewRequest class DownloadCallbackConnectionClassFactory(object): """Creates a class that can override an httplib2 connection. This is used to provide progress callbacks, disable dumping the download payload during debug statements, and provide on-the-fly hash digestion during download. On-the-fly digestion is particularly important because httplib2 will decompress gzipped content on-the-fly, thus this class provides our only opportunity to calculate the correct hash for an object that has a gzip hash in the cloud. """ def __init__(self, bytes_downloaded_container, buffer_size=TRANSFER_BUFFER_SIZE, total_size=0, progress_callback=None, digesters=None): self.buffer_size = buffer_size self.total_size = total_size self.progress_callback = progress_callback self.digesters = digesters self.bytes_downloaded_container = bytes_downloaded_container def GetConnectionClass(self): """Returns a connection class that overrides getresponse.""" class DownloadCallbackConnection(httplib2.HTTPSConnectionWithTimeout): """Connection class override for downloads.""" outer_total_size = self.total_size outer_digesters = self.digesters outer_progress_callback = self.progress_callback outer_bytes_downloaded_container = self.bytes_downloaded_container processed_initial_bytes = False callback_processor = None def __init__(self, *args, **kwargs): kwargs['timeout'] = SSL_TIMEOUT_SEC httplib2.HTTPSConnectionWithTimeout.__init__(self, *args, **kwargs) def getresponse(self, buffering=False): """Wraps an HTTPResponse to perform callbacks and hashing. In this function, self is a DownloadCallbackConnection. Args: buffering: Unused. This function uses a local buffer. Returns: HTTPResponse object with wrapped read function. """ orig_response = http_client.HTTPConnection.getresponse(self) if orig_response.status not in (http_client.OK, http_client.PARTIAL_CONTENT): return orig_response orig_read_func = orig_response.read def read(amt=None): # pylint: disable=invalid-name """Overrides HTTPConnection.getresponse.read. This function only supports reads of TRANSFER_BUFFER_SIZE or smaller. Args: amt: Integer n where 0 < n <= TRANSFER_BUFFER_SIZE. This is a keyword argument to match the read function it overrides, but it is required. Returns: Data read from HTTPConnection. """ if not amt or amt > TRANSFER_BUFFER_SIZE: raise BadRequestException( 'Invalid HTTP read size %s during download, expected %s.' % (amt, TRANSFER_BUFFER_SIZE)) else: amt = amt or TRANSFER_BUFFER_SIZE if not self.processed_initial_bytes: self.processed_initial_bytes = True if self.outer_progress_callback: self.callback_processor = ProgressCallbackWithTimeout( self.outer_total_size, self.outer_progress_callback) self.callback_processor.Progress( self.outer_bytes_downloaded_container.bytes_transferred) data = orig_read_func(amt) read_length = len(data) if self.callback_processor: self.callback_processor.Progress(read_length) if self.outer_digesters: for alg in self.outer_digesters: self.outer_digesters[alg].update(data) return data orig_response.read = read return orig_response return DownloadCallbackConnection def WrapDownloadHttpRequest(download_http): """Overrides download request functions for an httplib2.Http object. Args: download_http: httplib2.Http.object to wrap / override. Returns: Wrapped / overridden httplib2.Http object. """ # httplib2 has a bug (https://github.com/httplib2/httplib2/issues/75) where # custom connection_type is not respected after redirects. This function is # copied from httplib2 and overrides the request function so that the # connection_type is properly passed through (everything here should be # identical to the _request method in httplib2, with the exception of the line # below marked by the "BUGFIX" comment). # pylint: disable=protected-access,g-inconsistent-quotes,unused-variable # pylint: disable=g-equals-none,g-doc-return-or-yield # pylint: disable=g-short-docstring-punctuation,g-doc-args # pylint: disable=too-many-statements # yapf: disable def OverrideRequest(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey): """Do the actual request using the connection object. Also follow one level of redirects if necessary. """ auths = ([(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)]) auth = auths and sorted(auths)[0][1] or None if auth: auth.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers) if auth: if auth.response(response, body): auth.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers) response._stale_digest = 1 if response.status == 401: for authorization in self._auth_from_challenge( host, request_uri, headers, response, content): authorization.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers) if response.status != 401: self.authorizations.append(authorization) authorization.response(response, body) break if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303): if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: # Pick out the location header and basically start from the beginning # remembering first to strip the ETag header and decrement our 'depth' if redirections: if 'location' not in response and response.status != 300: raise httplib2.RedirectMissingLocation( "Redirected but the response is missing a Location: header.", response, content) # Fix-up relative redirects (which violate an RFC 2616 MUST) if 'location' in response: location = response['location'] (scheme, authority, path, query, fragment) = parse_uri(location) if authority is None: response['location'] = urllib.parse.urljoin(absolute_uri, location) if response.status == 301 and method in ["GET", "HEAD"]: response['-x-permanent-redirect-url'] = response['location'] if 'content-location' not in response: response['content-location'] = absolute_uri httplib2._updateCache(headers, response, content, self.cache, cachekey) if 'if-none-match' in headers: del headers['if-none-match'] if 'if-modified-since' in headers: del headers['if-modified-since'] if ('authorization' in headers and not self.forward_authorization_headers): del headers['authorization'] if 'location' in response: location = response['location'] old_response = copy.deepcopy(response) if 'content-location' not in old_response: old_response['content-location'] = absolute_uri redirect_method = method if response.status in [302, 303]: redirect_method = "GET" body = None (response, content) = self.request( location, redirect_method, body=body, headers=headers, redirections=redirections-1, # BUGFIX (see comments at the top of this function): connection_type=conn.__class__) response.previous = old_response else: raise httplib2.RedirectLimit( "Redirected more times than redirection_limit allows.", response, content) elif response.status in [200, 203] and method in ["GET", "HEAD"]: # Don't cache 206's since we aren't going to handle byte range # requests if 'content-location' in response: response['content-location'] = absolute_uri httplib2._updateCache(headers, response, content, self.cache, cachekey) return (response, content) # Wrap download_http so we do not use our custom connection_type # on POSTS, which are used to refresh oauth tokens. We don't want to # process the data received in those requests. request_orig = download_http.request def NewRequest(uri, method='GET', body=None, headers=None, redirections=httplib2.DEFAULT_MAX_REDIRECTS, connection_type=None): if method == 'POST': return request_orig(uri, method=method, body=body, headers=headers, redirections=redirections, connection_type=None) else: return request_orig(uri, method=method, body=body, headers=headers, redirections=redirections, connection_type=connection_type) # Replace the request methods with our own closures. download_http._request = types.MethodType(OverrideRequest, download_http) download_http.request = NewRequest return download_http class HttpWithNoRetries(httplib2.Http): """httplib2.Http variant that does not retry. httplib2 automatically retries requests according to httplib2.RETRIES, but in certain cases httplib2 ignores the RETRIES value and forces a retry. Because httplib2 does not handle the case where the underlying request body is a stream, a retry may cause a non-idempotent write as the stream is partially consumed and not reset before the retry occurs. Here we override _conn_request to disable retries unequivocally, so that uploads may be retried at higher layers that properly handle stream request bodies. """ def _conn_request(self, conn, request_uri, method, body, headers): # pylint: disable=too-many-statements try: if hasattr(conn, 'sock') and conn.sock is None: conn.connect() conn.request(method, request_uri, body, headers) except socket.timeout: raise except socket.gaierror: conn.close() raise httplib2.ServerNotFoundError('Unable to find the server at %s' % conn.host) except httplib2.ssl.SSLError: conn.close() raise except socket.error as e: err = 0 if hasattr(e, 'args'): err = getattr(e, 'args')[0] else: err = e.errno if err == httplib2.errno.ECONNREFUSED: # Connection refused raise except http_client.HTTPException: conn.close() raise try: response = conn.getresponse() except (socket.error, http_client.HTTPException): conn.close() raise else: content = '' if method == 'HEAD': conn.close() else: content = response.read() response = httplib2.Response(response) if method != 'HEAD': # pylint: disable=protected-access content = httplib2._decompressContent(response, content) return (response, content) class HttpWithDownloadStream(httplib2.Http): """httplib2.Http variant that only pushes bytes through a stream. httplib2 handles media by storing entire chunks of responses in memory, which is undesirable particularly when multiple instances are used during multi-threaded/multi-process copy. This class copies and then overrides some httplib2 functions to use a streaming copy approach that uses small memory buffers. Also disables httplib2 retries (for reasons stated in the HttpWithNoRetries class doc). """ def __init__(self, *args, **kwds): self._stream = None self._logger = logging.getLogger() super(HttpWithDownloadStream, self).__init__(*args, **kwds) @property def stream(self): return self._stream @stream.setter def stream(self, value): self._stream = value # pylint: disable=too-many-statements def _conn_request(self, conn, request_uri, method, body, headers): try: if hasattr(conn, 'sock') and conn.sock is None: conn.connect() conn.request(method, request_uri, body, headers) except socket.timeout: raise except socket.gaierror: conn.close() raise httplib2.ServerNotFoundError('Unable to find the server at %s' % conn.host) except httplib2.ssl.SSLError: conn.close() raise except socket.error as e: err = 0 if hasattr(e, 'args'): err = getattr(e, 'args')[0] else: err = e.errno if err == httplib2.errno.ECONNREFUSED: # Connection refused raise except http_client.HTTPException: # Just because the server closed the connection doesn't apparently mean # that the server didn't send a response. conn.close() raise try: response = conn.getresponse() except (socket.error, http_client.HTTPException) as e: conn.close() raise else: content = '' if method == 'HEAD': conn.close() response = httplib2.Response(response) elif method == 'GET' and response.status in (http_client.OK, http_client.PARTIAL_CONTENT): content_length = None if hasattr(response, 'msg'): content_length = response.getheader('content-length') http_stream = response bytes_read = 0 while True: new_data = http_stream.read(TRANSFER_BUFFER_SIZE) if new_data: if self.stream is None: raise apitools_exceptions.InvalidUserInputError( 'Cannot exercise HttpWithDownloadStream with no stream') text_util.write_to_fd(self.stream, new_data) bytes_read += len(new_data) else: break if (content_length is not None and long(bytes_read) != long(content_length)): # The input stream terminated before we were able to read the # entire contents, possibly due to a network condition. Set # content-length to indicate how many bytes we actually read. self._logger.log( logging.DEBUG, 'Only got %s bytes out of content-length %s ' 'for request URI %s. Resetting content-length to match ' 'bytes read.', bytes_read, content_length, request_uri) # Failing to delete existing headers before setting new values results # in the header being set twice, see https://docs.python.org/3/library/email.compat32-message.html#email.message.Message.__setitem__. # This trips apitools up when executing a retry, so the line below is # essential: del response.msg['content-length'] response.msg['content-length'] = str(bytes_read) response = httplib2.Response(response) else: # We fall back to the current httplib2 behavior if we're # not processing download bytes, e.g., it's a redirect, an # oauth2client POST to refresh an access token, or any HTTP # status code that doesn't include object content. content = response.read() response = httplib2.Response(response) # pylint: disable=protected-access content = httplib2._decompressContent(response, content) return (response, content) # pylint: enable=too-many-statements utils/__pycache__/text_util.cpython-39.pyc 0000644 00000031171 15025013616 0014570 0 ustar 00 a +(Wg�3 � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlmZ ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lm Z ej r�e Z ddddddd�Z edd� �Z!dd� Z"dd� Z#dd� Z$d d � Z%d d � Z&d d!� Z'd"d#� Z(d$d%� Z)d&d'� Z*d(d)� Z+d*d+� Z,d,d-� Z-d.d/� Z.d0d1� Z/d2d3� Z0d4d5� Z1d8d6d7�Z2dS )9z<Shared utility structures and methods for manipulating text.� )�absolute_import)�print_function)�division)�unicode_literalsN)�urllib)�range��CommandException)� LazyWrapper)�UTF8)� WINDOWS_1252)� IS_CP1252ZCOLDLINEZ DURABLE_REDUCED_AVAILABILITYZNEARLINEZSTANDARDZARCHIVE)ZCLZDRA�NL�SZSTD�Ac C s t �d�S )Nz,^(?P<maj>\d+)(\.(?P<min>\d+)(?P<suffix>.*))?)�re�compile� r r �$/opt/gsutil/gslib/utils/text_util.py�<lambda>8 � r c C sb t j�| �\}}}}}t jj|dd�}|�||f� d�dd� |D ��} t j�|||| |f�} | S )a� Adds a query parameter to a URL string. Appends a query parameter to the query string portion of a url. If a parameter with the given name was already present, it is not removed; the new name/value pair will be appended to the end of the query string. It is assumed that all arguments will be of type `str` (either ASCII or UTF-8 encoded) or `unicode`. Note that this method performs no URL-encoding. It is the caller's responsibility to ensure proper URL encoding of the entire URL; i.e. if the URL is already URL-encoded, you should pass in URL-encoded values for param_name and param_value. If the URL is not URL-encoded, you should not pass in URL-encoded parameters; instead, you could perform URL-encoding using the URL string returned from this function. Args: url_str: (str or unicode) String representing the URL. param_name: (str or unicode) String key of the query parameter. param_value: (str or unicode) String value of the query parameter. Returns: (str or unicode) A string representing the modified url, of type `unicode` if the url_str argument was a `unicode`, otherwise a `str` encoded in UTF-8. T)�keep_blank_values�&c S s g | ]\}}d ||f �qS )z%s=%sr )�.0�k�vr r r � <listcomp>W r z&AddQueryParamToUrl.<locals>.<listcomp>)r �parse�urlsplit� parse_qsl�append�join� urlunsplit) �url_strZ param_nameZ param_value�scheme�netloc�pathZ query_str�fragmentZ query_paramsZ new_query_strZnew_urlr r r �AddQueryParamToUrl; s �r( c C s� t � �t| ��}t � �t|��}|r(|s,dS t|�d��}|�d�rRt|�d��nd}|�d�}t|�d��}|�d�r�t|�d��nd}|�d�} ||kr�dS ||kr�||kr�dS ||kr�t| �o�| dfS dS ) al Compares the first and second gsutil version strings. For example, 3.33 > 3.7, and 4.1 is a greater major version than 3.33. Does not handle multiple periods (e.g. 3.3.4) or complicated suffixes (e.g., 3.3RC4 vs. 3.3RC5). A version string with a suffix is treated as less than its non-suffix counterpart (e.g. 3.32 > 3.32pre). Args: first: First gsutil version string. second: Second gsutil version string. Returns: (g, m): g is True if first known to be greater than second, else False. m is True if first known to be greater by at least 1 major version, else False. )FFZmaj�minr �suffix)TT)TFF)�VERSION_MATCHER�match�str�int�group�bool) �first�secondZm1Zm2Z major_ver1Z minor_ver1Z suffix_ver1Z major_ver2Z minor_ver2Z suffix_ver2r r r �CompareVersions^ s$ r3 c c s | D ]}d| V qdS )z:A generator that adds '**' to each url string in url_strs.z%s**Nr )Zurl_strsr# r r r � ConvertRecursiveToFlatWildcard� s r4 c C s& t �t| �dd� �d��}t�|�S )a8 Decodes an encoded python long into an ASCII string. This is used for modeling S3 version_id's as apitools generation. Args: long_to_convert: long to convert to ASCII string. If this is already a string, it is simply returned. Returns: String decoded from the input long. � N�L)�binascii� unhexlify�hex�rstrip�six� ensure_str)Zlong_to_convertZunhexedr r r �DecodeLongAsString� s r= c C s t �t�| �d�}t|d�S )a) Encodes an ASCII string as a python long. This is used for modeling S3 version_id's as apitools generation. Because python longs can be arbitrarily large, this works. Args: string_to_convert: ASCII string to convert to a long. Returns: Long that represents the input string. � hex_codec� )�codecs�encoder; � ensure_binary�long)Zstring_to_convertZ hex_bytestrr r r �EncodeStringAsLong� s rD c C s t rt�| t�S t�| t�S dS )a� Attempts to detect Windows CP1252 encoding and convert to UTF8. Windows doesn't provide a way to set UTF-8 for string encodings; you can set the system locale (see http://windows.microsoft.com/en-us/windows/change-system-locale#1TC=windows-7) but that takes you to a "Change system locale" dropdown that just lists languages (e.g., "English (United States)". Instead, we're forced to check if a encoding as UTF8 raises an exception and if so, try converting from CP1252 to Unicode. Args: input_str: (str or bytes) The input string. Returns: (unicode) The converted string or the original, if conversion wasn't needed. N)r r; � ensure_textr r �Z input_strr r r �FixWindowsEncodingIfNeeded� s rG c C s t �| ��t�pt �| j�S )z8Returns a short Unicode string describing the exception.)r; � text_typerA r � __class__)�excr r r �GetPrintableExceptionString� s rK c C s t dd� | D ��st|��dS )ae Ensures that the string passed in consists of only ASCII values. Args: string: Union[str, unicode, bytes] Text that will be checked for ASCII values. message: Union[str, unicode, bytes] Error message, passed into the exception, in the event that the check on `string` fails. Returns: None Raises: CommandException c s s | ]}t |�d k V qdS )� N)�ord)r �cr r r � <genexpr>� r z InsistAscii.<locals>.<genexpr>N)�allr )�string�messager r r � InsistAscii� s rS c C s t | d| � dS )z�Checks for ASCII-only characters in `header`. Also constructs an error message using `header` if the check fails. Args: header: Union[str, binary, unicode] Text being checked for ASCII values. Returns: None z Invalid non-ASCII header (%s).N)rS )�headerr r r �InsistAsciiHeader� s rU c C s t |dt|�| f � dS )a; Checks for ASCII-only characters in `value`. Also constructs an error message using `header` and `value` if the check fails. Args: header: Header name, only used in error message in case of an exception. value: Union[str, binary, unicode] Text being checked for ASCII values. Returns: None z�Invalid non-ASCII value (%s) was provided for header %s. Only ASCII characters are allowed in headers other than x-goog-meta- and x-amz-meta- headersN)rS �repr)rT �valuer r r �InsistAsciiHeaderValue� s ��rX c C s | dkr| dkrt |��dS )aO Ensures that the value passed in consists of only "on" or "off" Args: value: (unicode) Unicode string that will be checked for correct text. message: Union[str, unicode, bytes] Error message passed into the exception in the event that the check on value fails. Returns: None Raises: CommandException �on�offNr )rW rR r r r � InsistOnOrOff s r[ c C s | � � } | tv rt| } | S )a� Returns a normalized form of the given storage class name. Converts the given string to uppercase and expands valid abbreviations to full storage class names (e.g 'std' would return 'STANDARD'). Note that this method does not check if the given storage class is valid. Args: sc: (str) String representing the storage class's full name or abbreviation. Returns: (str) A string representing the full name of the given storage class. )�upper�$STORAGE_CLASS_SHORTHAND_TO_FULL_NAME)Zscr r r �NormalizeStorageClass s r^ c C s | S )a� Return an UTF8-encoded string type, or None if `input_val` is None. Args: input_val: (unicode, str, or None) A string-like object or None. This method simply calls encode() on `input_val` if it is not None; if `input_val` is not of type "unicode", this will implicitly call decode() with the default encoding for strings (for Python 2, this is ASCII), then call encode(). Returns: (str) A UTF-8 encoded string, or None. r )Z input_valr r r � PrintableStr' s r_ c O s` dd� }dd� }|f i |��\}}}t �|�}t �|�}|| � }|�|�}||7 }t||� dS )a A Python 2/3 compatible analogue to the print function. This function writes text to a file descriptor as the builtin print function would, favoring unicode encoding. Aguments and return values are the same as documented in the Python 2 print function. c [ s` t �dddtjfg�}| �� D ]8\}}||vrNd}t|�|d�|�� ����q |||< q |� � S )a Validates keyword arguments that would be used in Print Valid keyword arguments, mirroring print(), are 'sep', 'end', and 'file'. These must be of types string, string, and file / file interface respectively. Returns the above kwargs of the above types. )�sep� )�end� �filez9{} is not a valid keyword argument. Please use one of: {}ra ) � collections� OrderedDict�sys�stdout�items�KeyError�formatr! �keys�values)�kwargsZexpected_keywords�keyrW Z error_msgr r r � _get_argsA s � � z print_to_fd.<locals>._get_argsc W sT g }| D ]F}t |tjtjf�s&t|�}t |tj�r>|�|� q|�t�|�� q|S )zCGets a `bytes` string for each item in a list of printable objects.)� isinstancer; � binary_typerH r- r rB )�objectsZ byte_objects�itemr r r �_get_byte_stringsX s z&print_to_fd.<locals>._get_byte_stringsN)r; rB r! � write_to_fd)rs rn rp ru r` rb rd �datar r r � print_to_fd7 s rx c C s� t jr| �|� dS t|t�rtt| d�r2d| jv s>t| tj�rJ| �|� q�t| d�rb| j �|� q�| �t � |�� n&d| jv r�| �t � |�� n | �|� dS )zGWrite given data to given file descriptor, doing any conversions neededN�mode�b�buffer) r; �PY2�writerq �bytes�hasattrry �io�BytesIOr{ rE rB )�fdrw r r r rv r s rv c C s t �dd| �S )z4Returns the input string with all \n and \r removed.z[\r\n]� )r �subrF r r r �RemoveCRLFFromString� s r� c C s6 t �|� tdd� t| �D ��}t�|�}t �� |S )af Generates binary string representation of a list of ASCII characters. Args: size: Integer quantity of characters to generate. seed: A seed may be specified for deterministic behavior. Int 0 is used as the default value. Returns: Binary encoded string representation of a list of characters of length equal to size argument. c S s g | ]}t �tj��qS r )�random�choicerQ � ascii_letters)r �_r r r r � r z*get_random_ascii_chars.<locals>.<listcomp>)r� �seedr- r r; rB )�sizer� �contentsr r r �get_random_ascii_chars� s r� )r )3�__doc__� __future__r r r r r7 r@ �osrg r� r �localere r� r; rQ � six.movesr r �gslib.exceptionr Zgslib.lazy_wrapperr �gslib.utils.constantsr r Zgslib.utils.system_utilr �PY3r. rC r] r+ r( r3 r4 r= rD rG rK rS rU rX r[ r^ r_ rx rv r� r� r r r r �<module> sf � �#*; utils/__pycache__/constants.cpython-39.pyc 0000644 00000005604 15025013616 0014565 0 ustar 00 a +(WgS � @ sF d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd l m Z d Z d Zd e Zd Zej�ej�dd��ZdZdZd ZdZdZde ZdZde Z ej Z dZ dZ dZ!dZ"d Z#e#e Z$dZ%dZ&e%e&gZ'd Z(d e Z)d e*ej+��,� v �r d e Z-nd e Z-d Z.d!Z/dZ0G d"d#� d#e1�Z2dS )$a� Shared, hard-coded constants. A constant should not be placed in this file if: - it requires complicated or conditional logic to initialize. - it requires importing any modules outside of the Python standard library. This helps reduce dependency graph complexity and the chance of cyclic deps. - it is only used in one file (in which case it should be defined within that module). - it semantically belongs somewhere else (e.g. 'BYTES_PER_KIB' would belong in unit_util.py). � )�absolute_import)�print_function)�division)�unicode_literalsN)�ONE_GIB)�ONE_KIB)�ONE_MIB� � � Zv1�~z.gsutilzgs://pub/gsutil.tar.gzzgs://pub/gsutil4.tar.gz� � � i� �d z:https://pub.storage.googleapis.com/gsutil_ReleaseNotes.txtZ CLOUDSDK_CORE_REQUEST_REASONzx-goog-request-reasonz13b89a6b5-b55a-4900-8c44-0b0a2f5eab43-s3-AclMarkerz4eadeeee8-fa8c-49bb-8a7d-0362215932d8-s3-DeleteMarker�< � �win32�@ zutf-8�cp1252c @ s( e Zd ZdZdZdZdZdZdZdZ dS ) �Scopesz'Enum class for auth scopes, as unicode.z.https://www.googleapis.com/auth/cloud-platformz8https://www.googleapis.com/auth/cloud-platform.read-onlyz7https://www.googleapis.com/auth/devstorage.full_controlz4https://www.googleapis.com/auth/devstorage.read_onlyz5https://www.googleapis.com/auth/devstorage.read_writez/https://www.googleapis.com/auth/accounts.reauthN) �__name__� __module__� __qualname__�__doc__ZCLOUD_PLATFORMZCLOUD_PLATFORM_READ_ONLYZ FULL_CONTROLZ READ_ONLYZ READ_WRITEZREAUTH� r r �$/opt/gsutil/gslib/utils/constants.pyr z s �r )3r � __future__r r r r �os�sys�sixZgslib.utils.unit_utilr r r �DEBUGLEVEL_DUMP_REQUESTS�%DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADSZDEFAULT_FILE_BUFFER_SIZEZ DEFAULT_GCS_JSON_API_VERSION�path� expanduser�joinZDEFAULT_GSUTIL_STATE_DIRZGSUTIL_PUB_TARBALLZGSUTIL_PUB_TARBALL_PY2ZIAM_POLICY_VERSION�IMPERSONATE_SERVICE_ACCOUNT�LONG_RETRY_WARN_SECZ"MAX_UPLOAD_COMPRESSION_BUFFER_SIZEZ MIN_ACCEPTABLE_OPEN_FILES_LIMITZMIN_SIZE_COMPUTE_LOGGING�MAXSIZEZNO_MAXZNUM_OBJECTS_PER_LIST_PAGEZRELEASE_NOTES_URLZREQUEST_REASON_ENV_VARZREQUEST_REASON_HEADER_KEYZRESUMABLE_THRESHOLD_MIBZRESUMABLE_THRESHOLD_BZS3_ACL_MARKER_GUIDZS3_DELETE_MARKER_GUIDZS3_MARKER_GUIDSZSSL_TIMEOUT_SECZSTART_CALLBACK_PER_BYTES�str�platform�lowerZTRANSFER_BUFFER_SIZE�UTF8� WINDOWS_1252ZXML_PROGRESS_CALLBACKS�objectr r r r r �<module> sT utils/__pycache__/acl_helper.cpython-39.pyc 0000644 00000022604 15025013616 0014646 0 ustar 00 a +(Wg�2 � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl m Z G d d � d e �Z G d d � d e �ZG d d� de �Zdd� ZdS )z7Contains helper objects for changing and deleting ACLs.� )�absolute_import)�print_function)�division)�unicode_literalsN)�CommandException)�storage_v1_messagesc @ s e Zd ZdZdZdZdS )� ChangeType�User�Group�ProjectN)�__name__� __module__� __qualname__�USER�GROUP�PROJECT� r r �%/opt/gsutil/gslib/utils/acl_helper.pyr s r c @ s� e Zd ZdZddgZddgZddgZdgZd gZee e e e Z d Z d Z e e fZ d Z d ZdZdZdZdddddddd�Zdd� Zdd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'S )(� AclChangez6Represents a logical change to an access control list.�AllAuthenticatedUsers�AllUsers�UserById� GroupById� UserByEmail� GroupByEmail� GroupByDomainr �allUsers�allAuthenticatedUsers)zproject-editors-zproject-owners-zproject-viewers-zgroup-zuser-zdomain-zproject-�READER�WRITER�OWNER)�R�WZFC�O�READ�WRITE� FULL_CONTROLc C s$ d| _ || _| �||� | �� dS )a< Creates an AclChange object. Args: acl_change_descriptor: An acl change as described in the "ch" section of the "acl" command's help. scope_type: Either ChangeType.USER or ChangeType.GROUP or ChangeType.PROJECT, specifying the extent of the scope. � N)� identifier�raw_descriptor�_Parse� _Validate)�selfZacl_change_descriptor� scope_typer r r �__init__@ s zAclChange.__init__c C s d� | j| j| j�S )NzAclChange<{0}|{1}|{2}>)�formatr- �permr( �r, r r r �__str__O s �zAclChange.__str__c C s� dd� }|� d�dkr$td�|���|�d�\}}|�� }|| jv rR| j| | _n|| _||�}|dkr|d�|�| _|| _nZ|dv r�d �||�| _|| _n<|d kr�d | _n,|d kr�d | _n |d kr�d | _|| _n|| _d S )z Parses an ACL Change descriptor.c S s@ ddddddd�}|� � D ] \}}t�|| tj�r| S qd S )Nz!^(AllAuthenticatedUsers|AllAuth)$z^(AllUsers|All)$z ^.+@.+\..+$z^[0-9A-Fa-f]{64}$z^[^@]+\.[^@]+$z (owners|editors|viewers)\-.+$)r r �Email�Id�Domainr )�items�re�match� IGNORECASE)�textZre_mapZ type_string�regexr r r �_ClassifyScopeIdentifierV s �z2AclChange._Parse.<locals>._ClassifyScopeIdentifier�:� z%{0} is an invalid change description.r5 z {0}ByDomain)r3 r4 z{0}By{1}r r r N) �countr r/ �split�upper� permission_shorthand_mappingr0 r- r( )r, Zchange_descriptorr- r< Z scope_stringZ perm_tokenZ scope_classr r r r* S s2 � zAclChange._Parsec s� � fdd�}� j � jvr(|d�� j �� � j � jv rJ� jrJ|d�� j �� � j � jv rl� jsl|d�� j �� � j � jv r�� js�|d�� j �� � j � jv r�� js�|d�� j �� � j� j � � vr�d� t � j � � ��}|d �|�� d S ) z$Validates a parsed AclChange object.c s t d�� j| ���d S )Nz!{0} is not a valid ACL change {1})r r/ r) )�msgr1 r r � _ThrowError� s �z(AclChange._Validate.<locals>._ThrowErrorz {0} is not a valid scope typez{0} requires no argumentsz{0} requires an idz {0} requires an email addressz{0} requires domainz, zAllowed permissions are {0}N) r- � scope_typesr/ � public_scopesr( � id_scopes� email_scopes� domain_scopesr0 rB �values�join�set)r, rD Zpermsr r1 r r+ � s zAclChange._Validatec c s� |D ]�}| j dv r,|jr,| j|jkr,|V q| j dv rP|jrP| j|jkrP|V q| j dkrt|jrt| j|jkrt|V q| j dkr�|jr�| jd|jj|jjf kr�|V q| j dkr�|j� � | j � � kr�|V q| j dkr|j� � | j � � kr|V qdS ) aR Generator that yields entries that match the change descriptor. Args: current_acl: A list of apitools_messages.BucketAccessControls or ObjectAccessControls which will be searched for matching entries. Yields: An apitools_messages.BucketAccessControl or ObjectAccessControl. )r r �r r r r �%s-%sr r N) r- �entityIdr( �email�domain� projectTeam�team� projectNumber�entity�lower�public_entity_all_users� public_entity_all_auth_users�r, � current_acl�entryr r r �_YieldMatchingEntries� s: � � � ��� � �z AclChange._YieldMatchingEntriesc C s� | j dv r| j| j S | j dv r,| j| j S | j dkrB| j| j S | j dkrX| j| j S | j dkrh| jS | j dkrx| jS td| j ��dS ) z3Gets an appropriate entity string for an ACL grant.)r r )r r r r r r z.Add entry to ACL got unexpected scope type %s.N) r- �user_entity_prefixr( �group_entity_prefix�project_entity_prefix�domain_entity_prefixrX rW r r1 r r r � GetEntity� s �zAclChange.GetEntityc C s^ | � � }|| j|d�}| jdv r*| j|_n&| jdv r>| j|_n| jdkrP| j|_|�|� dS )z Adds an entry to current_acl.)�rolerU )r r r rM r N)ra r0 r- r( rO rP rQ �append)r, rZ � entry_classrU r[ r r r � _AddEntry� s zAclChange._AddEntryc C s |D ] }|j S t�� j S )N)� __class__�apitools_messages�ObjectAccessControl)r, rZ � acl_entryr r r �_GetEntriesClass� s zAclChange._GetEntriesClassc C s� |� d|| j|� | jdkrP|dkr@|�� r@|�d| j|� dS |dkrPtd��| �|�}t| �|��}d}|r�|D ] }|j | jkrt| j|_ |d7 }qtn| � ||� d}|� d t |�� |S ) a� Executes the described change on an ACL. Args: storage_url: StorageUrl representing the object to change. current_acl: A list of ObjectAccessControls or BucketAccessControls to permute. command_name: String name of comamnd being run (e.g., 'acl'). logger: An instance of logging.Logger. Returns: The number of changes that were made. �Executing %s %s on %sr �aclz6Skipping %s on %s, as WRITER does not apply to objectsr �defaclzUWRITER cannot be set as a default object ACL because WRITER does not apply to objectsr> � New Acl: %s) �debugr) r0 �IsObject�warningr rj �listr\ rb re �str) r, � storage_urlrZ � command_name�loggerrd �matching_entriesZ change_countr[ r r r �Execute� s. � � zAclChange.ExecuteN) r r r �__doc__rF rG rH rI Zproject_scopesrE rW rX Zpublic_entity_typesZproject_entity_prefixesr^ r] r` r_ rB r. r2 r* r+ r\ ra re rj rx r r r r r " sD �� 1 r c @ s2 e Zd ZdZddd�Zdd� Zdd� Zd d � Zd S ) �AclDelz8Represents a logical change from an access control list.r r )z All(Users)?$zAllAuth(enticatedUsers)?$c C sN d� |�| _|| _| j�� D ] \}}t�|| jtj�r || _q d| _d| _ d S )Nz-d {0}�Any�NONE) r/ r) r( � scope_regexesr6 r7 r8 r9 r- r0 )r, r( r; �scoper r r r. s zAclDel.__init__c c s� |D ]�}|j r*| j�� |j �� kr*|V q|jrL| j�� |j�� krL|V q|jrn| j�� |j�� krn|V q|jr�| j�� d�� |jj|jjf kr�|V q|j�� dkr�| jdkr�|V q|j�� dkr| jdkr|V qdS )aW Generator that yields entries that match the change descriptor. Args: current_acl: An instance of apitools_messages.BucketAccessControls or ObjectAccessControls which will be searched for matching entries. Yields: An apitools_messages.BucketAccessControl or ObjectAccessControl. rN Zallusersr Zallauthenticatedusersr N) rO r( rV rP rQ rR rS rT rU rY r r r r\ s" ��z AclDel._YieldMatchingEntriesc C sL |� d|| j|� t| �|��}|D ]}|�|� q$|� dt|�� t|�S )Nrk rn )ro r) rr r\ �removers �len)r, rt rZ ru rv rw r[ r r r rx 8 s � zAclDel.ExecuteN)r r r ry r} r. r\ rx r r r r rz s � rz c C s� g }t jt jt jd�}| D ]r\}}||v r\t||| d�}d�|�� |j�}|�||f� q|dkr~t |�}|�d|j f� q|�||f� q|S )N)z-gz-pz-u)r- zentity={},role={}z-d) r r r r r r/ ra r0 rc rz r( )�sub_optsZtranslated_sub_optsZscope_type_from_flag�flag�value�changeZ new_valuer r r �translate_sub_opts_for_shimB s � r� )ry � __future__r r r r r7 �gslib.exceptionr �"gslib.third_party.storage_apitoolsr rg �objectr r rz r� r r r r �<module> s l5 utils/__pycache__/wrapped_credentials.cpython-39.pyc 0000644 00000013473 15025013616 0016573 0 ustar 00 a +(WgT � @ s� d Z ddlZddlZddlZddlZddlmZ ddlmZ ddlmZ ddlm Z ddlm Z ddlm Z dd lm Z dd l mZ dd lmZ ddlZejjejjejjejjejjgZG d d � d ejj�Zdd� Z dd� Z dd� Z dd� Z dd� Z dS )a Classes and functions to allow google.auth credentials to be used within oauth2client. In particular, the External Account credentials don't have an equivalent in oauth2client, so we create helper methods to allow variants of this particular class to be used in oauth2client workflows. � N)�aws)� credentials)� exceptions)�external_account)� external_account_authorized_user)� identity_pool)� pluggable)�requests)� constantsc s� e Zd ZdZeeejjj �dg �Z � fdd�Z dd� Z e dd� �Z e jd d� �Z e d d � �Zejd d � �Z� fd d�Zedd� �Zedd� �Zedd� �Z� ZS )�WrappedCredentialszWA utility class to use Google Auth credentials in place of oauth2client credentials. �_basec s| || _ t|tj�r$| j j}d}d}n.t|tj�rJ| j j}| j j}| j j}nt d��t t | �j | j j |||| j jddd� dS )z�Initializes oauth2client credentials based on underlying Google Auth credentials. Args: external_account_creds: subclass of google.auth.external_account.Credentials NzInvalid Credentials)� access_token� client_id� client_secret� refresh_token� token_expiry� token_uri� user_agent)r � isinstancer � CredentialsZ _audiencer r r r � TypeError�superr �__init__�token�expiry)�self� base_credsr r r �� __class__� �./opt/gsutil/gslib/utils/wrapped_credentials.pyr 7 s$ �zWrappedCredentials.__init__c C s* | j �t�� � | jd ur&| j�| � d S �N)r �refreshr �Request�store� locked_put)r �httpr r r �_do_refresh_requestQ s z&WrappedCredentials._do_refresh_requestc C s | j jS r! �r r �r r r r r V s z WrappedCredentials.access_tokenc C s || j _d S r! r( �r �valuer r r r Z s c C s | j jS r! �r r r) r r r r ^ s z WrappedCredentials.token_expiryc C s || j _d S r! r, r* r r r r b s c sJ t � �� }t�|�}t�| jj�|d< | jj|d< t| j �|d< t� |�S )z�Utility function that creates JSON repr. of a Credentials object. Returns: string, a JSON representation of this instance, suitable to pass to from_json(). r r r ) r �to_json�json�loads�copyr �infor � _parse_expiryr �dumps)r Zserialized_dataZdeserialized_datar r r r- f s zWrappedCredentials.to_jsonc C s t |�}| |�S r! )�+_get_external_account_credentials_from_file��cls�filename�credsr r r �for_external_accountu s z'WrappedCredentials.for_external_accountc C s t |�}| |�S r! )�;_get_external_account_authorized_user_credentials_from_filer5 r r r �$for_external_account_authorized_userz s �z7WrappedCredentials.for_external_account_authorized_userc C s� t �|�}|�d�}d}|�d�dkr0t|�}n|�d�dkrFt|�}| |�}|�d�|_|�d�r�t|d tj�s�z tj�|d t j j �|d< W n t y� d|d< Y n0 |�d�|_ |S )a Instantiate a Credentials object from a JSON description of it. The JSON should have been produced by calling .to_json() on the object. Args: data: dict, A deserialized JSON object. Returns: An instance of a Credentials subclass. r N�typer r r r )r. r/ �get�+_get_external_account_credentials_from_info�;_get_external_account_authorized_user_credentials_from_infor r �datetime�strptime� oauth2client�client� EXPIRY_FORMAT� ValueErrorr )r6 � json_data�data�baser r8 r r r � from_json� s, � � � z WrappedCredentials.from_json)�__name__� __module__� __qualname__�__doc__� frozenset�listrB rC �OAuth2Credentials�NON_SERIALIZED_MEMBERSr r' �propertyr �setterr r- � classmethodr9 r; rI � __classcell__r r r r r 0 s. �� r c C sd | � d�dkr tjj| td�S | � d�d urP| � d�� d�d urPtjj| td�S tjj| td�S d S )N�subject_token_typez+urn:ietf:params:aws:token-type:aws4_request)�scopes�credential_source� executable)r= r r � from_info�DEFAULT_SCOPESr r �r1 r r r r> � s ���r> c C sF t j| ddd��"}t�|�}t|�W d � S 1 s80 Y d S �N�rzutf-8)�encoding)�io�openr. �loadr> �r7 Z json_filerG r r r r4 � s r4 c C s t j�| �S r! )r r rZ r\ r r r r? � s r? c C sF t j| ddd��"}t�|�}t|�W d � S 1 s80 Y d S r] )r` ra r. rb r? rc r r r r: � s r: c C s& | r t | tj�r | �tjj�S d S d S r! )r r@ �strftimerB rC rD )r r r r r2 � s r2 )!rM r0 r@ r` r. � google.authr r r r r r r Zgoogle.auth.transportr � gslib.utilsr rB �Scopes�CLOUD_PLATFORM�CLOUD_PLATFORM_READ_ONLY� FULL_CONTROL� READ_ONLY� READ_WRITEr[ rC rP r r> r4 r? r: r2 r r r r �<module> s4 � t utils/__pycache__/update_util.cpython-39.pyc 0000644 00000006623 15025013616 0015072 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZ dd lmZ dd lmZ dd lmZ e jfd d �Zdd� Zdd� ZdS )z<Shared utility methods for the update command and its tests.� )�absolute_import)�print_function)�division)�unicode_literalsN)�IS_OSX)�CommandException)�StorageUrlFromString)�GSUTIL_PUB_TARBALL)�GSUTIL_PUB_TARBALL_PY2c C s ddg}z~t tj�| d�d��V}|D ]@}|�d�rF|�|�� d � q$t�d|�r$|�|�� d � q$W d � n1 sz0 Y W n$ t y� t � � � d | � Y dS 0 d }t� | �D ]D}|�d �s�||v r�q�||vr�td �t�d tj�tj|� ����q�dS )aI Disallows the update command if files not in the gsutil distro are found. This prevents users from losing data if they are in the habit of running gsutil from the gsutil directory and leaving data in that directory. This will also detect someone attempting to run gsutil update from a git repo, since the top-level directory will contain git files and dirs (like .git) that are not distributed with gsutil. Args: directory: (str) The directory to use this functionality on. Raises: CommandException: if files other than those distributed with gsutil found. z MANIFEST.in� third_party�rzinclude ���zrecursive-include \w+ \*� Nz7MANIFEST.in not found in %s. Skipping user data check. )z .coveragez .DS_Store�.githubz .style.yapfz .yapfignore� __pycache__r z.pyc� z�A file (%s) that is not distributed with gsutil was found in the gsutil directory. The update command cannot run with user data in the gsutil directory.)�open�os�path�join� startswith�append�split�re�match�IOError�logging� getLogger�warn�listdir�endswithr �textwrap�wrap�gslib� GSUTIL_DIR)� directoryZmanifest_lines�fp�lineZ addl_excludes�filename� r) �&/opt/gsutil/gslib/utils/update_util.py� DisallowUpdateIfDataInGsutilDir' s4 6 � ���r+ c C s^ t |�}|�� rZ| j|j|j|jdgd�}|jrZ|jjrZ|jjD ]}|jdkr@|j S q@dS )a� Looks up the gsutil version of the specified gsutil tarball URL. Version is specified in the metadata field set on that object. Args: gsutil_api: gsutil Cloud API to use when retrieving gsutil tarball. url_str: tarball URL to retrieve (such as 'gs://pub/gsutil.tar.gz'). Returns: Version string if URL is a cloud URL containing x-goog-meta-gsutil-version metadata, else None. �metadata)�provider�fieldsZgsutil_versionN) r � IsCloudUrl�GetObjectMetadata� bucket_name� object_name�schemer, �additionalProperties�key�value)� gsutil_api�url_str�url�obj�propr) r) r* �LookUpGsutilVersione s � r< c C s t jjdkrtS tS )z�Returns the appropriate gsutil pub tarball based on the Python version. Returns: The storage_uri of the appropriate pub tarball. � )�sys� version_info�majorr r r) r) r) r* �GsutilPubTarball~ s rA )�__doc__� __future__r r r r r r r r! r> r# Zgslib.utils.system_utilr �gslib.exceptionr �gslib.storage_urlr �gslib.utils.constantsr r r$ r+ r< rA r) r) r) r* �<module> s$ > utils/__pycache__/retry_util.cpython-39.pyc 0000644 00000005001 15025013616 0014742 0 ustar 00 a +(Wgu � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd lmZ ejZdd d �ZdS )zCShared utility structures and methods for handling request retries.� )�absolute_import)�print_function)�division)�unicode_literalsN)� http_wrapper)�thread_message)� constants)�retry_decoratorFc s$ � fdd�}� fdd�}| r |S |S )a Higher-order function allowing retry handler to access global status queue. Args: is_data_transfer: If True, disable retries in apitools. status_queue: The global status queue. Returns: A retry function for retryable errors in apitools. c sX | j dur$| j tjkr$t�d| j� � rJ� �tj| j t � � | j| j d�� t � | � dS )a� Exception handler for http failures in apitools. If the user has had to wait several seconds since their first request, print a progress message to the terminal to let them know we're still retrying, then perform the default retry logic and post a gslib.thread_message.RetryableErrorMessage to the global status queue. Args: retry_args: An apitools ExceptionRetryArgs tuple. Nz Retrying request, attempt #%d...�� num_retries�total_wait_sec) r r �LONG_RETRY_WARN_SEC�logging�infor �putr �RetryableErrorMessage�exc�timer �)HandleExceptionsAndRebuildHttpConnections�� retry_args�� status_queue� �%/opt/gsutil/gslib/utils/retry_util.py�WarnAfterManyRetriesHandler, s ���z8LogAndHandleRetries.<locals>.WarnAfterManyRetriesHandlerc s4 � r&� � tj| jt�� | j| jd�� t�| � dS )a- Exception handler that disables retries in apitools data transfers. Post a gslib.thread_message.RetryableErrorMessage to the global status queue. We handle the actual retries within the download and upload functions. Args: retry_args: An apitools ExceptionRetryArgs tuple. r N) r r r r r r r r �RethrowExceptionHandlerr r r r � RetriesInDataTransferHandlerC s ��z9LogAndHandleRetries.<locals>.RetriesInDataTransferHandlerr )�is_data_transferr r r r r r �LogAndHandleRetries! s r )FN)�__doc__� __future__r r r r r r �apitools.base.pyr �gslibr � gslib.utilsr r �retry�Retryr r r r r �<module> s utils/__pycache__/boto_util.cpython-39.pyc 0000644 00000040724 15025013616 0014553 0 ustar 00 a +(Wgh � @ sT d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddl Z ddlmZ dd lmZ dd lmZ dd lmZ ddlZdd lmZ dd lmZ ddlmZ ddl m Z ddl m Z ddl m Z ddl m Z ddl!m"Z" ddl!m#Z# ddl$Z$ddl%m&Z& e j'�r<e(Z)e �*ddd�Z+e �*ddd�Z,da-da.dd� Z/d d � Z0d d � Z1d d!� Z2d"d#� Z3d$d%� Z4d&d'� Z5d(d)� Z6d*d+� Z7d,d-� Z8d.d/� Z9d0d1� Z:d2d3� Z;d4d5� Z<d6d7� Z=e$j>fd8d9�Z?d:d;� Z@d<d=� ZAd>d?� ZBd@dA� ZCdBdC� ZDdDdE� ZEdFdG� ZFdHdI� ZGdJdK� ZHdLdM� ZIdUdOdP�ZJdQdR� ZKdSdT� ZLdS )Vz�Shared utility structures and methods that require importing boto. This module also imports httplib2 (as it is Boto's http transport and closely tied to some of Boto's core functionality) and oauth2client. � )�absolute_import)�print_function)�division)�unicode_literalsN)�config)�NoAuthHandlerFound)� GSConnection)�Provider)�BotoConfigLocations)�context_config)�CommandException)� system_util)� DEFAULT_GCS_JSON_API_VERSION)�DEFAULT_GSUTIL_STATE_DIR)�SSL_TIMEOUT_SEC)�UTF8)�HumanReadableToBytes)�ONE_MIB)� HAS_CRYPTO�Boto� is_secureT�https_validate_certificatesc C s� t j�ddd�} | dkr dS | s�ts�tj�tj�tj dd��atj� t�s�t � dd�}|sbt d ��t�|�}tjd d d �\}}t�|d �}|�|� |�� |atat} | S )z�Configures and returns the CA Certificates file. If one is already configured, use it. Otherwise, use the cert roots distributed with gsutil. Returns: string filename of the certs file to use. r Zca_certificates_fileN�system�dataz cacerts.txt�gslibzdata/cacerts.txtzACertificates file not found. Please reinstall gsutil from scratchz.txtzgsutil-cacerts)�suffix�prefix�w)�botor �get�configured_certs_file�os�path�abspath�joinr � GSLIB_DIR�exists�pkgutil�get_datar �six� ensure_str�tempfile�mkstemp�fdopen�write�close�temp_certs_file)Z certs_fileZ certs_data�fd�fname�f� r4 �$/opt/gsutil/gslib/utils/boto_util.py�ConfigureCertsFileB s* � r6 c C sZ t � sVt�dd�rJtsJt�� r4td�t� d����qVtd�t� d����n ddl m } dS ) zASets up no-op auth handler if no boto credentials are configured.� Credentials�gs_service_client_id� aI Your gsutil is configured with an OAuth2 service account, but you do not have PyOpenSSL or PyCrypto 2.6 or later installed. Service account authentication requires one of these libraries; please reactivate your service account via the gcloud auth command and ensure any gcloud packages necessary for service accounts are present.a( Your gsutil is configured with an OAuth2 service account, but you do not have PyOpenSSL or PyCrypto 2.6 or later installed. Service account authentication requires one of these libraries; please install either of them to proceed, or configure a different type of credentials with "gsutil config".r ��no_op_auth_pluginN) �HasConfiguredCredentialsr � has_optionr r �InvokedViaCloudSdkr r$ �textwrap�wrapr r; r: r4 r4 r5 �ConfigureNoOpAuthIfNeededl s ��� �� rA c C s t S �N)r r4 r4 r4 r5 � GetCertsFile� s rC c C s t r t gS g S )zHReturns a list of temp files to delete (if possible) when program exits.)r0 r4 r4 r4 r5 �GetCleanupFiles� s rD c C s� t } dtjv r | �tjd � tj�tj�d�dd�}tj�|�rL| �|� g }| D ]P}z8t|d�� |�|� W d � n1 s�0 Y W qT t y� Y qT0 qT|S )zFReturns a list of the path(s) to the boto config file(s) to be loaded.�AWS_CREDENTIAL_FILE�~z.aws� credentials�rN) r r! �environ�appendr" r$ � expanduser�isfile�open�IOError)Zpotential_config_pathsZ aws_cred_file�readable_config_pathsr" r4 r4 r5 �GetConfigFilePaths� s , rP c C s t � } t| �dkr | �d� | S )zGLike GetConfigFilePaths but returns a not-found message if paths empty.r zNo config found)rP �lenrJ )rO r4 r4 r5 �GetFriendlyConfigFilePaths� s rR c C s t �ddt�} t�| � | S )a� Returns the location of the directory for gsutil state files. Certain operations, such as cross-process credential sharing and resumable transfer tracking, need a known location for state files which are created by gsutil as-needed. This location should only be used for storing data that is required to be in a static location. Returns: Path to directory for gsutil static state files. �GSUtilZ state_dir)r r r r �CreateDirIfNeeded)Zconfig_file_dirr4 r4 r5 �GetGsutilStateDir� s rU c C s t j�t� d�S )NZ credstore2�r! r" r$ rU r4 r4 r4 r5 �GetCredentialStoreFilename� s rW c C s t j�t� d�S )NZ gcecredcacherV r4 r4 r4 r5 � GetGceCredentialCacheFilename� s rX c C s t �ddt�S )NrS �json_api_version)r r r r4 r4 r4 r5 �GetGcsJsonApiVersion� s rZ c C sP t �ddtd��} | dkr$td�} n(| td� dkrL| td�| td� 7 } | S )NrS �json_resumable_chunk_sizei @r i )r �getint�long)� chunk_sizer4 r4 r5 �GetJsonResumableChunkSize� s � r_ c C s t j�t� d�S )Nz.last_software_update_checkrV r4 r4 r4 r5 �*GetLastCheckedForGsutilUpdateTimestampFile� s r` c C sH t � } dt }| | d d|d d d }t� | }|dkrDd}|S )z�Gets the max concurrent transport compressed uploads allowed in parallel. Returns: The max number of concurrent transport compressed uploads allowed in parallel without exceeding the max_upload_compression_buffer_size. � � � � i�? r )r_ r �!GetMaxUploadCompressionBufferSize)Zupload_chunk_sizeZcompression_chunk_sizeZtotal_upload_sizeZmax_concurrent_uploadsr4 r4 r5 �!GetMaxConcurrentCompressedUploads� s ��rf c C s t �ddd�S )Nr �max_retry_delay� �r r\ r4 r4 r4 r5 �GetMaxRetryDelay� s rj c C s t t�ddd��S )zEGet the max amount of memory compressed transport uploads may buffer.rS Z"max_upload_compression_buffer_sizeZ2GiB)r r r r4 r4 r4 r5 re � s �re c K s� t �ddd�}|t �ddd�t �dd�t �ddd�t �ddd�t �dd |rNd nd�d �}t|�}t� |d < t|d < | f d|i|��}t �dd� |_t� � }|r�|j r�|j |j |j d|j d� |S )z�Creates and returns a new httplib2.Http instance. Args: http_class: Optional custom Http class to use. **kwargs: Arguments to pass to http_class constructor. Returns: An initialized httplib2.Http instance. r �proxyN� proxy_type�http� proxy_port� proxy_user� proxy_pass� proxy_rdnsT�� proxy_hostrl rn ro rp rq �ca_certs�timeout� proxy_infor � )�key�cert�domain�password)r r r\ � SetProxyInforC r �getbool�"disable_ssl_certificate_validationr �get_context_config�use_client_certificate�add_certificate�client_cert_path�client_cert_password)� http_class�kwargsrs �boto_proxy_configrv rm Zglobal_context_configr4 r4 r5 � GetNewHttp s. � � �r� c C s t �ddd�S )Nr � num_retries� ri r4 r4 r4 r5 � GetNumRetries8 s r� c C s t j�t� d�S )Nztab-completion-logsrV r4 r4 r4 r5 �GetTabCompletionLogFilename< s r� c C s, t j�t� d�} tj| dd� t j�| d�S )Nztab-completioni� )�mode�cache)r! r" r$ rU r rT )Ztab_completion_dirr4 r4 r5 � GetTabCompletionCacheFilename@ s r� c C s� t �dd�ot �dd�} t �dd�o.t �dd�}t �dd�}t �dd�}t �dd�}tont �dd �ont �dd �}| s�|s�|s�|s�|s�|r�d S d }z<tjjtjt td �dgd�}dt t |dd �dd �kr�d }W n t y� Y n0 |S )z1Determines if boto credential/config file exists.r7 �gs_access_key_id�gs_secret_access_key�aws_access_key_id�aws_secret_access_key�gs_oauth2_refresh_tokenZgs_external_account_fileZ(gs_external_account_authorized_user_filer8 �gs_service_key_fileTN�google�s3��requested_capabilityZNoOpAuth� __class__�__name__) r r= r r �authZget_auth_handlerr � DefaultHostr �getattrr )Zhas_goog_credsZhas_amzn_credsZhas_oauth_credsZhas_external_credsZ*has_external_account_authorized_user_credsZhas_service_account_credsZvalid_auth_handlerr4 r4 r5 r< G sX � ���� � � ��� � � r< c C s t �ddd �} | d uS )NrS r[ )r r )Zchunk_size_definedr4 r4 r5 � JsonResumableChunkSizeDefinedq s r� c sh ddl �tjj�d��fdd� } | tj_ddl�ddl�tjj� tjj�� ���fdd�}|tjj _ dS )a� Apply gsutil-specific patches to Boto. Here be dragons. Sorry. Note that this method should not be used as a replacement for contributing fixes to the upstream Boto library. However, the Boto library has historically not been consistent about release cadence, so upstream fixes may not be available immediately in a version which we can pin to. Also, some fixes may only be applicable to gsutil. In such cases, patches should be applied to the Boto library here (and removed if/when they are included in the upstream repository and included in an official new release that we pull in). This method should be invoked after all other Boto-related initialization has been completed. r Nc s\ �| |d�}�j j�j jf� t� fdd�|D �dd� d�t� fdd�|D �dd� d� }|S ) Nr� c s g | ]}|� vr|�qS r4 r4 ��.0rH �Zxml_oauth2_handlersr4 r5 � <listcomp>� � zDMonkeyPatchBoto.<locals>._PatchedGetPluginMethod.<locals>.<listcomp>c S s | j S rB �r� �Z handler_tr4 r4 r5 �<lambda>� r� zBMonkeyPatchBoto.<locals>._PatchedGetPluginMethod.<locals>.<lambda>)rx c s g | ]}|� v r|�qS r4 r4 r� r� r4 r5 r� � r� c S s | j S rB r� r� r4 r4 r5 r� � r� )Z oauth2_pluginZOAuth2ServiceAccountAuthZ OAuth2Auth�sorted)�clsr� Zhandler_subclassesZ new_result)�gcs_oauth2_boto_plugin�orig_get_plugin_methodr� r5 �_PatchedGetPluginMethod� s" ������ z0MonkeyPatchBoto.<locals>._PatchedGetPluginMethodc s` t | d�r"��| j| jf| j�}n��| j| jf�}d}| jrN|d| j 7 }n|d7 }tj�|� t �d��rt �dd��r�� �j �}�j |_ | jr�|�| j� | jr�|�| j| j� |j|| jd�| _| j| j_| j| j_|j | j_�j | j_| j| j_d | j_n �j|| j| j�j | jd �| _| j�� }| j�d d �d }�||��s\� ||d | ��d S ) Nru zwrapping ssl socket; zCA certificate file=%sz using system provided SSL certs� SSLContext�HAS_SNIF)�server_hostname)�keyfile�certfile� cert_reqsrt �:r z/remote hostname "%s" does not match certificate)�hasattr�create_connection�host�portru rt r �log�debugr� r� �PROTOCOL_SSLv23� CERT_REQUIRED� verify_mode�load_verify_locations� cert_file�load_cert_chain�key_file� wrap_socket�sockr� r� r� � ssl_version�ciphers� getpeercert�split)�selfr� �msg�contextry �hostname)�InvalidCertificateException�ValidateCertificateHostname�socket�sslr4 r5 �_PatchedConnectMethod� sH � ��z.MonkeyPatchBoto.<locals>._PatchedConnectMethod)N) r� r �plugin� get_pluginr� r� Zhttps_connectionr� r� Z CertValidatingHTTPSConnection�connect)r� r� r4 )r� r� r� r� r� r� r5 �MonkeyPatchBotov s ��-�r� c C sj t j�| �}|r | �� �d�s0t�tjjdd�S | �� � d�d }|�� �d�s\|d | }tj ||d�S )a Reads proxy info from the environment and converts to httplib2.ProxyInfo. Args: proxy_env_var: Environment variable string to read, such as http_proxy or https_proxy. Returns: httplib2.ProxyInfo constructed from the environment string. rm Nr �_z://)�method) r! rI r �lower� startswith�httplib2� ProxyInfo�socks�PROXY_TYPE_HTTPr� �proxy_info_from_url)� proxy_env_varZ proxy_urlZproxy_protocolr4 r4 r5 �ProxyInfoFromEnvironmentVar� s r� c C s t �dddt �S )NrS �resumable_threshold� )r r\ r r4 r4 r4 r5 �ResumableThreshold s r� c C s� ddddd�}|� | � d��� |d �}| � d�}| � d�}| � d �}| � d �}t| � d ��}tj||||||d �}|j|d ks�d |_|jr�|js�dD ]8} | t j v r�t j | r�t | �}| � d �dkr�d|_ q�q�|S )z�Sets proxy info from boto and environment and converts to httplib2.ProxyInfo. Args: dict: Values read from the .boto file Returns: httplib2.ProxyInfo constructed from boto or environment variable string. rd � � )Zsocks4Zsocks5rm �httpsrl rm rs rn ro rp rq rr F)� http_proxy� https_proxy� HTTPS_PROXYNT) r r� �boolr� r� rl rq rs rn r! rI r� ) r� Zproxy_type_specrl rs rn ro rp rq rv r� r4 r4 r5 r| s4 � � r| c C s> t j�ddd �} | d ur | S tdt� t� dgd�}t|dd�S )NrS Ztest_assume_fast_crcmodz crcmod.crcmodZ_usingExtensionr F)r r r � __import__�globals�localsr� )Zboto_optZ nested_crcmodr4 r4 r5 �UsingCrcmodExtension5 s �r� Fc C s | j jj}|s*|jdv r dS |�d�r*dS d|j kr@dkr�n nv|�d�| _| j}t|t�rj|� t �}|�dd�}|�d d�}|du r�|du r�| jd | kr�|� d | j| jf ��dS |jd kr�|� � }|� |j|j|�}|jd v r�tjjd|j |d��dS )zFReplaces boto.s3.key's should_retry() to handle KMS-encrypted objects.)i� i� T�location�� i+ �etagz/x-amz-server-side-encryption-customer-algorithmNz x-goog-encryption-kms-key-namez"%s"z2ETag from S3 did not match computed MD5. %s vs. %si� )ZRequestTimeoutzSaw %s, retrying)�responseF)�bucket� connection�provider�status� getheaderr� �md5� isinstance�bytes�decoder Zstorage_data_error�readZstorage_response_error�reason� error_coder � exception�PleaseRetryException) r� r� Zchunked_transferr� r� Z-amz_server_side_encryption_customer_algorithmZ goog_customer_managed_encryption�body�errr4 r4 r5 �_PatchedShouldRetryMethodJ sP ��� �� � �r c C s< t j�ddd �} t�t jjjj�}| d ur8|t�| �kS dS )Nr7 �gs_hostF) r r r r) r* �gsr� r r� )r Z default_hostr4 r4 r5 �HasUserSpecifiedGsHost� s r c C sR t j} | �dd�}| �dd�o(| �dd�}| �dd�o@| �dd�}| oP| oP|S )Nr7 r� r8 r� r� r� )r r r= )r Zhas_refresh_tokenZ has_service_account_credentialsZhas_hmac_credentialsr4 r4 r5 � UsingGsHmac� s � � �r )F)M�__doc__� __future__r r r r r! r' r+ r? r) r r Z boto.auth�boto.exceptionr �boto.gs.connectionr Z boto.providerr �boto.pyami.configr r r �gslib.exceptionr � gslib.utilsr �gslib.utils.constantsr r r r �gslib.utils.unit_utilr r r� �oauth2client.clientr �PY3�intr] r �BOTO_IS_SECURE� CERTIFICATE_VALIDATION_ENABLEDr r0 r6 rA rC rD rP rR rU rW rX rZ r_ r` rf rj re �Httpr� r� r� r� r< r� r� r� r� r| r� r r r r4 r4 r4 r5 �<module> s� �* 5*x/ ; utils/__pycache__/translation_helper.cpython-39.pyc 0000644 00000063260 15025013616 0016450 0 ustar 00 a +(Wg�� � @ s> d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z ddlZddlmZ dd lmZ ddlZdd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddlm Z ddlm Z ddl m Z ddl m!Z! ddl"m#Z# ddl"m$Z$ ddl"m%Z% ddl"m&Z& ddl'm(Z( ddl)m*Z+ dd l,m-Z- dd l,m.Z. ej/�r�e0Z1e�2d ej3�Z4e�2d ej3�Z5e�2d ej3�Z6e�2d!ej3�Z7e�2d"ej3�Z8e�2d#ej3�Z9e�2d$ej3�Z:e�2d%ej3�Z;e�2d&ej3�Z<e�2d'ej3�Z=e�2d(ej3�Z>e�2d)ej3�Z?e�2d*ej3�Z@eAe4e5e6e7e8e9e:e;e<e=e>e?h �ZBd+ZCd,ZDe+jEjFd-d.gd/�gZGe+jHd0d1�ZId2d3� ZJd4d5� ZKd6d7� ZLdUd9d:�ZMdVd;d<�ZNd=d>� ZOdWd?d@�ZPdAdB� ZQdXdCdD�ZRdEdF� ZSG dGdH� dHeT�ZUG dIdJ� dJeT�ZVdKdL� ZWdMdN� ZXdOdP� ZYG dQdR� dReT�ZZG dSdT� dTeT�Z[dS )YzDUtility module for translating XML API objects to/from JSON objects.� )�absolute_import)�print_function)�division)�unicode_literalsN)� ParseError)�decode_datetime)�encoding)�ACL)�ALL_AUTHENTICATED_USERS)� ALL_USERS)�Entries)�Entry)�GROUP_BY_DOMAIN)�GROUP_BY_EMAIL)� GROUP_BY_ID)� USER_BY_EMAIL)� USER_BY_ID)�Tags)�TagSet)�ArgumentException��BucketNotFoundException)�NotFoundException)� Preconditions)�CommandException)�storage_v1_messages)�S3_ACL_MARKER_GUID)�S3_MARKER_GUIDSz^cache-controlz^content-dispositionz^content-encodingz^content-languagez ^content-md5z ^content-typez ^custom-timez^x-goog-api-versionz^x-goog-if-generation-matchz ^x-goog-if-metageneration-matchz ^x-goog-meta-(?P<header_key>.*)z ^x-amz-meta-(?P<header_key>.*)z^x-amz-(?P<header_key>.*)zcustom-amz-headerzapplication/octet-stream����REMOVE_CORS_CONFIG)� maxAgeSeconds�method�PRIVATE_DEFAULT_OBJ_ACL)�idc s8 i }| � � D ]&\� }t� fdd�tD ��s ||� < q |S )Nc 3 s | ]}|� � �V qd S �N)�match)�.0�regex��header� �-/opt/gsutil/gslib/utils/translation_helper.py� <genexpr>y � z(GetNonMetadataHeaders.<locals>.<genexpr>)�items�any�metadata_header_regexes)�headersZarbitrary_headers�valuer* r( r+ �GetNonMetadataHeadersv s r3 c C s� t �� }| �� D �]�\}}t�|�r0|�� |_qt�|�rF|�� |_qt �|�r\|�� |_ qt �|�rr|�� |_ qt �|�r�|�� |_qt�|�r�|s�t|_n |�� |_qt�|�r�t|�� �|_qt�|�r�qqt�|�r�qqt�|�r�qqt�|�}t�|�}t�|�}d}|�r"|�d�}n&|�r4|�d�}n|�rHt |�d� }|r|� � dk�r\q|j �spt j� � |_ |j j �s�g |j _ |j j �!t jj j"||d�� q|S )a� Creates object metadata according to the provided headers. gsutil -h allows specifiying various headers (originally intended to be passed to boto in gsutil v3). For the JSON API to be compatible with this option, we need to parse these headers into gsutil_api Object fields. Args: headers: Dict of headers passed via gsutil -h Raises: ArgumentException if an invalid header is encountered. Returns: apitools Object with relevant fields populated from headers. N� header_keyzx-goog-content-language��keyr2 )#�apitools_messages�Objectr. �CACHE_CONTROL_REGEXr% �strip� cacheControl�CONTENT_DISPOSITION_REGEX�contentDisposition�CONTENT_ENCODING_REGEX�contentEncoding�CONTENT_MD5_REGEX�md5Hash�CONTENT_LANGUAGE_REGEX�contentLanguage�CONTENT_TYPE_REGEX�DEFAULT_CONTENT_TYPE� contentType�CUSTOM_TIME_REGEXr � customTime�GOOG_API_VERSION_REGEX�GOOG_GENERATION_MATCH_REGEX� GOOG_METAGENERATION_MATCH_REGEX�CUSTOM_GOOG_METADATA_REGEX�CUSTOM_AMZ_METADATA_REGEX�CUSTOM_AMZ_HEADER_REGEX�group�S3_HEADER_PREFIX�lower�metadata� MetadataValue�additionalProperties�append�AdditionalProperty)r1 � obj_metadatar) r2 Zcustom_goog_metadata_matchZcustom_amz_metadata_matchZcustom_amz_header_matchr4 r* r* r+ �ObjectMetadataFromHeaders~ sb � ��rX c C s2 i }| s dS | j dur4| j s&d|d< n| j �� |d< | jrX| jsJd|d< n| j�� |d< | jr|| jsnd|d< n| j�� |d< | jr�| js�d|d< n| j�� |d< | jr�| js�d|d< n| j�� |d< | jdur�| js�d|d< n| j�� |d< | jdu�r| j�s d|d< n| j�� |d< | j�rfd }|d k�r8d | }n |d k�rLd | }n t d| ��| j�� ||< | j �r.| j j �r.| j j D ]�}|j dk�r��q�|j t v �r��q�|d k�r�d|j }nH|d k�r�|j �t��r�d |j tt�d� }n d|j }n t d| ��|jdu�r |j�s d||< n |j||< �q�|S )z�Creates a header dictionary based on existing object metadata. Args: dst_obj_metadata: Object metadata to create the headers from. provider: Provider string ('gs' or 's3'). Returns: Headers dictionary. Nz cache-controlzcontent-dispositionzcontent-encodingzcontent-languagez Content-MD5z content-typez custom-timez storage-class�gszx-goog-�s3zx-amz-z Invalid provider specified: %sz x-goog-meta-z x-amz-meta-)r; r: r= r? rC rA rF rH � storageClassr rR rT r6 r � startswithrP �lenr2 )�dst_obj_metadata�providerr1 � header_nameZadditional_propertyr* r* r+ �HeadersFromObjectMetadata� s~ � � � � ra Fc C s� |s |j s| j |_ |s |js$| j|_|s.|js6| j|_|s@|jsH| j|_|sR|jsZ| j|_|sd|jsl| j|_|sv|js~| j|_t| ||d� dS )ab Copies metadata from src_obj_metadata to dst_obj_metadata. Args: src_obj_metadata: Metadata from source object. dst_obj_metadata: Initialized metadata for destination object. override: If true, will overwrite metadata in destination object. If false, only writes metadata for values that don't already exist. )�overrideN)r; r= r? rC rF rH rA �CopyCustomMetadata)�src_obj_metadatar^ rb r* r* r+ �CopyObjectMetadata s re c C s� | j r�| j jr�|j s tj�� |_ |j js0g |j _i }|j jD ]}|j||j< q<| j jD ]R}|j|v r�|r�|jdur�|js�d||j< q�|j||j< qV|jdkrV|j||j< qVg |j _t�|�D ]$\}}|j j� tjjj ||d�� q�dS )ai Copies custom metadata from src_obj_metadata to dst_obj_metadata. Args: src_obj_metadata: Metadata from source object. dst_obj_metadata: Initialized metadata for destination object. override: If true, will overwrite metadata in destination object. If false, only writes metadata for values that don't already exist. N� r5 ) rR rT r7 r8 rS r2 r6 �six� iteritemsrU rV )rd r^ rb Zdst_metadata_dictZdst_propZsrc_prop�k�vr* r* r+ rc 9 s2 � ��rc c C sv t � }z>| �� D ]0\}}t�|�r,t|�|_t�|�rt|�|_qW n, typ } zt d��W Y d}~n d}~0 0 |S )a Creates bucket or object preconditions acccording to the provided headers. Args: headers: Dict of headers passed via gsutil -h Returns: gsutil Cloud API Preconditions object fields populated from headers, or None if no precondition headers are present. z�Invalid precondition header specified. x-goog-if-generation-match and x-goog-if-metageneration match must be specified with a positive integer value.N) r r. rJ r% �long� gen_matchrK �meta_gen_match� ValueErrorr )r1 Zreturn_preconditionsr) r2 �_r* r* r+ �PreconditionsFromHeadersc s rp c C sR d| |f }|rF|rFd|||f }|r6|dt |� 7 }td||f �S td| �S )a� Creates a NotFoundException for an object upload or copy. This is necessary because 404s don't necessarily specify which resource does not exist. Args: dst_provider: String abbreviation of destination provider, e.g., 'gs'. dst_bucket_name: Destination bucket name for the write operation. src_provider: String abbreviation of source provider, i.e. 'gs', if any. src_bucket_name: Source bucket name, if any (for the copy case). src_object_name: Source object name, if any (for the copy case). src_generation: Source object generation, if any (for the copy case). Returns: NotFoundException with appropriate message. z%s://%s� %s://%s/%s�#%szAThe source object %s or the destination bucket %s does not exist.zZThe destination bucket %s does not exist or the write to the destination must be restarted��strr )Z dst_provider�dst_bucket_name� src_provider�src_bucket_name�src_object_name�src_generationZdst_url_stringZsrc_url_stringr* r* r+ �%CreateNotFoundExceptionForObjectWrite| s" �����rz c C s t d||f || d�S )Nz %s://%s bucket does not exist.��statusr )�coder_ � bucket_namer* r* r+ � CreateBucketNotFoundException� s ��r c C s2 d|||f }|r"|dt |� 7 }td| | d�S )Nrq rr z%s does not exist.r{ rs )r} r_ r~ � object_name� generation� uri_stringr* r* r+ � CreateObjectNotFoundException� s r� c C s^ z6t jj�t|�� td�t�d� | | � � �����W n t yH Y n0 td| |f ��dS )z=Checks a JSON parse exception for provided XML configuration.� a XML {0} data provided; Google Cloud Storage {0} configuration now uses JSON format. To convert your {0}, set the desired XML ACL using 'gsutil {1} set ...' with gsutil version 3.x. Then use 'gsutil {1} get ...' with gsutil version 4 or greater to get the corresponding JSON {0}.z)JSON %s data could not be loaded from: %sN) �xml�etree� ElementTreeZ fromstringrt r �join�textwrap�wrap�formatrQ � XmlParseError)Zconfig_type_string�json_txtr* r* r+ � CheckForXmlConfigurationAndRaise� s ��� �r� c @ sL e Zd ZdZedd� �Zedd� �Zedd� �Zedd � �Zed d � �Z d S ) �LifecycleTranslationz�Functions for converting between various lifecycle formats. This class handles conversation to and from Boto Cors objects, JSON text, and apitools Message objects. c C sN t jj�� }|�rJ|jD �].}t jj�� }|jr||jjr||jj�� dkrVt jjj |_n&|jj�� dkr|t jjj |_|jj |_ |j �r>|j jdur�t|j j�|jt jjj< |j jr�t|j j�|jt jjj< |j jdur�t|j j��� |jt jjj< |j j�rdd� |j jD �|jt jjj< |j jdu�r>t|j j�|jt jjj< |�|� q|S )z:Translates an apitools message to a boto lifecycle object.�deleteZsetstorageclassNc S s g | ] }t |��qS r* )rt )r& �scr* r* r+ � <listcomp>� s zALifecycleTranslation.BotoLifecycleFromMessage.<locals>.<listcomp>)�botorY � lifecycle�LifecycleConfig�rule�Rule�action�typerQ �DELETE�SET_STORAGE_CLASSr[ � action_text� condition�agert � conditions�AGE� createdBefore�CREATED_BEFORE�isLive�IS_LIVE�matchesStorageClass�MATCHES_STORAGE_CLASS�numNewerVersions�NUM_NEWER_VERSIONSrU )�cls�lifecycle_message�boto_lifecycleZ rule_message� boto_ruler* r* r+ �BotoLifecycleFromMessage� s@ �� � �� z-LifecycleTranslation.BotoLifecycleFromMessagec C s� d}|�r�t j�� }|D �]�}t jj�� }t jjj�� |_|jr�|jtjj j krft jjjj dd�|_n(|jtjj j kr�t jjjj d|j d�|_tjj j|jv r�t|jtjj j �|j_tjj j|jv r�t�|jtjj j �|j_tjj j|jv �rF|jtjj j �� }|dk�rd|j_n,|dk�r.d |j_ntd |jtjj j ��tjj j|jv �r~|jtjj j D ]}|jj� |� �qhtjj j |jv �r�t|jtjj j �|j_ |j � |� q|S ) z:Translates a boto lifecycle object to an apitools message.N�Delete)r� �SetStorageClass)r� r[ �trueT�falseFzRGot an invalid Boto value for IsLive condition ("%s"), expected "true" or "false".) r7 �Bucket�LifecycleValue�RuleValueListEntry�ConditionValuer� r� r� rY r� r� � ActionValuer� r� r� r� �intr� r� r� � TranslateBotoLifecycleTimestampr� r� rQ r� r r� r� rU r� r� r� )r� r� r� r� Zlifecycle_ruleZboto_is_live_str� storage_classr* r* r+ �BotoLifecycleToMessage� s` � � ����� ����z+LifecycleTranslation.BotoLifecycleToMessagec C s t t�|��d S )z1Translates an apitools message to lifecycle JSON.r� )rt r � MessageToJson)r� r� r* r* r+ �JsonLifecycleFromMessage! s z-LifecycleTranslation.JsonLifecycleFromMessagec C sV z4t �|�}d|v r |d }t�|p&i tjj�}|W S tyP td|� Y n0 dS )z1Translates lifecycle JSON to an apitools message.r� N) �json�loadsr � DictToMessager7 r� r� rn r� )r� r� Zdeserialized_lifecycler� r* r* r+ �JsonLifecycleToMessage& s � z+LifecycleTranslation.JsonLifecycleToMessagec C s t j �|d��� S )zDParses the timestamp from the boto lifecycle into a datetime object.z%Y-%m-%d)�datetime�strptime�date)r� Zlifecycle_datetimer* r* r+ r� 8 s z4LifecycleTranslation.TranslateBotoLifecycleTimestampN) �__name__� __module__� __qualname__�__doc__� classmethodr� r� r� r� r� r* r* r* r+ r� � s " 1 r� c @ s@ e Zd ZdZedd� �Zedd� �Zedd� �Zedd � �Zd S ) �CorsTranslationz�Functions for converting between various CORS formats. This class handles conversation to and from Boto Cors objects, JSON text, and apitools Message objects. c C s t jj�� }g |_|D ]�}g }|jr>|�t jjjt|j�f� |jr|g }|jD ]}|�t jjj |f� qN|�t jjj |f� |j r�g }|j D ]}|�t jjj |f� q�|�t jjj |f� |jr�g } |jD ]} | �t jjj| f� q�|�t jjj| f� |j�|� q|S )z5Translates an apitools message to a boto Cors object.)r� rY �cors�Corsr rU � MAXAGESECrt r! �METHOD�METHODS�origin�ORIGIN�ORIGINS�responseHeader�HEADER�HEADERS) r� � cors_messager� �collection_messageZcollection_elementsZmethod_elementsr! Zorigin_elementsr� Zheader_elementsr) r* r* r+ �BotoCorsFromMessageE s2 � z#CorsTranslation.BotoCorsFromMessagec C s� g }|j r�|j D ]�}|rtj�� }|D ]�}|d tjj jkrJt|d �|_|d tjj j krz|d D ]}|j � |d � qd|d tjj j kr�|d D ]}|j � |d � q�|d tjj jkr&|d D ]}|j� |d � q�q&|� |� q|S )z5Translates a boto Cors object to an apitools message.r � )r� r7 r� �CorsValueListEntryr� rY r� r� r r� r! rU r� r� r� r� ) r� � boto_corsZ message_corsZcors_collectionr� Z element_tupleZ method_tupleZ origin_tupleZ header_tupler* r* r+ �BotoCorsToMessagea s& z!CorsTranslation.BotoCorsToMessagec C sj d}zt �|�}W n ty. td|� Y n0 t|t�sBtd��g }|D ]}|�t� |t j j �� qJ|S )a0 Translates CORS JSON to an apitools message. Args: json_cors: JSON string representing CORS configuration. Raises: ArgumentException on invalid CORS JSON data. Returns: List of apitools Bucket.CorsValueListEntry. An empty list represents no CORS configuration. N�CORSzdCORS JSON should be formatted as a list containing one or more JSON objects. See "gsutil help cors".) r� r� rn r� � isinstance�listr rU r r� r7 r� r� )r� Z json_corsZdeserialized_corsr� � cors_entryr* r* r+ �JsonCorsToMessageEntriesx s" ���z(CorsTranslation.JsonCorsToMessageEntriesc C sF d}|d7 }d}|D ]$}|r&|d7 }nd}|t �|�7 }q|d7 }|S )z,Translates an apitools message to CORS JSON.rf �[F�,Tz] )r r� )r� r� Z json_textZ printed_oner� r* r* r+ �MessageEntriesToJson� s z$CorsTranslation.MessageEntriesToJsonN) r� r� r� r� r� r� r� r� r� r* r* r* r+ r� > s r� c C s8 | r4| j r4| j jr4| j jD ]}|jtkr|j S qdS )z�Retrieves GUID-marked S3 ACL from object metadata, if present. Args: object_metadata: Object metadata to check. Returns: S3 ACL text, if present, None otherwise. N)rR rT r6 r r2 )�object_metadata�propr* r* r+ � S3MarkerAclFromObjectMetadata� s � r� c C sB | j stj�� | _ | j js"g | j _| j j�tjjjt|d�� dS )z�Adds a GUID-marked S3 ACL to the object metadata. Args: object_metadata: Object metadata to add the acl to. acl_text: S3 ACL text to add. r5 N)rR r7 r8 rS rT rU rV r )r� �acl_textr* r* r+ � AddS3MarkerAclToObjectMetadata� s ��r� c C st t | �dkr tdt | � ��| �� d \}}tjj�|�}t|�� �D ] \}}tjj�||�}||_ qFtjj� |�S )a~ Generates XML representation of a nested dict. This dict contains exactly one top-level entry and an arbitrary number of 2nd-level entries, e.g. capturing a WebsiteConfiguration message. Args: message: The dict encoding the message. Returns: XML string representation of the input dict. Raises: Exception: if dict contains more than one top-level entry. r� z$Expected dict of size 1, got size %dr ) r] � Exceptionr. r� r� r� �Element�sortedZ SubElement�textZtostring)�message�name�contentZ element_typeZelement_propertyr2 �noder* r* r+ �UnaryDictToXml� s r� c @ sB e Zd ZdZedd� �Zedd� �Zed dd��Zed d � �Zd S ) �LabelTranslationz�Functions for converting between various Label(JSON)/Tags(XML) formats. This class handles conversion to and from Boto Tags objects, JSON text, and apitools LabelsValue message objects. c C s8 i }|D ] }|� tdd� |D ��� q|r4| �|�S d S )Nc s s | ]}|j |jfV qd S r$ r5 )r& �ir* r* r+ r, � r- z5LabelTranslation.BotoTagsToMessage.<locals>.<genexpr>)�update�dictr� )r� �tags� label_dict�tag_setr* r* r+ �BotoTagsToMessage� s z"LabelTranslation.BotoTagsToMessagec C sN t �| �|��}t� }t�|�D ]\}}|r |�||� q t� }|�|� |S r$ ) r� r� �JsonFromMessager rg rh �add_tagr � add_tag_set)r� r� r� r� r6 r2 r� r* r* r+ �BotoTagsFromMessage� s z$LabelTranslation.BotoTagsFromMessageFc C s* t �|�}|r&tjt�|�dddd�S |S )NT� �r� z: �� sort_keys�indent� separators)r r� r� �dumpsr� )r� r� Z pretty_print�json_strr* r* r+ r� � s �z LabelTranslation.JsonFromMessagec C s t �|tjj�S r$ )r r� r7 r� � LabelsValue)r� r� r* r* r+ r� s �z LabelTranslation.DictToMessageN)F) r� r� r� r� r� r� r� r� r� r* r* r* r+ r� � s r� c @ s� e Zd ZdZdddd�Zdddd �Zed d � �Zed d � �Zedd� �Z edd� �Z edd� �Z edd� �Z edd� �Z edd� �Zedd� �Zed d � �Zed d � �Zd S )!�AclTranslationz�Functions for converting between various ACL formats. This class handles conversion to and from Boto ACL objects, JSON text, and apitools Message objects. �READ�WRITE� FULL_CONTROL)�READER�WRITER�OWNERr r r )r r r c C s t � }d |_| �||�|_|S r$ )r �parent�BotoEntriesFromJson�entries)r� �acl_json�aclr* r* r+ �BotoAclFromJson s z AclTranslation.BotoAclFromJsonc C s4 g }|D ] }|t kr q*|�t�|�� q| �|�S r$ )r" rU r � MessageToDictr )r� Z acl_messageZ acl_dictsr� r* r* r+ �BotoAclFromMessage& s z!AclTranslation.BotoAclFromMessagec C s t |d�r| �|j�S g S )Nr )�hasattr�BotoEntriesToJsonr )r� r r* r* r+ � BotoAclToJson2 s z AclTranslation.BotoAclToJsonc c s. | � |�D ] }t�|tj�}d|_|V q d S )N�storage#objectAccessControl)r r r� r7 �ObjectAccessControl�kind�r� r �entryr� r* r* r+ �BotoObjectAclToMessage8 s �z%AclTranslation.BotoObjectAclToMessagec c s. | � |�D ] }t�|tj�}d|_|V q d S )Nzstorage#bucketAccessControl)r r r� r7 �BucketAccessControlr r r* r* r+ �BotoBucketAclToMessage@ s �z%AclTranslation.BotoBucketAclToMessagec s& t |�}||_� fdd�|D �|_|S )Nc s g | ]}� � |��qS r* )�BotoEntryFromJson)r& � entry_json�r� r* r+ r� L s z6AclTranslation.BotoEntriesFromJson.<locals>.<listcomp>)r r � entry_list)r� r r r r* r# r+ r H s �z"AclTranslation.BotoEntriesFromJsonc s � fdd�|j D �S )Nc s g | ]}� � |��qS r* )�BotoEntryToJson)r& r r# r* r+ r� S r- z4AclTranslation.BotoEntriesToJson.<locals>.<listcomp>)r$ )r� r r* r# r+ r Q s z AclTranslation.BotoEntriesToJsonc C s |d }| j |d }|�� t�� kr2tt|d�S |�� t�� krNtt|d�S |�d�rbtd��n�d|v r�|�d�rzt}n|�d�r�t}t||d |d �S d |v r�|�d�r�t }n|�d�r�t }t||d |d �S d |v � r�|�d �r�t }t||d |d �S td��dS )z,Converts a JSON entry into a Boto ACL entry.�entity�role)r� � permission�projectz>XML API does not support project scopes, cannot translate ACL.�email�userrO )r� � email_addressr( �entityId)r� r# r( �domain)r� r. r( z$Failed to translate JSON ACL to XML.N) �JSON_TO_XML_ROLESrQ r r r r\ r r r r r r )r� r" r& r( Z scope_typer* r* r+ r! U sD � � �z AclTranslation.BotoEntryFromJsonc C s0 i }|j j�� }|t�� kr&d|d< n�|t�� kr<d|d< n�|t�� krfd|j j |d< |j j|d< n�|t�� kr�d|j j |d< |j j|d< n�|t �� kr�d|j j |d< |j j|d< nb|t �� kr�d|j j |d< |j j|d< n8|t �� k�rd|j j |d< |j j |d < n t d | ��| j|j |d < |S ) z5Converts a Boto ACL entry to a valid JSON dictionary.ZallUsersr& ZallAuthenticatedUserszuser-%sr* r- zgroup-%sz domain-%sr. z#ACL contains invalid scope type: %sr' )�scoper� rQ r r r r, r r# r r r r. r �XML_TO_JSON_ROLESr( )r� r Zacl_entry_jsonZscope_type_lowerr* r* r+ r% y s4 �z AclTranslation.BotoEntryToJsonc C sR z0t �|�}g }|D ]}|�t�||�� q|W S tyL td|� Y n0 dS )a� Converts the input JSON data into list of Object/BucketAccessControls. Args: json_data: String of JSON to convert. message_type: Which type of access control entries to return, either ObjectAccessControl or BucketAccessControl. Raises: ArgumentException on invalid JSON data. Returns: List of ObjectAccessControl or BucketAccessControl elements. r N)r� r� rU r r� rn r� )r� � json_data� message_typeZdeserialized_aclr � acl_entryr* r* r+ � JsonToMessage� s z AclTranslation.JsonToMessagec C sl g }|durZ|D ]H}|j dkr*d|_d|_d|_ d|_d|_d|_d|_|�t� |�� qt j |dddd�S )z�Strips unnecessary fields from an ACL message and returns valid JSON. Args: acl: iterable ObjectAccessControl or BucketAccessControl Returns: ACL JSON string. Nr Tr� r� r ) r �objectr� �bucketr# �selfLink�etagrU r r r� r )r� r Zserializable_aclr4 r* r* r+ r� � s" �z AclTranslation.JsonFromMessageN)r� r� r� r� r/ r1 r� r r r r r r r r! r% r5 r� r* r* r* r+ r s> �� # r )F)F)NNNN)N)\r� � __future__r r r r r� r� �rer� Zxml.etree.ElementTreer� r r� rg Z apitools.base.protorpclite.utilr �apitools.base.pyr r� � boto.gs.aclr r r r r r r r r r �boto.s3.taggingr r �gslib.cloud_apir r r r �gslib.exceptionr �"gslib.third_party.storage_apitoolsr r7 �gslib.utils.constantsr r �PY3r� rk �compile�Ir9 r< r> rB r@ rD rG rI rJ rK rL rM rN � frozensetr0 rP rE r� r� r r r" r3 rX ra re rc rp rz r r� r� r6 r� r� r� r� r� r� r r* r* r* r+ �<module> s� ��� �� �IW * � % � yk) utils/__pycache__/posix_util.cpython-39.pyc 0000644 00000030725 15025013616 0014752 0 ustar 00 a +(Wg�J � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ es�ddlZddlZe jr�e Z dZ dZ dZ dZ!dZ"dZ#dZ$dZ%e �&d�Z'dZ(dZ)dZ*dZ+dZ,dZ-dZ.d Z/d Z0da1e2� a3G d d � d e4�Z5G d d!� d!e4�Z6d"d#� Z7d$d%� Z8dAd'd(�Z9d)d*� Z:d+d,� Z;d-d.� Z<e$e$e%fd/d0�Z=dBd1d2�Z>d3d4� Z?d5d6� Z@d7d8� ZAd9d:� ZBd;d<� ZCd=d>� ZDd?d@� ZEdS )Cz Helper file for POSIX methods.� )�absolute_import)�print_function)�division)�unicode_literals)�timegmN)�CommandException)�UTC)�CreateCustomMetadata)� GetValueFromObjectCustomMetadata)� IS_WINDOWS)�SECONDS_PER_DAYzgoog-reserved-file-atimezgoog-reserved-posix-gidzgoog-reserved-posix-modezgoog-reserved-file-mtimezgoog-reserved-posix-uid���z ^[0-7]{3}$� � �@ � � � � � � c @ s$ e Zd ZdZeeeedfdd�ZdS )�POSIXAttributesz1Class to hold POSIX attributes for a file/object.Nc C s. || _ || _|| _|| _t|r"|nt�| _dS )a6 Constructor for POSIXAttributes class which holds relevant data. Args: atime: The access time of the file/object. mtime: The modification time of the file/object. uid: The user ID that owns the file. gid: The group ID that the user is in. mode: An instance of POSIXMode. N)�atime�mtime�uid�gid� POSIXMode�NA_MODE�mode)�selfr r r r r � r �%/opt/gsutil/gslib/utils/posix_util.py�__init__V s zPOSIXAttributes.__init__)�__name__� __module__� __qualname__�__doc__�NA_TIME�NA_IDr" r r r r! r S s �r c @ s e Zd Zdd� ZdS )r c C s || _ d S )N)� permissions)r r) r r r! r" n s zPOSIXMode.__init__N)r# r$ r% r" r r r r! r l s r c C s t t| �dd� �S )z7Converts a base-10 mode integer from os.stat to base-8.���N)�int�oct�r r r r! �ConvertModeToBase8r s r. c C s� t � }t| tt�\}}zJt|�}|r<|tkr<td|� t}n"|tt�� �t kr^td|� t}W n t y� t d|� t}Y n0 ||_ t | t ||� t | t||� t| tt�\}}|r�t�|�r�ztt|��|_W n t y� t d|� Y n0 |S )aA Parses the POSIX attributes from the supplied metadata. Args: obj_metadata: The metadata for an object. url_str: File/object path that provides context if a warning is thrown. Returns: A POSIXAttribute object with the retrieved values or a default value for any attribute that could not be found. r r )r r � ATIME_ATTRr' �long�WarnNegativeAttribute�timer �WarnFutureTimestamp� ValueError�WarnInvalidValuer �DeserializeIDAttribute�GID_ATTR�UID_ATTR� MODE_ATTRr � MODE_REGEX�matchr r+ r )� obj_metadata�url_str� posix_attrs�foundr r r r r! �+DeserializeFileAttributesFromObjectMetadatay s6 � � r@ Fc C s� | j tkr tt| j i|d� |r�| jtkr<tt| ji|d� | jtkrXtt| ji|d� | j tkrttt | j i|d� | j j t kr�tt| j j i|d� dS )a+ Takes a POSIXAttributes object and serializes it into custom metadata. Args: posix_attrs: A POSIXAttributes object. custom_metadata: A custom metadata object to serialize values into. preserve_posix: Whether or not to preserve POSIX attributes other than mtime. )�entries�custom_metadataN)r r' r � MTIME_ATTRr r/ r r( r8 r r7 r r) r r9 )r> rB �preserve_posixr r r! �'SerializeFileAttributesToObjectMetadata� s* � � � � �rE c C sv |� d�d }t| |t�\}}z&t|�}|rB|tkrBt||� t}W n tyd t||� t}Y n0 t|||� dS )a Parses the POSIX attributes from the supplied metadata into posix_attrs. Args: obj_metadata: The metadata for an object. attr: Either GID_ATTR or UID_ATTR. url_str: File/object path that provides context if a warning is thrown. posix_attrs: POSIXAttribute object. �-r N)�splitr r( r+ r1 r4 r5 �setattr)r< �attrr= r> � attr_namer? �valr r r! r6 � s r6 c C s� t � } | tk} |tk} |tk} |tk}|tk}|tk}|tk}|tk}|tk}| tk}| rd| sd| | _| rr|sr|| _|r�|s�|| _|r�|s�|| _|r�|s�|| j_ | | r�| p�| r�| p�|r�| p�|r�| p�|o�| fS )a; Checks whether an update for any POSIX attribute is needed. Args: src_atime: The source access time. dst_atime: The destination access time. src_mtime: The source modification time. dst_mtime: The destination modification time. src_uid: The source user ID. dst_uid: The destination user ID. src_gid: The source group ID. dst_gid: The destination group ID. src_mode: The source mode. dst_mode: The destination mode. Returns: A tuple containing a POSIXAttribute object and a boolean for whether an update was needed. ) r r' r( r r r r r r r) )Z src_atimeZ dst_atimeZ src_mtimeZ dst_mtimeZsrc_uidZdst_uidZsrc_gidZdst_gidZsrc_modeZdst_moder> Z has_src_atimeZ has_dst_atimeZ has_src_mtimeZ has_dst_mtimeZ has_src_uidZ has_dst_uidZ has_src_gidZ has_dst_gidZ has_src_modeZ has_dst_moder r r! �NeedsPOSIXAttributeUpdate� s< ����rL c C s. t �t| �dd� �o,| t@ p,| t@ p,| t@ S )z�Validates whether the mode is valid. In order for the mode to be valid either the user, group, or other byte must be >= 4. Args: mode: The mode as a 3-digit, base-8 integer. Returns: True/False r* N)r: r; r, �U_R�G_R�O_Rr- r r r! �ValidatePOSIXMode s &�rP c C s� t rdS |tk}t|�tk}|tk}|s4|s4|s4dS t�� dkrDdS ttt|�d��}|rl|stdd| fS ntt�}|r�zt � |� W n& t t fy� dd| |f f Y S 0 |r�zt �|� W n& t t fy� dd| |f f Y S 0 |t�� k}tt|�d�}|�s@|�r@|�r@t|t@ ��s@dd| |t|�d d � f fS |�rzt|t@ �} | | �r^d nd | |t|�d d � f fS t|�tv �r�t|t@ �} | | �r�d nd| |t|�d d � f fS |t@ �r�dS |�s�|�s�|�r�dS dd | fS )a� Validates that the user has file access if uid, gid, and mode are applied. Args: url_str: The path to the object for which this is validating. uid: A POSIX user ID. gid: A POSIX group ID. mode: A 3-digit, number representing POSIX permissions, must be in base-8. Returns: A (bool, str) tuple, True if and only if it's safe to copy the file, and a string containing details for the error. )T� r r Fz$Mode for %s won't allow read access.z3UID for %s doesn't exist on current system. uid: %dz3GID for %s doesn't exist on current system. gid: %dz?Insufficient access with uid/gid/mode for %s, gid: %d, mode: %sr* NrQ z?Insufficient access with uid/gid/mode for %s, uid: %d, mode: %sz"There was a problem validating %s.)r r( r+ r �os�geteuidrP �str�SYSTEM_POSIX_MODE�pwd�getpwuid�KeyError� OverflowError�grp�getgrgid�getuid�boolrM r, � USER_GROUPSrN rO ) r= r r r Z uid_presentZ gid_presentZ mode_presentZ mode_validZuid_is_current_user�validr r r! � ValidateFilePermissionAccess sj � � � �� �� r` c C sN |du rt d| ���zt|ttd�\}}t|ttd�\}}t|ttd�\}} t|ttd�\} } t|tt d�\} } |r�t |�}|s�t � | �j }t �| ||f� W dS n |r�t|j�}t �| ||f� |s�W dS |r�t |�}|tkr�|tkr�t �| ||f� nb|tk�r.|tk�r.t � | �j}t �| ||f� n0|tk�r^|tk�r^t � | �j }t �| ||f� t�rjW dS |�r�t �� dk�r�t| �} nt} | �r�t| �} | tk�r�| tk�r�t �| | | � nF| tk�r�| tk�r�t �| | d� n"| tk�r| tk�rt �| d| � | �r$tt| �d�} t �| | � W n" t�yH t d|j ��Y n0 dS )a� Parses POSIX attributes from obj_metadata and sets them. Attributes will only be set if they exist in custom metadata. This function should only be called after ValidateFilePermissionAccess has been called for the specific file/object so as not to orphan files. Args: path: The local filesystem path for the file. Valid metadata attributes will be set for the file located at path, some attributes will only be set if preserve_posix is set to True. obj_metadata: The metadata for the file/object. is_rsync: Whether or not the caller is the rsync command. Used to determine if timeCreated should be used. preserve_posix: Whether or not all POSIX attributes should be set. Nz"obj_metadata cannot be None for %s)� default_valuer r r z#Check POSIX attribute values for %s)r r r/ r' rC r8 r( r7 r9 r r0 rR �stat�st_atime�utime�ConvertDatetimeToPOSIX� timeCreated�st_mtimer rS r+ �chownrT �chmodr4 �name)�pathr< �is_rsyncrD Zfound_atr Zfound_mtr Z found_uidr Z found_gidr Z found_moder Z atime_tmpZ mtime_tmpr r r! �ParseAndSetPOSIXAttributeso s� � � � � � �rm c C s t �� �d|| � dS )z�Logs if an attribute has a negative value. Args: attr_name: The name of the attribute to log. url_str: The path of the file for context. z$%s has a negative %s in its metadataN��logging� getLogger�warn�rJ r= r r r! r1 � s �r1 c C s t �� �d|| � dS )��Logs if an attribute has an invalid value. Args: attr_name: The name of the attribute to log. url_str: The path of the file for context. z$%s has an invalid %s in its metadataNrn rr r r r! r5 � s �r5 c C s t �� �d|| � dS )rs z5%s has an %s more than 1 day from current system timeNrn rr r r r! r3 � s �r3 c C s t t| jt� d��� ��S )a Converts a datetime object to UTC and formats as POSIX. Sanitize the timestamp returned in dt, and put it in UTC format. For more information see the UTC class. Args: dt: A Python datetime object. Returns: A POSIX timestamp according to UTC. )�tzinfo)r0 r �replacer � timetuple)�dtr r r! re � s re c C sD t r dadS d} t�d�}t�|� | | }t|d@ �dd� adS )z.Records the default POSIX mode using os.umask.Z666Ni� � i� r* )r rU rR �umaskr, )Zmax_permissionsZ current_umaskr r r r! �InitializeDefaultMode� s rz c sH t rdS t�� } t�| �j� tt�| �jg� fdd�t� � D � �a dS )z�Initializes the set of groups that the user is in. Should only be called if the flag for preserving POSIX attributes is set. Nc s g | ]}� |j v r|j�qS r )�gr_mem�gr_gid)�.0�g�Z user_namer r! � <listcomp> � z(InitializeUserGroups.<locals>.<listcomp>) r rR r\ rV rW �pw_name�set�pw_gidrZ �getgrallr^ )�user_idr r r! �InitializeUserGroups s ��r� c C s t � t� dS )z<Initializes POSIX data. Run once at the beginning of a copy.N)rz r� r r r r! �InitializePreservePosixData" s r� )F)FF)Fr& � __future__r r r r �calendarr �getpassro rR �rer2 �six�gslib.exceptionr Z gslib.tz_utcr �gslib.utils.metadata_utilr r �gslib.utils.system_utilr �gslib.utils.unit_utilr rZ rV �PY3r+ r0 r/ r7 r9 rC r8 r' r( r �compiler: rM �U_WZU_XrN ZG_WZG_XrO ZO_WZO_XrU r� r^ �objectr r r. r@ rE r6 rL rP r` rm r1 r5 r3 re rz r� r� r r r r! �<module> s| , � 1X � _ utils/__pycache__/shim_util.cpython-39.pyc 0000644 00000047463 15025013616 0014557 0 ustar 00 a +(Wg�l � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z ddl mZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ G dd� dej�ZG dd� dej�Ze �d�Ze g d��Z e e g d��B Z e g d��Z e g d��Z e g d��Z!ddddd d �d d d d!d"d#d$d%d&d'd(� d)d*d+d,d-d.d/d0�d1d2d3d4d5�d6�Z"e d7d8g�Z#dPd:d;�Z$d<d=� Z%d>d?� Z&G d@dA� dAe'�Z(G dBdC� dCe'�Z)dDdE� Z*dFdG� Z+dHdI� Z,dJdK� Z-dLdM� Z.G dNdO� dOe'�Z/dS )QzCHelper for shim used to translate gsutil command to gcloud storage.� )�absolute_import)�print_function)�division)�unicode_literalsN)�config)� exception)� ApiSelector)�CommandException)� boto_util)� constants)� system_utilc @ s e Zd ZdZdZdZdS )�HIDDEN_SHIM_MODEZ no_fallback�dry_run�noneN)�__name__� __module__� __qualname__� NO_FALLBACK�DRY_RUN�NONE� r r �$/opt/gsutil/gslib/utils/shim_util.pyr % s r c @ s e Zd ZdZdZdS )�RepeatFlagTyper � N)r r r �LIST�DICTr r r r r + s r z(^decryption_key([1-9]$|[1-9][0-9]$|100$))�cp�mv�rsyncZsetmeta)Zls�rewrite�stat�cat�compose)r" r Zrm� retention)z cache-controlzcontent-dispositionzcontent-encoding� content-md5zcontent-languagez content-typez custom-time)zx-goog-generation-matchzx-goog-if-generation-matchzx-goog-metageneration-matchz x-goog-if-metageneration-match�AWS_ACCESS_KEY_ID�AWS_SECRET_ACCESS_KEYZ%CLOUDSDK_STORAGE_GS_XML_ACCESS_KEY_IDZ)CLOUDSDK_STORAGE_GS_XML_SECRET_ACCESS_KEYZ-CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE)�aws_access_key_id�aws_secret_access_key�gs_access_key_id�gs_secret_access_key�use_client_certificateZCLOUDSDK_PROXY_ADDRESSZCLOUDSDK_PROXY_TYPEZCLOUDSDK_PROXY_PORTZCLOUDSDK_PROXY_USERNAMEZCLOUDSDK_PROXY_PASSWORDZCLOUDSDK_PROXY_RDNSZCLOUDSDK_CORE_HTTP_TIMEOUTZ"CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILEZ!CLOUDSDK_STORAGE_BASE_RETRY_DELAYZ CLOUDSDK_STORAGE_MAX_RETRIES) �proxy� proxy_type� proxy_port� proxy_user� proxy_pass� proxy_rdns�http_socket_timeout�ca_certificates_file�max_retry_delay� num_retriesZ CLOUDSDK_STORAGE_CHECK_HASHESZCLOUDSDK_CORE_PROJECTZ%CLOUDSDK_CORE_DISABLE_USAGE_REPORTINGZ CLOUDSDK_STORAGE_USE_MAGICFILEZ4CLOUDSDK_STORAGE_PARALLEL_COMPOSITE_UPLOAD_THRESHOLDZ$CLOUDSDK_STORAGE_RESUMABLE_THRESHOLDZ&CLOUDSDK_STORAGE_RSYNC_LIST_CHUNK_SIZE)� check_hashes�default_project_id�disable_analytics_prompt� use_magicfile�#parallel_composite_upload_threshold�resumable_threshold�rsync_buffer_linesZCLOUDSDK_AUTH_CLIENT_IDZCLOUDSDK_AUTH_CLIENT_SECRETZCLOUDSDK_AUTH_AUTH_HOSTZCLOUDSDK_AUTH_TOKEN_HOST)� client_id� client_secretZprovider_authorization_uri�provider_token_uri)� Credentials�Boto�GSUtil�OAuth2Zstet_binary_pathZstet_config_pathFc C s� | � � }|tv r:| td�d� }|�d�s4d| }qL|}n|tv rH|}nd}|dur�|rt|tv sh|dkrldS d| S d�||�S dD ]>}|�|�r�| t|�d� }|r�d| S d �||� S q�dS ) a� Returns the gcloud storage flag for the given gsutil header. Args: raw_header_key: The header key. header_value: The header value unset: If True, the equivalent clear/remove flag is returned instead of the setter flag. This only applies to setmeta. Returns: A string representing the equivalent gcloud storage flag and value, if translation is possible, else returns None. Examples: >> get_flag_from_header('Cache-Control', 'val') --cache-control=val >> get_flag_from_header('x-goog-meta-foo', 'val') --update-custom-metadata=foo=val >> get_flag_from_header('x-goog-meta-foo', 'val', unset=True) --remove-custom-metadata=foo zx-goog-Nzif-r$ z--clear-z--{}={})z x-goog-meta-z x-amz-meta-z--remove-custom-metadata=z --update-custom-metadata={}={})�lower�PRECONDITIONS_HEADERS�len� startswith�DATA_TRANSFER_HEADERS�format)�raw_header_key� header_value�unset�lowercase_header_keyZproviderless_flagZ flag_name� header_prefixZ metadata_keyr r r �get_flag_from_header� s0 � rO c C s t jr dS dS )Nz^^^^�^�r � IS_WINDOWSr r r r �get_format_flag_caret� s rS c C s t jr dS dS )Nz^\^n� rQ r r r r �get_format_flag_newline� s rU c @ s e Zd Zddd�ZdS )�GcloudStorageFlagNFc C s || _ || _|| _dS )a Initializes GcloudStorageFlag. Args: gcloud_flag (str|dict): The name of the gcloud flag or a dictionary for when the gcloud flag depends on a gsutil value. gsutil "--pap off" -> gcloud "--no-public-access-prevention" repeat_type (RepeatFlagType|None): Gsutil sometimes handles list and dictionary inputs by accepting a flag multiple times. support_output_translation (bool): If True, this flag in gcloud storage supports printing gsutil formatted output. N)� gcloud_flag� repeat_type�supports_output_translation)�selfrW rX rY r r r �__init__� s zGcloudStorageFlag.__init__)NF)r r r r[ r r r r rV � s �rV c @ s e Zd ZdZddd�ZdS )�GcloudStorageMapzEMapping to translate gsutil command to its gcloud storage equivalent.Fc C s || _ || _|| _dS )a- Intalizes GcloudStorageMap. Args: gcloud_command (dict|str): The corresponding name of the command to be called in gcloud. If this command supports sub-commands, then this field must be a dict of sub-command-name:GcloudStorageMap pairs. flag_map (dict): A dict of str to GcloudStorageFlag. Mapping of gsutil flags to their equivalent gcloud storage flag names. supports_output_translation (bool): Indicates if the corresponding gcloud storage command supports the printing gsutil formatted output. N)�gcloud_command�flag_maprY )rZ r] r^ rY r r r r[ � s zGcloudStorageMap.__init__N)F)r r r �__doc__r[ r r r r r\ � s �r\ c C s t j�| dtjrdnd�S )N�binz gcloud.cmd�gcloud)�os�path�joinr rR )� cloudsdk_rootr r r �_get_gcloud_binary_path� s �rf c C s: t j�d�} | r| S t j�d�}|d u r2t�d��t|�S )NZGCLOUD_BINARY_PATHZCLOUDSDK_ROOT_DIRa� Requested to use "gcloud storage" but the gcloud binary path cannot be found. This might happen if you attempt to use gsutil that was not installed via Cloud SDK. You can manually set the `CLOUDSDK_ROOT_DIR` environment variable to point to the google-cloud-sdk installation directory to resolve the issue. Alternatively, you can set `use_gcloud_storage=False` to disable running the command using gcloud storage.)rb �environ�getr � GcloudStorageTranslationErrorrf )�gcloud_binary_pathre r r r �!_get_validated_gcloud_binary_path� s �rk c C sL | � dd�}|rH| � dd�}|r(d| nd}| � ddd�}d�|||�S d S ) Nr@ � gs_json_host� gs_json_port�:� �json_api_version�v1zhttps://{}{}/storage/{}�rh rI )r rl rm �portrp r r r �'_get_gcs_json_endpoint_from_boto_config s �rt c C s< | � dd�}|r8| � dd�}|r(d| nd}d�||�S d S )Nr@ �s3_host�s3_portrn ro z https://{}{}rr )r ru rv rs r r r �!_get_s3_endpoint_from_boto_config s rw c C s| g }t �t�}d}|t| �k �r| | |jvrF|�| | � |d7 }q|j| | }|sb|d7 }q|jr�|| �| |d � |d7 }qt|jt �r�|�|j� |d7 }q| |d |jvr�t d� | | | |d ���|j| |d }|r�|�|� |d7 }q|� � D ]l\}}|jt ju �r,d�|�}n4|jt ju �rPd�dd� |D ��}nt d� |j���|�d � |j|�� �q |S ) Nr r � z.Flag value not in translation map for "{}": {}�,c S s g | ]}d j |�dd�� �qS )�{}={}rn r )rI �split)�.0�sr r r � <listcomp>O � z2_convert_args_to_gcloud_values.<locals>.<listcomp>z'Shim cannot handle repeat flag type: {}rz )� collections� defaultdict�listrF r^ �appendrX � isinstancerW �str� ValueErrorrI �itemsr r rd r ) �args�gcloud_storage_mapZ gcloud_argsZrepeat_flag_data�iZgcloud_flag_objectZtranslated_flag_and_value�valuesZcondensed_flag_valuesr r r � _convert_args_to_gcloud_values$ sR �� �� � r� c @ sz e Zd ZdZdZdd� Zdd� Zdd� Zd d � Zd d d �Z dd� Z d dd�Z d dd�Z dd� Z dd� Zdd� Zdd� ZdS ) �GcloudStorageCommandMixinz�Provides gcloud storage translation functionality. The command.Command class must inherit this class in order to support converting the gsutil command to it's gcloud storage equivalent. Nc C s d | _ d | _d | _d S �N)�"_translated_gcloud_storage_command�_translated_env_variables�_gcloud_has_active_account)rZ r r r r[ b s z"GcloudStorageCommandMixin.__init__c C st t � }| jdu rntj|dddgtjtjdd�}|jrFt�d�|j ���|j � � dk| _| j � d �|j |j �� | jS ) z8Returns True if gcloud has an active account configured.Nr rh �accountzutf-8)�stdout�stderr�encodingzJError occurred while trying to retrieve gcloud's active account. Error: {}ro zHResult for "gcloud config get account" command: STDOUT: {}. STDERR: {})rk r� � subprocess�run�PIPE� returncoder ri rI r� r� �strip�logger�debug)rZ Z gcloud_path�processr r r �gcloud_has_active_accountg s$ ��� �z3GcloudStorageCommandMixin.gcloud_has_active_accountc C s� |d u rt �d�| j���g }t|jt�r8|�|j� njt|jt�r�|j rZt d�| j���|d }| j |dd � dd�\}}| � |||j� |��S t d�| j���|r�|D ]@\}}||j vr�t �d�|���q�|�|� |d kr�|�|� q�t|| |�S ) Nz_Command "{}" cannot be translated to gcloud storage because the translation mapping is missing.zcFlags mapping should not be present at the top-level command if a sub-command is used. Command: {}.r r F)r� � should_update_sub_opts_and_argsz(Incorrect mapping found for "{}" commandz:Command option "{}" cannot be translated to gcloud storagero )r ri rI � command_namer� r] r� �extend�dictr^ r� � ParseSubOpts�_get_gcloud_storage_argsrh r� r� ) rZ �sub_optsZ gsutil_argsr� r� Z sub_command� parsed_args�option�valuer r r r� { sR �� �� � �� �� �z2GcloudStorageCommandMixin._get_gcloud_storage_argsc C s� g }ddd�}| j dkr&|�ddg� | j dkr:|�d� | jrJ|�d � | jr`|�d | j � | jrv|�d | j � tjr�|�d tj � | jd ko�t � dd�dko�t � dd�dk }| j s�|s�d|d< d|d< ||fS )a� Translates gsutil's top level flags. Gsutil specifies the headers (-h) and boto config (-o) as top level flags as well, but we handle those separately. Returns: A tuple. The first item is a list of top level flags that can be appended to the gcloud storage command. The second item is a dict of environment variables that can be set for the gcloud storage command execution. Z gsutil_shim�True)Z CLOUDSDK_METRICS_ENVIRONMENTZ#CLOUDSDK_STORAGE_RUN_BY_GSUTIL_SHIM� z --verbosityr� � z --log-httpz--no-user-output-enabledz--billing-project=z--trace-token=z --impersonate-service-account=r rB �parallel_process_count�1Zthread_process_countZ CLOUDSDK_STORAGE_THREAD_COUNTZ CLOUDSDK_STORAGE_PROCESS_COUNT) r� r� r� � quiet_mode� user_project� trace_tokenr �IMPERSONATE_SERVICE_ACCOUNTr� r rh �parallel_operations)rZ �top_level_flags� env_variablesZshould_use_rsync_overrider r r �_translate_top_level_flags� s4 � � � z4GcloudStorageCommandMixin._translate_top_level_flagsFc C s� g }|dur|n| j }g }|�� D ]�\}}|�� }|dkr<q"t|||d�} | jtv rd| r�|�| � n | jtv r�|tv r�|�| � | s"| j � d� ||�� |�d� ||�� q"|r�|�dd� |� � |S )ak Translates gsutil headers to equivalent gcloud storage flags. Args: headers (dict|None): If absent, extracts headers from command instance. unset (bool): Yield metadata clear flags instead of setter flags. Returns: List[str]: Translated flags for gcloud. Raises: GcloudStorageTranslationError: Could not translate flag. Nzx-goog-api-version)rL zzHeader {}:{} cannot be translated to a gcloud storage equivalent flag. It is being treated as an arbitrary request header.rz z--additional-headers=ry ) �headersr� rD rO r� � COMMANDS_SUPPORTING_ALL_HEADERSr� �$PRECONDITONS_ONLY_SUPPORTED_COMMANDSrE r� �warnrI rd ) rZ r� rL �flagsZheaders_to_translate�additional_headersrJ rK rM �flagr r r �_translate_headers� s0 � �z,GcloudStorageCommandMixin._translate_headersc C sb g }i }t t�}|r ||d< tt�}|r0||d< g }t�� D �] \}}|�� D ]�\}} |dkrx| jtv rx|�d| � qNt�|�r�| jtv r�|�| � qN|dkr�| jt v r�|�d| � qN|t v r�| j � d� ||�� qN|dkr�| s�d |d < qN|d v �rt�� �s| j �d � ||�� qNt�|i ��|d �} | d urN| || < qNq<|�rZ|�dd�|� � ||fS )a Translates boto config options to gcloud storage properties. Returns: A tuple where first element is a list of flags and the second element is a dict representing the env variables that can be set to set the gcloud storage properties. Z'CLOUDSDK_API_ENDPOINT_OVERRIDES_STORAGEZ CLOUDSDK_STORAGE_S3_ENDPOINT_URL�encryption_keyz--encryption-key=�content_languagez--content-language=zNThe boto config field {}:{} cannot be translated to gcloud storage equivalent.�https_validate_certificatesTZ$CLOUDSDK_AUTH_DISABLE_SSL_VALIDATION)r) r* zxThe boto config field {}:{} skipped translation to the gcloud storage equivalent as it would have been unused in gsutil.Nz--decryption-keys=ry )rt r rw r� r� � ENCRYPTION_SUPPORTED_COMMANDSr� �DECRYPTION_KEY_REGEX�matchr� �'_REQUIRED_BOTO_CONFIG_NOT_YET_SUPPORTEDr� �errorrI r � UsingGsHmacr� �_BOTO_CONFIG_MAPrh rd ) rZ r� Zenv_varsZgcs_json_endpointZ s3_endpointZdecryption_keys� section_name�section�keyr� �env_varr r r �_translate_boto_config� sR � � � � � � z0GcloudStorageCommandMixin._translate_boto_configc C s | � | j| j|p| j�S )a� Translates the gsutil command flags to gcloud storage flags. It uses the command_spec.gcloud_storage_map field that provides the translation mapping for all the flags. Args: gcloud_storage_map (GcloudStorageMap|None): Command surface may pass a custom translation map instead of using the default class constant. Useful for when translations change based on conditional logic. Returns: A list of all the options and arguments that can be used with the equivalent gcloud storage command. Raises: GcloudStorageTranslationError: If a flag or command cannot be translated. ValueError: If there is any issue with the mapping provided by GcloudStorageMap. )r� r� r� r� )rZ r� r r r �get_gcloud_storage_args* s �z1GcloudStorageCommandMixin.get_gcloud_storage_argsc C sV |r | j jn| j j}|d�d�|��� |rR|d� |�� D ]\}}|d||� q<d S )NzGcloud Storage Command: {}� z)Environment variables for Gcloud Storage:z%s=%s)r� �infor� rI rd r� )rZ r] r� r Z logger_func�k�vr r r �"_print_gcloud_storage_command_infoA s z<GcloudStorageCommandMixin._print_gcloud_storage_command_infoc C sN | � � \}}| �� }| �� \}}|�|� t� }|g| | | | }||fS r� )r� r� r� �updaterk ) rZ r� r� r� Z header_flagsZflags_from_botoZenv_vars_from_botorj �gcloud_storage_commandr r r �._get_full_gcloud_storage_execution_informationL s ��zHGcloudStorageCommandMixin._get_full_gcloud_storage_execution_informationc C sp | j dks| j dkrdS t�ddd�}ztt�ddd��}W n2 tyn t�d�d � d d � tD �����Y n0 |�rlz�| � | � � �\}}| � � s�d |d < |tj kr�| j||dd� nftj�d�s�t�d�t� ���nFt�� r�tj| jjvr�td�| j ���n | �||� || _|| _W dS W nL tj�yj } z0|tjk�rHt�|��| j � d|� W Y d}~n d}~0 0 dS )a\ Translates the gsutil command to gcloud storage equivalent. The translated commands get stored at self._translated_gcloud_storage_command. This command also translate the boto config, which gets stored as a dict at self._translated_env_variables Returns: True if the command was successfully translated, else False. �version�testFrB �use_gcloud_storage�hidden_shim_moder zYInvalid option specified for GSUtil:hidden_shim_mode config setting. Should be one of: {}z | c S s g | ] }|j �qS r )r� )r| �xr r r r~ s r zVGcloudStorageCommandMixin.translate_to_gcloud_storage_if_requested.<locals>.<listcomp>r� Z!CLOUDSDK_AUTH_DISABLE_CREDENTIALST)r �(CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTILz�Requested to use "gcloud storage" but gsutil is not using the same credentials as gcloud. You can make gsutil use the same credentials by running: {} config set pass_credentials_to_gsutil Truez�Requested to use "gcloud storage" with Cloud Storage XML API HMAC credentials but the "{}" command can only be used with the Cloud Storage JSON API.zYCannot translate gsutil command to gcloud storage. Going to run gsutil command. Error: %sN) r� r �getboolr rh r� r r rI rd r� r� r� r r� rb rg ri rk r r� r �XML� command_spec�gs_api_supportr� r� r r� r� )rZ r� r� r� r� �er r r �(translate_to_gcloud_storage_if_requestedY sj � �� � � �� ���� �zBGcloudStorageCommandMixin.translate_to_gcloud_storage_if_requestedc C s t j�� }|�| j� |S r� )rb rg �copyr� r� )rZ Zsubprocess_envsr r r �'_get_shim_command_environment_variables� s zAGcloudStorageCommandMixin._get_shim_command_environment_variablesc C s t j| j| �� d�}|jS )N)�env)r� r� r� r� r� )rZ r� r r r �run_gcloud_storage� s �z,GcloudStorageCommandMixin.run_gcloud_storage)NF)N)F)r r r r_ r� r[ r� r� r� r� r� r� r� r� r� r� r� r r r r r� Y s +' '6 � Dr� )F)0r_ � __future__r r r r r� �enumrb �rer� �botor �gslibr �gslib.cs_api_mapr �gslib.exceptionr � gslib.utilsr r r �Enumr r �compiler� � frozensetr� r� r� rH rE r� r� rO rS rU �objectrV r\ rf rk rt rw r� r� r r r r �<module> s� �� � ����1� : 5 utils/__pycache__/stet_util.cpython-39.pyc 0000644 00000006547 15025013616 0014574 0 ustar 00 a +(WgU � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd l mZ G d d � d e�Zd d� Zdd� Zdd� Zdd� ZdS )z=Helper functions for Split Trust Encryption Tool integration.� )�absolute_import)�print_function)�division)�unicode_literalsN)� storage_url)�execution_util)�temporary_file_util)�configc @ s e Zd ZdZdZdZdS )�StetSubcommandNamez*Enum class for available STET subcommands.�encrypt�decryptN)�__name__� __module__� __qualname__�__doc__�ENCRYPT�DECRYPT� r r �$/opt/gsutil/gslib/utils/stet_util.pyr s r c C s@ t �d��t jj�D ]&} t j�| d�}t j�|�r| S qdS )zBRetrieves STET binary from path if available. Python 2 compatible.�PATH�stetN)�os�getenv�split�path�pathsep�join�exists)Zpath_directory� binary_pathr r r �_get_stet_binary_from_path& s r c C s� t �ddt� �}|s td��tj�|�| g}t �ddd�}|rT|�dtj�|� � |�d| ||g� t � |�\}} |� | � dS )a Runs a STET transform on a file. Encrypts for uploads. Decrypts for downloads. Automatically populates flags for the STET binary. Args: subcommand (StetSubcommandName): Subcommand to call on STET binary. blob_id (str): Cloud URL that binary uses for validation. in_file_path (str): File to be transformed source. out_file_path (str): Where to write result of transform. logger (logging.Logger): For logging STET binary output. Raises: KeyError: STET binary or config could not be found. �GSUtil�stet_binary_pathz2Could not find STET binary in boto config or PATH.�stet_config_pathNz--config-file=z --blob-id=) r �getr �KeyErrorr r � expanduser�append�extendr �ExecuteExternalCommand�debug) � subcommand�blob_idZ in_file_pathZ out_file_path�loggerr Z command_argsZ config_path�_�stderrr r r �_stet_transform. s �r/ c C s2 | j }t�| �}|j}ttj||||� t�|�S )a) Encrypts a file with STET binary before upload. Args: source_url (StorageUrl): Copy source. destination_url (StorageUrl): Copy destination. logger (logging.Logger): For logging STET binary output. Returns: stet_temporary_file_url (StorageUrl): Path to STET-encrypted file. ) � object_namer �GetStetTempFileName� url_stringr/ r r r �StorageUrlFromString)� source_url�destination_urlr, �in_file�out_filer+ r r r �encrypt_uploadM s �r8 c C s6 |}t �|�}| j}ttj||||� t�||� dS )a STET-decrypts downloaded file. Args: source_url (StorageUrl): Copy source. destination_url (StorageUrl): Copy destination. temporary_file_name (str): Path to temporary file used for download. logger (logging.Logger): For logging STET binary output. N)r r1 r2 r/ r r �shutil�move)r4 r5 �temporary_file_namer, r6 r7 r+ r r r �decrypt_downloada s �r< )r � __future__r r r r r r9 �gslibr � gslib.utilsr r �botor �objectr r r/ r8 r< r r r r �<module> s utils/__pycache__/retention_util.cpython-39.pyc 0000644 00000021427 15025013616 0015616 0 ustar 00 a +(Wg�) � @ s4 d Z ddlmZ ddlmZ ddlmZ ddlZddlm Z ddl m Z ddl m Z d Zd e Zed e �Zd Ze d d� �Ze dd� �Ze dd� �Ze dd� �Zdd� Zdd� Zdd� Zdd� Zdd� Z d4d d �Z d5d d �Z d6d d!�Z d7d"d#�Z d$d%� Z!d&d'� Z"d(d)� Z#d*d+� Z$d,d-� Z%d.d/� Z&d0d1� Z'd2d3� Z(dS )8zEImplementation of Retention Policy configuration command for buckets.� )�absolute_import)�input)�DecimalN)�CommandException)� LazyWrapper)�storage_v1_messagesi�Q � g �v@znThis will PERMANENTLY set the Retention Policy on gs://{} to: {} This setting cannot be reverted! Continue?c C s t �d�S )Nz(?P<number>\d+)y$��re�compile� r r �)/opt/gsutil/gslib/utils/retention_util.py�<lambda>" � r c C s t �d�S )Nz(?P<number>\d+)m$r r r r r r $ r c C s t �d�S )Nz(?P<number>\d+)d$r r r r r r & r c C s t �d�S )Nz(?P<number>\d+)s$r r r r r r ( r c C sz d}|rd| ddf }nd| ddf }t |��� }|s:|S |dvrLtd� q&|d ks\|dkr`d S |d ksp|dkr&d S q&d S )a Prompts user to confirm an action with yes or no response. Args: question: Yes/No question to be used for the prompt. default_response: Default response to the question: True, False Returns: Returns the rough equivalent duration in seconds. � z %s [%s|%s]: �Y�n�y�N)r �yesr �noz+ Please respond with 'yes'/'y' or 'no'/'n'.r Tr FN)r �lower�print)ZquestionZdefault_response�prompt�responser r r �_ConfirmWithUserPrompt+ s r c C s� t | �}d}|t |t kr,d�|t �}nd|t |t krLd�|t �}nD|t |t krld�|t �}n$|tkr�d�| |t �}n d�| �}d�|�S )z�Converts Retention Period to Human readable format. Args: retention_period: Retention duration in seconds (integer value). Returns: Returns a string representing retention duration in human readable format. Nz {} Year(s)z {} Month(s)z {} Day(s)z{} Seconds (~{} Day(s))z {} Second(s)z Duration: {})r �SECONDS_IN_YEAR�format�SECONDS_IN_MONTH�SECONDS_IN_DAY)�retention_periodZperiodZ duration_strr r r �_RetentionPeriodToStringH s � r! c C sV d}| rH| j rH| jrdnd}t| j �}d�| j�d��}d�|||�}n d�|�}|S )z3Converts Retention Policy to Human readable format.r z(LOCKED)z (UNLOCKED)z Effective Time: {}z%a, %d %b %Y %H:%M:%S GMTz Retention Policy {}: {} {}z{} has no Retention Policy.)�retentionPeriod�isLockedr! r � effectiveTime�strftime)�retention_policy� bucket_urlZretention_policy_strZ locked_stringr Zretention_effective_timer r r �RetentionPolicyToStringe s � �� r( c C s t || �}t�| |�}t|d�S )NF)r( � _LOCK_PROMPTr r )r' r& Z lock_promptr r r �ConfirmLockRequestv s r* c C s | j �|� d| _dS )zDException handler that maintains state about post-completion status.FN)�logger�error�everything_set_okay)�cls�er r r �$UpdateObjectMetadataExceptionHandler| s r0 c C s& d}t jdd�}| j||||d� d S )Nz Setting Temporary Hold on %s...T�� temporaryHold�� thread_state��apitools_messages�ObjectZObjectUpdateMetadataFunc�r. �name_expansion_resultr4 Z log_templateZobject_metadata_updater r r �SetTempHoldFuncWrapper� s �r: c C s& d}t jdd�}| j||||d� d S )Nz!Releasing Temporary Hold on %s...Fr1 r3 r5 r8 r r r �ReleaseTempHoldFuncWrapper� s �r; c C s& d}t jdd�}| j||||d� d S )Nz!Setting Event-Based Hold on %s...T��eventBasedHoldr3 r5 r8 r r r �SetEventHoldFuncWrapper� s �r> c C s& d}t jdd�}| j||||d� d S )Nz#Releasing Event-Based Hold on %s...Fr< r3 r5 r8 r r r �ReleaseEventHoldFuncWrapper� s �r? c C s | t S )z�Converts duration specified in days to equivalent seconds. Args: days: Retention duration in number of days. Returns: Returns the equivalent duration in seconds. )r ��daysr r r � DaysToSeconds� s rB c C s | t S )a� Converts duration specified in months to equivalent seconds. GCS bucket lock API uses following duration equivalencies to convert durations specified in terms of months or years to seconds: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Args: months: Retention duration in number of months. Returns: Returns the rough equivalent duration in seconds. )r ��monthsr r r �MonthsToSeconds� s rE c C s | t S )a� Converts duration specified in years to equivalent seconds. GCS bucket lock API uses following duration equivalencies to convert durations specified in terms of months or years to seconds: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Args: years: Retention duration in number of years. Returns: Returns the rough equivalent duration in seconds. )r �Zyearsr r r �YearsToSeconds� s rG c C s t � �| �S )aa Test whether the string matches retention in years pattern. Args: years: string to match for retention specified in years format. Returns: Returns a match object if the string matches the retention in years pattern. The match object will contain a 'number' group for the duration in number of years. Otherwise, None is returned. )�_RETENTION_IN_YEARS�matchrF r r r �RetentionInYearsMatch� s rJ c C s t � �| �S )af Test whether the string matches retention in months pattern. Args: months: string to match for retention specified in months format. Returns: Returns a match object if the string matches the retention in months pattern. The match object will contain a 'number' group for the duration in number of months. Otherwise, None is returned. )�_RETENTION_IN_MONTHSrI rC r r r �RetentionInMonthsMatch� s rL c C s t � �| �S )a\ Test whether the string matches retention in days pattern. Args: days: string to match for retention specified in days format. Returns: Returns a match object if the string matches the retention in days pattern. The match object will contain a 'number' group for the duration in number of days. Otherwise, None is returned. )�_RETENTION_IN_DAYSrI r@ r r r �RetentionInDaysMatch� s rN c C s t � �| �S )ak Test whether the string matches retention in seconds pattern. Args: seconds: string to match for retention specified in seconds format. Returns: Returns a match object if the string matches the retention in seconds pattern. The match object will contain a 'number' group for the duration in number of seconds. Otherwise, None is returned. )�_RETENTION_IN_SECONDSrI )�secondsr r r �RetentionInSecondsMatch� s rQ c C s� d}t | �}t| �}t| �}t| �}|r<tt|�d���}nL|rTtt|�d���}n4|rltt|�d���}n |r�t|�d��}nt d��|S )z�Converts a retention period string pattern to equivalent seconds. Args: pattern: a string pattern that represents a retention period. Returns: Returns the retention period in seconds. If the pattern does not match N�numberz�Incorrect retention period specified. Please use one of the following formats to specify the retention period : <number>y, <number>m, <number>d, <number>s.) rJ rL rN rQ rG �int�grouprE rB r )�patternrP Z year_matchZ month_matchZ day_matchZ second_matchr r r �RetentionInSeconds s rV )N)N)N)N))�__doc__� __future__r � six.movesr �decimalr r �gslib.exceptionr �gslib.lazy_wrapperr �"gslib.third_party.storage_apitoolsr r6 r r rS r r) rH rK rM rO r r! r( r* r0 r: r; r> r? rB rE rG rJ rL rN rQ rV r r r r �<module> sB � utils/__pycache__/hashing_helper.cpython-39.pyc 0000644 00000037324 15025013616 0015535 0 ustar 00 a +(WgHI � @ sv d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl m Z ddl Z ddlmZ dd lmZ dd lmZ dd lmZ dd lmZ dd lmZ dZdZdZdZdZdZ dZ dZ dZ g d�Z dZ!dZ"dd� Z#d d � Z$d d � Z%d d!� Z&d:d"d#�Z'd$d%� Z(d&d'� Z)d(d)� Z*d*d+� Z+d,d-� Z,d.d/� Z-d0d1� Z.d;d3d4�Z/d<d6d7�Z0G d8d9� d9e1�Z2dS )=z+Helper functions for hashing functionality.� )�absolute_import)�print_function)�division)�unicode_literalsN)�config)�CommandException)�UsingCrcmodExtension)�DEFAULT_FILE_BUFFER_SIZE)�MIN_SIZE_COMPUTE_LOGGING)�TRANSFER_BUFFER_SIZE)�UTF8z� WARNING: You have requested checksumming but your crcmod installation isn't using the module's C extension, so checksumming will run very slowly. For help installing the extension, please see "gsutil help crcmod". a� WARNING: gsutil rsync uses hashes when modification time is not available at both the source and destination. Your crcmod installation isn't using the module's C extension, so checksumming will run very slowly. If this is your first rsync since updating gsutil, this rsync can take significantly longer than usual. For help installing the extension, please see "gsutil help crcmod". a| WARNING: Downloading this composite object requires integrity checking with CRC32c, but your crcmod installation isn't using the module's C extension, so the hash computation will likely throttle download performance. For help installing the extension, please see "gsutil help crcmod". To disable slow integrity checking, see the "check_hashes" option in your boto config file. a: Downloading this composite object requires integrity checking with CRC32c, but your crcmod installation isn't using the module's C extension, so the hash computation will likely throttle download performance. For help installing the extension, please see "gsutil help crcmod". To download regardless of crcmod performance or to skip slow integrity checks, see the "check_hashes" option in your boto config file. NOTE: It is strongly recommended that you not disable integrity checks. Doing so could allow data corruption to go undetected during uploading/downloading.aT WARNING: This download will not be validated since your crcmod installation doesn't use the module's C extension, so the hash computation would likely throttle download performance. For help in installing the extension, please see "gsutil help crcmod". To force integrity checking, see the "check_hashes" option in your boto config file. �if_fast_else_fail�if_fast_else_skip�always�never) � � � � i iAo� ivE�:i Wi�P�Yl T/U l ?O i��Ni.�%i'��hi��nTi��\Fib�&l �"�, iGWL;i�gl �b ! l <�c l �!� l �N,Q l �2�! l =X� i��D3iK�S}i�2�=l �m�y i�x�Bl Ao�= � c C s |s| S t | d| �|A S )a� Computes CRC32C for concat(A, B) given crc(A), crc(B) and len(B). An explanation of the algorithm can be found at crcutil.googlecode.com/files/crc-doc.1.0.pdf. Args: crc_a: A 32-bit integer representing crc(A) with least-significant coefficient first. crc_b: Same as crc_a. num_bytes_in_b: Length of B in bytes. Returns: CRC32C for concat(A, B) � )�_ExtendByZeros)Zcrc_aZcrc_bZnum_bytes_in_b� r �)/opt/gsutil/gslib/utils/hashing_helper.py� ConcatCrc32cj s r c C sN d}dt > }tt �D ]4}| d@ r(||N }|dK }||@ r@|tN }| dL } q|S )z�Multiplies two polynomials together modulo CASTAGNOLI_POLY. Args: p: The first polynomial. q: The second polynomial. Returns: Result of the multiplication. r � )�DEGREE�range�CASTAGNOLI_POLY)�p�q�resultZtop_bit�_r r r � _CrcMultiply s r# c C sX dd� }|| �} d}|dkrL|d@ r:t | t|tt� �} |d7 }|dL }q|| �} | S )z�Given crc representing polynomial P(x), compute P(x)*x^num_bits. Args: crc: crc respresenting polynomial P(x). num_bits: number of bits in crc. Returns: P(x)*x^num_bits c S s t dj| dd�d d d� d�S )Nz{0:032b}r )�width���r )�int�format)�crcr r r �_ReverseBits32� s z&_ExtendByZeros.<locals>._ReverseBits32r r )r# �X_POW_2K_TABLE�len)r( �num_bitsr) �ir r r r � s r c C s2 d|i}| � d� t| |� | � d� |d �� S )a& Calculates a base64 digest of the contents of a seekable stream. This function resets the file pointer to position 0. Args: fp: An already-open file object. hash_alg: Instance of hashing class initialized to start state. Returns: Hash of the stream in hex string format. � placeholderr )�seek�CalculateHashesFromContents� hexdigest)�fp�hash_alg� hash_dictr r r �_CalculateHashFromContents� s r5 c C s\ | � t�}|sqXtjr*t|t�r*|�t�}t�|�D ]}|� |� q4|r |� t |�� q dS )a} Calculates hashes of the contents of a file. Args: fp: An already-open file object (stream will be consumed). hash_dict: Dict of (string alg_name: initialized hashing class) Hashing class will be populated with digests upon return. callback_processor: Optional callback processing class that implements Progress(integer amount of bytes processed). N) �readr �six�PY3� isinstance�str�encoder � itervalues�updateZProgressr+ )r2 r4 Zcallback_processor�datar3 r r r r0 � s r0 c C s t | tj�d��S )a Calculates a base64 CRC32c checksum of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. Returns: CRC32c checksum of the file in base64 format. �crc-32c)�$_CalculateB64EncodedHashFromContents�crcmod� predefined�Crc�r2 r r r �%CalculateB64EncodedCrc32cFromContents� s �rE c C s t | t� �S )z�Calculates a base64 MD5 digest of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. Returns: MD5 digest of the file in base64 format. )r@ �GetMd5rD r r r �"CalculateB64EncodedMd5FromContents� s rG c C s t | t� �S )z�Calculates a base64 MD5 digest of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. Returns: MD5 digest of the file in hex format. )r5 rF rD r r r �CalculateMd5FromContents� s rH c C s t �t�| ��}|�d��t�S )zAReturns the base64-encoded version of the input hex digest value.� )�base64� b64encode�binascii� unhexlify�rstrip�decoder )Z digest_valueZ encoded_bytesr r r �Base64EncodeHash s rP c C s t �| �d��t��}t�|�S )z�Returns the hex digest value of the input base64-encoded hash. Args: base64_hash: Base64-encoded hash, which may contain newlines and single or double quotes. Returns: Hex digest of the input argument. z "')rJ � b64decode�stripr; r rL �hexlify)Z base64_hashZ decoded_bytesr r r �Base64ToHexHash s rT c C s t t| |��S )a4 Calculates a base64 digest of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. hash_alg: Instance of hashing class initialized to start state. Returns: Hash of the stream in base64 format. )rP r5 )r2 r3 r r r r@ s r@ c C s" t �ddt�} | dkri S dtiS )a? Returns a dict of hash algorithms for validating an uploaded object. This is for use only with single object uploads, not compose operations such as those used by parallel composite uploads (though it can be used to validate the individual components). Returns: dict of (algorithm_name: hash_algorithm) �GSUtil� check_hashesr �md5)r �get� CHECK_HASH_IF_FAST_ELSE_FAILrF )�check_hashes_configr r r �GetUploadHashAlgs( s �r[ Fc C s� t �ddt�}|tkri S i }|r,t|d< nj|r�t� rDdd� |d< nR|s�|tkrZtt��n<|tkrn| � t � n(|t kr�| � t � dd� |d< ntd��|S ) a� Returns a dict of hash algorithms for validating an object. Args: logger: logging.Logger for outputting log messages. consider_md5: If True, consider using a md5 hash. consider_crc32c: If True, consider using a crc32c hash. Returns: Dict of (string, hash algorithm). Raises: CommandException if hash algorithms satisfying the boto config file cannot be returned. rU rV rW c S s t j�d�S �Nr? �rA rB rC r r r r �<lambda>U � z%GetDownloadHashAlgs.<locals>.<lambda>�crc32cc S s t j�d�S r\ r] r r r r r^ ] r_ z8Your boto config 'check_hashes' option is misconfigured.) r rX rY �CHECK_HASH_NEVERrF r r �_SLOW_CRC_EXCEPTION_TEXT� CHECK_HASH_IF_FAST_ELSE_SKIP�warn�_NO_HASH_CHECK_WARNING�CHECK_HASH_ALWAYS� _SLOW_CRCMOD_DOWNLOAD_WARNING)�loggerZ consider_md5Zconsider_crc32crZ � hash_algsr r r �GetDownloadHashAlgs9 s. � �rj r_ c C s2 z t �| �W S ty, t j| dd� Y S 0 dS )aN Returns md5 object, avoiding incorrect FIPS error on Red Hat systems. Examples: GetMd5(b'abc') GetMd5(bytes('abc', encoding='utf-8')) Args: byte_string (bytes): String in bytes form to hash. Don't include for empty hash object, since md5(b'').digest() == md5().digest(). Returns: md5 hash object. F)�usedforsecurityN)�hashlibrW � ValueError)� byte_stringr r r rF e s rF c @ sT e Zd ZdZdd� Zedd� �Zddd�Zd d � Zd d � Z e j fd d�Z dd� Z dS )�HashingFileUploadWrappera� Wraps an input stream in a hash digester and exposes a stream interface. This class provides integrity checking during file uploads via the following properties: Calls to read will appropriately update digesters with all bytes read. Calls to seek (assuming it is supported by the wrapped stream) using os.SEEK_SET will catch up / reset the digesters to the specified position. If seek is called with a different os.SEEK mode, the caller must return to the original position using os.SEEK_SET before further reads. Calls to seek are fast if the desired position is equal to the position at the beginning of the last read call (we only need to re-hash bytes from that point on). c C st |st d��n |st d��|| _|| _|| _|| _d| _i | _| jD ]}| j| �� | j|< qDd| _d| _ || _ dS )a� Initializes the wrapper. Args: stream: Input stream. digesters: dict of {string: hash digester} containing digesters, where string is the name of the hash algorithm. hash_algs: dict of {string: hash algorithm} for resetting and recalculating digesters. String is the name of the hash algorithm. src_url: Source FileUrl that is being copied. logger: For outputting log messages. z0HashingFileUploadWrapper used with no digesters.z0HashingFileUploadWrapper used with no hash_algs.Nr ) r �_orig_fp� _digesters�_src_url�_logger� _seek_away�_digesters_previous�copy�_digesters_previous_mark�_digesters_current_mark� _hash_algs)�self�stream� digestersri Zsrc_urlrh �algr r r �__init__� s z!HashingFileUploadWrapper.__init__c C s t | jdd�S )z<Returns the mode of the underlying file descriptor, or None.�modeN)�getattrrp �rz r r r r � s z HashingFileUploadWrapper.moder% c C s� | j durtd��| j�|�}t|tj�r4|�t�}| j | _ | j D ](}| j | � � | j |< | j | �|� qB| j t|�7 _ |S )ai "Reads from the wrapped file pointer and calculates hash digests. Args: size: The amount of bytes to read. If ommited or negative, the entire contents of the file will be read, hashed, and returned. Returns: Bytes from the wrapped stream. Raises: CommandException if the position of the wrapped stream is unknown. Nz\Read called on hashing file pointer in an unknown position; cannot correctly compute digest.)rt r rp r6 r9 r7 � text_typer; r rx rw rq rv ru r= r+ )rz �sizer> r} r r r r6 � s z HashingFileUploadWrapper.readc C s | j �� S )z$Returns the current stream position.)rp �tellr� r r r r� � s z HashingFileUploadWrapper.tellc C s | j �� S )z'Returns true if the stream is seekable.)rp �seekabler� r r r r� � s z!HashingFileUploadWrapper.seekablec C s |t jkr| j�� | _n�d| _|| jk rd| jD ]}| j| � | j|< q.d| _| j� d� | � |� n�|| jkr�| j| _| jD ]}| j | | j|< q|nl|| jk r�| j| _| jD ]}| j | | j|< q�| j� | j� | � || j � n | j� | j� | � || j � | j� ||�S )z�Seeks in the wrapped file pointer and catches up hash digests. Args: offset: The offset to seek to. whence: os.SEEK_CUR, or SEEK_END, SEEK_SET. Returns: Return value from the wrapped stream's seek call. Nr ) �os�SEEK_SETrp r� rt rw rq ry rx r/ �_CatchUpru )rz �offset�whencer} r r r r/ � s, z HashingFileUploadWrapper.seekc C s� | j �� | jkr(td| j �� | jf ��| jD ]4}|tkrN| j�d|| jj � | j| � � | j |< q.| j| _ |}t |t�}|r�| j �|�}t|tj�r�|�t�}||8 }| jD ]}| j| �|� q�t |t�}qz| j|7 _dS )ak Catches up hashes, but does not return data and uses little memory. Before calling this function, digesters_current_mark should be updated to the current location of the original stream and the self._digesters should be current to that point (but no further). Args: bytes_to_read: Number of bytes to catch up from the original stream. zJInvalid mark when catching up hashes. Stream position %s, hash position %szCatching up %s for %s...N)rp r� rx r rq r rs �debugrr � url_stringrv ru rw �minr r6 r9 r7 r� r; r r= )rz Z bytes_to_readr} Zbytes_remainingZbytes_this_roundr> r r r r� s0 �� � z!HashingFileUploadWrapper._CatchUpN)r% )�__name__� __module__� __qualname__�__doc__r~ �propertyr r6 r� r� r� r� r/ r� r r r r ro | s 0ro )N)FF)r_ )3r� � __future__r r r r rJ rL rl r� r7 �botor rA �gslib.exceptionr �gslib.utils.boto_utilr �gslib.utils.constantsr r r r ZSLOW_CRCMOD_WARNINGZSLOW_CRCMOD_RSYNC_WARNINGrg rb re rY rc rf ra r* r r r r# r r5 r0 rE rG rH rP rT r@ r[ rj rF �objectro r r r r �<module> sX , utils/__pycache__/unit_util.cpython-39.pyc 0000644 00000016011 15025013616 0014557 0 ustar 00 a +(Wg� � @ s d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZej rVe Z g d�Z g d�Z d d � Zd Zd e Zd e Zd e Zd e Zde Ze d�Ze� \ZZdd� Zdd� Zdd� Zdd� Zdd� Z d'dd �Z d d � Z d d � Z d!d"� fd#d$�Z d%d&� Z!dS )(zCShared utility methods that calculate, convert, and simplify units.� )�absolute_import)�print_function)�division)�unicode_literalsN))r �B�bit)� ZKiBZKibit�K)� ZMiBZMibit�M)� ZGiBZGibit�G)�( ZTiBZTibit�T)�2 ZPiBZPibit�P)�< ZEiBZEibit�E))� �k)� �m)� �b)� �t)� �qc C st d} g }i }t t�D ]>\}}dd� t|�dd� D �}|D ] }|||< q:|�|� q| d�|�; } t�| �}||fS )z6Creates a suffix regex for human-readable byte counts.z'(?P<num>\d*\.\d+|\d+)\s*(?P<suffix>%s)?c S s g | ] }|� � �qS � )�lower)�.0�sr r �$/opt/gsutil/gslib/utils/unit_util.py� <listcomp>9 � z(_GenerateSuffixRegex.<locals>.<listcomp>� N�|)� enumerate� _EXP_STRINGS�list�extend�join�re�compile)Zhuman_bytes_re�suffixesZ suffix_to_si�i�siZ si_suffixes�suffixZmatcherr r r" �_GenerateSuffixRegex3 s r2 i � � r i�Q c C sX d}|d t t�k r6| dt|d d kr6|d7 }q|tt| �dt| d d�fS )Nr r% r3 � @)�lenr( �round�float)�numr/ r r r" �_RoundToNearestExponentQ s ( r: c C s |dk r d}t | �t |� S )a Calculates throughput and checks for a small total_elapsed_time. Args: total_bytes_transferred: Total bytes transferred in a period of time. total_elapsed_time: The amount of time elapsed in seconds. Returns: The throughput as a float. g{�G�z�?)r8 )Ztotal_bytes_transferredZtotal_elapsed_timer r r" �CalculateThroughputX s r; c C sF t t�D ]4\}}| d| krdt| �d| }|| S qt| �S )a� Creates a shorter string version for a given number of objects. Args: num: The number of objects to be shortened. Returns: shortened string version for this number. It takes the largest scale (thousand, million or billion) smaller than the number and divides it by that scale, indicated by a suffix with one decimal place. This will thus create a string of at most 6 characters, assuming num < 10^18. Example: 123456789 => 123.4m r z%.1lf)�reversed�_EXP_TEN_STRINGr8 �str)r9 Z divisor_expr1 �quotientr r r" � DecimalShortg s r@ c C s | | }| | dkr |d7 }|S )a Returns ceil(dividend / divisor). Takes care to avoid the pitfalls of floating point arithmetic that could otherwise yield the wrong result for large numbers. Args: dividend: Dividend for the operation. divisor: Divisor for the operation. Returns: Quotient. r r% r )ZdividendZdivisorr? r r r" � DivideAndCeilz s rA c C sp | � � } t�| �}|r`t|�d��}|�d�rPtt|�d� d }|d| 9 }tt|��}|S t d| ��dS )z�Tries to convert a human-readable string to a number of bytes. Args: human_string: A string supplied by user, e.g. '1M', '3 GiB'. Returns: An integer containing the number of bytes. Raises: ValueError: on an invalid string. r9 r1 r r5 z!Invalid byte string specified: %sN) r �MATCH_HUMAN_BYTES�matchr8 �groupr( � SUFFIX_TO_SI�intr7 � ValueError)Z human_stringr r9 Zpowerr r r" �HumanReadableToBytes� s rH r% c C s� t | ��� }tttd| t|d � ���}|dkrRd|rFdd| nd |d< n8t|�}|r�|d|| � d ||| d� }||d< d�|�S )a Creates a human readable format for bytes with fixed decimal places. Args: number: The number of bytes. decimal_places: The number of decimal places. Returns: String representing a readable format for number with decimal_places decimal places. r r �0�.� N� )�MakeHumanReadable�splitr> rF r7 r8 r6 r+ )ZnumberZdecimal_placesZ number_formatr9 Z num_lengthr r r" � HumanReadableWithDecimalPlaces� s" ��� �rO c C s t | �\}}d|t| d f S )z�Generates human readable string for a number of bits. Args: num: The number, in bits. Returns: A string form of the number using bit size abbreviations (kbit, Mbit, etc.) �%g %sr3 �r: r( �r9 r/ Z rounded_valr r r" �MakeBitsHumanReadable� s rS c C s t | �\}}d|t| d f S )z�Generates human readable string for a number of bytes. Args: num: The number, in bytes. Returns: A string form of the number using size abbreviations (KiB, MiB, etc.). rP r% rQ rR r r r" rM � s rM c C s | S )Nr )�xr r r" �<lambda>� r$ rU c C s| | sdS t | �d | }t�|�}t�|�}||krD|| t|� �S || t|� �|| }|| t|� �|| }|| S )a� Find the percentile of a list of values. Taken from: http://code.activestate.com/recipes/511478/ Args: values: a list of numeric values. Note that the values MUST BE already sorted. percent: a float value from 0.0 to 1.0. key: optional key function to compute value from each element of the list of values. Returns: The percentile of the values. Nr% )r6 �math�floor�ceilrF )�values�percent�keyr �f�cZd0Zd1r r r" � Percentile� s r^ c C sz t t| ��} | d }|dkr*dt|d� S | d| 8 } | d }| d| 8 } | }td| �d td| � d td| � S )aD Creates a standard version for a given remaining time in seconds. Created over using strftime because strftime seems to be more suitable for a datetime object, rather than just a number of seconds remaining. Args: remaining_time: The number of seconds remaining as a float, or a string/None value indicating time was not correctly calculated. Returns: if remaining_time is a valid float, %H:%M:%D time remaining format with the nearest integer from remaining_time (%H might be higher than 23). Else, it returns the same message it received. i �d z%d+ hrsi� r z%02d�:)rF r7 �minr> )Zremaining_time�hours�minutes�secondsr r r" � PrettyTime� s �re )r% )"�__doc__� __future__r r r r rV r, �six�PY3rF Zlongr( r= r2 �ONE_KIB�ONE_MIB�ONE_GIBZTWO_MIBZ EIGHT_MIBZTEN_MIBZSECONDS_PER_DAYrE rB r: r; r@ rA rH rO rS rM r^ re r r r r" �<module> s< utils/__pycache__/ls_helper.cpython-39.pyc 0000644 00000030635 15025013616 0014530 0 ustar 00 a +(Wg�G � @ sD d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd l mZ dd lmZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m Z ddgZ g d�Z d*dd�Z dd� Z!dd � Z"d d � Z#d d � Z$d!d"� Z%d+d$d%�Z&d&d'� Z'G d(d)� d)e(�Z)dS ),zCUtility functions and class for listing commands such as ls and du.� )�absolute_import)�print_function)�division)�unicode_literalsN)�EncryptionException)�CommandException)�PluralityCheckableIterator)�GenerationFromUrlAndString)�S3_ACL_MARKER_GUID)�S3_DELETE_MARKER_GUID)�S3_MARKER_GUIDS)�UTF8)� IS_WINDOWS)�AclTranslation)� text_util)�StorageUrlFromString�md5Hash�crc32c)�acl� cacheControl�componentCount�contentDisposition�contentEncoding�contentLanguage� contentType� customTime� kmsKeyName�customerEncryption�etag�eventBasedHold� generation�metadata�metageneration�retentionExpirationTime�size� storageClass� temporaryHold� timeCreated� timeDeleted�timeStorageClassUpdated�updated� c C s" d� d| d | d �d�|�S )a� Returns a string with a vertically aligned label and value. Labels of the same indentation level will start at the same column. Values will all start at the same column (unless the combined left-indent and label length is excessively long). If a value spans multiple lines, indentation will only be applied to the first line. Example output from several calls: Label1: Value (default indent of 1 was used) Sublabel1: Value (used indent of 2 here) Label2: Value Args: label: The label to print in the first column. value: The value to print in the second column. indent: (4 * indent) spaces will be placed before the label. Returns: A string with a vertically aligned label and value. z{}{}� � �:� )�format�ljust)�label�value�indent� r5 �$/opt/gsutil/gslib/utils/ls_helper.py�MakeMetadataLineF s r7 c C s dS )z�Default function for printing headers for buckets. Header is printed prior to listing the contents of the bucket. Args: bucket_listing_ref: BucketListingRef of type BUCKET. Nr5 ��bucket_listing_refr5 r5 r6 �PrintBucketHeader] s r: c C s t �| j� dS )z�Default function for printing buckets or prefixes. Args: bucket_listing_ref: BucketListingRef of type BUCKET or PREFIX. N)r � print_to_fd� url_stringr8 r5 r5 r6 �PrintDirh s r= c C s dS )z�Off-by-default function for printing buckets or prefix size summaries. Args: num_bytes: Number of bytes contained in the directory. bucket_listing_ref: BucketListingRef of type BUCKET or PREFIX. Nr5 )� num_bytesr9 r5 r5 r6 �PrintDirSummaryr s r? c C s t �d�| j�� dS )z�Default function for printing headers for prefixes. Header is printed prior to listing the contents of the prefix. Args: bucket_listing_ref: BucketListingRef of type PREFIX. �{}:N)r r; r0 r< r8 r5 r5 r6 �PrintDirHeader| s rA c C s t �� dS )z<Default function for printing new lines between directories.N)r r; r5 r5 r5 r6 � PrintNewLine� s rB Tc C s | j }t|�}| j}|jr8t|jjv r8d}d}|d7 }n |j}d}t�d� |�� |j rpt�t d|j � d��� |j r�t�t d|j � d��� |jr�|j|j kr�t�t d|j� d��� |jr�t�t d |j�� |jr�t�t d d �� |jr�t�t d d �� |j�r t�t d |j� d��� |j�r6t�t d|j�� |j�rPt�t d|j�� |j�rjt�t d|j�� |j�r�t�t d|j�� |j�r�t�t d|j�� t�t d|j�� t�t d|j�� |j�r�t�t d|j�� |j�r�t�t d|j�� |j�rt�t d|j� d��� i }|j�r�|jj�r�g }|jjD ]*} | j t v�rT|� | � n | j || j < �q8|�r�t�t dd�� |D ]6} d� | j �} d� | j �} t | | dd �} t�| � �q~|j �r|j!�s�t�t d d �� |j"�s�t�t d d �� t�t d |j j#�� t�t d!|j j$�� |j!�r0t�t d |j!�� |j"�rJt�t d |j"�� t�t d"|j%�&d#��� |j'�r�t(||j'�}t�t d$|�� |j)�r�t�t d%|j)�� |�r�|j*�r�t�t d&t+�,|j*��� n0t-|v �r�t�t d&|t- �� nt�t d&d'�� ||fS )(a� Print full info for given object (like what displays for gsutil ls -L). Args: bucket_listing_ref: BucketListingRef being listed. Must have ref_type OBJECT and a populated root_object with the desired fields. incl_acl: True if ACL info should be output. Returns: Tuple (number of objects, object_length) Raises: Exception: if calling bug encountered. r z<DeleteMarker>r+ r@ z Creation timez%a, %d %b %Y %H:%M:%S GMTz Update timezStorage class update timez Storage classzTemporary Hold�EnabledzEvent-Based HoldzRetention ExpirationzKMS keyz Cache-ControlzContent-DispositionzContent-EncodingzContent-LanguagezContent-Lengthz Content-TypezComponent-Countz Custom-TimezNoncurrent timeZMetadata� z{}� )r4 z Hash (crc32c)� encryptedz Hash (md5)zEncryption algorithmzEncryption key SHA256�ETagz"'� GenerationZMetageneration�ACLz[]).r< r � root_objectr! r �additionalPropertiesr$ r r; r0 r' r7 �strftimer* r) r% r& r r# r r r r r r r r r( �keyr �appendr3 r r r �encryptionAlgorithm� keySha256r �stripr r r" r r �JsonFromMessager )r9 Zincl_acl�url_str� storage_url�objr> �num_objsZ marker_propsZnon_marker_propsZadd_propZapZap_keyZap_valueZmeta_data_line�generation_strr5 r5 r6 �PrintFullInfoAboutObject� s � �� �� � �� �� � � � � �� ���� �� �rX c C sJ zt �| j� W n4 tyD } z tr.|jdks0� W Y d}~n d}~0 0 dS )z�Default printing function for objects. Args: bucket_listing_ref: BucketListingRef of type OBJECT. Returns: (num_objects, num_bytes). r N)r+ r )r r; r< �IOErrorr �errno)r9 �er5 r5 r6 � PrintObject s r\ c @ sJ e Zd ZdZeeeeee dddddf dd�Z dd � Z dd d �Z d d � Z dS )�LsHelperzHelper class for ls and du.FN)�nameTc C sR || _ || _|| _|| _|| _|| _|| _|| _| | _| | _ | | _ | | _ | | _ dS )a� Initializes the helper class to prepare for listing. Args: iterator_func: Function for instantiating iterator. Inputs- url_string- Url string to iterate on. May include wildcards. all_versions=False- If true, iterate over all object versions. logger: Logger for outputting warnings / errors. print_object_func: Function for printing objects. print_dir_func: Function for printing buckets/prefixes. print_dir_header_func: Function for printing header line for buckets or prefixes. print_bucket_header_func: Function for printing header line for buckets or prefixes. print_dir_summary_func: Function for printing size summaries about buckets/prefixes. print_newline_func: Function for printing new lines between dirs. all_versions: If true, list all object versions. should_recurse: If true, recursively listing buckets/prefixes. exclude_patterns: Patterns to exclude when listing. fields: Fields to request from bucket listings; this should include all fields that need to be populated in objects so they can be listed. Can be set to None to retrieve all object fields. Defaults to short listing fields. list_subdir_contents: If true, return the directory and any contents, otherwise return only the directory itself. N) �_iterator_func�logger�_print_object_func�_print_dir_func�_print_dir_header_func�_print_bucket_header_func�_print_dir_summary_func�_print_newline_func� all_versions�should_recurse�exclude_patterns�bucket_listing_fields�list_subdir_contents)�selfZ iterator_funcr` �print_object_func�print_dir_func�print_dir_header_funcZprint_bucket_header_func�print_dir_summary_func�print_newline_funcrg rh ri �fieldsrk r5 r5 r6 �__init__' s ,zLsHelper.__init__c C s� d}d}d}d}|� � s | jr@|� � r0| �|� | j|jdd�S |�� rP|j}n|�� }t| j|| j d�j d| j d��}|� � }z |� � W n@ ty� t| j|jdd�| j d�j dtd��}|� � }Y n0 |D ]�} | �| �r�q�| �� �rd} | �| �\} } d}n�| �� �r�|�r | �� nd}|�r<| j�r<| �| � n|�rN| j�sNd}t| j�j| j�rdd ndd�} | �| �\} } } | �| | � ntd ��|| 7 }|| 7 }|| 7 }q�|||fS dS ) z�Iterates over the given URL and calls print functions. Args: url: StorageUrl to iterate over. Returns: (num_objects, num_bytes) total number of objects and bytes iterated. r F)�print_initial_newline�rg T��expand_top_level_bucketsrj N��wildcard_suffix�*z?Sub-level iterator returned a CsBucketListingRef of type Bucket)�IsBucketrh rd �_RecurseExpandUrlAndPrintr< � HasGeneration�CreatePrefixUrlr r_ rg �IterAllrj � HasPlurality� PeekExceptionr � UNENCRYPTED_FULL_LISTING_FIELDS�_MatchesExcludedPattern�IsObjectra �IsPrefixrf rk rc r re r )rl �url� num_objects�num_dirsr> Z print_newlineZ iteration_urlZtop_level_iteratorZ plurality�blr�nd�no�nb�expansion_url_strr5 r5 r6 �ExpandUrlAndPrinta s� ���� ��� ��� zLsHelper.ExpandUrlAndPrintc C s� d}d}d}| j d| | jd�jd| jd�D ]�}| �|�r<q,|�� rXd}| �|�\}} nx|�� r�| jr�|rt| � � nd}| � |� t |j �j dd�} | �| �\}}} | �| |� q�d\}}} | �|� ntd ��||7 }||7 }|| 7 }q,|||fS ) a� Iterates over the given URL string and calls print functions. Args: url_str: String describing StorageUrl to iterate over. Must be of depth one or higher. print_initial_newline: If true, print a newline before recursively expanded prefixes. Returns: (num_objects, num_bytes) total number of objects and bytes iterated. r z%sru Trv rz rx )r+ r r z=Sub-level iterator returned a bucketListingRef of type Bucket)r_ rg r rj r� r� ra r� rh rf rc r r< r~ r| re rb r ) rl rS rt r� r� r> r� r� r� r� r� r5 r5 r6 r| � sH �� �� � z"LsHelper._RecurseExpandUrlAndPrintc C s: | j r6t�|j�}| j D ] }t�|t�|��r dS qdS )z�Checks bucket listing reference against patterns to exclude. Args: blr: BucketListingRef to check. Returns: True if reference matches a pattern and should be excluded. TF)ri �six� ensure_strr< �fnmatch)rl r� Ztomatch�patternr5 r5 r6 r� � s z LsHelper._MatchesExcludedPattern)T)�__name__� __module__� __qualname__�__doc__r\ r= rA r: r? rB rs r� r| r� r5 r5 r5 r6 r] $ s � :N 2r] )r+ )T)*r� � __future__r r r r r� �sysr� �gslib.cloud_apir �gslib.exceptionr �"gslib.plurality_checkable_iteratorr �gslib.storage_urlr �gslib.utils.constantsr r r r Zgslib.utils.system_utilr � gslib.utils.translation_helperr � gslib.utilsr Zgslib.wildcard_iteratorr ZENCRYPTED_FIELDSr� r7 r: r= r? rA rB rX r\ �objectr] r5 r5 r5 r6 �<module> sB � utils/__pycache__/__init__.cpython-39.pyc 0000644 00000000176 15025013616 0014307 0 ustar 00 a +(Wg � @ s d S )N� r r r �#/opt/gsutil/gslib/utils/__init__.py�<module> � utils/__pycache__/metadata_util.cpython-39.pyc 0000644 00000006151 15025013616 0015364 0 ustar 00 a +(Wg� � @ sz d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ dd d �Z dd d �Z dd d�Z dd� Z dd� ZdS )zKShared utility methods for manipulating metadata of requests and resources.� )�absolute_import)�print_function)�division)�unicode_literalsN)�storage_v1_messagesFc C s |r d| d< d S )N�gzipzaccept-encoding� )Z headers_dict�compressed_encodingr r �(/opt/gsutil/gslib/utils/metadata_util.py� AddAcceptEncodingGzipIfNeeded s r c C s\ |du rt jjg d�}| du r"i } t�| �D ]*\}}|j�t jjjt|�t|�d�� q,|S )a Creates a custom MetadataValue object. Inserts the key/value pairs in entries. Args: entries: (Dict[str, Any] or None) The dictionary containing key/value pairs to insert into metadata. Both the key and value must be able to be casted to a string type. custom_metadata (apitools_messages.Object.MetadataValue or None): A pre-existing custom metadata object to add to. If one is not provided, a new one will be constructed. Returns: An apitools_messages.Object.MetadataValue. N)�additionalProperties��key�value) �apitools_messages�Object� MetadataValue�six� iteritemsr �append�AdditionalProperty�str)�entries�custom_metadatar r r r r �CreateCustomMetadata, s � ��r c sX z8t � fdd�| jjD �d�}|du r0d|fW S d|fW S tyR d|f Y S 0 dS )aE Filters a specific element out of an object's custom metadata. Args: obj_metadata: (apitools_messages.Object) The metadata for an object. search_key: (str) The custom metadata key to search for. default_value: (Any) The default value to use for the key if it cannot be found. Returns: (Tuple(bool, Any)) A tuple indicating if the value could be found in metadata and a value corresponding to search_key (the value at the specified key in custom metadata, or the default value if the specified key does not exist in the custom metadata). c 3 s | ]}|j � kr|jV qd S )Nr )�.0�attr�� search_keyr r � <genexpr>Z s �z3GetValueFromObjectCustomMetadata.<locals>.<genexpr>NFT)�next�metadatar �AttributeError)� obj_metadatar � default_valuer r r r � GetValueFromObjectCustomMetadataH s �� r% c C s | � d�p| � d�S )zDReturns true if header (which must be lowercase) is a custom header.z x-goog-meta-z x-amz-meta-)� startswith)�headerr r r �IsCustomMetadataHeaderd s r( c C s | j o| j �� �d�S )zGReturns true if the apitools_messages.Object has gzip content-encoding.r )�contentEncoding�lower�endswith)r# r r r �ObjectIsGzipEncodedi s �r, )F)NN)N)�__doc__� __future__r r r r r �"gslib.third_party.storage_apitoolsr r r r r% r( r, r r r r �<module> s � utils/__pycache__/cloud_api_helper.cpython-39.pyc 0000644 00000006454 15025013616 0016053 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl m Z ddl m Z dd d �Z dd d �Zdd d�Zdd� ZdS )z/Helper functions for Cloud API implementations.� )�absolute_import)�print_function)�division)�unicode_literalsN)�ArgumentException)�AddQueryParamToUrlc C s |p| j S )a� Gets a gsutil Cloud API instance. Since Cloud API implementations are not guaranteed to be thread-safe, each thread needs its own instance. These instances are passed to each thread via the thread pool logic in command. Args: cls: Command class to be used for single-threaded case. thread_state: Per thread state from this thread containing a gsutil Cloud API instance. Returns: gsutil Cloud API instance. )� gsutil_api)�clsZ thread_state� r �+/opt/gsutil/gslib/utils/cloud_api_helper.py�GetCloudApiInstance s r c C sJ | j }|rt|d|�}tjr0t|t�r0|�d�}d|| j|d�}t� |�S )a� Returns download serialization data. There are five entries: auto_transfer: JSON-specific field, always False. progress: How much of the download has already been completed. total_size: Total object size. url: Implementation-specific field used for saving a metadata get call. For JSON, this the download URL of the object. For XML, this is a pickled boto key. user_project: Project to be billed to, added as query param. Args: src_obj_metadata: Object to be downloaded. progress: See above. user_project: User project to add to query string. Returns: Serialization data for use with Cloud API GetObjectMedia. � userProject�ascii�False)� auto_transfer�progress� total_size�url) � mediaLinkr �six�PY3� isinstance�bytes�decode�size�json�dumps)�src_obj_metadatar � user_projectr �serialization_dictr r r � GetDownloadSerializationData1 s �r c C s: | r6t � }| D ]"}|dv r q|�t�dd|�� q|S dS )a Removes 'items/' from the input fields and converts it to a set. Args: list_fields: Iterable fields usable in ListBuckets/ListObjects calls. Returns: Set of fields usable in GetBucket/GetObjectMetadata calls (None implies all fields should be returned). )�kind� nextPageToken�prefixeszitems/� N)�set�add�re�sub)� list_fields� get_fields�fieldr r r �ListToGetFieldsZ s r, c C s, | s t d��| jst d��| js(t d��dS )z�Ensures dst_obj_metadata supplies the needed fields for copy and insert. Args: dst_obj_metadata: Metadata to validate. Raises: ArgumentException if metadata is invalid. z3No object metadata supplied for destination object.zCObject metadata supplied for destination object had no object name.zCObject metadata supplied for destination object had no bucket name.N)r �name�bucket)�dst_obj_metadatar r r �ValidateDstObjectMetadatao s ���r0 )N)r N)N)�__doc__� __future__r r r r r r' r �gslib.cloud_apir �gslib.utils.text_utilr r r r, r0 r r r r �<module> s � ) utils/__pycache__/signurl_helper.cpython-39.pyc 0000644 00000010324 15025013616 0015566 0 ustar 00 a +(Wg\ � @ s� d Z ddlZddlmZ ddlmZ ddlZddlmZ ddlZddlm Z dZ dZ d Z d Z d Zd d � Zddd�Zdd� Zefdd�ZdS )z&Utility functions for signurl command.� N)�datetime)�timezone)�UTF8)�urllibzN{method} {resource} {query_string} {headers} {signed_headers} {hashed_payload}zGOOG4-RSA-SHA256zA{signing_algo} {request_time} {credential_scope} {hashed_request}z;https://{host}/{path}?x-goog-signature={sig}&{query_string}zUNSIGNED-PAYLOADc C s t jtjd�jd d�S )N)�tz)�tzinfo)r �nowr �utc�replace� r r �)/opt/gsutil/gslib/utils/signurl_helper.py�_NowUTC$ s r Fc sh t � } | �d�} | �d�} dj| |d�} t| d | | d�t� �� ��d|�� d��|d urf|�d < |d urv|�d < d �|�}d ��fdd�t��� �D ��}d�� fdd�t� �� �D ��d }d�t� �� ��}tj|||||t d�}t j r�|� t �}t�� }|�|� t�|�� ��� �t �}tjt| | |d�}| �r`|�r`|�d| � |�d| � ||fS )a Create a string that needs to be signed. Args: client_id: Client ID signing this URL. method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. path: String path to the bucket of object for signing, in the form 'bucket' or 'bucket/object'. generation: If not None, specifies a version of an object for signing. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. signed_headers: Dict containing the header info like host content-type etc. billing_project: Specify a user project to be billed for the request. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. Returns: A tuple where the 1st element is the string to sign. The second element is the query string. z%Y%m%dz%Y%m%dT%H%M%SZz%{date}/{region}/storage/goog4_request)�date�region�/�;z%d)zx-goog-algorithmzx-goog-credentialz x-goog-datezx-goog-signedheaderszx-goog-expiresN� userProject� generationz/{}�&c s$ g | ] }d � |tj�� | ���qS )z{}={})�formatr �parse� quote_plus)�.0�param)�signed_query_paramsr r � <listcomp>^ s �z!CreatePayload.<locals>.<listcomp>� c s g | ]}d � |�� � | ��qS )z{}:{})r �lower)r �header)�signed_headersr r r b s �)�method�resource� query_string�headersr Zhashed_payload)Z signing_algoZ request_time�credential_scopeZhashed_requestz=Canonical request (ignore opening/closing brackets): [[[%s]]]z:String to sign (ignore opening/closing brackets): [[[%s]]])r �strftimer � _SIGNING_ALGO�join�sorted�keys� total_seconds�_CANONICAL_REQUEST_FORMAT�_UNSIGNED_PAYLOAD�six�PY3�encoder �hashlib�sha256�update�base64� b16encode�digestr �decode�_STRING_TO_SIGN_FORMAT�debug)� client_idr �duration�pathr �loggerr r Zbilling_project�string_to_sign_debugZ signing_timeZ canonical_dayZcanonical_timeZcanonical_scopeZcanonical_resource�canonical_query_string�canonical_headersZcanonical_signed_headers�canonical_requestZcanonical_request_hasherZhashed_canonical_request�string_to_signr )r r r � CreatePayload( sv � � � ��� ��� ���rB c C s$ t �| ��� �� }tj||||d�S )zGet the final signed url.)�hostr; �sigr" )r3 r4 r r6 �_SIGNED_URL_FORMATr )� raw_signaturerC r; r> � signaturer r r � GetFinalUrl� s �rH c C s t | t�r| S | �|�S )N)� isinstance�bytesr/ )�value�encodingr r r �to_bytes� s rM )NF)�__doc__r3 r r r0 �gslib.utils.constantsr r- � six.movesr r+ r& r7 rE r, r rB rH rM r r r r �<module> s$ � ` utils/__pycache__/copy_helper.cpython-39.pyc 0000644 00000313216 15025013616 0015063 0 ustar 00 a +(Wg�� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZ ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddlZdd l m Z dd l m Z dd l!m"Z" dd l#m$Z$ ddl%Z%ddl&Z&dd l'm(Z( dd l'm)Z) ddl'm*Z* ddl'm+Z+ ddl'm,Z, ddl'm-Z- ddl'm.Z. ddl'm/Z/ ddl'm0Z0 ddl'm1Z1 ddl'm2Z2 ddl'm3Z3 ddl4m5Z5 ddl6m7Z7 ddl6m8Z8 ddl6m9Z9 dd l6m:Z: dd l6m;Z; dd l6m<Z< dd l=m>Z> dd l?m@Z@ dd!lAmBZB dd"lAmCZC dd#lAmDZD dd$lEmFZF dd%lGmHZH dd&lGmIZI dd'lGmJZJ dd(lGmKZK dd)lGmLZL dd*lMmNZN dd+lMmOZO dd,lPmQZQ dd-l&mRZR dd.lSmTZT dd/lSmUZU dd0lSmVZV dd1lSmWZW dd2lXmYZZ dd3l[m\Z\ dd4l[m]Z] dd5l^m_Z_ dd6l^m`Z` dd7l^maZa dd8l^mbZb dd9l^mcZc dd:l^mdZd dd;l^meZe dd<l^mfZf dd=l^mgZg dd>l^mhZh dd?l^miZi dd@l^mjZj ddAlkmlZl ddBlkmmZm ddClkmnZn ddDlkmoZo ddElpmqZq ddFlpmrZr ddGlpmsZs ddHlpmtZt ddIlpmuZu ddJlvmwZw ddKlvmxZx ddLlymzZz ddMlym{Z{ ddNlym|Z| ddOl}m~Z~ ddPl}mZ ddQl}m�Z� ddRl}m�Z� ddSl�m�Z� ddTl�m�Z� ddUl�m�Z� ddVl�m�Z� ddWl�m�Z� ddXl�m�Z� ddYl�m�Z� ddZl�m�Z� dd[l�m�Z� dd\l�m�Z� dd]l�m�Z� dd^l�m�Z� dd_l�m�Z� dd`l�m�Z� ddal�m�Z� ddbl�m�Z� ddcl�m�Z� dddl�m�Z� ddel�m�Z� ddfl�m�Z� ddgl�m�Z� ddhl�m�Z� ddil�m�Z� ddjl�m�Z� ddkl�m�Z� ddll�m�Z� ddml�m�Z� ddnl�m�Z� ddol�m�Z� ddpl�m�Z� ddql�m�Z� ddrl�m�Z� ddsl�m�Z� ddtl�m�Z� ddul�m�Z� ddvl�m�Z� ddwl�m�Z� ej��rXe�Z�e��rfddl�Z�e�e�� j��rxelj�nddx�a�el��� a�dyZ�dzZ�d{Z�d|Z�g d}�Z�ed~d�Z�ed�d��Z�ed�d��Z�d�Z�d�Z�e�Z�d�Z�d�Z�e�e�� j��r�elj�nddx�a�el��� a�d�d�d��Z�G d�d�� d�eȃZ�d�d�� Z�d�d�� Zːdd�d��Z�ed�g d���Z͐dd�d��Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d�� ZԐdd�d��Z�d�d�� Z�d�d�� Z�d�d�� Zؐdd�d��Z�d�d�� Z�d�d�� Zېdd�d��Z�d�d�� Zݐdd�d��Zސdd�d��Zߐdd�d��Z�d�d�� Z�d�d�� Z�d�d�� Z�d�d Z�dd�dĄZ�d�dƄ Z�dd�dȄZ�dd�dʄZ�dd�d̄Z�d�d΄ Z�d�dЄ Z�d d�dӄZ�d�dՄ Z�d�dׄ Z�d d�dلZ�d�dۄ Z�G d�d݄ d�e�Z�d d�d߄Z�d d�d�Z�d d�d�Z��d!d�d�Z��d"d�d�Z��d#d�d�Z��d$d�d�Z�d�d� Z��d%d�d�Z��d&d�d�Z�d�e� d�e� d�e� d��Z�d�d�� Z�d�d�� Z�d�d�� �Z �d'd�d���ZG d�d�� d�e��ZG �d �d� �deȃ�ZG �d�d� �deȃ�ZG �d�d� �d�e��Z�d�d� �Z�d�d � �Z�d �d � �Z�d(�d �d ��Z �d�d� �Z dS () z(Helper functions for copy functionality.� )�absolute_import)�print_function)�division)�unicode_literalsN)� namedtuple)� attrgetter)�range)� protojson)�config)�AccessDeniedException)�ArgumentException)�CloudApi)�EncryptionException)�NotFoundException)�PreconditionException)� Preconditions)�ResumableDownloadException)� ResumableUploadAbortException)�ResumableUploadException)�!ResumableUploadStartOverException)�ServiceException)�MAX_COMPOSE_ARITY)�0DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE)�+DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD)�-DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE)�-DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS)�(DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD)� DEFAULT_GZIP_COMPRESSION_LEVEL)� ApiSelector)�DaisyChainWrapper)�CommandException)�HashMismatchException)�InvalidUrlError)�FilePart)� GenerateComponentObjectPrefix)� ReadParallelUploadTrackerFile)�$ValidateParallelCompositeTrackerData)�)WriteComponentToParallelUploadTrackerFile)� WriteParallelUploadTrackerFile)�FileProgressCallbackHandler)�ProgressCallbackWithTimeout)�#ResumableStreamingJsonUploadWrapper)� storage_url)�ContainsWildcard)�GenerationFromUrlAndString)�IsCloudSubdirPlaceholder)�StorageUrlFromString)�storage_v1_messages)� FileMessage)�RetryableErrorMessage)�DeleteDownloadTrackerFiles)�DeleteTrackerFile)� ENCRYPTION_UPLOAD_TRACKER_ENTRY)�GetDownloadStartByte)�GetTrackerFilePath)�GetUploadTrackerData)�#RaiseUnwritableTrackerFileException)� ReadOrCreateDownloadTrackerFile)�"SERIALIZATION_UPLOAD_TRACKER_ENTRY)�TrackerFileType)�!WriteDownloadComponentTrackerFile)�WriteJsonDataToTrackerFile)�parallelism_framework_util)� stet_util)�temporary_file_util)� text_util)�GetJsonResumableChunkSize)�GetMaxRetryDelay)� GetNumRetries)�ResumableThreshold)�UsingCrcmodExtension)�GetCloudApiInstance)� GetDownloadSerializationData)�DEFAULT_FILE_BUFFER_SIZE)�MIN_SIZE_COMPUTE_LOGGING)�UTF8)� CryptoKeyType)�CryptoKeyWrapperFromKey)� FindMatchingCSEKInBotoConfig)�GetEncryptionKeyWrapper)�Base64EncodeHash)�"CalculateB64EncodedMd5FromContents)�CalculateHashesFromContents)� CHECK_HASH_IF_FAST_ELSE_FAIL)�CHECK_HASH_NEVER)� ConcatCrc32c)�GetDownloadHashAlgs)�GetMd5)�GetUploadHashAlgs)�HashingFileUploadWrapper)�ObjectIsGzipEncoded)� AtomicDict)�$CheckMultiprocessingAvailableAndInit)�PutToQueueWithTimeout)� ATIME_ATTR)�ConvertDatetimeToPOSIX)�GID_ATTR)� MODE_ATTR)� MTIME_ATTR)�ParseAndSetPOSIXAttributes)�UID_ATTR)�CheckFreeSpace)� GetFileSize)�GetStreamFromFileUrl)� IS_WINDOWS)� AddS3MarkerAclToObjectMetadata)�CopyObjectMetadata)�DEFAULT_CONTENT_TYPE)�ObjectMetadataFromHeaders)�PreconditionsFromHeaders)� S3MarkerAclFromObjectMetadata)� DivideAndCeil)�HumanReadableToBytes)�MakeHumanReadable)�SECONDS_PER_DAY)�TEN_MIB)�CreateWildcardIterator)�managerFZ_corruptzF/gsutil/tmp/parallel_composite_uploads/for_details_see/gsutil_help_cp/ad PARALLEL_UPLOAD_SALT_TO_PREVENT_COLLISIONS. The theory is that no user will have prepended this to the front of one of their object names and then done an MD5 hash of the name, and then prepended PARALLEL_UPLOAD_TEMP_NAMESPACE to the front of their object name. Note that there will be no problems with object name length since we hash the original name. )�crc32c�customerEncryption�etag� generation�md5Hash�size�%PerformParallelUploadFileToObjectArgsz�filename file_start file_length src_url dst_url canned_acl content_type storage_class tracker_file tracker_file_lock encryption_key_sha256 gzip_encoded�%PerformSlicedDownloadObjectToFileArgszicomponent_num src_url src_obj_metadata_json dst_url download_file_name start_byte end_byte decryption_key�!PerformSlicedDownloadReturnValueszKcomponent_num crc32c bytes_transferred component_total_size server_encodingi �GZIP_ALL_FILESi ` l z text/markdownzapplication/gzip)�mdZtgzc @ s e Zd ZdZdS )�FileConcurrencySkipErrorz@Raised when skipping a file due to a concurrent, duplicate copy.N��__name__� __module__� __qualname__�__doc__� r� r� �&/opt/gsutil/gslib/utils/copy_helper.pyr� + s r� c C s | j �t|�� dS )�9Simple exception handler to allow post-completion status.N)�logger�error�str��cls�er� r� r� �_RmExceptionHandler/ s r� c C s4 | j �t|�� | jd7 _| j �dt�� � dS )r� � z* Encountered exception while copying: %s N)r� r� r� �op_failure_count�debug� traceback� format_excr� r� r� r� � _ParallelCopyExceptionHandler4 s �r� c C s� t |j|j|j�}t| |d�}|�� d}tj|jj|jj |j |j d�}|j }zLt jr\tj|_ t|j||j|j|||| j| tddd|jd�}W t jr�||_ nt jr�||_ 0 W d � n1 s�0 Y |d } t|j|j| | j|jd� |S ) aO Function argument to Apply for performing parallel composite uploads. Args: cls: Calling Command class. args: PerformParallelUploadFileToObjectArgs tuple describing the target. thread_state: gsutil Cloud API instance to use for the operation. Returns: StorageUrl representing a successfully uploaded component. �� thread_stateN)�name�bucket� contentType� storageClassFT)� gzip_exts�allow_splitting� is_component� gzip_encoded� ��encryption_key_sha256)r# �filename� file_start� file_lengthrI �apitools_messages�Object�dst_url� object_name� bucket_name� content_type� storage_class� prefer_api�global_copy_helper_opts� canned_aclr �XML�_UploadFileToObject�src_urlr� r� r� r' � tracker_file�tracker_file_lockr� ) r� �argsr� �fp� gsutil_api� preconditionsZdst_object_metadata�orig_prefer_api�ret� componentr� r� r� �"_PerformParallelUploadFileToObject< sT ���&�r� �CopyHelperOpts� � perform_mv� no_clobber� daisy_chain�read_args_from_stdin� print_ver� use_manifest� preserve_aclr� �skip_unsupported_objects�test_callback_file�dest_storage_classc C s" t | ||||||||| | d� atS )z9Creates CopyHelperOpts for passing options to CopyHelper.r� )r� r� r� r� r� r� �CreateCopyHelperOpts� s � r� c C s t S )z.Returns namedtuple holding CopyHelper options.)r� r� r� r� r� �GetCopyHelperOpts� s r� c C sh d}| � � rP| jtjkr d}z t�| j�j}t�|�r:d}W n tyN Y n0 |r\tj j S tj j S dS )z�Get download strategy based on the destination object. Args: dst_url: Destination StorageUrl. Returns: gsutil Cloud API DownloadStrategy. FTN) � IsFileUrlr� �os�devnull�stat�st_mode�S_ISCHR�OSErrorr �DownloadStrategy�ONE_SHOT� RESUMABLE)r� Zdst_is_special�moder� r� r� �_SelectDownloadStrategy� s r� c C s4 | � � r| �� r$| �� r0| �� r0|s0td| ��dS )a� Ensures the destination URL names a container. Acceptable containers include directory, bucket, bucket subdir, and non-existent bucket subdir. Args: exp_dst_url: Wildcard-expanded destination StorageUrl. have_existing_dst_container: bool indicator of whether exp_dst_url names a container (directory, bucket, or existing bucket subdir). command_name: Name of command making call. May not be the same as the calling class's self.command_name in the case of commands implemented atop other commands (like mv command). Raises: CommandException: if the URL being checked does not name a container. zuDestination URL must name a directory, bucket, or bucket subdirectory for the multiple source form of the %s command.N)r� � IsDirectory� IsCloudUrl�IsBucketr )� exp_dst_url�have_existing_dst_container� command_namer� r� r� �InsistDstUrlNamesContainer� s ����r� c C s |rdS |� � r | p|o|S dS )a" Checks whether dst_url should be treated as a bucket "sub-directory". The decision about whether something constitutes a bucket "sub-directory" depends on whether there are multiple sources in this request and whether there is an existing bucket subdirectory. For example, when running the command: gsutil cp file gs://bucket/abc if there's no existing gs://bucket/abc bucket subdirectory we should copy file to the object gs://bucket/abc. In contrast, if there's an existing gs://bucket/abc bucket subdirectory we should copy file to gs://bucket/abc/file. And regardless of whether gs://bucket/abc exists, when running the command: gsutil cp file1 file2 gs://bucket/abc we should copy file1 to gs://bucket/abc/file1 (and similarly for file2). Finally, for recursive copies, if the source is a container then we should copy to a container as the target. For example, when running the command: gsutil cp -r dir1 gs://bucket/dir2 we should copy the subtree of dir1 to gs://bucket/dir2. Note that we don't disallow naming a bucket "sub-directory" where there's already an object at that URL. For example it's legitimate (albeit confusing) to have an object called gs://bucket/dir and then run the command gsutil cp file1 file2 gs://bucket/dir Doing so will end up with objects gs://bucket/dir, gs://bucket/dir/file1, and gs://bucket/dir/file2. Args: have_multiple_srcs: Bool indicator of whether this is a multi-source operation. dst_url: StorageUrl to check. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. src_url_names_container: bool indicator of whether the source URL is a container. recursion_requested: True if a recursive operation has been requested. Returns: bool indicator. TN)r� )�have_multiple_srcsr� �have_existing_dest_subdir�src_url_names_container�recursion_requestedr� r� r� � _ShouldTreatDstUrlAsBucketSubDir� s ,�r� c C s6 |r | r dS |� � r |�� S | o0| o0|�� S dS )av Checks that dst_url names a single file/object after wildcard expansion. It is possible that an object path might name a bucket sub-directory. Args: src_url_names_container: Bool indicator of whether the source for the operation is a container (bucket, bucket subdir, or directory). have_multiple_srcs: Bool indicator of whether this is a multi-source operation. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. dst_url: StorageUrl to check. recursion_requested: True if a recursive operation has been requested. Returns: bool indicator. FN)r� r� �IsObject)r� r� r� r� r� r� r� r� � _ShouldTreatDstUrlAsSingleton s �r� c sN � � � s dS � j�� j�}|�� j�\}}}|tjvoL|� fdd�tjD �vS )ac Returns True if not FileUrl ending in relative path symbols. A URL is invalid if it is a FileUrl and the parent directory of the file is a relative path symbol. Unix will not allow a file itself to be named with a relative path symbol, but one can be the parent. Notably, "../obj" can lead to unexpected behavior at the copy destination. We examine the pre-recursion "url", which might point to "..", to see if the parent is valid. If the user does a recursive copy from a URL, it may not end up the final parent of the copied object. For example, see: "dir/nested_dir/obj". If you ran "cp -r dir gs://bucket" from the parent of "dir", then the "url" arg would be "dir", but "nested_dir" would be the parent of "obj". This actually doesn't matter since recursion won't add relative path symbols to the path. However, we still return if "url" is valid because there are cases where we need to copy every parent directory up to "dir" to prevent file name conflicts. Args: url: StorageUrl before recursion. Returns: Boolean indicating if the "url" is valid as a parent directory. Tc s g | ]}� j d | �qS )z://)�scheme)�.0�symbol��urlr� r� � <listcomp>U s �z(_IsUrlValidParentDir.<locals>.<listcomp>)r� �versionless_url_string�rstrip�delim� rpartitionr, �RELATIVE_PATH_SYMBOLS)r� Z#url_string_minus_trailing_delimiter�_Zafter_last_delimiterr� r� r� �_IsUrlValidParentDir3 s �� ��r� c C s |� � r |�� r |r td��t|||||�r0|S |� � rl|�� sH|�� rl|rh|�� rXdnd} td| ��|S |s�|s�|j�| j�d } td|j � |j�|j| f �S |s�|s�|� � r�|� � r�|j � |j�s�td|j |jf �}t | �} | s�|r�td�| ���d | jv�o| �o|�p|} | �s8|�rt|�� �s8|� � �rtt| |�} |jt| �d � �| j�}| �s�|�| j�d }n|j�| j�d }|� � �s�t|||||��r�|j�r�|j� |j��r�d|j� |j�|j|f }n(|j�r�|jnd }d|j�p�d ||f }|�� }|�| j|j�|_|S ) aQ Constructs the destination URL for a given exp_src_url/exp_dst_url pair. Uses context-dependent naming rules that mimic Linux cp and mv behavior. Args: src_url: Source StorageUrl to be copied. exp_src_url: Single StorageUrl from wildcard expansion of src_url. src_url_names_container: True if src_url names a container (including the case of a wildcard-named bucket subdir (like gs://bucket/abc, where gs://bucket/abc/* matched some objects). have_multiple_srcs: True if this is a multi-source request. This can be true if src_url wildcard-expanded to multiple URLs or if there were multiple source URLs in the request. has_multiple_top_level_srcs: Same as have_multiple_srcs but measured before recursion. exp_dst_url: the expanded StorageUrl requested for the cp destination. Final written path is constructed from this plus a context-dependent variant of src_url. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. recursion_requested: True if a recursive operation has been requested. preserve_posix: True if preservation of posix attributes has been requested. Returns: StorageUrl to use for copy. Raises: CommandException if destination object name not specified for source and source is a stream. z/Cannot preserve POSIX attributes with a stream.�stream� named pipez2Destination object name needed when source is a %s���z%s%s%sz%s%szmPresence of multiple top-level sources and invalid expanded URL make file name conflicts possible for URL: {}z**N� )r� �IsStreamr r� �IsFifor� r� r� r0 � url_stringr� r� �endswithr� r" �formatr� r� �GetPathBeforeFinalDir�len�lstrip� partitionr� �Clone�replace)r� � exp_src_urlr� r� Zhas_multiple_top_level_srcsr� r� r� �preserve_posixZ type_textZsrc_final_compZsrc_url_is_valid_parentZpreserve_src_top_level_dirs�src_url_path_sans_final_dirZ dst_key_namer� Znew_exp_dst_urlr� r� r� �ConstructDstUrl[ s� '��� ���� ����) �� �� �� ���� ��r c C s8 i }| r4| D ]&}t �| | �� ��d��d�||< q |S )N� �ascii)�base64� b64encode�digestr� �decode)� digesters�digests�algr� r� r� �_CreateDigestsFromDigesters s ��� r c C s� i }d|v rt � |d< d|v r.tj�d�|d< t|d��2}t||t|jt| |dd�j �d� W d � n1 sp0 Y i }t � |�D ]\}} t | � � �||< q�|S ) a� Creates a base64 CRC32C and/or MD5 digest from file_name. Args: status_queue: Queue for posting progress messages for UI/Analytics. algs: List of algorithms to compute. file_name: File to digest. src_url: StorageUrl for local object. Used to track progress. src_obj_metadata: Metadata of source object. Returns: Dict of algorithm name : base 64 encoded digest �md5rx zcrc-32c�rb�Hashing)r� �operation_name)�callback_processorN)rY �crcmod� predefined�Crc�openrT r* r} r) �call�six� iteritemsrR � hexdigest) � status_queueZalgs� file_namer� �src_obj_metadata� hash_dictr� r �alg_namer r� r� r� �_CreateDigestsFromLocalFile s* ���$r$ c C s t �ddt�}|tkrdS d}i }i }|jr@t�|j�} | |d< |jrZt�|j�} | |d< |jrtt�|j�} | |d< |jr�t�|j�} | |d< t�|�D ]f\} }| |vr�q�|| }tj r�t |t �r�|� d�}| � d| |||� ||kr�td | |||f ��d }q�|�s| �d ||� dS ) a� Validates integrity of two cloud objects copied via daisy-chain. Args: logger: for outputting log messages. src_url: CloudUrl for source cloud object. dst_url: CloudUrl for destination cloud object. src_obj_metadata: Cloud Object metadata for object being downloaded from. dst_obj_metadata: Cloud Object metadata for object being uploaded to. Raises: CommandException: if cloud digests don't match local digests. �GSUtil� check_hashesNFr rx r z;Comparing source vs destination %s-checksum for %s. (%s/%s)zn%s signature for source object (%s) doesn't match destination object digest (%s). Object (%s) will be deleted.Tz~WARNING: Found no hashes to validate object downloaded from %s and uploaded to %s. Integrity cannot be assured without hashes.)r �getrU rV r| r � ensure_binaryrx r �PY3� isinstancer� �encoder� r! �warn)r� r� r� r! �dst_obj_metadata�check_hashes_config� checked_oneZdownload_hashesZ upload_hashesZ src_md5hashZsrc_crc32c_hashZ dst_md5hashZdst_crc32c_hashr Zupload_b64_digestZdownload_b64_digestr� r� r� �_CheckCloudHashes. sT � � ���r0 c C s� |}i }|j r(t�|j �}|�d�|d< |jrHt�|j�} | �d�|d< d} |D ]h} | |vr^qPt�|| �} || } | �d| || | � | | kr�td| | | |r�dnd|r�|n|f ��d } qP| s�|r�| �d |� n | �d |� d S ) a Validates integrity by comparing cloud digest to local digest. Args: logger: for outputting log messages. obj_url: CloudUrl for cloud object. obj_metadata: Cloud Object being downloaded from or uploaded to. file_name: Local file name on disk being downloaded to or uploaded from (used only for logging). digests: Computed Digests for the object. is_upload: If true, comparing for an uploaded object (controls logging). Raises: CommandException: if cloud digests don't match local digests. r r rx Fz4Comparing local vs cloud %s-checksum for %s. (%s/%s)zl%s signature computed for local file (%s) doesn't match cloud-supplied digest (%s). %s (%s) will be deleted.z Cloud objectz Local fileTzgWARNING: Found no hashes to validate object uploaded to %s. Integrity cannot be assured without hashes.ziWARNING: Found no hashes to validate object downloaded to %s. Integrity cannot be assured without hashes.N)r| r r( r� rx r� r! r, )r� Zobj_url� obj_metadatar r � is_upload� local_hashes� cloud_hashesZmd5_b64_digestZcrc32c_b64_hashr/ r Zlocal_b64_digestZcloud_b64_digestr� r� r� � _CheckHashesn sL �� �����r5 c C s t | t�p t | t�o d| jv S )aR Checks to see if the server attempted to clobber a file. In this case we specified via a precondition that we didn't want the file clobbered. Args: e: The Exception that was generated by a failed copy operation Returns: bool indicator - True indicates that the server did attempt to clobber an existing file. Z412)r* r r �message)r� r� r� r� �IsNoClobberServerException� s �r7 c C s^ |� � r dS |j}tj�|�}tj�|�r<td| j|f ��tj�|�rZtd| j|f ��dS )a� Checks whether copying exp_src_url into dst_url is not possible. This happens if a directory exists in local file system where a file needs to go or vice versa. In that case we print an error message and exits. Example: if the file "./x" exists and you try to do: gsutil cp gs://mybucket/x/y . the request can't succeed because it requires a directory where the file x exists. Note that we don't enforce any corresponding restrictions for buckets, because the flat namespace semantics for buckets doesn't prohibit such cases the way hierarchical file systems do. For example, if a bucket contains an object called gs://bucket/dir and then you run the command: gsutil cp file1 file2 gs://bucket/dir you'll end up with objects gs://bucket/dir, gs://bucket/dir/file1, and gs://bucket/dir/file2. Args: exp_src_url: Expanded source StorageUrl. dst_url: Destination StorageUrl. Raises: CommandException: if errors encountered. NzTCannot retrieve %s because a file exists where a directory needs to be created (%s).zVCannot retrieve %s because a directory exists (%s) where the file needs to be created.) r� r� r� �path�dirname�isfiler r� �isdir)r r� Zdst_pathZ final_dirr� r� r� �CheckForDirFileConflict� s � �r< c C s� t t�ddt��} t|t| �\} }i }g }t| �D ]�}t�t |j �}t � }|� |� |� � }|t | d t|� }|�� }||_|| d k r�|}n|| d | }|| }t|j ||||| |||| | | � }|�|� |||< q2|S )a� Partitions a file into FilePart objects to be uploaded and later composed. These objects, when composed, will match the original file. This entails splitting the file into parts, naming and forming a destination URL for each part, and also providing the PerformParallelUploadFileToObjectArgs corresponding to each part. Args: canned_acl: The user-provided canned_acl, if applicable. content_type: content type for the component and final objects. dst_bucket_url: CloudUrl for the destination bucket. file_size: The size of fp, in bytes. fp: The file object to be partitioned. random_prefix: The randomly-generated prefix used to prevent collisions among the temporary component names. src_url: Source FileUrl from the original command. storage_class: storage class for the component and final objects. tracker_file: The path to the parallel composite upload tracker file. tracker_file_lock: The lock protecting access to the tracker file. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: dst_args: The destination URIs for the temporary component objects. r% �(parallel_composite_upload_component_sizer� r� )rr r r' r �_GetPartitionInfor r r r( �PARALLEL_UPLOAD_STATIC_SALTr� rY �updater � PARALLEL_UPLOAD_TEMP_NAMESPACEr� r r� r~ �append)r� r� �dst_bucket_url� file_sizer� � random_prefixr� r� r� r� r� r� r= �num_components�component_size�dst_argsZ file_names�i� encoded_name� content_md5r Ztemp_file_nameZ tmp_dst_urlZfile_part_length�offsetZ func_argsr� r� r� �_PartitionFile� sD %��� �� � rM c C s t | j| j�d�d d� �S )z�Gets component number from component CloudUrl. Used during parallel composite upload. Args: component: CloudUrl representing component. Returns: component number r� r� N)�intr� �rfind)r� r� r� r� �_GetComponentNumber7 s rP c ' C s t � � } t|j�} |j|jd�}tt�}|r2|jnd}t|t j ||�}t || �\}}}t |||||| || t t� \}}|dur�|�d�nd}|dur�|nt|d�}t||||d� t�� }i }t||j| || |||j|||| d� }t||| |�\}}}|D ]H}tj|jf||jj < t |j t|j |jt � � |jdt |j�tjd�� q�|D ]V}|d j!} tj"|d f|| < t |j t||d t � � d|d t |d �tj"d �� �qF|D ]&}t |j t||t � � dtj#d �� �q�|j$t%|| d t&j'j(|j)j*d d�} g } | D ]} | �+| d � �q�| dd� |D � } t,| �t,|�k�r�t-| t d�} g }!| D ]6}"t.j/j0|"j1d�}#|"�2� �rjt3|"j4�|#_4|!�+|#� �q@|j5|!|||jg d�|d�}$�z*z�| | }%|j$t |%tt&j'j(|j)j*d� | D ]p}|j!} z:t |j t||t � � d t |�|| d || d d�� W n( t |j t||t � � d d�� Y n0 �q�|D ]&}t |j t||t � � d tj#d �� �q8W n, t6�y� | �7dd�8|�9� � � Y n0 W |� t:|� W d � n1 �s�0 Y n0|� t:|� W d � n1 �s�0 Y 0 nt;d��t � � | }&|&|$fS ) a Uploads a local file to a cloud object using parallel composite upload. The file is partitioned into parts, and then the parts are uploaded in parallel, composed to form the original destination object, and deleted. Args: fp: The file object to be uploaded. src_url: FileUrl representing the local file. dst_url: CloudUrl representing the destination file. dst_obj_metadata: apitools Object describing the destination object. canned_acl: The canned acl to apply to the object, if any. file_size: The size of the source file in bytes. preconditions: Cloud API Preconditions for the final object. gsutil_api: gsutil Cloud API instance to use. command_obj: Command object (for calling Apply). copy_exception_handler: Copy exception handler (for use in Apply). logger: logging.Logger for outputting log messages. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, crc32c, and size fields populated. ��providerNr r� )r� r� F�r} �finished� component_num� message_typer r� )rT r} rU rV )rT rV )r� �total_bytes_transferredT�� arg_checker� parallel_operations_override�should_return_resultsr� c S s g | ] }|d �qS )r r� )r� rI r� r� r� r� � � z._DoParallelCompositeUpload.<locals>.<listcomp>��key)r� )rx r{ r} )r� rR �fields�encryption_tuple)rY rZ )rT rU r} rV )rT z:Failed to delete some of the following temporary objects: � zSSome temporary components were not uploaded successfully. Please retry this upload.)<�timer0 �bucket_url_string�GetApiSelectorr� rQ r �crypto_key_sha256r8 r= �PARALLEL_UPLOADr% r& � _DeleteTempComponentObjectFnr� r r$ r( r@ � CreateLockrM r� r� �FilterExistingComponentsr2 �COMPONENT_TO_UPLOADr� r� r� r_ r r� rP r� �EXISTING_COMPONENT�EXISTING_OBJECT_TO_DELETE�Applyr� �gslib�command�DummyArgChecker�ParallelOverrideReason�SLICErB r� �sortedr� �ComposeRequest�SourceObjectsValueListEntryr� � HasGeneration�longr{ � ComposeObject� Exceptionr, �join�keysr5 r )'r� r� r� r- r� rD r� r� � command_obj�copy_exception_handlerr� r� � start_timerC � api_selector�encryption_keywrapperr� �tracker_file_nameZexisting_enc_key_sha256Zexisting_prefix�existing_componentsrE r� Zcomponents_inforH �components_to_upload�existing_objects_to_deleter� Z component_str� cp_results�uploaded_components� cp_result� componentsZrequest_componentsZ component_urlr! Zcomposed_objectZobjects_to_delete� elapsed_timer� r� r� �_DoParallelCompositeUploadE sh # ������� � ������� �� � �� �� � �� �� �� ��� �� *�,� r� c C s� t t�ddt��}|o8|�� o8|�� o8|jdko8| }|r�|dkr�|tkr�t�< t �d�s�| � d� t � d��d � dt d< W d � n1 s�0 Y |o�|dko�||kS ) a� Determines whether parallel composite upload strategy should be used. Args: logger: for outputting log messages. allow_splitting: If false, then this function returns false. src_url: FileUrl corresponding to a local file. dst_url: CloudUrl corresponding to destination cloud object. file_size: The size of the source file, in bytes. gsutil_api: CloudApi that may be used to check if the destination bucket has any metadata attributes set that would discourage us from using parallel composite uploads. canned_acl: Canned ACL to apply to destination object, if any. Returns: True iff a parallel upload should be performed on the source file. r% �#parallel_composite_upload_threshold�gsr � suggestedra a� ==> NOTE: You are uploading one or more large file(s), which would run significantly faster if you enable parallel composite uploads. This feature can be enabled by editing the "parallel_composite_upload_threshold" value in your .boto configuration file. However, note that if you do this large files will be uploaded as `composite objects <https://cloud.google.com/storage/docs/composite-objects>`_,which means that any user who downloads such objects will need to have a compiled crcmod installed (see "gsutil help crcmod"). This is because without a compiled crcmod, computing checksums on composite objects is so slow that gsutil disables downloads of composite objects.TN)rr r r' r r� r� r� �'PARALLEL_COMPOSITE_SUGGESTION_THRESHOLD� suggested_sliced_transfers_lock�suggested_sliced_transfers�inforz �textwrap�wrap) r� r� r� r� rD r� r� r� Zall_factors_but_sizer� r� r� � _ShouldDoParallelCompositeUpload( s< ������ � �� �& �r� c C sR |p t �� }t| �r\tt| |||d��}t|�dkr@td| ��|d }t|j�|� � fS t| �}|� � rx||� � fS |� � r�|dfS t |�r�|dfS |j�d�}|j|j|d|jddgd �} | D ]�} | jtjjkr�| j|d kr�|df S | jtjjk�r | jj|jd k�r |df S | jtjjkr�| jj|jkr�|d f S q�||fS ) a- Expands wildcard if present in url_str. Args: url_str: String representation of requested url. gsutil_api: gsutil Cloud API instance to use. project_id: project ID to use (for iterators). treat_nonexistent_object_as_subdir: indicates if should treat a non-existent object as a subdir. logger: logging.Logger instance to use for output. If None, the root Logger will be used. Returns: (exp_url, have_existing_dst_container) where exp_url is a StorageUrl and have_existing_dst_container is a bool indicating whether exp_url names an existing directory, bucket, or bucket subdirectory. In the case where we match a subdirectory AND an object, the object is returned. Raises: CommandException: if url_str matched more than 1 URL. )� project_idr� r� z)Destination (%s) must match exactly 1 URLr T�/�prefixesz items/name)�prefix� delimiterrR r_ z _$folder$F)�logging�Loggerr- �listrv r� r r0 r� r� r� r� r� r/ r� r� � ListObjectsr� r� �datatyper �CsObjectOrPrefixType�PREFIX�data�OBJECTr� ) �url_strr� r� Z"treat_nonexistent_object_as_subdirr� Z blr_expansion�blrr, r� � list_iterator� obj_or_prefixr� r� r� �ExpandUrlToSingleBlrh sV �� � � � � �r� c C s� t �ddd�sdS | �� s dS t| j�r,dS |dkr8dS z|j| jdg| jd� W n4 ty� } z t |t j j �rr� W Y d}~n d}~0 0 dS )al Makes a request to the destination API provider to trigger reauth. Addresses https://github.com/GoogleCloudPlatform/gsutil/issues/1639. If an API call occurs in a child process, the library that handles reauth will fail. We need to make at least one API call in the main process to allow a user to reauthorize. For cloud source URLs this already happens because the plurality of the source name expansion iterator is checked in the main thread. For cloud destination URLs, only some situations result in a similar API call. In these situations, this function exits without performing an API call. In others, this function performs an API call to trigger reauth. Args: destination_url (StorageUrl): The destination of the transfer. gsutil_api (CloudApiDelegator): API to use for the GetBucket call. worker_count (int): If greater than 1, assume that parallel execution is used. Technically, reauth challenges can be answered in the main process, but they may be triggered multiple times if multithreading is used. Returns: None, but performs an API call if necessary. r% Z0trigger_reauth_challenge_for_parallel_operationsFNr� �location)r_ rR ) r �getboolr� r- r� � GetBucketr� r� ry r* �pyu2f�errors� PluginError)�destination_urlr� Z worker_countr� r� r� r� �.TriggerReauthForDestinationProviderIfNecessary� s& � � r� c C s6 | � � r2| jdkr2|�� r2t�dd|j�}t|�}|S )a Translates Windows pathnames to cloud pathnames. Rewrites the destination URL built by ConstructDstUrl(). Args: src_url: Source StorageUrl to be copied. dst_url: The destination StorageUrl built by ConstructDstUrl(). Returns: StorageUrl to use for copy. �\z\\r� )r� r� r� �re�subr� r0 )r� r� Z trans_url_strr� r� r� �FixWindowsNaming s r� c C sP | � � r4|� � r4tj�| j�}tj�|j�}||kS | j|jkoJ| j|jkS dS )z�Checks if src_url and dst_url represent the same object or file. We don't handle anything about hard or symbolic links. Args: src_url: Source StorageUrl. dst_url: Destination StorageUrl. Returns: Bool indicator. N)r� r� r8 �normpathr� r� r{ )r� r� Z new_src_pathZ new_dst_pathr� r� r� � SrcDstSame s �r� c C sn |� � r |r |jr d|j }nd}|�� rZ|�� s:|�� rZ|�� rFdnd}| �d||� n| �d|j|� dS )a* Logs copy operation, including Content-Type if appropriate. Args: logger: logger instance to use for output. src_url: Source StorageUrl. dst_url: Destination StorageUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. z [Content-Type=%s]r� z<STDIN>r� zCopying from %s%s...zCopying %s%s...N)r� r� r� r� r� r� r� )r� r� r� r- Zcontent_type_msgZsrc_textr� r� r� �_LogCopyOperation5 s � r� c C s� t |�}tt�}t�� } t|j| |dd�j} tjrnt tjd�� } t � | � � �j} W d � n1 sd0 Y |j ||| jtj|| |jt||d� } t�� } |�� }t|| j�|_t|jt| || tj|jdd�� | | |j|| jfS )a� Performs copy-in-the cloud from specified src to dest object. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata for source object; must include etag and size. dst_url: Destination CloudUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API instance to use for the copy. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. ZCopying�r� r� r r N)�src_generationr� r� �progress_callbackrR r_ �decryption_tupler` T�rV r} rT )rO rQ r rb r) r r r� r� r �pickle�loads�read� CopyObjectr{ r� r� �UPLOAD_RETURN_FIELDSr r. r_ r2 �FILE_CLOUD_COPYr} r| )r� r! r� r- r� r� �decryption_key�decryption_keywrapperr� r~ r� �test_fpZdst_obj�end_time� result_urlr� r� r� �_CopyObjToObjInTheCloudL sP �.� ��� �r� c C sT |j du �rP| �� �rP| �� �sP| �� �sP| j}d}|dk�r@tj�|�}t� ddd��rt �sztt j ddd|gt j t j d �}|�� \}}|j�� |j�� |jd ks�|r�td ||j|f ��|�� }t�|�}W n6 t�y } z td ||f ��W Y d}~n d}~0 0 n2|�d �\} } } | tv �r2t| }nt�|�\}} |�sJt }||_ dS )z�Detects and sets Content-Type if src_url names a local file. Args: src_url: Source StorageUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. N�-r% � use_magicfileF�filez-bz--mime)�stdout�stderrr zAEncountered error running "file -b --mime %s" (returncode=%d). %sz2Encountered OSError running "file -b --mime %s" %s�.) r� r� r� r� r� r� r8 �realpathr r� rj � subprocess�Popen�PIPE� communicater� �closer� � returncoder r� r � ensure_strr� r� �COMMON_EXTENSION_RULES� mimetypes� guess_typerm ) r� r- r� r� Zreal_file_path�p�outputr� r� r� � extensionr� r� r� �_SetContentTypeFromFile� sN �� � ���� r� c C s� t |j| |dd�j}tjrVttjd�� } t�| �� �j}W d � n1 sL0 Y t � � } t t �} | � � sv| � � r�|j||tj||| |jt|d� } n"|j||tj|||| |jt|d� } t � � } | | }|| fS )a� Uploads the file using a non-resumable strategy. This function does not support component transfers. Args: src_url: Source StorageUrl to upload. src_obj_filestream: File pointer to uploadable bytes. src_obj_size (int or None): Size of the source object. dst_url: Destination StorageUrl for the upload. dst_obj_metadata: Metadata for the target object. preconditions: Preconditions for the upload, if any. gsutil_api: gsutil Cloud API instance to use for the upload. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, md5, and size fields populated. � Uploadingr� r N)�object_metadatar� r� r� r` rR r_ r� ) r� r� r} r� r� r` rR r_ r� )r) r r r� r� r r� r� r� rb rQ r r� r� �UploadObjectStreamingr� r� r� � UploadObject)r� �src_obj_filestream� src_obj_sizer� r- r� r� r� r� r� r~ r� �uploaded_objectr� r� r� r� r� � _UploadFileToObjectNonResumable� sL �.� � r� c s� t |tj|j|jd���tt�} | r6| jr6| j�d�nd� � �fdd�} t �|� d�} | rh|� d| j � d} |rxt |�nd}t |j| ||d d �j}tjr�ttjd �� }t�|�� �j}W d � n1 s�0 Y t�� }d }| �r��z�z0|j||tj||j|| | t| || d � }d} W �n~ t�y� } �zJ|� d| j � |d7 }|t� k } | �sX� zpz&|� d|j � |j |j |jd� W n( t �y� Y n t �y� � Y n0 W t �� |� d�| j f � n t �� |� d�| j f � 0 |� d| j ||f � d} |�!d � |�rt |�nd}t |j| ||d d �j}t"|jt#|t�� |d�� t�$t%t&�&� d| t'� �� W Y d}~n"d}~0 t(�y� d} � Y n0 W | �s�t �� q�| �s�t �� 0 q�t�� }|| }||fS )a� Uploads the file using a resumable strategy. Args: src_url: Source FileUrl to upload. Must not be a stream. src_obj_filestream: File pointer to uploadable bytes. src_obj_size (int or None): Size of the source object. dst_url: Destination StorageUrl for the upload. dst_obj_metadata: Metadata for the target object. preconditions: Preconditions for the upload, if any. gsutil_api: gsutil Cloud API instance to use for the upload. logger: for outputting log messages. is_component: indicates whether this is a single component or whole file. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, md5, and size fields populated. rQ r Nc s t � tt| �i}t�|� dS )a Creates a new tracker file for starting an upload from scratch. This function is called by the gsutil Cloud API implementation and the the serialization data is implementation-specific. Args: serialization_data: Serialization data used in resuming the upload. N)r6 r< r� r? )�serialization_datar� �r� r� r� r� �_UploadTrackerCallback/ s �z<_UploadFileToObjectResumable.<locals>._UploadTrackerCallbackr� zResuming upload for %sTr� )r� rU r� r r r ) r� r� r� rR r} r� r` r_ �tracker_callbackr� r� Fz:Caught ResumableUploadStartOverException for upload of %s.r� z8Checking that bucket %s exists before retrying upload...zCDeleted tracker file %s for resumable upload of %s before retrying.z�Restarting upload of %s from scratch (retry #%d) after exception indicating we need to start over with a new resumable upload ID: %s)� num_retriesr� ))r8 r= �UPLOADrd r� rQ r re r r9 r� r� rP r) r r r� r� r r� r� r� rb �UploadObjectResumabler� r� r rF r� r� r r r5 �seekr_ r3 �sleep�min�randomrE r )r� r� r� r� r- r� r� r� r� r� r� r� � tracker_data� retryablerU r� r� r~ Znum_startover_attemptsr� r� r� r� r� r� r� � _UploadFileToObjectResumable s� ������.� � � ��� �� ����� � r� c C sV d}d}| � d�}|r |r d}n.|tksD|rNt|�dkrN|d |v rN| }|}||fS )a� Selects how an upload should be compressed. This is a helper function for _UploadFileToObject. Args: object_name: The object name of the source FileUrl. is_component: indicates whether this is a single component or whole file. gzip_exts: List of file extensions to gzip prior to upload, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts for selecting which files will be encoded. Streaming files compressed is only supported on the JSON GCS API. Returns: A tuple: (If the file should be gzipped locally, if the file should be gzip transport encoded). Fr� Tr� r� )�splitr� r� )r� r� r� r� � zipped_file�gzip_encoded_fileZ fname_partsr� r� r� � _SelectUploadCompressionStrategy� s � � �r� c C s |dur |t kr |�d| � t�� \}}d}z�| �� s>| �� rFtd��|durnt|�dt|� k rntd| ��t � ddt �}t j |d|d �}|�t�}|r�|�|� |�t�}q�W |r�|�� t�|� |�� n |r�|�� t�|� |�� 0 tj�|�} t |d �} t|�| | fS ) a� Compresses a to-be-uploaded local file to save bandwidth. This is a helper function for _UploadFileToObject. Args: src_url: Source FileUrl. src_obj_filestream: Read stream of the source file - will be consumed and closed. src_obj_size (int or None): Size of the source file. logger: for outputting log messages. Returns: StorageUrl path to compressed file, read stream of the compressed file, compressed file size. NzCompressing %s (to tmp)...z�gzip compression is not currently supported on streaming uploads. Remove the compression flag or save the streamed output temporarily to a file before uploading.r� z|Inadequate temp space available to compress %s. See the CHANGING TEMP DIRECTORIES section of "gsutil help cp" for more info.r% Zgzip_compression_level�wb)� compresslevelr )rL r� �tempfile�mkstempr� r� r rg rN r �getintr �gzipr r� �GZIP_CHUNK_SIZE�writer� r� r8 �getsizer0 ) r� r� r� r� Zgzip_fhZ gzip_path�gzip_fpZcompression_levelr� Z gzip_sizeZcompressed_filestreamr� r� r� � _ApplyZippedUploadCompression� sD � ��� � r c C s� d}d}z�|rF|sFt jj� | � \}}W d � qP1 s:0 Y n | � \}}W |�� |r�zt�|j� W q� ty� |�d|j� Y q�0 n@|�� |r�zt�|j� W n ty� |�d|j� Y n0 0 ||fS )a� Handles setup and tear down logic for uploads. This is a helper function for _UploadFileToObject. Args: upload_delegate: Function that handles uploading the file. upload_url: StorageURL path to the file. upload_stream: Read stream of the file being uploaded. This will be closed after the upload. zipped_file: Flag for if the file is locally compressed prior to calling this function. If true, the local temporary file is deleted after the upload. gzip_encoded_file: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. parallel_composite_upload: Set to true if this upload represents a top-level parallel composite upload (not an upload of a component). If true, resource locking is skipped. logger: For outputting log messages. Returns: The elapsed upload time, the uploaded object. NzZCould not delete %s. This can occur in Windows because the temporary file is still locked.) rn ro �!concurrent_compressed_upload_lockr� r� �unlinkr� ry �warning) Zupload_delegate� upload_url� upload_streamr� r� �parallel_composite_uploadr� r� r� r� r� r� �_DelegateUploadFileToObject s4 * � � � r Tc s �r �j s t�dd�}|r |�_ | � |� |� t| j�| | �\}��rV�sV��d| � nH|r�t| ||��\� � � d�_�js�d�_n d�j� � vr�� jd7 _�s�t �j t � �t � � t j� dd�� d }d }t� �t�fd d ��p�i D ��}t�| � �|�tjd �}� d u �rd n� t� k �p.| �� �p.| �� }| �� �sD| �� �rj�j�jd�tjk�rj� }t|t � �� |�s�t ���r�t � |�� ��� n� � � ������� � � � f dd�}����� � � � fdd�}������� � � � f dd�}|�r�|}n|�r |}n|}t |� � |�|��\}}|�s�z"t |�}t!��|| j|dd� W nj t"�y� t#�r�t$j%�j&�j'|j(d�}�jt) �_&�j*|��jd� �j+�j,�j|j-�jd� � Y n0 ��.� } |j-| _-t/| |j-�| _-��s�t �j t � �t � � t j� dd�� ||j0| |j1fS )a| Uploads a local file to an object. Args: src_url: Source FileUrl. src_obj_filestream: Read stream of the source file to be read and closed. src_obj_size (int or None): Size of the source file. dst_url: Destination CloudUrl. dst_obj_metadata: Metadata to be applied to the destination object. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API to use for the copy. logger: for outputting log messages. command_obj: command object for use in Apply in parallel composite uploads. copy_exception_handler: For handling copy exceptions during Apply. gzip_exts: List of file extensions to gzip prior to upload, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. allow_splitting: Whether to allow the file to be split into component pieces for an parallel composite upload. is_component: indicates whether this is a single component or whole file. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts for selecting which files will be encoded. Streaming files compressed is only supported on the JSON GCS API. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. r% �content_languagez+Using compressed transport encoding for %s.r� z no-transformz ,no-transformFr� Nc 3 s | ]}|� | � fV qd S �Nr� �r� r �� hash_algsr� r� � <genexpr>� r\ z&_UploadFileToObject.<locals>.<genexpr>)r� r rQ c s" t � � ��tj���� ���d� S �N)r� )r� r� r� r� ) r| r} r- r� r� r� r� r� � upload_sizer r r� r� �CallParallelCompositeUpload� s �z8_UploadFileToObject.<locals>.CallParallelCompositeUploadc s t ����� ���d�S r )r� r� )r- r� r� r� r� r r �wrapped_filestreamr� r� �CallNonResumableUpload� s �z3_UploadFileToObject.<locals>.CallNonResumableUploadc s t �� ��� �����d� S )N)r� r� )r� r� ) r- r� r� r� r� r� r� r r r r� r� �CallResumableUpload� s �z0_UploadFileToObject.<locals>.CallResumableUploadT)r2 �r� r� rz �r{ rR )2�contentLanguager � get_valuer� r� r� r �contentEncoding� cacheControl�lowerr_ r r2 rb � FILE_UPLOADrZ �dictr� r� r� rG r� r� rd r� r �JSONr+ rD r� r[ r r r5 r! �_RENAME_ON_HASH_MISMATCHr� r� r� r� rz � _RENAME_ON_HASH_MISMATCH_SUFFIXr� � DeleteObjectr� r{ r r. r} r| ) r� r� r� r� r- r� r� r� r| r} r� r� r� r� r r� r� r� r r Znon_resumable_uploadZ orig_streamr r r Zdelegater �corrupted_obj_metadatar� r� )r| r} r- r� r� r� r r� r� r� r r r r r� r� E s� , �� ��������� �� ������� �r� c C s� t j�| j�}|r`t j�|�s`zt �|� W n2 ty^ } z|jtjkrJ� W Y d}~n d}~0 0 d}t |�r�d}t � | �}|� d|� n t � | �}t j�| j�r�t �| j� t j�|�s�t|d�}|�� ||fS )a� Creates a new download file, and deletes the file that will be replaced. Names and creates a temporary file for this download. Also, if there is an existing file at the path where this file will be placed after the download is completed, that file will be deleted. Args: dst_url: Destination FileUrl. src_obj_metadata: Metadata from the source object. logger: for outputting log messages. Returns: (download_file_name, need_to_unzip) download_file_name: The name of the temporary file to which the object will be downloaded. need_to_unzip: If true, a temporary zip file was used and must be uncompressed as part of validation. NFTz$Downloading to temp gzip filename %s�w)r� r8 r9 r� �exists�makedirsr� �errno�EEXISTr\ rB ZGetTempZipFileNamer� �GetTempFileNamer r r� )r� r! r� �dir_namer� � need_to_unzip�download_file_namer� r� r� r� �_GetDownloadFile s& r, c C s� t t�ddt��}t�ddt�}t�ddt�}|jo8t� }|pD|t k}|or| t j j uor|dkor|or|dkor|j |k} | s�|j tkr�t� s�|t kr�t�< t�d�s�|�d�t�d ��d � d td< W d � n1 s�0 Y | S ) a| Determines whether the sliced download strategy should be used. Args: download_strategy: CloudApi download strategy. src_obj_metadata: Metadata from the source object. allow_splitting: If false, then this function returns false. logger: logging.Logger for log message output. Returns: True iff a sliced download should be performed on the source file. r% � sliced_object_download_threshold�%sliced_object_download_max_componentsr& r� r r� ra z�==> NOTE: You are downloading one or more large file(s), which would run significantly faster if you enabled sliced object downloads. This feature is enabled by default but requires that compiled crcmod be installed (see "gsutil help crcmod").TN)rr r r' r r� r rU rx rH rV r r� r� r} r� r� r� r� rz r� r� ) �download_strategyr! r� r� r- �max_componentsr. Zparallel_hashingZ hashing_okayZ use_slicer� r� r� �_ShouldDoSlicedDownloadR sV ���� �������� ���&r1 c s� t | |d�}t�tj|j�}t| j|jd�� t � fdd�� p>i D ��}t |j ||j |j || j||j|j|j|jd� \}}d}d|v r�|d j}t|j|||j|j d |�S ) a" Function argument to Apply for performing sliced downloads. Args: cls: Calling Command class. args: PerformSlicedDownloadObjectToFileArgs tuple describing the target. thread_state: gsutil Cloud API instance to use for the operation. Returns: PerformSlicedDownloadReturnValues named-tuple filled with: component_num: The component number for this download. crc32c: CRC32C hash value (integer) of the downloaded bytes. bytes_transferred: The number of bytes transferred, potentially less than the component size if the download was resumed. component_total_size: The number of bytes corresponding to the whole component size, potentially more than bytes_transferred if the download was resumed. r� )�consider_crc32cc 3 s | ]}|� | � fV qd S r r� r r r� r� r � r\ z5_PerformSlicedDownloadObjectToFile.<locals>.<genexpr>)rU � start_byte�end_byter� Nrx r� )rI r �decode_messager� r� �src_obj_metadata_jsonrX r� rx r � _DownloadObjectToFileResumabler� r� r+ rU r3 r4 r� �crcValuer� ) r� r� r� r� r! r �bytes_transferred�server_encodingZ crc32c_valr� r r� �"_PerformSlicedDownloadObjectToFile� s: ��� �r; c C s� | j s J �d}| jt� k r dS t|tj|�}d}�zz�t|d�}t|�} | | jkr�t|d�}t� |�} | d | j kr�| d | j kr�| d |kr�W W |r�|� � |r�|� � dS |� � |� d|j � W nX ttf�y } z:t| t�s�| jtjk�r |� d|t| �f � W Y d} ~ n d} ~ 0 0 W |�r2|� � |�r`|� � n |�rP|� � |�r^|� � 0 t||� zPt|d ��0}| j | j |d �} |�t�| �� W d � n1 �s�0 Y W n2 t�y� } zt|| j� W Y d} ~ n d} ~ 0 0 dS ) a� Maintains sliced download tracker files in order to permit resumability. Reads or creates a sliced download tracker file representing this object download. Upon an attempt at cross-process resumption, the contents of the sliced download tracker file are verified to make sure a resumption is possible and appropriate. In the case that a resumption should not be attempted, existing component tracker files are deleted (to prevent child processes from attempting resumption), and a new sliced download tracker file is created. Args: src_obj_metadata: Metadata from the source object. Must include etag and generation. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for the download. logger: for outputting log messages. api_selector: The Cloud API implementation used. num_components: The number of components to perform this download with. Nr �rrz r{ rF z`Sliced download tracker file doesn't match for download of %s. Restarting download from scratch.zVCouldn't read sliced download tracker file (%s): %s. Restarting download from scratch.r# )rz r{ rF )rz r} rG r8 r= �SLICED_DOWNLOADr rh �json�loadr{ r� r, r� �IOError� ValueErrorr* r& �ENOENTr� r4 r� �dumpsr: �strerror) r! r� r+ r� r rF r� r� r� �existing_file_sizeZtracker_file_datar� r� r� r� �#_MaintainSlicedDownloadTrackerFiles� sj � � ��� � � �4rF c @ sR e Zd ZdZdd� Zedd� �Zdd� Zej fdd �Z d d � Z d d � Z dd� Z dS )�SlicedDownloadFileWrappera� Wraps a file object to be used in GetObjectMedia for sliced downloads. In order to allow resumability, the file object used by each thread in a sliced object download should be wrapped using SlicedDownloadFileWrapper. Passing a SlicedDownloadFileWrapper object to GetObjectMedia will allow the download component tracker file for this component to be updated periodically, while the downloaded bytes are normally written to file. c C s( || _ || _|| _d| _|| _|| _dS )a; Initializes the SlicedDownloadFileWrapper. Args: fp: The already-open file object to be used for writing in GetObjectMedia. Data will be written to file starting at the current seek position. tracker_file_name: The name of the tracker file for this component. src_obj_metadata: Metadata from the source object. Must include etag and generation. start_byte: The first byte to be downloaded for this parallel component. end_byte: The last byte to be downloaded for this parallel component. N)�_orig_fp�_tracker_file_name�_src_obj_metadata�_last_tracker_file_byte� _start_byte� _end_byte)�selfr� r� r! r3 r4 r� r� r� �__init__ s z"SlicedDownloadFileWrapper.__init__c C s t | jdd�S )z<Returns the mode of the underlying file descriptor, or None.r� N)�getattrrH �rN r� r� r� r� ' s z SlicedDownloadFileWrapper.modec C s� | j �� }| j|kr*|t|� | jd ks.J �t�| j |� | j �� }t}| jd u sp|| j |ksp|| jd kr�t | j | j |� || _d S �Nr� ) rH �tellrL r� rM rC � write_to_fd� TRACKERFILE_UPDATE_THRESHOLDrK r> rI rJ )rN r� �current_file_pos� thresholdr� r� r� r� , s" � � ��z SlicedDownloadFileWrapper.writec C s\ |t jkr"| j�|| j d � n| j�||� | j| j�� krR| jd ksXn J �d S rR )r� �SEEK_ENDrH r� rM rL rS )rN rL �whencer� r� r� r� = s z SlicedDownloadFileWrapper.seekc C s | j �� S r )rH rS rQ r� r� r� rS D s z SlicedDownloadFileWrapper.tellc C s | j �� d S r )rH �flushrQ r� r� r� rZ G s z SlicedDownloadFileWrapper.flushc C s | j r| j �� d S r )rH r� rQ r� r� r� r� J s z SlicedDownloadFileWrapper.closeN)r� r� r� r� rO �propertyr� r� r� �SEEK_SETr� rS rZ r� r� r� r� r� rG s rG c C s� t t�ddt��}t�ddt�}t|j||�\}}g } g } t|�D ]`} | | } t | d | d |jd �} | � | | d � t � |�}| � t | | |||| | |�� qB| | fS )a� Partitions an object into components to be downloaded. Each component is a byte range of the object. The byte ranges of the returned components are mutually exclusive and collectively exhaustive. The byte ranges are inclusive at both end points. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object with non-pickleable fields removed. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for the download. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: components_to_download: A list of PerformSlicedDownloadObjectToFileArgs to be used in Apply for the sliced download. r% �%sliced_object_download_component_sizer. r� )rr r r' r r� r r> r} r r� rB r �encode_messager )r� r! r� r+ r� Z sliced_download_component_sizer0 rF rG �components_to_download�component_lengthsrI r3 r4 r6 r� r� r� �_PartitionObjectO s6 ���� ��ra c C s� d|_ t| ||||�\} } t| �} t|||||| � t|d�� } | �|j� W d � n1 s`0 Y t| �D ]Z\}}|j|j d }t ||||j ||�}||j }t | t | |t � � |d|t j|d�� qr|jt| |tjj|jjdd�}t|�| k �rtd|j ��t|td �d �}|d j}|du�rTtd| �D ] }t ||| j| | �}�q6d }t |�}|D ]h}t | t | |t � � |j d|j t jd �� ||j 7 }|j!�o�|j!�"� �#d �}|�rd|�sdtd|j ���qd||fS )a� Downloads a cloud object to a local file using sliced download. Byte ranges are decided for each thread/process, and then the parts are downloaded in parallel. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. command_obj: command object for use in Apply in parallel composite uploads. logger: for outputting log messages. copy_exception_handler: For handling copy exceptions during Apply. api_selector: The Cloud API implementation used. decryption_key: Base64-encoded decryption key for the source object, if any. status_queue: Queue for posting file messages for UI/Analytics. Returns: (bytes_transferred, crc32c) bytes_transferred: Number of bytes transferred from server this call. crc32c: a crc32c hash value (integer) for the downloaded bytes, or None if crc32c hashing wasn't performed. N�r+br� F)r} rT rU rV �bytes_already_downloadedTrX zSSome components of %s were not downloaded successfully. Please retry this download.rU r] r rS r� zTDownload of %s failed because the server sent back data with an unexpected encoding.)$ry ra r� rF r �truncater} � enumerater4 r3 r7 r_ r2 rb �COMPONENT_TO_DOWNLOADrm r; rn ro rp rq rr r r� rs r rx r rW r\ Zcomponent_total_sizerU r9 r: r r� )r� r! r� r+ r| r� r} r r� r r_ r` rF r� rI r� r} �download_start_byterc r� rx r9 Z expect_gzipr� � server_gzipr� r� r� �_DoSlicedDownload� s� $ �� *� �� ��� �� � ��ri c " C s� | du r|j d } | | d } |du} |j| jd�} d}|j}| rN|d| 7 }d}�z��z6t|d�}|�|� |j| jd�} t|�}t|||| |||�\}}||k s�|| d kr�t|� t d| ��||| k}||ko�| }|r�|� d|� n|�r|� d|� t |||j d �}|�s |�r�d }|| }t |t|j|| |d d �j�}||k �r�tt|| �}|�|�}||7 }|D ]} || �t�|�� �qt|�t|�� �qFn| �s�|�d � d }t|j|| | ||d d�j} tj�rttjd�� } t � | �� �j} W d � n1 �s0 Y | �r<|j t � k�r<t ||||| �}t |�} |�s�|�|� |j!| j"| j||| | | j#|j t$j%j&| j||| t'| �d�}W n8 t(�y� } z |�)d| j*|� � W Y d} ~ n d} ~ 0 0 W |�r�|�+� n|�r�|�+� 0 | | d }!|!|fS )a] Downloads an object to a local file using the resumable strategy. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. gsutil_api: gsutil Cloud API instance to use for the download. logger: for outputting log messages. digesters: Digesters corresponding to the hash algorithms that will be used for validation. component_num: Which component of a sliced download this call is for, or None if this is not a sliced download. start_byte: The first byte of a byte range for a sliced download. end_byte: The last byte of a byte range for a sliced download. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (bytes_transferred, server_encoding) bytes_transferred: Number of bytes transferred from server this call. server_encoding: Content-encoding string if it was detected that the server sent encoded bytes during transfer, None otherwise. Nr� rQ z component %drb z�Resumable download start point for %s is not in the correct byte range. Deleting tracker file, so if you re-try this download it will start from scratchzResuming download for %szRDownload already complete for %s, skipping download but will run integrity checks.)�progress� user_projectr r )rU r� r� r � Downloading)r3 �override_total_sizer� r� rU r r ) r3 r4 �compressed_encodingr{ � object_sizer/ rR r� r r� r� z:Caught ResumableDownloadException (%s) for download of %s.),r} rd r� r� r r� rh r; r5 r r� rJ rk r* r) r r r� rK r� r@ r r( �Progressr� rd r� r� r� r� rG rG r\ �GetObjectMediar� r{ r r� r� rO r r �reasonr� )"r� r! r� r+ r� r� r rU r3 r4 r� Z download_sizeZ is_slicedr r: � download_namer� rE r� rg �download_completeZresumingr� Zbytes_digestedZtotal_bytes_to_digestZ hash_callback� bytes_to_readr� r# r� r� rn r� r9 r� r� r� r7 � s� " � �� �� �� � � 0� � � � r7 c C s� d}z�t |d�}t|d|jd�}t|j| |dd�j} tjrvt tjd�� } t� | � � �j} W d � n1 sl0 Y |j | j | j || j|jtjj| j||| t|�d� } W |r�|�� n|r�|�� 0 |j| fS ) a^ Downloads an object to a local file using the non-resumable strategy. This function does not support component transfers. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. gsutil_api: gsutil Cloud API instance to use for the download. digesters: Digesters corresponding to the hash algorithms that will be used for validation. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (bytes_transferred, server_encoding) bytes_transferred: Number of bytes transferred from server this call. server_encoding: Content-encoding string if it was detected that the server sent encoded bytes during transfer, None otherwise. Nr# r )rk rl r� r )r{ ro r/ rR r� r r� r� )r rJ rk r) r r r� r� r� r� r� rq r� r� r{ r} r r� r� r� rO r� ) r� r! r� r+ r� r r� r� r� r� r� r: r� r� r� �!_DownloadObjectToFileNonResumable� sB ��.� � rv c sD |j �|j�r8|�d�t�d|j ��� td|j ��|j| j d�} t |�} t | |||�}t |||�\}}t �( t�|d�r�t�dt|< W d � n1 s�0 Y |jo�| }t|||jd�� t� fd d �� p�i D ��}d}|jd k}d }t�� }|�s�|�r<t| ||||||| ||jd � \}}d |v �r�||d _nf| tjj u �rft | ||||||d�\}}n<| tjj u �r�t | |||||||d�\}}nt d| |f ��t�� }|�o�|�!� �d�}t"|| |||||� || ||| | | d�}t � t�#|� W d � n1 �s 0 Y t$|jt%| ||t%j&|jdd�� || |||fS )a� Downloads an object to a local file. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. gsutil_api: gsutil Cloud API instance to use for the download. logger: for outputting log messages. command_obj: command object for use in Apply in sliced downloads. copy_exception_handler: For handling copy exceptions during Apply. allow_splitting: Whether or not to allow sliced download. decryption_key: Base64-encoded decryption key for the source object, if any. is_rsync: Whether or not the caller is the rsync command. preserve_posix: Whether or not to preserve POSIX attributes. use_stet: Decrypt downloaded file with STET binary if available on system. Returns: (elapsed_time, bytes_transferred, dst_url, md5), where time elapsed excludes initial GET. Raises: FileConcurrencySkipError: if this download is already in progress. CommandException: if other errors encountered. ra z�Skipping attempt to download to filename ending with slash (%s). This typically happens when using gsutil to download from a subdirectory created by the Cloud Console (https://cloud.google.com/console)z Invalid destination path: %srQ FTN)� consider_md5r2 c 3 s | ]}|� | � fV qd S r r� r r r� r� r # r\ z(_DownloadObjectToFile.<locals>.<genexpr>r )r� r rx �r� z.Invalid download strategy %s chosen forfile %sr� )�is_rsyncr �use_stet)� message_timerV r} rT )'r� r� r� r, rz r� r� r" rd r� r� r1 r, �open_files_lock�open_files_mapr' r� r| rX rx r r} rb ri r r8 r r� r� rv r� r7 r r � _ValidateAndCompleteDownload�deleter_ r2 � FILE_DOWNLOAD)r� r! r� r� r� r| r} r� r� ry r rz r r/ �sliced_downloadr+ r* rw r r: rt r9 r~ rx r� rh Z local_md5r� r r� �_DownloadObjectToFile� s� %����� & � �� � � ��*�� r� c C s� |j }d}|D ]}|| sd} q$q|r2t|�}nt| j||||�}d}d}z t| ||||� t|| � W n� ty� } zj|r�| �d� d}nJ| tj kr�| �d� |}d}n,t|| � t r�t � ||t � n t �|� � W Y d}~n d}~0 0 |s�|s�|}n�t�|�}| tk�r | �d|� d}z�zdt�|d�}t|d��8}|�t�}|�rd|�|� |�t�}�qFW d � n1 �sz0 Y W n> t�y� } z$d t|�v �r�|�r�|�W Y d}~n d}~0 0 W |�r�|�� n|�r�|�� 0 t �|� |�slz0t| j||||�}t| ||||� t|| � W nB t�yj t|| � t �rZt � ||t � n t �|� � Y n0 |�r�t� |||| � t � ||� t ||| | d � d |v �r�|d S dS ) a� Validates and performs necessary operations on a downloaded file. Validates the integrity of the downloaded file using hash_algs. If the file was compressed (temporarily), the file will be decompressed. Then, if the integrity of the file was successfully validated, the file will be moved from its temporary download location to its permanent location on disk. Args: logger: For outputting log messages. src_url: StorageUrl for the source object. src_obj_metadata: Metadata for the source object, potentially containing hash values. dst_url: StorageUrl describing the destination file. need_to_unzip: If true, a temporary zip file was used and must be uncompressed as part of validation. server_gzip: If true, the server gzipped the bytes (regardless of whether the object metadata claimed it was gzipped). digesters: dict of {string, hash digester} that contains up-to-date digests computed during the download. If a digester for a particular algorithm is None, an up-to-date digest is not available and the hash must be recomputed from the local file. hash_algs: dict of {string, hash algorithm} that can be used if digesters don't have up-to-date digests. temporary_file_name: Temporary file name that was used for download. api_selector: The Cloud API implementation used (used tracker file naming). bytes_transferred: Number of bytes downloaded (used for logging). gsutil_api: Cloud API to use for service and status. is_rsync: Whether or not the caller is the rsync function. Used to determine if timeCreated should be used. preserve_posix: Whether or not to preserve the posix attributes. use_stet: If True, attempt to decrypt downloaded files with the STET binary if it's present on the system. Returns: An MD5 of the local file, if one was calculated as part of the integrity check. TFNzDHash did not match but server gzipped the content, will recalculate.zMHash did not match but server may have gzipped the content, will recalculate.z/Uncompressing temporarily gzipped file to %s...r r� zNot a gzipped file�ry r r ) r� r r$ r r5 r4 r! r� r r� r r� �renamer r rB r( ru r� r� r r� r� r� r@ r� r� rA Zdecrypt_downloadre )r� r� r! r� r* rh r r Ztemporary_file_namer r9 r� ry r rz Zfinal_file_nameZdigesters_succeededr r3 Zdigest_verifiedZhash_invalid_exceptionr� Z unzipped_temporary_file_namer Zf_outr� r� r� r� r~ u s� 4 � � � � � 2 � � � � � � r~ c C s� t | �}tj�|j�}|r\zt�|� W n2 tyZ } z|jtjkrF� W Y d}~n d}~0 0 t |jd��$}t � � }t � ||� W d � n1 s�0 Y | � � s�|�� t � � } t|t| || tj|r�|jnddd�� | | tj�|j�|dfS )a Copies a local file to a local file. Args: src_url: Source FileUrl. dst_url: Destination FileUrl. status_queue: Queue for posting file messages for UI/Analytics. src_obj_metadata: An apitools Object that may contain file size, or None. Returns: (elapsed_time, bytes_transferred, dst_url, md5=None). Raises: CommandException: if errors encountered. Nr� Tr� )ri r� r8 r9 r� r% r� r& r' r rb �shutil� copyfileobjr� r� r_ r2 �FILE_LOCAL_COPYr} r� ) r� r� r r! Zsrc_fpr) r� Zdst_fpr~ r� r� r� r� �_CopyFileToFile s6 * �� �r� c C s d S r r� )r� r� r� r� �_DummyTrackerCallbackK s r� c C s� t jr| j|jkrtd��t js&g |_d}t jrltt jd�� } t�| � � �j }W d � n1 sb0 Y t |�} t t �} t�� } t| |j|| ||d�} d}|jdkr�|j| |t j||jt|j| d�}n4|j| |t j||jt|jt|j| |dd�j t| d � }t�� }zt|| |||� W nv t�y� t�rptj |j |j |j d �}|j t! |_ t"|�}|j#|||j|| d � |j$|j%|j |j&|jd � � Y n0 |�'� }t(||j&�|_&t)|jt*| ||t*j+|jd d�� || |j||j,fS )a Copies from src_url to dst_url in "daisy chain" mode. See -D OPTION documentation about what daisy chain mode is. Args: src_url: Source CloudUrl src_obj_metadata: Metadata from source object dst_url: Destination CloudUrl dst_obj_metadata: Object-specific metadata that should be overidden during the copy. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API to use for the copy. logger: For outputting log messages. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. z"Cross-provider cp -p not supportedNr )rn r� r� r )r� r� r� rR r_ r} r` r� r� ) r� r� r� rR r_ r} r� r� r` r )rR r� r` r Tr� )-r� r� r� �NotImplementedError�aclr� r r� r� r� r r\ rQ r rb r r} r� r� r� r� r) r r� r0 r! r r� r� r� r� rz r� r rO r� r! r� r{ r r. r_ r2 �FILE_DAISY_COPYr| )r� r! r� r- r� r� r� r� r� r� rn r� r~ Z upload_fpr� r� r" r� r� r� r� r� �_CopyObjToObjDaisyChainModeP s� �.� ��� �������� �r� c C s� t � }| r:|�g d�� |r(|�dg� |s�|�dg� nT|�g d�� |r^|�dt dg� |r�dt dt dt dt dt g}|�|� |r�|�ddg� |r�|�dg� t|�S )a Determines the metadata fields needed for a copy operation. This function returns the fields we will need to successfully copy any cloud objects that might be iterated. By determining this prior to iteration, the cp command can request this metadata directly from the iterator's get/list calls, avoiding the need for a separate get metadata HTTP call for each iterated result. As a trade-off, filtering objects at the leaf nodes of the iteration (based on a remaining wildcard) is more expensive. This is because more metadata will be requested when object name is all that is required for filtering. The rsync command favors fast listing and comparison, and makes the opposite trade-off, optimizing for the low-delta case by making per-object get metadata HTTP call so that listing can return minimal metadata. It uses this function to determine what is needed for get metadata HTTP calls. Args: dst_is_cloud: if true, destination is a Cloud URL. skip_unsupported_objects: if true, get metadata for skipping unsupported object types. preserve_acl: if true, get object ACL. is_rsync: if true, the calling function is rsync. Determines if metadata is needed to verify download. preserve_posix: if true, retrieves POSIX attributes into user metadata. delete_source: if true, source object will be deleted after the copy (mv command). file_size_will_change: if true, do not try to record file size. Returns: List of necessary field metadata field names. )r �componentCount�contentDispositionr r r� rx ry rz r{ r| � mediaLink�metadata�metagenerationr� � timeCreatedr� r} ) rx r r� ry rz r� r| r} r{ z metadata/%sr� r� )�setr@ rd r` rb rc rf r� ) Z dst_is_cloudr� r� ry r � delete_source�file_size_will_changeZsrc_obj_fields_setZ posix_fieldsr� r� r� � GetSourceFieldsNeededForCopy� s4 ' � � r� � �Z im )ZnearlineZcoldline�archivec C sh | j dkrd|rd|jrd|jrd|j�� }t�|d�}|rd|t|j� }t�� |k rd|�d|| j |t � dS )a� Warns when deleting a gs:// object could incur an early deletion charge. This function inspects metadata for Google Cloud Storage objects that are subject to early deletion charges (such as Nearline), and warns when performing operations like mv that would delete them. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with necessary fields (per GetSourceFieldsNeededForCopy). logger: logging.Logger for outputting warning. r� Nz�Warning: moving %s object %s may incur an early deletion charge, because the original object is less than %s days old according to the local system time.) r� r� r� r � EARLY_DELETION_MINIMUM_LIFETIMEr' ra rb r, r� rt )r� r! r� Zobject_storage_classZ early_deletion_cutoff_secondsZminimum_delete_ager� r� r� �"WarnIfMvEarlyDeletionChargeAppliesI s( �� ��� �r� c C s$ | j dkr tjr |jdkr t� �dS )a Skips unsupported object types if requested. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with storageClass field (per GetSourceFieldsNeededForCopy). Raises: SkipGlacierError: if skipping a s3 Glacier object. �s3�GLACIERN)r� r� r� r� �SkipGlacierError)r� r! r� r� r� �MaybeSkipUnsupportedObjecth s ��r� c C s4 |j r0t|j jt�}|s,td|j j| f ��|S dS )a� Ensures a matching decryption key is available for the source object. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with optional customerEncryption field. Raises: EncryptionException if the object is encrypted and no matching key is found. Returns: Base64-encoded decryption key string if the object is encrypted and a matching key is found, or None if object is not encrypted. zOMissing decryption key with SHA256 hash %s. No decryption key matches object %sN)ry rP � keySha256r r )r� r! r� r� r� r� �GetDecryptionCSEKy s � ��r� c C s |r|� � }ni }t|�}|�� r6|jdkr6t|�}nt� }d}d}d}|�� r�|�� rn|j|jkrntjsnd}tjr�t ||| � t ||� t ||�}|j }|j |_ tjr�|�� r�|jdkr�|js�td| ��|j|_|jdkr�t|�}|r�t||� n�|�r t�||| �}n|}z t|�}W nl t�y� } zRd|t|�f }|j�rl| jd7 _| �|� W Y d}~dS t|��W Y d}~n d}~0 0 |�� �s�|� � �r�d}n*|�r�|j �r�|�s�|j }nt j � |j �}tj!�r�| �"|j#d |� |jdk�r$|dk�r$|du�r$|t$k�r$td |t%t$�f ��t&�rT|�'� �rT|�� �rTt(�)t|��*� t j+� tj,�r�|j-�rnt.d ��nd |_-|�'� �r�t j �/|j ��r�t0� �nJ|�� �r�z|j1|j2|j |jd �}W n t3�y� d}Y n0 |�r�t0� �|�� �rP|j |_4|j2|_5|�� �r|j |_4|j2|_5n t6||� t7t8�}|�rP|j9t:j;k�rP|jdk�rP|j<|_=|�rdt>||dd� tj?�rttj?|_@t8�Add�tBk�r�d|_CtD| |||� |�� �rj|�'� �r�tE|jFtG||tH�H� tGjI|dd�� tJ||||| ||||| | |d� S |�r.tE|jFtG||tH�H� tGjK|dd�� tL|||||||d�S tE|jFtG||tH�H� tGjM|dd�� tN||||||| |d�S n�|�� �r�tO|||||||| ||| || d� } |�r�t �P|j4� | S tE|jFtG||tH�H� tGjQ|dd�� tR|||jF|d�} |�� �s|�� �stS|j || | d� | S dS )a� Performs copy from src_url to dst_url, handling various special cases. Args: logger: for outputting log messages. src_url: Source StorageUrl. dst_url: Destination StorageUrl. gsutil_api: gsutil Cloud API instance to use for the copy. command_obj: command object for use in Apply in parallel composite uploads and sliced object downloads. copy_exception_handler: for handling copy exceptions during Apply. src_obj_metadata: If source URL is a cloud object, source object metadata with all necessary fields (per GetSourceFieldsNeededForCopy). Required for cloud source URLs. If source URL is a file, an apitools Object that may contain file size, or None. allow_splitting: Whether to allow the file to be split into component pieces for an parallel composite upload or download. headers: optional headers to use for the copy operation. manifest: optional manifest for tracking copy operations. gzip_exts: List of file extensions to gzip, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. is_rsync: Whether or not the caller is the rsync command. preserve_posix: Whether or not to preserve posix attributes. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts. Streaming files compressed is only supported on the JSON GCS API. use_stet: If True, will perform STET encryption or decryption using the binary specified in the boto config or PATH. Returns: (elapsed_time, bytes_transferred, version-specific dst_url) excluding overhead like initial GET. Raises: ItemExistsError: if no clobber flag is specified and the destination object already exists. SkipUnsupportedObjectError: if skip_unsupported_objects flag is specified and the source is an unsupported type. CommandException: if other errors encountered. r� NFTzZNo OWNER permission found for object %s. OWNER permission is required for preserving ACLs.r� z Error opening file "%s": %s.r� r} z�"%s" exceeds the maximum gsutil-supported size for an S3 upload. S3 objects greater than %s in size require multipart uploads, which gsutil does not support.zASpecifying x-goog-if-generation-match is not supported with cp -nr rQ )�overrider% r& r� )r� r� ry r rz rx )r� r� r� )r r! r� )T�copyrn r� r� ro r r� r� r� r� r� r� r} r� r� r� r rp rk rA Zencrypt_uploadri ry r� �continue_on_errorr� r� r� r� r� r8 r� r� r� �Setr� �S3_MAX_UPLOAD_SIZErs rj r� �msvcrtZsetmode�fileno�O_BINARYr� � gen_matchr r$ �ItemExistsError�GetObjectMetadatar� r r� r� r� rQ r � crypto_typerN �CMEK� crypto_key� kmsKeyNamerl r� r� r' rV r| r� r_ r r2 rb r� r� r� r� r� r� r� r r� r� re ) r� r� r� r� r| r} r! r� �headers�manifestr� ry r r� rz Zdst_obj_headersr- r� r� r� Zcopy_in_the_cloudr� �acl_textZsource_stream_urlr� r6 Z dst_objectr� Zuploaded_metadata�resultr� r� r� � PerformCopy� s� 7 � �� �� �� � �� ��� ������ � ����r� c @ sZ e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z ddd�Z dd� Z dd� Z dS )�Manifestz2Stores the manifest items for the CpCommand class.c C s8 i | _ i | _t�� | _tj�|�| _| � � | � � d S r ) �items�manifest_filterr@ rh �lockr� r8 � expanduser� manifest_path�_ParseManifest�_CreateManifestFile)rN r8 r� r� r� rO � s zManifest.__init__c C s� z�t j�| j�r�t| jd���}d}t�|�}|D ]h}|rrz|�d�}|�d�}W n typ t d| j ��Y n0 d}|| }|| }|dv r0|| j |< q0W d � n1 s�0 Y W n t y� t d | j ��Y n0 dS ) z|Load and parse a manifest file. This information will be used to skip any files that have a skip or OK status. r< T�Source�Resultz$Missing headers in manifest file: %sF)�OK�skipNzCould not parse %s) r� r8 r$ r� r �csv�reader�indexrA r r� r@ ) rN �fZ first_rowr� �rowZ source_indexZ result_index�sourcer� r� r� r� r� � s* � . zManifest._ParseManifestc C s || j v S )z?Returns whether the specified src url was marked as successful.)r� )rN �srcr� r� r� � WasSuccessful� s zManifest.WasSuccessfulc C s� z�t j�| j�r"t �| j�jdkr�tjrrt| jddd��(}t � |�}|� g d�� W d � q�1 sf0 Y nFt| jdd��(}t � |�}|� g d�� W d � n1 s�0 Y W n t y� t d ��Y n0 dS ) z;Opens the manifest file and assigns it to the file pointer.r r# r� ��newline) r� Z DestinationZStartZEndZMd5�UploadIdz Source SizezBytes Transferredr� Z DescriptionNr� r� z Could not create manifest file.)r� r8 r$ r� r� �st_sizer r) r r� �writer�writerowr@ r )rN r� r� r� r� r� r� � s � . 0 z Manifest._CreateManifestFilec C s8 |d u r d S || j v r&|| j | |< n||i| j |< d S r �r� )rN r� r^ �valuer� r� r� r� s z Manifest.Setc C sD | � |d|� | � |d|� | � |dtjjtjjd�jd d�� d S )N� source_uri�destination_urir~ ��tz��tzinfo)r� �datetime�now�timezone�utcr )rN � source_urlr� r� r� r� � Initialize) s zManifest.Initializer� c C sf | � |d|� | � |d|� | � |d|� | � |dtjjtjjd�jd d�� | �|� | �|� d S )N�bytesr� � descriptionr� r� r� )r� r� r� r� r� r �_WriteRowToManifestFile�_RemoveItemFromManifest)rN r� r9 r� r� r� r� r� � SetResult/ s $ zManifest.SetResultc C s | j | }|d |d d|d �� d|d �� d|v rB|d ndd|v rT|d ndd |v rjt|d �ndd |v r�t|d �nd|d |d g }d d� |D �}| j�X tr�tjr�t| jdddd�}nt| jdd�}t � |�}|� |� |� � W d � n1 �s0 Y dS )zBWrites a manifest entry to the manifest file for the url argument.r� r� z%sZr~ r� r r� � upload_idr} r� r� r� c S s g | ]}t �|��qS r� )r r� )r� r� r� r� r� r� G r\ z4Manifest._WriteRowToManifestFile.<locals>.<listcomp>�ar� r� N) r� � isoformatr� r� rj r r) r r� r� r� r� r� )rN r� Zrow_itemr� r� r� r� r� r� r� 7 s( � z Manifest._WriteRowToManifestFilec C s | j |= d S r r� )rN r� r� r� r� r� T s z Manifest._RemoveItemFromManifestN)r� ) r� r� r� r� rO r� r� r� r� r� r� r� r� r� r� r� r� r� � s ' r� c @ s e Zd ZdZdS )r� zHException class for objects that are skipped because they already exist.Nr� r� r� r� r� r� [ s r� c s e Zd ZdZ� fdd�Z� ZS )�SkipUnsupportedObjectError�CException for objects skipped because they are an unsupported type.c s t t| ��� d| _d S )N�Unknown)�superr� rO �unsupported_typerQ �� __class__r� r� rO c s z#SkipUnsupportedObjectError.__init__�r� r� r� r� rO � __classcell__r� r� r� r� r� ` s r� c s e Zd ZdZ� fdd�Z� ZS )r� r� c s t t| ��� d| _d S )Nr� )r� r� rO r� rQ r� r� r� rO k s zSkipGlacierError.__init__r� r� r� r� r� r� h s r� c C s| | j }| �� rJ| jtd�d� }|�|�dkr2dS d|�|��|�d S | �� r\d| j S | j�|��|�d }t ||�S )a? Returns the path section before the final directory component of the URL. This handles cases for file system directories, bucket, and bucket subdirectories. Example: for gs://bucket/dir/ we'll return 'gs://bucket', and for file://dir we'll return file:// Args: url: StorageUrl representing a filesystem directory, cloud bucket or bucket subdir. exp_src_url: StorageUrl representing the fully expanded object to-be-copied; used for resolving cloud wildcards. Returns: String name of above-described path, sans final path separator. zfile://Nr� z file://%sr z%s://) r� r� r� r� �findr� r� r� r� �$ResolveWildcardsInPathBeforeFinalDir)r� r �sepZ past_schemeZpath_sans_final_dirr� r� r� r� p s r� c C sp t | �s | S t| �j�d�}|j�d�}tt|��D ] }t || �r4|| ||< q4d�|�}|�� }||_|jS )af Returns the path section for a bucket subdir with wildcards resolved. This handles cases for bucket subdirectories where the initial source URL contains a wildcard. In this case, src_url must be wildcard-expanded before calculating the final directory. Example: A bucket containing: gs://bucket/dir1/subdir/foo gs://bucket/dir2/subdir/foo and source URL gs://bucket/*/subdir and src_url_path_sans_final dir gs://bucket/* should yield final path gs://bucket/dir1 or gs://bucket/dir2 according to the expanded source URL. Args: src_url_path_sans_final_dir: URL string with wildcards representing a bucket subdir as computed from GetPathBeforeFinalDir. exp_src_url: CloudUrl representing the fully expanded object to-be-copied. Returns: String name of above-described path, sans final path separator. r� ) r- r0 r� r� r r� rz r r� )r r Zwildcarded_src_obj_pathZexpanded_src_obj_pathZpath_segment_indexZresolved_src_pathZfinal_path_urlr� r� r� r� � s �� � r� c C s, t | |�}tt||�d�}t | |�}||fS )a Gets info about a file partition for parallel file/object transfers. Args: file_size: The number of bytes in the file to be partitioned. max_components: The maximum number of components that can be composed. default_component_size: The size of a component, assuming that max_components is infinite. Returns: The number of components in the partitioned file, and the size of each component (except the last, which will have a different size iff file_size != 0 (mod num_components)). r� )rq �maxr� )rD r0 Zdefault_component_sizerF rG r� r� r� r> � s r> c C s@ t | |�}z |j|j|j|j|jd� W n ty: Y n0 dS )zGWrapper func to be used with command.Apply to delete temporary objects.r N)rI r! r� r� r{ r� r )r� Z url_to_deleter� r� r� r� r� rg � s � rg c C s� g }dd� |D �}| D ]}||vr|� | | � qg }g }g } |D �] } | j| �� vs`| j|v r�|�� } | j| _| j| _|� | � qB| | j } t| j| j| j�} t | �}z0| j }|j |j |j|j|j g d�d�}|j}W n ty� d}Y n0 ||k�r6|� | � |� | j� | j�rd| j �� }| j|_|� |� qB| j �� } | j| _| � | | jf� |� | j� qB| �r|t�dt| �� || |fS )a] Determines course of action for component objects. Given the list of all target objects based on partitioning the file and the list of objects that have already been uploaded successfully, this function determines which objects should be uploaded, which existing components are still valid, and which existing components should be deleted. Args: dst_args: The map of file_name -> PerformParallelUploadFileToObjectArgs calculated by partitioning the file. existing_components: A list of ObjectFromTracker objects that have been uploaded in the past. bucket_url: CloudUrl of the bucket in which the components exist. gsutil_api: gsutil Cloud API instance to use for retrieving object metadata. Returns: components_to_upload: List of components that need to be uploaded. uploaded_components: List of components that have already been uploaded and are still valid. Each element of the list contains the dst_url for the uploaded component and its size. existing_objects_to_delete: List of components that have already been uploaded, but are no longer valid and are in a versioned bucket, and therefore should be deleted. c S s g | ] }|j �qS r� )r� )r� r� r� r� r� r� s z,FilterExistingComponents.<locals>.<listcomp>)ry rz r| )r{ rR r_ Nz0Found %d existing temporary components to reuse.)rB r� r{ r r{ r# r� r� r� rS r� r� r� r� r| ry r� r� r� )rH r� � bucket_urlr� r� Zexisting_component_namesZcomponent_nameZobjects_already_chosenr� r� Ztracker_objectr� Zdst_argZ file_partrK r� Z dst_metadataZ cloud_md5Z!invalid_component_with_generationr� r� r� ri � sh � � �� �ri )N) FFFFFFFNFNN)F)F)NF)F)N)FN)N)F)FF)FFF)NTFF)N)N)NN)Nr NN)N)TNFFF)FFF)NN)N)FFFF) NTNNNFFFF)N( r� � __future__r r r r r � collectionsr r� r� r& r� r> r� r� �operatorr r� r� r� r� r� r� r r� r� r� r� rb r� � six.movesr �apitools.base.protorpcliter �botor r rn �gslib.cloud_apir r r r r r r r r r r r Zgslib.commands.composer Zgslib.commands.configr r r r r r �gslib.cs_api_mapr Zgslib.daisy_chain_wrapperr �gslib.exceptionr r! r" Zgslib.file_partr# Zgslib.parallel_tracker_filer$ r% r& r' r( �gslib.progress_callbackr) r* Z gslib.resumable_streaming_uploadr+ r, �gslib.storage_urlr- r. r/ r0 �"gslib.third_party.storage_apitoolsr1 r� �gslib.thread_messager2 r3 �gslib.tracker_filer4 r5 r6 r7 r8 r9 r: r; r< r= r> r? � gslib.utilsr@ rA rB rC Zgslib.utils.boto_utilrD rE rF rG rH � gslib.utils.cloud_api_helperrI rJ �gslib.utils.constantsrK rL rM � gslib.utils.encryption_helperrN rO rP rQ �gslib.utils.hashing_helperrR rS rT rU rV rW rX rY rZ r[ �gslib.utils.metadata_utilr\ �&gslib.utils.parallelism_framework_utilr] r^ r_ �gslib.utils.posix_utilr` ra rb rc rd re rf �gslib.utils.system_utilrg rh ri rj � gslib.utils.translation_helperrk rl rm rn ro rp �gslib.utils.unit_utilrq rr rs rt ru Zgslib.wildcard_iteratorrv r) rN rw r� � is_available�top_level_managerr} rh r| r r rA r? r� r~ r r� r� r� rU r� r� r� r� r� ry r� r� r� r� r� r� r� r� r� r� r� r� r r r$ r0 r5 r7 r< rM rP r� r� r� r� r� r� r� r� r� r� r� r� r r r� r, r1 r; rF �objectrG ra ri r7 rv r� r~ r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r> rg ri r� r� r� r� �<module> s� � �� ��� � �� < � 3 0� ) "E� C2 � M� j� C � j; � D:� N � & � '7D � T:3 0SK� = � z � 0� G � " � + , � � q� " � 3 / utils/__pycache__/version_check.cpython-39.pyc 0000644 00000002624 15025013616 0015372 0 ustar 00 a +(Wg� � @ s, d Z ddlZddlmZ ddiZdd� ZdS )zBUtility functions to ensure the correct version of Python is used.� N)� version_info� � c C sT t jj} t jj}| tvr(ddj| d�fS |t| k rPt| }ddj| ||d�fS dS )a� Return an exception if running in an unsupported version of Python. This function compares the running version of cPython and against the list of supported python version. If the running version is less than any of the supported versions, return a Tuple of (False, Str(error message)) for the caller to handle. Minor versions of Python greater than those listed in the supported versions are allowed. Args: None Returns: Tuple(Boolean, String) A Tuple containing a Boolean and a String. The boolean represents if the version is supported, and the String will either be empty, or contain an error message. Fz3Gsutil does not support running under Python{major})�majorziFor Python{major}, gsutil requires Python{major}.{lowest_minor}+, but you are using Python{major}.{minor}�r �minorZ lowest_minor)T� )�sysr r r � MIN_SUPPORTED_PYTHON_VERSION�formatr � r �(/opt/gsutil/gslib/utils/version_check.py� check_python_version_support s � ��r )�__doc__r r r r r r r r �<module> s utils/__pycache__/iam_helper.cpython-39.pyc 0000644 00000021156 15025013616 0014656 0 ustar 00 a +(WgI+ � @ s d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl Z dd l m Z dd l m Z dd lmZ eg d ��Zeg d ��ZdZeddg�Zedddg�Zedddg�ZdZdd� Zdd� Zdd� Zd d � Z d d � Z d d!� Z d"d#� Z d$d%� Z d&d'� Z!d(d)� Z"dS )*z"Helper module for the IAM command.� )�absolute_import)�print_function)�division)�unicode_literals)� defaultdict)� namedtupleN)� protojson)�CommandException)�storage_v1_messages) �userz deleted:userZserviceAccountzdeleted:serviceAccount�groupz deleted:group�domain� principalZ principalSetZprincipalHierarchy)Z projectOwnerZ projectEditorZ projectViewera� Assigning roles (e.g. objectCreator, legacyBucketOwner) for project convenience groups is not supported by gsutil, as it goes against the principle of least privilege. Consider creating and using more granular groups with which to assign permissions. See https://cloud.google.com/iam/docs/using-iam-securely for more information. Assigning a role to a project group can be achieved by setting the IAM policy directly (see gsutil help iam for specifics).�allUsers�allAuthenticatedUsers� BindingsTuple�is_grant�bindings�BindingsDictTuple� c C s | j dd� | jD �fS )aS Serializes the BindingsValueListEntry instances in a BindingsTuple. This is necessary when passing instances of BindingsTuple through Command.Apply, as apitools_messages classes are not by default pickleable. Args: bindings_tuple: A BindingsTuple instance to be serialized. Returns: A serialized BindingsTuple object. c S s g | ]}t �|��qS � )r �encode_message��.0�tr r �%/opt/gsutil/gslib/utils/iam_helper.py� <listcomp>[ � z*SerializeBindingsTuple.<locals>.<listcomp>�r r )Zbindings_tupler r r �SerializeBindingsTupleN s �r c C s | \}}t |dd� |D �d�S )Nc S s g | ]}t �tjj|��qS r )r �decode_message�apitools_messages�Policy�BindingsValueListEntryr r r r r a s ��z,DeserializeBindingsTuple.<locals>.<listcomp>r )r )Zserialized_bindings_tupler r r r r �DeserializeBindingsTuple^ s ��r$ c C s( t t�}| D ]}||j �|j� q |S )z�Reformats policy bindings metadata. Args: bindings: A list of BindingsValueListEntry instances. Returns: A {role: set(members)} dictionary. )r �set�role�update�members�r Z tmp_bindings�bindingr r r �BindingsMessageToUpdateDicth s r+ c C s, t t�}| D ]}||d �|d � q |S )a Reformats policy bindings metadata. Args: bindings: List of dictionaries representing BindingsValueListEntry instances. e.g.: { "role": "some_role", "members": ["allAuthenticatedUsers", ...] } Returns: A {role: set(members)} dictionary. r& r( )r r% r' r) r r r �BindingsDictToUpdateDictx s r, c C s t | |�\}}|j o |j S )N)� DiffBindingsr )�a�b�granted�removedr r r �IsEqualBindings� s r2 c C s� t | �}t |�}t g �}t g �}t�|�D ] \}}|| �|�|| �� q*t�|�D ] \}}|| �|�|| �� qVdd� t�|�D �}dd� t�|�D �}td|�td|�fS )a] Computes the difference between two BindingsValueListEntry lists. Args: old: The original list of BindingValuesListEntry instances new: The updated list of BindingValuesListEntry instances Returns: A pair of BindingsTuple instances, one for roles granted between old and new, and one for roles removed between old and new. c S s( g | ] \}}|rt jj|t|�d ��qS �)r& r( �r! r" r# �list�r �r�mr r r r � s �z DiffBindings.<locals>.<listcomp>c S s( g | ] \}}|rt jj|t|�d ��qS r3 r4 r6 r r r r � s �TF)r+ �six� iteritemsr' � differencer )�old�newZtmp_oldZtmp_newr0 r1 r& r( r r r r- � s ��r- c C sv |r4t �|�D ]"\}}|s"td��| | �|� qn.| D ](}| | �|| � | | �|t � q8dd� t �| �D �S )a� Patches a diff list of BindingsValueListEntry to the base. Will remove duplicate members for any given role on a grant operation. Args: base (dict): A dictionary returned by BindingsMessageToUpdateDict or BindingsDictToUpdateDict representing a resource's current IAM policy. diff (dict): A dictionary returned by BindingsMessageToUpdateDict or BindingsDictToUpdateDict representing the IAM policy bindings to add/remove from `base`. is_grant (bool): True if `diff` should be added to `base`, False if it should be removed from `base`. Returns: A {role: set(members)} dictionary created by applying `diff` to `base`. z+Role must be specified for a grant request.c S s i | ]\}}|r||�qS r r )r r& r( r r r � <dictcomp>� r z!PatchBindings.<locals>.<dictcomp>)r9 r: r r' �difference_update�DROP_ALL)�base�diffr r& r( r r r � PatchBindings� s rC c sz |� d�s|d7 }|�d�}dd� tD �}dd� tD �}dd� tD �}|d �� }d|d �� |d �� f }||v r�|| |d< nN||v r�|| |d< n8||v r�|| |d< n"||v r�|| �d�\|d< |d< d�|�}| o�|d tv }|� d�dk�rjd|d |d f tv �r&td | ��nB|d tv �r>|\� } n*|d tv �sR|�r\|� t} n td | ��n�|� d�d k�r�d|d |d f tv �r�|� t} n>|�r�|\} } } d| | f � n |\} } } t | |� d| | f � nR|� d�d k�r$|�d�\} }} } d| |f } t | |� d| | f � n td | ��| �rD| �sDtd ��dd� | �d�D �} � fdd�t | �D �}t | |d�S )a� Parses an iam ch bind string to a list of binding tuples. Args: is_grant: If true, binding is to be appended to IAM policy; else, delete this binding from the policy. input_str: A string representing a member-role binding. e.g. user:foo@bar.com:objectAdmin user:foo@bar.com:objectAdmin,objectViewer user:foo@bar.com allUsers deleted:user:foo@bar.com?uid=123:objectAdmin,objectViewer deleted:serviceAccount:foo@bar.com?uid=123 Raises: CommandException in the case of invalid input. Returns: A BindingsDictTuple instance. �:c S s i | ]}|� � |�qS r ��lower�r �sr r r r> � r z(BindingStringToTuple.<locals>.<dictcomp>c S s i | ]}|� � |�qS r rE rG r r r r> � r c S s i | ]}|� � |�qS r rE rG r r r r> � r r z%s:%s� z+Incorrect public member type for binding %s� � zInvalid ch format %sz Must specify a role to grant.c S s g | ] }t |��qS r )� ResolveRole�r r7 r r r r ) r z(BindingStringToTuple.<locals>.<listcomp>�,c s g | ]}|� gd ��qS r3 r rM ��memberr r r + r r ) �count�split�PUBLIC_MEMBERS�TYPES�DISCOURAGED_TYPESrF �joinr r@ �_check_member_typer% r )r � input_str�tokensZpublic_members�typesZdiscouraged_typesZ possible_public_member_or_type� possible_typeZremoving_discouraged_typeZroles� member_type� project_idZ member_idZmember_type_p1Zmember_type_p2r r rO r �BindingStringToTuple� sh � � r^ c C s* | t v rtt��n| tvr&td| ��d S )Nz$Incorrect member type for binding %s)rU r �DISCOURAGED_TYPES_MSGrT )r\ rX r r r rW / s rW c C s | st S d| v r| S d| S )Nzroles/zroles/storage.%s)r@ )r& r r r rL 6 s rL )#�__doc__� __future__r r r r � collectionsr r r9 �apitools.base.protorpcliter �gslib.exceptionr �"gslib.third_party.storage_apitoolsr r! r% rT rU r_ rS r r r@ r r$ r+ r, r2 r- rC r^ rW rL r r r r �<module> s> � � $!X utils/__pycache__/rsync_util.cpython-39.pyc 0000644 00000003144 15025013616 0014741 0 ustar 00 a +(Wg] � @ sX d Z ddlmZ ddlmZ ddlmZ ddlmZ G dd� de�ZG dd � d e�Zd S ) z>Shared utility structures and methods for rsync functionality.� )�absolute_import)�print_function)�division)�unicode_literalsc @ s e Zd ZdZdZdZdZdZdS )� DiffActionzCEnum class representing possible actions to take for an rsync diff.�copy�removeZmtime_src_to_dstZposix_src_to_dstN)�__name__� __module__� __qualname__�__doc__�COPYZREMOVEZMTIME_SRC_TO_DSTZPOSIX_SRC_TO_DST� r r �%/opt/gsutil/gslib/utils/rsync_util.pyr s r c @ s e Zd ZdZdd� ZdS )�RsyncDiffToApplyzAClass that encapsulates info needed to apply diff for one object.c C s" || _ || _|| _|| _|| _dS )a� Constructor. Args: src_url_str: (str or None) The source URL string, or None if diff_action is REMOVE. dst_url_str: (str) The destination URL string. src_posix_attrs: (posix_util.POSIXAttributes) The source POSIXAttributes. diff_action: (DiffAction) DiffAction to be applied. copy_size: (int or None) The amount of bytes to copy, or None if diff_action is REMOVE. N)� src_url_str� dst_url_str�src_posix_attrs� diff_action� copy_size)�selfr r r r r r r r �__init__" s zRsyncDiffToApply.__init__N)r r r r r r r r r r s r N) r � __future__r r r r �objectr r r r r r �<module> s utils/__pycache__/temporary_file_util.cpython-39.pyc 0000644 00000001623 15025013616 0016624 0 ustar 00 a +(Wg� � @ sP d Z ddlmZ ddlmZ ddlmZ ddlmZ dd� Zdd � Zd d � Zd S ) zUtils for temporary files.� )�absolute_import)�print_function)�division)�unicode_literalsc C s d| j S )z2Returns temporary file name for uncompressed file.z %s_.gstmp�� object_name�� storage_url� r �./opt/gsutil/gslib/utils/temporary_file_util.py�GetTempFileName s r c C s d| j S )z9Returns temporary name for a temporarily compressed file.z %s_.gztmpr r r r r �GetTempZipFileName s r c C s d| j S )z9Returns temporary file name for result of STET transform.z %s_.stet_tmpr r r r r �GetStetTempFileName! s r N) �__doc__� __future__r r r r r r r r r r r �<module> s utils/__pycache__/arg_helper.cpython-39.pyc 0000644 00000002006 15025013616 0014652 0 ustar 00 a +(Wg� � @ s, d Z ddlZddlZddlmZ dd� ZdS )z:Contains helper for parsing command arguments and options.� N)�CommandExceptionc C sT z t � tjdd� dg d��W S t jyN } zt| j��W Y d} ~ n d} ~ 0 0 dS )a8 Gets the list of arguments and options from the command input. Returns: The return value consists of two elements: the first is a list of (option, value) pairs; the second is the list of program arguments left after the option list was stripped (this is a trailing slice of the first argument). � NzdDvo:?h:i:u:mq) �debugZ detailedDebug�version�option�help�headerz impersonate-service-account=Z multithreaded�quietZtestexceptiontracesz trace-token=zperf-trace-token=)�getopt�sys�argv� GetoptErrorr �msg)�e� r �%/opt/gsutil/gslib/utils/arg_helper.py�GetArgumentsAndOptions s r )�__doc__r r �gslib.exceptionr r r r r r �<module> s utils/__pycache__/parallelism_framework_util.cpython-39.pyc 0000644 00000027452 15025013616 0020175 0 ustar 00 a +(Wg�> � @ sl d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddl mZ dd lmZ zddlZd ZW n( ey� Z zd ZW Y dZ[n dZ[0 0 d Zd Zd ZdZdadada e� dddg�Z ze � d�Z W n e!e"f�y e Z Y n0 G dd� de#�Z$G dd� de#�Z%dd� Z&dd� Z'd!dd �Z(d d � Z)efd d �Z*dS )"z:Utility classes and methods for the parallelism framework.� )�absolute_import)�print_function)�division)�unicode_literalsN)� constants)� system_util)�queueTF�< � )z There were noz tasks to do� MultiprocessingIsAvailableResult� is_available� stack_trace�forkc @ sN e Zd ZdZddd�Zdd� Zdd� Zdd d �Zd d � Zd d� Z ddd�Z dS )� AtomicDictz�Thread-safe (and optionally process-safe) dictionary protected by a lock. If a multiprocessing.Manager is supplied on init, the dictionary is both process and thread safe. Otherwise, it is only thread-safe. Nc C s. |r|� � | _|�� | _nt� � | _i | _dS )z�Initializes the dict. Args: manager: (multiprocessing.Manager or None) Manager instance (required for cross-process safety), or none if cross-process safety is not needed. N)�Lock�lock�dict� threading)�self�manager� r �5/opt/gsutil/gslib/utils/parallelism_framework_util.py�__init__U s zAtomicDict.__init__c C s4 | j � | j| W d � S 1 s&0 Y d S �N�r r �r �keyr r r � __getitem__c s zAtomicDict.__getitem__c C s4 | j � || j|< W d � n1 s&0 Y d S r r )r r �valuer r r � __setitem__g s zAtomicDict.__setitem__c C s8 | j � | j�||�W d � S 1 s*0 Y d S r �r r �get)r r � default_valuer r r r! l s zAtomicDict.getc C s2 | j � | j|= W d � n1 s$0 Y d S r r r r r r �deletep s zAtomicDict.deletec C s4 | j � | j�� W d � S 1 s&0 Y d S r )r r �values�r r r r r$ t s zAtomicDict.valuesr c C sJ | j �0 | j�||�| }|| j|< |W d � S 1 s<0 Y dS )a� Atomically updates the stored value associated with the given key. Performs the atomic equivalent of dict[key] = dict.get(key, default_value) + inc. Args: key: lookup key for the value of the first operand of the "+" operation. inc: Second operand of the "+" operation. default_value: Default value if there is no existing value for the key. Returns: Incremented value. Nr )r r Zincr" �valr r r � Incrementx s zAtomicDict.Increment)N)N)r ) �__name__� __module__� __qualname__�__doc__r r r r! r# r$ r' r r r r r N s r c @ s: e Zd ZdZdd� Zddd�Zdd� Zd d � Zd d � Zd S )�ProcessAndThreadSafeInta� This class implements a process and thread-safe integer. It is backed either by a multiprocessing Value of type 'i' or an internal threading lock. This simplifies the calling pattern for global variables that could be a Multiprocessing.Value or an integer. Without this class, callers need to write code like this: global variable_name if isinstance(variable_name, int): return variable_name else: return variable_name.value c C s0 || _ | j r t�dd�| _nt�� | _d| _d S )N�ir )� multiprocessing_is_available�multiprocessing_context�Valuer r r r )r r. r r r r � s z ProcessAndThreadSafeInt.__init__r c C s@ | j r|| j_n,| j� || _W d � n1 s20 Y d S r �r. r r )r Z reset_valuer r r �Reset� s z ProcessAndThreadSafeInt.Resetc C sP | j r| j jd7 _n4| j� | jd7 _W d � n1 sB0 Y d S �N� r1 r% r r r r' � s z!ProcessAndThreadSafeInt.Incrementc C sP | j r| j jd8 _n4| j� | jd8 _W d � n1 sB0 Y d S r3 r1 r% r r r � Decrement� s z!ProcessAndThreadSafeInt.Decrementc C s> | j r| jjS | j� | jW d � S 1 s00 Y d S r r1 r% r r r �GetValue� s z ProcessAndThreadSafeInt.GetValueN)r ) r( r) r* r+ r r2 r' r5 r6 r r r r r, � s r, c C s� zt �| �\}}W n t jtfy, Y dS 0 ||krfzt �| ||f� |W S t jtfyd Y n0 ||k r�zt �| ||f� |W S t jtfy� | Y S 0 n|S dS )a Sets a new soft limit for the maximum number of open files. The soft limit is used for this process (and its children), but the hard limit is set by the system and cannot be exceeded. We will first try to set the soft limit to the hard limit's value; if that fails, we will try to set the soft limit to the fallback_value iff this would increase the soft limit. Args: resource_name: Name of the resource to increase the soft limit for. fallback_value: Fallback value to be used if we couldn't set the soft value to the hard value (e.g., if the hard value is "unlimited"). Returns: Current soft limit for the resource (after any changes we were able to make), or -1 if the resource doesn't exist. ���N)�resourceZ getrlimit�error� ValueErrorZ setrlimit)� resource_nameZfallback_valueZ soft_limitZ hard_limitr r r � _IncreaseSoftLimitForResource� s" r< c C s2 t jr dS t jrdS z�tdd��f} | �� �� D ]D}d|v r.|�d�d �d�}d |�� v |f W d � W S q.W d � W d S 1 s�0 Y W n� t � y� } z@|j t j kr�t � d |j t|�f � W Y d }~d S � W Y d }~nDd }~0 t�y, } z"t � d �t|��� W Y d }~d S d }~0 0 d S )a Determines if the OS doesn't support multiprocessing. There are two cases we currently know about: - Multiple processes are not supported on Windows. - If an error is encountered while using multiple processes on Alpine Linux gsutil hangs. For this case it's possible we could do more work to find the root cause but after a fruitless initial attempt we decided instead to fall back on multi-threading w/o multiprocesing. Returns: (bool indicator if multiprocessing should be prohibited, OS name) )T�Windows)F�macOSz/etc/os-release�rzNAME=�=r4 �"z alpine linuxN)FZUnknownzeUnable to open /etc/os-release to determine whether OS supports multiprocessing: errno=%d, message=%szYSomething went wrong while trying to determine multiprocessing capabilities. Message: {0})r � IS_WINDOWS�IS_OSX�open�read� splitlines�split�strip�lower�IOError�errno�ENOENT�logging�debug�str� Exception�format)�f�line�os_name�e�excr r r � ShouldProhibitMultiprocessing� s. && ��rW c C sp t dur,| r | �t� | �t� tt td�S t� \}}|r\d| }| rP| �|� tddd�S d}d}d}z�zt�dd� W n |d 7 }� Y n0 t� � a d }t r�zt |t tjtj��}W n ty� Y n0 zt |t tjtj��}W n ty� Y n0 |tjk �r |d | 7 }td | ��W n6 t�� }d}| du�rR| �|� | �|� Y n0 |a |a|att td�S ) aq Checks if multiprocessing is available, and if so performs initialization. There are some environments in which there is no way to use multiprocessing logic that's built into Python (e.g., if /dev/shm is not available, then we can't create semaphores). This simply tries out a few things that will be needed to make sure the environment can support the pieces of the multiprocessing module that we need. See gslib.command.InitializeMultiprocessingVariables for an explanation of why this is necessary. Args: logger: (logging.Logger) Logger to use for debug output. Returns: (MultiprocessingIsAvailableResult) A namedtuple with the following attrs: - multiprocessing_is_available: True iff the multiprocessing module is available for use. - stack_trace: The stack trace generated by the call we tried that failed. N)r r z� Multiple processes are not supported on %s. Operations requesting parallelism will be executed with multiple threads in a single process only. FTz� You have requested multiple processes for an operation, but the required functionality of Python's multiprocessing module is not available. Operations requesting parallelism will be executed with multiple threads in a single process only. r- r zI Please ensure that you have write access to both /dev/shm and /run/shm. r7 a� Your max number of open files, %s, is too low to allow safe multiprocessing. On Linux you can fix this by adding something like "ulimit -n 10000" to your ~/.bashrc or equivalent file and opening a new terminal. On macOS, you may also need to run a command like this once (in addition to the above instructions), which might require a restart of your system to take effect: launchctl limit maxfiles 10000 Alternatively, edit /etc/launchd.conf with something like: limit maxfiles 10000 10000 z)Max number of open files, %s, is too low.)�$_cached_multiprocessing_is_availablerN �)_cached_multiprocessing_check_stack_trace�warn�,_cached_multiprocessing_is_available_messager rW r/ r0 �Manager�top_level_manager�_HAS_RESOURCE_MODULE�maxr< r8 Z RLIMIT_NOFILEr � MIN_ACCEPTABLE_OPEN_FILES_LIMIT�AttributeErrorZ RLIMIT_OFILErP � traceback� format_exc)�loggerZ should_prohibit_multiprocessingrT �messager r. �limitr r r �$CheckMultiprocessingAvailableAndInit s� � � ��� �� � �rg c C s t � jrt�� S t�� S dS )a Returns either a multiprocessing lock or a threading lock. Use Multiprocessing lock iff we have access to the parts of the multiprocessing module that are necessary to enable parallelism in operations. Returns: Multiprocessing or threading lock. N)rg r r] r r r r r r � CreateLock� s rh c C s: d}|s6z| j ||d� d}W q tjy2 Y q0 qdS )a7 Puts an item to the status queue. If the queue is full, this function will timeout periodically and repeat until success. This avoids deadlock during shutdown by never making a fully blocking call to the queue, since Python signal handlers cannot execute in between instructions of the Python interpreter (see https://docs.python.org/2/library/signal.html for details). Args: queue: Queue class (typically the global status queue) msg: message to post to the queue. timeout: (optional) amount of time to wait before repeating put request. F)�timeoutTN)�put�Queue�Full)r �msgri Z put_successr r r �PutToQueueWithTimeout� s rn )N)+r+ � __future__r r r r � collectionsrK rM �multiprocessingr rb � gslib.utilsr r � six.movesr rk r8 r^ � ImportErrorrU ZSEEK_AHEAD_JOIN_TIMEOUTZSTATUS_QUEUE_OP_TIMEOUTZUI_THREAD_JOIN_TIMEOUTZZERO_TASKS_TO_DO_ARGUMENTrX rY r[ � namedtupler � get_contextr/ ra r: �objectr r, r< rW rg rh rn r r r r �<module> sN � >52- } utils/__pycache__/user_agent_helper.cpython-39.pyc 0000644 00000002771 15025013616 0016246 0 ustar 00 a +(Wg� � @ sN d Z ddlZddlZddlZddlmZ ddlmZ ddlm Z d dd�Z dS ) z5Contains helper for appending user agent information.� N)� system_util)�StorageUrlFromString)�InvalidUrlErrorTc C s. dt j }|dtj 7 }|d|r$dnd 7 }|dt�� 7 }t| �dkr�|d| d 7 }t| �d kr�| d d v r�zLtt� | d ��}tt� | d ��}|� � r�|� � r�|j |j kr�|d 7 }W q� t y� Y q�0 n,| d dkr�d| v r�|d7 }d| v r�|d7 }t� � �r*|d7 }t�� �r*|dt�� 7 }|S )a% Using the command arguments return a suffix for the UserAgent string. Args: args: str[], parsed set of arguments entered in the CLI. metrics_off: boolean, whether the MetricsCollector is disabled. Returns: str, A string value that can be appended to an existing UserAgent. z gsutil/%sz (%s)z analytics/%s�disabledZenabledz interactive/%sr z command/%s� )Zcp�mv�rsync������z -DaisyChainZrewritez-kz-sz google-cloud-sdkz/%s)�gslib�VERSION�sys�platformr �IsRunningInteractively�lenr �six� ensure_textZ IsCloudUrl�schemer �InvokedViaCloudSdkZCloudSdkVersion)�argsZ metrics_offZ user_agent�src�dst� r �,/opt/gsutil/gslib/utils/user_agent_helper.py� GetUserAgent s2 r )T) �__doc__r r r � gslib.utilsr Zgslib.storage_urlr �gslib.exceptionr r r r r r �<module> s utils/__pycache__/encryption_helper.cpython-39.pyc 0000644 00000013535 15025013616 0016304 0 ustar 00 a +(Wg � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl Z ddl Z ddl Z ddl mZ dd lmZ d Zed d � �ZG d d� de�ZG dd� de�Zdd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd d � Z dS ) zGHelper functions for dealing with encryption keys used with cloud APIs.� )�absolute_import)�print_function)�division)�unicode_literalsN)�sha256)�CommandException)� LazyWrapper�d c C s t �d�S )Nzqprojects/([^/]+)/locations/([a-zA-Z0-9_-]{1,63})/keyRings/([a-zA-Z0-9_-]{1,63})/cryptoKeys/([a-zA-Z0-9_-]{1,63})$)�re�compile� r r �,/opt/gsutil/gslib/utils/encryption_helper.py�<lambda>" � r c @ s e Zd ZdZdZdZdS )� CryptoKeyTypezDEnum of valid types of encryption keys used with cloud API requests.�CSEK�CMEKN)�__name__� __module__� __qualname__�__doc__r r r r r r r ( s r c @ s e Zd ZdZdd� ZdS )�CryptoKeyWrapperz�Class describing a crypto key used with cloud API requests. This class should be instantiated via the `CryptoKeyWrapperFromKey` method. c C s� || _ t|�dkrFtj| _d| _zt|�| _W q� td��Y q�0 nTz t |� W n2 ty� } ztd|j ��W Y d}~n d}~0 0 tj | _d| _d| _dS )a Initialize the CryptoKeyWrapper. Args: crypto_key: Base64-encoded string of a CSEK, or the name of a Cloud KMS CMEK. Raises: CommandException: The specified crypto key was neither a CMEK key name nor a valid base64-encoded string of a CSEK. �, �AES256z�Configured encryption_key or decryption_key looked like a CSEK, but it was not a valid 44-character base64 string. Please double-check your configuration and ensure the key is correct.zaConfigured encryption_key or decryption_key looked like a CMEK, but the key failed validation: %sN) � crypto_key�lenr r � crypto_type� crypto_alg�#Base64Sha256FromBase64EncryptionKey�crypto_key_sha256r � ValidateCMEK�reasonr )�selfr �er r r �__init__4 s* � ��zCryptoKeyWrapper.__init__N)r r r r r$ r r r r r . s r c C s | r t | �S dS )z>Returns a CryptoKeyWrapper for crypto_key, or None for no key.N)r )r r r r �CryptoKeyWrapperFromKeyX s r% c C s� t jrt| t�s| �d�} t|�ddd��}|durP|jtj krP|j | krP|j S t t �D ]T}|d }t|�ddt|� d��}|du r� q�qX|jtj krX|j | krX|j S qXdS )a~ Searches boto_config for a CSEK with the given base64-encoded SHA256 hash. Args: key_sha256: (str) Base64-encoded SHA256 hash of the AES256 encryption key. boto_config: (boto.pyami.config.Config) The boto config in which to check for a matching encryption key. Returns: (str) Base64-encoded encryption key string if a match is found, None otherwise. �ascii�GSUtil�encryption_keyN� zdecryption_key%s)�six�PY3� isinstance�bytes�encoder% �getr r r r r �range�MAX_DECRYPTION_KEYS�str)� key_sha256� boto_configZ keywrapper�iZ key_numberr r r � FindMatchingCSEKInBotoConfig] s. � �� � �r6 c C s | � ddd�}|rt|�S dS )a� Returns a CryptoKeyWrapper for the configured encryption key. Reads in the value of the "encryption_key" attribute in boto_config, and if present, verifies it is a valid base64-encoded string and returns a CryptoKeyWrapper for it. Args: boto_config: (boto.pyami.config.Config) The boto config in which to check for a matching encryption key. Returns: CryptoKeyWrapper for the specified encryption key, or None if no encryption key was specified in boto_config. r' r( N)r/ r �r4 r( r r r �GetEncryptionKeyWrapper� s r8 c C sL t jrt| t�s| �d�} t�| �}t|�}t� |�}t� |�}|� dd�S )Nr& � r ) r* r+ r, r- r. �base64� b64decode�_CalculateSha256FromString�binascii� unhexlify� b64encode�replace)Zcsek_encryption_key� decoded_bytesr3 Z sha256_bytesZ sha256_base64r r r r � s r c C s> | s t d��| �d�r"t d| ��t� �| �s:t d| ��d S )NzKMS key is empty.�/z5KMS key should not start with leading slash (/): "%s"z�Invalid KMS key name: "%s". KMS keys should follow the format "projects/<project-id>/locations/<location>/keyRings/<keyring>/cryptoKeys/<key-name>")r � startswith� VALID_CMEK_RE�match)�keyr r r r � s � ��r c C s t � }|�| � |�� S )N)r �update� hexdigest)Z input_stringZ sha256_hashr r r r< � s r< c C s: | � ddd�}|r6zt�|� W n td��Y n0 |S )aC Reads the encryption key from boto_config and ensures it is base64-encoded. Args: boto_config: (boto.pyami.config.Config) The boto config in which to check for a matching encryption key. Returns: (str) Base64-encoded encryption key string, or None if no encryption key exists in configuration. r' r( Nz�Configured encryption_key is not a valid base64 string. Please double-check your configuration and ensure the key is valid and in base64 format.)r/ r: r; r r7 r r r � _GetAndVerifyBase64EncryptionKey� s � rI ) r � __future__r r r r r: r= �hashlibr r �sysr* �gslib.exceptionr �gslib.lazy_wrapperr r1 rD �objectr r r% r6 r8 r r r< rI r r r r �<module> s2 �*$ utils/__pycache__/execution_util.cpython-39.pyc 0000644 00000002073 15025013616 0015606 0 ustar 00 a +(Wg5 � @ sT d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ dd � Z dS ) z(Helper functions for executing binaries.� )�absolute_import)�print_function)�division)�unicode_literalsN)� exceptionc C sp t j| t jt jd�}|�� \}}|dur:t|t�s:|�� }|durTt|t�sT|�� }|jdkrht� |��||fS )z�Runs external terminal command. Args: command_and_flags (List[str]): Ordered command and flag strings. Returns: (stdout (str|None), stderr (str|None)) from running command. Raises: OSError for any issues running the command. )�stdout�stderrNr ) � subprocess�Popen�PIPE� communicate� isinstance�str�decode� returncoder �ExternalBinaryError)Zcommand_and_flagsZcommand_processZcommand_stdoutZcommand_stderr� r �)/opt/gsutil/gslib/utils/execution_util.py�ExecuteExternalCommand s � r ) �__doc__� __future__r r r r r �gslibr r r r r r �<module> s utils/__pycache__/cat_helper.cpython-39.pyc 0000644 00000010333 15025013616 0014652 0 ustar 00 a +(Wg� � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm Z ddl m Z dd l m Z dd l mZ dd lmZ dd lmZ dd lmZ ddlmZ ddlmZ g d�ZG dd� de�ZdS )z)Helper for cat and cp streaming download.� )�absolute_import)�print_function)�division)�unicode_literalsN)�config)�EncryptionException)�CommandException)�NO_URLS_MATCHED_TARGET)�StorageUrlFromString)�CryptoKeyWrapperFromKey)� FindMatchingCSEKInBotoConfig)�ObjectIsGzipEncoded)� text_util)�bucket�contentEncoding�crc32c�customerEncryption� generation�md5Hash�name�sizec @ s* e Zd ZdZdd� Zdd� Zd d d �ZdS ) � CatHelperzDProvides methods for the "cat" command and associated functionality.c C s || _ dS )zmInitializes the helper object. Args: command_obj: gsutil command instance of calling command. N)� command_obj)�selfr � r �%/opt/gsutil/gslib/utils/cat_helper.py�__init__3 s zCatHelper.__init__c C s$ |� tj�}|sq t�||� q dS )a� Copies contents of the source to the destination via buffered IO. Buffered reads are necessary in the case where you're reading from a source that produces more data than can fit into memory all at once. This method does not close either file when finished. Args: src_fd: The already-open source file to read from. dst_fd: The already-open destination file to write to. N)�read�io�DEFAULT_BUFFER_SIZEr � write_to_fd)r Zsrc_fdZdst_fd�bufr r r � _WriteBytesBufferedFileToFile; s z'CatHelper._WriteBytesBufferedFileToFileFr Nc C s� d}|du rt j}t j}t jt _�z�|rF|d dv rF| �t j|� �nt|D �]l}d} | j�|�jtd�D �]<} d} | j r�| j j r�| j j j r�t | j j j t �} | s�td| j j j | jf ��t| �} d} |r�|r�t� td| � d}| j } dt| d d � k �r|k�rn n W |t _dS t| j�}|�� �rlt| �}| jjj| j| j||||| j|j| |j d � |� � qht |j d �� }| �||� W d � qh1 �s�0 Y qh| sJt t!| ��qJW |t _n|t _0 dS ) a� Prints each of the url strings to stdout. Args: url_strings: String iterable. show_header: If true, print a header per file. start_byte: Starting byte of the file to print, used for constructing range requests. end_byte: Ending byte of the file to print; used for constructing range requests. If this is negative, the start_byte is ignored and and end range is sent over HTTP (such as range: bytes -9) cat_out_fd: File descriptor to which output should be written. Defaults to stdout if no file descriptor is supplied. Returns: 0 on success. Raises: CommandException if no URLs can be found. FNr )�-zfile://-)�bucket_listing_fieldszOMissing decryption key with SHA256 hash %s. No decryption key matches object %sTz ==> %s <==r ���)�compressed_encoding� start_byte�end_byte� object_sizer �decryption_tuple�provider�rb)"�sys�stdout�stderrr" �stdinr �WildcardIterator� IterObjects�_CAT_BUCKET_LISTING_FIELDS� root_objectr � keySha256r r r � url_stringr �print�getattrr � IsCloudUrlr � gsutil_api�GetObjectMediar r r r �scheme�flush�open� object_namer r )r Z url_strings� show_headerr' r( Z cat_out_fd� printed_oneZ old_stdout�url_strZ did_some_work�blrZdecryption_keywrapper�decryption_keyZ cat_object� storage_urlr& �fr r r � CatUrlStringsL s| � � ���� $� � .zCatHelper.CatUrlStrings)Fr NN)�__name__� __module__� __qualname__�__doc__r r" rG r r r r r 0 s �r )rK � __future__r r r r r r- �botor �gslib.cloud_apir �gslib.exceptionr r �gslib.storage_urlr � gslib.utils.encryption_helperr r �gslib.utils.metadata_utilr � gslib.utilsr r3 �objectr r r r r �<module> s" utils/__pycache__/system_util.cpython-39.pyc 0000644 00000017633 15025013616 0015137 0 ustar 00 a +(Wg5' � @ s� d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl Z ddl Z ddl Z ddl m Z dZee j��� Zd ev Zd ev Zd ev Zd ev Ze�rdd lmZ ddlmZ ddlmZ ddlmZ ddlm Z ddlm Z ddlm Z ddlm Z e� � d e kZ!ndZ!dd� Z"dd� Z#dd � Z$d9d d �Z%d d!� Z&d:d"d#�Z'd$d%� Z(d;d'd(�Z)d)d*� Z*d+d,� Z+d-d.� Z,d/d0� Z-d1d2� Z.d3d4� Z/d5d6� Z0G d7d8� d8e j1�Z2dS )<a( Shared utility structures and methods for interacting with the host system. The methods in this module should be limited to obtaining system information and simple file operations (disk info, retrieving metadata about existing files, creating directories, fetching environment variables, etc.). � )�absolute_import)�print_function)�division)�unicode_literalsN)� WINDOWS_1252� �win32�cygwinZlinux�darwin)�c_int)�c_uint64)�c_char_p)� c_wchar_p)�windll)�POINTER)� WINFUNCTYPE)�WinError� Fc C s� t r�z0ttttt�tt�tt��}|dtjfd�}W n> tyr ttt tt�tt�tt��}|dtjfd�}Y n0 dd� }||_ |t � d��S t � | �\ }}}}}}}}}}|| S dS )z(Return path/drive free space (in bytes).ZGetDiskFreeSpaceExW))r Z lpszPathName)� ZlpFreeUserSpace)r Z lpTotalSpace)r Z lpFreeSpaceZGetDiskFreeSpaceExAc S s | s t � �|d jS )Nr )r �value)�resultZ unused_func�args� r �&/opt/gsutil/gslib/utils/system_util.py�GetDiskFreeSpaceExErrCheckU s z2CheckFreeSpace.<locals>.GetDiskFreeSpaceExErrCheckZ SystemDriveN)� IS_WINDOWSr r r r r r Zkernel32�AttributeErrorr Zerrcheck�os�getenv�statvfs)�pathZget_disk_free_space_exr �_�f_frsize�f_bavailr r r �CheckFreeSpace< s( � � � � r$ c C s t j�d�dkS )NZ(CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL�1�r �environ�getr r r r �CloudSdkCredPassingEnabledb s r) c C s t j�dd�S )NZCLOUDSDK_VERSION� r& r r r r �CloudSdkVersionf s r+ � c C s` t j�| �s\zt �| |� W n> tyZ } z&|jtjkrF|jtjkrF� W Y d}~n d}~0 0 dS )z7Creates a directory, suppressing already-exists errors.N)r r �exists�makedirs�OSError�errno�EEXISTZEISDIR)Zdir_path�mode�er r r �CreateDirIfNeededj s r4 c C s. d} g }t dd��P}|�� dd� }|D ]*}|�� \}}}}|d �� r(|�|� q(W d � n1 sh0 Y i }t dd���}|D ]�}|�� dd� }|\ }}}} }} } } }} }||v r�t| �| } t| �| } t| �} t| �} t| �} t|�}| | | | | |f||< q�W d � n1 �s 0 Y |S ) a� Retrieves disk I/O statistics for all disks. Adapted from the psutil module's psutil._pslinux.disk_io_counters: http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py Originally distributed under under a BSD license. Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola. Returns: A dictionary containing disk names mapped to the disk counters from /disk/diskstats. i z/proc/partitions�rr N���z/proc/diskstats� )�open� readlines�split�isdigit�append�int)Z sector_sizeZ partitions�f�lines�liner! �nameZretdict�valuesZreadsZrbytesZrtimeZwritesZwbytesZwtimer r r �GetDiskCountersy s, * 6rC c C s4 |s | � � }| �dtj� | � � }|s0| �|� |S )z>Returns size of file, optionally leaving fp positioned at EOF.r )�tell�seekr �SEEK_END)�fpZposition_to_eofZcur_posZ cur_file_sizer r r � GetFileSize� s rH c C s t � rt� rdS dS )a� Returns a tuple of the gsutil OAuth2 client ID and secret. Google OAuth2 clients always have a secret, even if the client is an installed application/utility such as gsutil. Of course, in such cases the "secret" is actually publicly known; security depends entirely on the secrecy of refresh tokens, which effectively become bearer tokens. Returns: (str, str) A 2-tuple of (client ID, secret). )z&32555940559.apps.googleusercontent.comZZmssLNjJy2998hD4CTg2ejr2)z'909320924072.apps.googleusercontent.comzp3RlpR10xMFh9ZXBS/ZNLYUu)�InvokedViaCloudSdkr) r r r r �GetGsutilClientIdAndSecret� s rJ �rbc C s, | � � r tjrtjS tjjS t| j|�S d S �N)ZIsStream�six�PY2�sys�stdin�bufferr8 Z object_name)Z storage_urlr2 r r r �GetStreamFromFileUrl� s rR c s� zddl � ddl�W n ty* t Y S 0 � �fdd�} | d�pP| d�pP| d�}|s�z(t�t�� tj�}| |�}t�|� W n Y n0 |s�tj � dt�}t |�S )z!Returns number of terminal lines.r Nc s2 z t �d� �| �jd��d W S Y dS 0 d S )NZhhZ1234r )�struct�unpackZioctlZ TIOCGWINSZ)�fd��fcntl�termiosr r � ioctl_GWINSZ� s z"GetTermLines.<locals>.ioctl_GWINSZr r �LINES) rW rX � ImportError�_DEFAULT_NUM_TERM_LINESr r8 �ctermid�O_RDONLY�closer' r( r= )rY ZiocrU r rV r � GetTermLines� s" r` c C s t j�d�dkS )NZCLOUDSDK_WRAPPERr% r& r r r r rI � s rI c C s dt jv } dt jv }| p|S )z<Returns True if running in a CI environment, e.g. GitHub CI.ZCIZ KOKORO_ROOT)r r' )Z on_github_ciZ on_kokoror r r �IsRunningInCiEnvironment� s ra c C s t j�� o t j�� o t j�� S )z9Returns True if currently running interactively on a TTY.)rO �stdout�isatty�stderrrP r r r r �IsRunningInteractively� s re c C sL t j} | jdkrH| jdk sB| jdkr.| jdk sB| jdkrH| jdkrHt� d S )N� � � r )rO � version_info�major�minor�micro� _MonkeyPatchHttpForPython_3x)�verr r r �MonkeyPatchHttp� s ��ro c s, dd l } | jjj� � fdd�}|| jj_d S )Nr c s: � | � | j dkr6| j�� D ]\}}td|d |� q d S )Nr zheader:�:)� debuglevel�headers�items�print)�self�hdr�val�Z old_beginr r � PatchedBegin s z2_MonkeyPatchHttpForPython_3x.<locals>.PatchedBegin)�http�client� HTTPResponse�begin)rz ry r rx r rm s rm c c s t jD ]} | �� V qdS )z3A generator function that returns lines from stdin.N)rO rP �rstrip)r@ r r r � StdinIterator s r c @ s e Zd ZdZdd� Zdd� ZdS )�StdinIteratorClsz�An iterator that returns lines from stdin. This is needed because Python 3 balks at pickling the generator version above. c C s | S rL r )ru r r r �__iter__* s zStdinIteratorCls.__iter__c C s t j�� }|st� �|�� S rL )rO rP �readline� StopIterationr~ )ru r@ r r r �__next__- s zStdinIteratorCls.__next__N)�__name__� __module__� __qualname__�__doc__r� r� r r r r r� $ s r� )r, )F)rK )3r� � __future__r r r r r0 �localer rS rO rM Zgslib.utils.constantsr r\ �str�platform�lowerZPLATFORMr Z IS_CYGWINZIS_LINUXZIS_OSXZctypesr r r r r r r r �getdefaultlocaleZ IS_CP1252r$ r) r+ r4 rC rH rJ rR r` rI ra re ro rm r �Iteratorr� r r r r �<module> sX & * utils/temporary_file_util.py 0000644 00000002345 15025013616 0012337 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utils for temporary files.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals def GetTempFileName(storage_url): """Returns temporary file name for uncompressed file.""" return '%s_.gstmp' % storage_url.object_name def GetTempZipFileName(storage_url): """Returns temporary name for a temporarily compressed file.""" return '%s_.gztmp' % storage_url.object_name def GetStetTempFileName(storage_url): """Returns temporary file name for result of STET transform.""" return '%s_.stet_tmp' % storage_url.object_name utils/update_util.py 0000644 00000011612 15025013616 0010575 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility methods for the update command and its tests.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import re import textwrap import sys import gslib from gslib.utils.system_util import IS_OSX from gslib.exception import CommandException from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import GSUTIL_PUB_TARBALL from gslib.utils.constants import GSUTIL_PUB_TARBALL_PY2 # This function used to belong inside of update.py. However, it needed to be # moved here due to compatibility issues with Travis CI, because update.py is # not included with PyPI installations. def DisallowUpdateIfDataInGsutilDir(directory=gslib.GSUTIL_DIR): """Disallows the update command if files not in the gsutil distro are found. This prevents users from losing data if they are in the habit of running gsutil from the gsutil directory and leaving data in that directory. This will also detect someone attempting to run gsutil update from a git repo, since the top-level directory will contain git files and dirs (like .git) that are not distributed with gsutil. Args: directory: (str) The directory to use this functionality on. Raises: CommandException: if files other than those distributed with gsutil found. """ # Manifest includes recursive-includes of gslib. Directly add # those to the list here so we will skip them in os.listdir() loop without # having to build deeper handling of the MANIFEST file here. Also include # 'third_party', which isn't present in manifest but gets added to the # gsutil distro by the gsutil submodule configuration; and the MANIFEST.in # and CHANGES.md files. manifest_lines = ['MANIFEST.in', 'third_party'] try: with open(os.path.join(directory, 'MANIFEST.in'), 'r') as fp: for line in fp: if line.startswith('include '): manifest_lines.append(line.split()[-1]) elif re.match(r'recursive-include \w+ \*', line): manifest_lines.append(line.split()[1]) except IOError: logging.getLogger().warn( 'MANIFEST.in not found in %s.\nSkipping user data ' 'check.\n', directory) return # Look just at top-level directory. We don't try to catch data dropped into # subdirs (like gslib) because that would require deeper parsing of # MANFFEST.in, and most users who drop data into gsutil dir do so at the top # level directory. addl_excludes = ( '.coverage', '.DS_Store', '.github', '.style.yapf', '.yapfignore', '__pycache__', '.github', ) for filename in os.listdir(directory): if filename.endswith('.pyc') or filename in addl_excludes: continue if filename not in manifest_lines: raise CommandException('\n'.join( textwrap.wrap( 'A file (%s) that is not distributed with gsutil was found in ' 'the gsutil directory. The update command cannot run with user ' 'data in the gsutil directory.' % os.path.join(gslib.GSUTIL_DIR, filename)))) def LookUpGsutilVersion(gsutil_api, url_str): """Looks up the gsutil version of the specified gsutil tarball URL. Version is specified in the metadata field set on that object. Args: gsutil_api: gsutil Cloud API to use when retrieving gsutil tarball. url_str: tarball URL to retrieve (such as 'gs://pub/gsutil.tar.gz'). Returns: Version string if URL is a cloud URL containing x-goog-meta-gsutil-version metadata, else None. """ url = StorageUrlFromString(url_str) if url.IsCloudUrl(): obj = gsutil_api.GetObjectMetadata(url.bucket_name, url.object_name, provider=url.scheme, fields=['metadata']) if obj.metadata and obj.metadata.additionalProperties: for prop in obj.metadata.additionalProperties: if prop.key == 'gsutil_version': return prop.value def GsutilPubTarball(): """Returns the appropriate gsutil pub tarball based on the Python version. Returns: The storage_uri of the appropriate pub tarball. """ if sys.version_info.major == 2: return GSUTIL_PUB_TARBALL_PY2 return GSUTIL_PUB_TARBALL utils/parallelism_framework_util.py 0000644 00000037335 15025013616 0013707 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility classes and methods for the parallelism framework.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import errno import logging import multiprocessing import threading import traceback from gslib.utils import constants from gslib.utils import system_util from six.moves import queue as Queue # pylint: disable=g-import-not-at-top try: # This module doesn't necessarily exist on Windows. import resource _HAS_RESOURCE_MODULE = True except ImportError as e: _HAS_RESOURCE_MODULE = False # Maximum time to wait (join) on the SeekAheadThread after the ProducerThread # completes, in seconds. SEEK_AHEAD_JOIN_TIMEOUT = 60 # Timeout for puts/gets to the global status queue, in seconds. STATUS_QUEUE_OP_TIMEOUT = 5 # Maximum time to wait (join) on the UIThread after the Apply # completes, in seconds. UI_THREAD_JOIN_TIMEOUT = 60 ZERO_TASKS_TO_DO_ARGUMENT = ('There were no', 'tasks to do') # Multiprocessing manager used to coordinate across all processes. This # attribute is only present if multiprocessing is available, which can be # determined by calling CheckMultiprocessingAvailableAndInit(). global top_level_manager # pylint: disable=global-at-module-level # Cache the values from this check such that they're available to all callers # without needing to run all the checks again (some of these, such as calling # multiprocessing.Manager(), are expensive operations). _cached_multiprocessing_is_available = None _cached_multiprocessing_check_stack_trace = None _cached_multiprocessing_is_available_message = None # This must be defined at the module level for pickling across processes. MultiprocessingIsAvailableResult = collections.namedtuple( 'MultiprocessingIsAvailableResult', ['is_available', 'stack_trace']) # Explicitly set start method to 'fork' since this isn't always the default # in later versions of Python. try: multiprocessing_context = multiprocessing.get_context('fork') except (AttributeError, ValueError): multiprocessing_context = multiprocessing class AtomicDict(object): """Thread-safe (and optionally process-safe) dictionary protected by a lock. If a multiprocessing.Manager is supplied on init, the dictionary is both process and thread safe. Otherwise, it is only thread-safe. """ def __init__(self, manager=None): """Initializes the dict. Args: manager: (multiprocessing.Manager or None) Manager instance (required for cross-process safety), or none if cross-process safety is not needed. """ if manager: self.lock = manager.Lock() self.dict = manager.dict() else: self.lock = threading.Lock() self.dict = {} def __getitem__(self, key): with self.lock: return self.dict[key] def __setitem__(self, key, value): with self.lock: self.dict[key] = value # pylint: disable=invalid-name def get(self, key, default_value=None): with self.lock: return self.dict.get(key, default_value) def delete(self, key): with self.lock: del self.dict[key] def values(self): with self.lock: return self.dict.values() def Increment(self, key, inc, default_value=0): """Atomically updates the stored value associated with the given key. Performs the atomic equivalent of dict[key] = dict.get(key, default_value) + inc. Args: key: lookup key for the value of the first operand of the "+" operation. inc: Second operand of the "+" operation. default_value: Default value if there is no existing value for the key. Returns: Incremented value. """ with self.lock: val = self.dict.get(key, default_value) + inc self.dict[key] = val return val class ProcessAndThreadSafeInt(object): """This class implements a process and thread-safe integer. It is backed either by a multiprocessing Value of type 'i' or an internal threading lock. This simplifies the calling pattern for global variables that could be a Multiprocessing.Value or an integer. Without this class, callers need to write code like this: global variable_name if isinstance(variable_name, int): return variable_name else: return variable_name.value """ def __init__(self, multiprocessing_is_available): self.multiprocessing_is_available = multiprocessing_is_available if self.multiprocessing_is_available: # Lock is implicit in multiprocessing.Value self.value = multiprocessing_context.Value('i', 0) else: self.lock = threading.Lock() self.value = 0 def Reset(self, reset_value=0): if self.multiprocessing_is_available: self.value.value = reset_value else: with self.lock: self.value = reset_value def Increment(self): if self.multiprocessing_is_available: self.value.value += 1 else: with self.lock: self.value += 1 def Decrement(self): if self.multiprocessing_is_available: self.value.value -= 1 else: with self.lock: self.value -= 1 def GetValue(self): if self.multiprocessing_is_available: return self.value.value else: with self.lock: return self.value def _IncreaseSoftLimitForResource(resource_name, fallback_value): """Sets a new soft limit for the maximum number of open files. The soft limit is used for this process (and its children), but the hard limit is set by the system and cannot be exceeded. We will first try to set the soft limit to the hard limit's value; if that fails, we will try to set the soft limit to the fallback_value iff this would increase the soft limit. Args: resource_name: Name of the resource to increase the soft limit for. fallback_value: Fallback value to be used if we couldn't set the soft value to the hard value (e.g., if the hard value is "unlimited"). Returns: Current soft limit for the resource (after any changes we were able to make), or -1 if the resource doesn't exist. """ # Get the value of the resource. try: (soft_limit, hard_limit) = resource.getrlimit(resource_name) except (resource.error, ValueError): # The resource wasn't present, so we can't do anything here. return -1 # Try to set the value of the soft limit to the value of the hard limit. if hard_limit > soft_limit: # Some OS's report 0 for "unlimited". try: resource.setrlimit(resource_name, (hard_limit, hard_limit)) return hard_limit except (resource.error, ValueError): # We'll ignore this and try the fallback value. pass # Try to set the value of the soft limit to the fallback value. if soft_limit < fallback_value: try: resource.setrlimit(resource_name, (fallback_value, hard_limit)) return fallback_value except (resource.error, ValueError): # We couldn't change the soft limit, so just report the current # value of the soft limit. return soft_limit else: return soft_limit def ShouldProhibitMultiprocessing(): """Determines if the OS doesn't support multiprocessing. There are two cases we currently know about: - Multiple processes are not supported on Windows. - If an error is encountered while using multiple processes on Alpine Linux gsutil hangs. For this case it's possible we could do more work to find the root cause but after a fruitless initial attempt we decided instead to fall back on multi-threading w/o multiprocesing. Returns: (bool indicator if multiprocessing should be prohibited, OS name) """ if system_util.IS_WINDOWS: # Issues have been observed while trying to use multi-processing in Windows return (True, 'Windows') if system_util.IS_OSX: # macOS does not contain /etc/os-release, used in this method. This # shortcuts raising an exception and returning 'Unknown' as an OS. return (False, 'macOS') try: with open('/etc/os-release', 'r') as f: # look for line that contains 'NAME=' both PRETTY_NAME and NAME should # be acceptable to try to find if alpine linux is being used. for line in f.read().splitlines(): if 'NAME=' in line: os_name = line.split('=')[1].strip('"') return ('alpine linux' in os_name.lower(), os_name) # Unable to determine OS. NAME line not found in /etc/os-release file. return (False, 'Unknown') except IOError as e: if e.errno == errno.ENOENT: logging.debug('Unable to open /etc/os-release to determine whether OS ' 'supports multiprocessing: errno=%d, message=%s' % (e.errno, str(e))) return (False, 'Unknown') else: raise except Exception as exc: logging.debug('Something went wrong while trying to determine ' 'multiprocessing capabilities.\nMessage: {0}'.format( str(exc))) return (False, 'Unknown') def CheckMultiprocessingAvailableAndInit(logger=None): """Checks if multiprocessing is available, and if so performs initialization. There are some environments in which there is no way to use multiprocessing logic that's built into Python (e.g., if /dev/shm is not available, then we can't create semaphores). This simply tries out a few things that will be needed to make sure the environment can support the pieces of the multiprocessing module that we need. See gslib.command.InitializeMultiprocessingVariables for an explanation of why this is necessary. Args: logger: (logging.Logger) Logger to use for debug output. Returns: (MultiprocessingIsAvailableResult) A namedtuple with the following attrs: - multiprocessing_is_available: True iff the multiprocessing module is available for use. - stack_trace: The stack trace generated by the call we tried that failed. """ # pylint: disable=global-variable-undefined global _cached_multiprocessing_is_available global _cached_multiprocessing_check_stack_trace global _cached_multiprocessing_is_available_message if _cached_multiprocessing_is_available is not None: if logger: logger.debug(_cached_multiprocessing_check_stack_trace) logger.warn(_cached_multiprocessing_is_available_message) return MultiprocessingIsAvailableResult( is_available=_cached_multiprocessing_is_available, stack_trace=_cached_multiprocessing_check_stack_trace) should_prohibit_multiprocessing, os_name = ShouldProhibitMultiprocessing() if should_prohibit_multiprocessing: message = """ Multiple processes are not supported on %s. Operations requesting parallelism will be executed with multiple threads in a single process only. """ % os_name if logger: logger.warn(message) return MultiprocessingIsAvailableResult(is_available=False, stack_trace=None) stack_trace = None multiprocessing_is_available = True message = """ You have requested multiple processes for an operation, but the required functionality of Python\'s multiprocessing module is not available. Operations requesting parallelism will be executed with multiple threads in a single process only. """ try: # Fails if /dev/shm (or some equivalent thereof) is not available for use # (e.g., there's no implementation, or we can't write to it, etc.). try: multiprocessing_context.Value('i', 0) except: message += """ Please ensure that you have write access to both /dev/shm and /run/shm. """ raise # We'll handle this in one place below. global top_level_manager # pylint: disable=global-variable-undefined top_level_manager = multiprocessing_context.Manager() # Check that the max number of open files is reasonable. Always check this # after we're sure that the basic multiprocessing functionality is # available, since this won't matter unless that's true. limit = -1 if _HAS_RESOURCE_MODULE: # Try to set this with both resource names - RLIMIT_NOFILE for most Unix # platforms, and RLIMIT_OFILE for BSD. Ignore AttributeError because the # "resource" module is not guaranteed to know about these names. try: limit = max( limit, _IncreaseSoftLimitForResource( resource.RLIMIT_NOFILE, constants.MIN_ACCEPTABLE_OPEN_FILES_LIMIT)) except AttributeError: pass try: limit = max( limit, _IncreaseSoftLimitForResource( resource.RLIMIT_OFILE, constants.MIN_ACCEPTABLE_OPEN_FILES_LIMIT)) except AttributeError: pass if limit < constants.MIN_ACCEPTABLE_OPEN_FILES_LIMIT: message += (""" Your max number of open files, %s, is too low to allow safe multiprocessing. On Linux you can fix this by adding something like "ulimit -n 10000" to your ~/.bashrc or equivalent file and opening a new terminal. On macOS, you may also need to run a command like this once (in addition to the above instructions), which might require a restart of your system to take effect: launchctl limit maxfiles 10000 Alternatively, edit /etc/launchd.conf with something like: limit maxfiles 10000 10000 """ % limit) raise Exception('Max number of open files, %s, is too low.' % limit) except: # pylint: disable=bare-except stack_trace = traceback.format_exc() multiprocessing_is_available = False if logger is not None: logger.debug(stack_trace) logger.warn(message) # Set the cached values so that we never need to do this check again. _cached_multiprocessing_is_available = multiprocessing_is_available _cached_multiprocessing_check_stack_trace = stack_trace _cached_multiprocessing_is_available_message = message return MultiprocessingIsAvailableResult( is_available=_cached_multiprocessing_is_available, stack_trace=_cached_multiprocessing_check_stack_trace) def CreateLock(): """Returns either a multiprocessing lock or a threading lock. Use Multiprocessing lock iff we have access to the parts of the multiprocessing module that are necessary to enable parallelism in operations. Returns: Multiprocessing or threading lock. """ if CheckMultiprocessingAvailableAndInit().is_available: return top_level_manager.Lock() else: return threading.Lock() # Pylint gets confused by the mixed lower and upper-case method names in # AtomicDict. # pylint: disable=invalid-name def PutToQueueWithTimeout(queue, msg, timeout=STATUS_QUEUE_OP_TIMEOUT): """Puts an item to the status queue. If the queue is full, this function will timeout periodically and repeat until success. This avoids deadlock during shutdown by never making a fully blocking call to the queue, since Python signal handlers cannot execute in between instructions of the Python interpreter (see https://docs.python.org/2/library/signal.html for details). Args: queue: Queue class (typically the global status queue) msg: message to post to the queue. timeout: (optional) amount of time to wait before repeating put request. """ put_success = False while not put_success: try: queue.put(msg, timeout=timeout) put_success = True except Queue.Full: pass # pylint: enable=invalid-name utils/retention_util.py 0000644 00000024743 15025013616 0011333 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Retention Policy configuration command for buckets.""" from __future__ import absolute_import from six.moves import input from decimal import Decimal import re from gslib.exception import CommandException from gslib.lazy_wrapper import LazyWrapper from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages SECONDS_IN_DAY = 24 * 60 * 60 SECONDS_IN_MONTH = 31 * SECONDS_IN_DAY SECONDS_IN_YEAR = int(365.25 * SECONDS_IN_DAY) _LOCK_PROMPT = ( 'This will PERMANENTLY set the Retention Policy on gs://{} to:\n\n' '{}\n\nThis setting cannot be reverted! Continue?') # Regex to match retention period in years. _RETENTION_IN_YEARS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)y$')) # Regex to match retention period in months. _RETENTION_IN_MONTHS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)m$')) # Regex to match retention period in days. _RETENTION_IN_DAYS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)d$')) # Regex to match retention period in seconds. _RETENTION_IN_SECONDS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)s$')) def _ConfirmWithUserPrompt(question, default_response): """Prompts user to confirm an action with yes or no response. Args: question: Yes/No question to be used for the prompt. default_response: Default response to the question: True, False Returns: Returns the rough equivalent duration in seconds. """ prompt = '' if default_response: prompt = '%s [%s|%s]: ' % (question, 'Y', 'n') else: prompt = '%s [%s|%s]: ' % (question, 'y', 'N') while True: response = input(prompt).lower() if not response: return default_response if response not in ['y', 'yes', 'n', 'no']: print('\tPlease respond with \'yes\'/\'y\' or \'no\'/\'n\'.') continue if response == 'yes' or response == 'y': return True if response == 'no' or response == 'n': return False def _RetentionPeriodToString(retention_period): """Converts Retention Period to Human readable format. Args: retention_period: Retention duration in seconds (integer value). Returns: Returns a string representing retention duration in human readable format. """ # TODO: add link to public documentation regarding conversion rates. period = Decimal(retention_period) duration_str = None if period // SECONDS_IN_YEAR == period / SECONDS_IN_YEAR: duration_str = '{} Year(s)'.format(period // SECONDS_IN_YEAR) elif period // SECONDS_IN_MONTH == period / SECONDS_IN_MONTH: duration_str = '{} Month(s)'.format(period // SECONDS_IN_MONTH) elif period // SECONDS_IN_DAY == period / SECONDS_IN_DAY: duration_str = '{} Day(s)'.format(period // SECONDS_IN_DAY) elif period > SECONDS_IN_DAY: duration_str = '{} Seconds (~{} Day(s))'.format(retention_period, period // SECONDS_IN_DAY) else: duration_str = '{} Second(s)'.format(retention_period) return (' Duration: {}').format(duration_str) def RetentionPolicyToString(retention_policy, bucket_url): """Converts Retention Policy to Human readable format.""" retention_policy_str = '' if retention_policy and retention_policy.retentionPeriod: locked_string = '(LOCKED)' if retention_policy.isLocked else '(UNLOCKED)' retention_period = _RetentionPeriodToString( retention_policy.retentionPeriod) retention_effective_time = ' Effective Time: {}'.format( retention_policy.effectiveTime.strftime('%a, %d %b %Y %H:%M:%S GMT')) retention_policy_str = ' Retention Policy {}:\n{}\n{}'.format( locked_string, retention_period, retention_effective_time) else: retention_policy_str = '{} has no Retention Policy.'.format(bucket_url) return retention_policy_str def ConfirmLockRequest(bucket_url, retention_policy): retention_policy = RetentionPolicyToString(retention_policy, bucket_url) lock_prompt = _LOCK_PROMPT.format(bucket_url, retention_policy) return _ConfirmWithUserPrompt(lock_prompt, False) def UpdateObjectMetadataExceptionHandler(cls, e): """Exception handler that maintains state about post-completion status.""" cls.logger.error(e) cls.everything_set_okay = False def SetTempHoldFuncWrapper(cls, name_expansion_result, thread_state=None): log_template = 'Setting Temporary Hold on %s...' object_metadata_update = apitools_messages.Object(temporaryHold=True) cls.ObjectUpdateMetadataFunc(object_metadata_update, log_template, name_expansion_result, thread_state=thread_state) def ReleaseTempHoldFuncWrapper(cls, name_expansion_result, thread_state=None): log_template = 'Releasing Temporary Hold on %s...' object_metadata_update = apitools_messages.Object(temporaryHold=False) cls.ObjectUpdateMetadataFunc(object_metadata_update, log_template, name_expansion_result, thread_state=thread_state) def SetEventHoldFuncWrapper(cls, name_expansion_result, thread_state=None): log_template = 'Setting Event-Based Hold on %s...' object_metadata_update = apitools_messages.Object(eventBasedHold=True) cls.ObjectUpdateMetadataFunc(object_metadata_update, log_template, name_expansion_result, thread_state=thread_state) def ReleaseEventHoldFuncWrapper(cls, name_expansion_result, thread_state=None): log_template = 'Releasing Event-Based Hold on %s...' object_metadata_update = apitools_messages.Object(eventBasedHold=False) cls.ObjectUpdateMetadataFunc(object_metadata_update, log_template, name_expansion_result, thread_state=thread_state) def DaysToSeconds(days): """Converts duration specified in days to equivalent seconds. Args: days: Retention duration in number of days. Returns: Returns the equivalent duration in seconds. """ return days * SECONDS_IN_DAY def MonthsToSeconds(months): """Converts duration specified in months to equivalent seconds. GCS bucket lock API uses following duration equivalencies to convert durations specified in terms of months or years to seconds: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Args: months: Retention duration in number of months. Returns: Returns the rough equivalent duration in seconds. """ return months * SECONDS_IN_MONTH def YearsToSeconds(years): """Converts duration specified in years to equivalent seconds. GCS bucket lock API uses following duration equivalencies to convert durations specified in terms of months or years to seconds: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Args: years: Retention duration in number of years. Returns: Returns the rough equivalent duration in seconds. """ return years * SECONDS_IN_YEAR def RetentionInYearsMatch(years): """Test whether the string matches retention in years pattern. Args: years: string to match for retention specified in years format. Returns: Returns a match object if the string matches the retention in years pattern. The match object will contain a 'number' group for the duration in number of years. Otherwise, None is returned. """ return _RETENTION_IN_YEARS().match(years) def RetentionInMonthsMatch(months): """Test whether the string matches retention in months pattern. Args: months: string to match for retention specified in months format. Returns: Returns a match object if the string matches the retention in months pattern. The match object will contain a 'number' group for the duration in number of months. Otherwise, None is returned. """ return _RETENTION_IN_MONTHS().match(months) def RetentionInDaysMatch(days): """Test whether the string matches retention in days pattern. Args: days: string to match for retention specified in days format. Returns: Returns a match object if the string matches the retention in days pattern. The match object will contain a 'number' group for the duration in number of days. Otherwise, None is returned. """ return _RETENTION_IN_DAYS().match(days) def RetentionInSecondsMatch(seconds): """Test whether the string matches retention in seconds pattern. Args: seconds: string to match for retention specified in seconds format. Returns: Returns a match object if the string matches the retention in seconds pattern. The match object will contain a 'number' group for the duration in number of seconds. Otherwise, None is returned. """ return _RETENTION_IN_SECONDS().match(seconds) def RetentionInSeconds(pattern): """Converts a retention period string pattern to equivalent seconds. Args: pattern: a string pattern that represents a retention period. Returns: Returns the retention period in seconds. If the pattern does not match """ seconds = None year_match = RetentionInYearsMatch(pattern) month_match = RetentionInMonthsMatch(pattern) day_match = RetentionInDaysMatch(pattern) second_match = RetentionInSecondsMatch(pattern) if year_match: seconds = YearsToSeconds(int(year_match.group('number'))) elif month_match: seconds = MonthsToSeconds(int(month_match.group('number'))) elif day_match: seconds = DaysToSeconds(int(day_match.group('number'))) elif second_match: seconds = int(second_match.group('number')) else: raise CommandException('Incorrect retention period specified. ' 'Please use one of the following formats ' 'to specify the retention period : ' '<number>y, <number>m, <number>d, <number>s.') return seconds utils/copy_helper.py 0000644 00000553365 15025013616 0010607 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2011 Google Inc. All Rights Reserved. # Copyright 2011, Nexenta Systems Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for copy functionality.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import base64 from collections import namedtuple import csv import datetime import errno import gzip import json import logging import mimetypes from operator import attrgetter import os import pickle import pyu2f import random import re import shutil import six import stat import subprocess import tempfile import textwrap import time import traceback import six from six.moves import range from six.moves import range from apitools.base.protorpclite import protojson from boto import config import crcmod import gslib from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ArgumentException from gslib.cloud_api import CloudApi from gslib.cloud_api import EncryptionException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.cloud_api import ResumableDownloadException from gslib.cloud_api import ResumableUploadAbortException from gslib.cloud_api import ResumableUploadException from gslib.cloud_api import ResumableUploadStartOverException from gslib.cloud_api import ServiceException from gslib.commands.compose import MAX_COMPOSE_ARITY from gslib.commands.config import DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE from gslib.commands.config import DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD from gslib.commands.config import DEFAULT_GZIP_COMPRESSION_LEVEL from gslib.cs_api_map import ApiSelector from gslib.daisy_chain_wrapper import DaisyChainWrapper from gslib.exception import CommandException from gslib.exception import HashMismatchException from gslib.exception import InvalidUrlError from gslib.file_part import FilePart from gslib.parallel_tracker_file import GenerateComponentObjectPrefix from gslib.parallel_tracker_file import ReadParallelUploadTrackerFile from gslib.parallel_tracker_file import ValidateParallelCompositeTrackerData from gslib.parallel_tracker_file import WriteComponentToParallelUploadTrackerFile from gslib.parallel_tracker_file import WriteParallelUploadTrackerFile from gslib.progress_callback import FileProgressCallbackHandler from gslib.progress_callback import ProgressCallbackWithTimeout from gslib.resumable_streaming_upload import ResumableStreamingJsonUploadWrapper from gslib import storage_url from gslib.storage_url import ContainsWildcard from gslib.storage_url import GenerationFromUrlAndString from gslib.storage_url import IsCloudSubdirPlaceholder from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import FileMessage from gslib.thread_message import RetryableErrorMessage from gslib.tracker_file import DeleteDownloadTrackerFiles from gslib.tracker_file import DeleteTrackerFile from gslib.tracker_file import ENCRYPTION_UPLOAD_TRACKER_ENTRY from gslib.tracker_file import GetDownloadStartByte from gslib.tracker_file import GetTrackerFilePath from gslib.tracker_file import GetUploadTrackerData from gslib.tracker_file import RaiseUnwritableTrackerFileException from gslib.tracker_file import ReadOrCreateDownloadTrackerFile from gslib.tracker_file import SERIALIZATION_UPLOAD_TRACKER_ENTRY from gslib.tracker_file import TrackerFileType from gslib.tracker_file import WriteDownloadComponentTrackerFile from gslib.tracker_file import WriteJsonDataToTrackerFile from gslib.utils import parallelism_framework_util from gslib.utils import stet_util from gslib.utils import temporary_file_util from gslib.utils import text_util from gslib.utils.boto_util import GetJsonResumableChunkSize from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNumRetries from gslib.utils.boto_util import ResumableThreshold from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.cloud_api_helper import GetDownloadSerializationData from gslib.utils.constants import DEFAULT_FILE_BUFFER_SIZE from gslib.utils.constants import MIN_SIZE_COMPUTE_LOGGING from gslib.utils.constants import UTF8 from gslib.utils.encryption_helper import CryptoKeyType from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.encryption_helper import FindMatchingCSEKInBotoConfig from gslib.utils.encryption_helper import GetEncryptionKeyWrapper from gslib.utils.hashing_helper import Base64EncodeHash from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents from gslib.utils.hashing_helper import CalculateHashesFromContents from gslib.utils.hashing_helper import CHECK_HASH_IF_FAST_ELSE_FAIL from gslib.utils.hashing_helper import CHECK_HASH_NEVER from gslib.utils.hashing_helper import ConcatCrc32c from gslib.utils.hashing_helper import GetDownloadHashAlgs from gslib.utils.hashing_helper import GetMd5 from gslib.utils.hashing_helper import GetUploadHashAlgs from gslib.utils.hashing_helper import HashingFileUploadWrapper from gslib.utils.metadata_util import ObjectIsGzipEncoded from gslib.utils.parallelism_framework_util import AtomicDict from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit from gslib.utils.parallelism_framework_util import PutToQueueWithTimeout from gslib.utils.posix_util import ATIME_ATTR from gslib.utils.posix_util import ConvertDatetimeToPOSIX from gslib.utils.posix_util import GID_ATTR from gslib.utils.posix_util import MODE_ATTR from gslib.utils.posix_util import MTIME_ATTR from gslib.utils.posix_util import ParseAndSetPOSIXAttributes from gslib.utils.posix_util import UID_ATTR from gslib.utils.system_util import CheckFreeSpace from gslib.utils.system_util import GetFileSize from gslib.utils.system_util import GetStreamFromFileUrl from gslib.utils.system_util import IS_WINDOWS from gslib.utils.translation_helper import AddS3MarkerAclToObjectMetadata from gslib.utils.translation_helper import CopyObjectMetadata from gslib.utils.translation_helper import DEFAULT_CONTENT_TYPE from gslib.utils.translation_helper import ObjectMetadataFromHeaders from gslib.utils.translation_helper import PreconditionsFromHeaders from gslib.utils.translation_helper import S3MarkerAclFromObjectMetadata from gslib.utils.unit_util import DivideAndCeil from gslib.utils.unit_util import HumanReadableToBytes from gslib.utils.unit_util import MakeHumanReadable from gslib.utils.unit_util import SECONDS_PER_DAY from gslib.utils.unit_util import TEN_MIB from gslib.wildcard_iterator import CreateWildcardIterator if six.PY3: long = int # pylint: disable=g-import-not-at-top if IS_WINDOWS: import msvcrt # Declare copy_helper_opts as a global because namedtuple isn't aware of # assigning to a class member (which breaks pickling done by multiprocessing). # For details see # http://stackoverflow.com/questions/16377215/how-to-pickle-a-namedtuple-instance-correctly # pylint: disable=global-at-module-level global global_copy_helper_opts # In-memory map of local files that are currently opened for write. Used to # ensure that if we write to the same file twice (say, for example, because the # user specified two identical source URLs), the writes occur serially. global open_files_map, open_files_lock open_files_map = AtomicDict( manager=(parallelism_framework_util.top_level_manager if CheckMultiprocessingAvailableAndInit().is_available else None)) # We don't allow multiple processes on Windows, so using a process-safe lock # would be unnecessary. open_files_lock = parallelism_framework_util.CreateLock() # For debugging purposes; if True, files and objects that fail hash validation # will be saved with the below suffix appended. _RENAME_ON_HASH_MISMATCH = False _RENAME_ON_HASH_MISMATCH_SUFFIX = '_corrupt' PARALLEL_UPLOAD_TEMP_NAMESPACE = ( '/gsutil/tmp/parallel_composite_uploads/for_details_see/gsutil_help_cp/') PARALLEL_UPLOAD_STATIC_SALT = u""" PARALLEL_UPLOAD_SALT_TO_PREVENT_COLLISIONS. The theory is that no user will have prepended this to the front of one of their object names and then done an MD5 hash of the name, and then prepended PARALLEL_UPLOAD_TEMP_NAMESPACE to the front of their object name. Note that there will be no problems with object name length since we hash the original name. """ # When uploading a file, get the following fields in the response for # filling in command output and manifests. UPLOAD_RETURN_FIELDS = [ 'crc32c', 'customerEncryption', 'etag', 'generation', 'md5Hash', 'size', ] # This tuple is used only to encapsulate the arguments needed for # command.Apply() in the parallel composite upload case. # Note that content_type is used instead of a full apitools Object() because # apitools objects are not picklable. # filename: String name of file. # file_start: start byte of file (may be in the middle of a file for partitioned # files). # file_length: length of upload (may not be the entire length of a file for # partitioned files). # src_url: FileUrl describing the source file. # dst_url: CloudUrl describing the destination component file. # canned_acl: canned_acl to apply to the uploaded file/component. # content_type: content-type for final object, used for setting content-type # of components and final object. # tracker_file: tracker file for this component. # tracker_file_lock: tracker file lock for tracker file(s). # gzip_encoded: Whether to use gzip transport encoding for the upload. PerformParallelUploadFileToObjectArgs = namedtuple( 'PerformParallelUploadFileToObjectArgs', 'filename file_start file_length src_url dst_url canned_acl ' 'content_type storage_class tracker_file tracker_file_lock encryption_key_sha256 ' 'gzip_encoded') PerformSlicedDownloadObjectToFileArgs = namedtuple( 'PerformSlicedDownloadObjectToFileArgs', 'component_num src_url src_obj_metadata_json dst_url download_file_name ' 'start_byte end_byte decryption_key') # This tuple is used only to encapsulate the arguments returned by # _PerformSlicedDownloadObjectToFile. # component_num: Component number. # crc32c: CRC32C hash value (integer) of the downloaded bytes # bytes_transferred: The number of bytes transferred, potentially less # than the component size if the download was resumed. # component_total_size: The number of bytes corresponding to the whole # component size, potentially more than bytes_transferred # if the download was resumed. # server_encoding: Content-encoding string if it was detected that the server # sent encoded bytes during transfer, None otherwise. PerformSlicedDownloadReturnValues = namedtuple( 'PerformSlicedDownloadReturnValues', 'component_num crc32c bytes_transferred component_total_size ' 'server_encoding') # TODO: Refactor this file to be less cumbersome. In particular, some of the # different paths (e.g., uploading a file to an object vs. downloading an # object to a file) could be split into separate files. # Chunk size to use while zipping/unzipping gzip files. GZIP_CHUNK_SIZE = 8192 # Indicates that all files should be gzipped, in _UploadFileToObject GZIP_ALL_FILES = 'GZIP_ALL_FILES' # Number of bytes to wait before updating a sliced download component tracker # file. TRACKERFILE_UPDATE_THRESHOLD = TEN_MIB PARALLEL_COMPOSITE_SUGGESTION_THRESHOLD = 150 * 1024 * 1024 # S3 requires special Multipart upload logic (that we currently don't implement) # for files > 5GiB in size. S3_MAX_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024 # TODO: Create a message class that serializes posting this message once # through the UI's global status queue. global suggested_sliced_transfers, suggested_sliced_transfers_lock suggested_sliced_transfers = AtomicDict( manager=(parallelism_framework_util.top_level_manager if CheckMultiprocessingAvailableAndInit().is_available else None)) suggested_sliced_transfers_lock = parallelism_framework_util.CreateLock() COMMON_EXTENSION_RULES = { 'md': 'text/markdown', 'tgz': 'application/gzip', } class FileConcurrencySkipError(Exception): """Raised when skipping a file due to a concurrent, duplicate copy.""" def _RmExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) def _ParallelCopyExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) cls.op_failure_count += 1 cls.logger.debug('\n\nEncountered exception while copying:\n%s\n', traceback.format_exc()) def _PerformParallelUploadFileToObject(cls, args, thread_state=None): """Function argument to Apply for performing parallel composite uploads. Args: cls: Calling Command class. args: PerformParallelUploadFileToObjectArgs tuple describing the target. thread_state: gsutil Cloud API instance to use for the operation. Returns: StorageUrl representing a successfully uploaded component. """ fp = FilePart(args.filename, args.file_start, args.file_length) gsutil_api = GetCloudApiInstance(cls, thread_state=thread_state) with fp: # We take many precautions with the component names that make collisions # effectively impossible. Specifying preconditions will just allow us to # reach a state in which uploads will always fail on retries. preconditions = None # Fill in content type and storage class if one was provided. dst_object_metadata = apitools_messages.Object( name=args.dst_url.object_name, bucket=args.dst_url.bucket_name, contentType=args.content_type, storageClass=args.storage_class) orig_prefer_api = gsutil_api.prefer_api try: if global_copy_helper_opts.canned_acl: # No canned ACL support in JSON, force XML API to be used for # upload/copy operations. gsutil_api.prefer_api = ApiSelector.XML ret = _UploadFileToObject(args.src_url, fp, args.file_length, args.dst_url, dst_object_metadata, preconditions, gsutil_api, cls.logger, cls, _ParallelCopyExceptionHandler, gzip_exts=None, allow_splitting=False, is_component=True, gzip_encoded=args.gzip_encoded) finally: if global_copy_helper_opts.canned_acl: gsutil_api.prefer_api = orig_prefer_api component = ret[2] WriteComponentToParallelUploadTrackerFile( args.tracker_file, args.tracker_file_lock, component, cls.logger, encryption_key_sha256=args.encryption_key_sha256) return ret CopyHelperOpts = namedtuple('CopyHelperOpts', [ 'perform_mv', 'no_clobber', 'daisy_chain', 'read_args_from_stdin', 'print_ver', 'use_manifest', 'preserve_acl', 'canned_acl', 'skip_unsupported_objects', 'test_callback_file', 'dest_storage_class', ]) # pylint: disable=global-variable-undefined def CreateCopyHelperOpts(perform_mv=False, no_clobber=False, daisy_chain=False, read_args_from_stdin=False, print_ver=False, use_manifest=False, preserve_acl=False, canned_acl=None, skip_unsupported_objects=False, test_callback_file=None, dest_storage_class=None): """Creates CopyHelperOpts for passing options to CopyHelper.""" # We create a tuple with union of options needed by CopyHelper and any # copy-related functionality in CpCommand, RsyncCommand, or Command class. global global_copy_helper_opts global_copy_helper_opts = CopyHelperOpts( perform_mv=perform_mv, no_clobber=no_clobber, daisy_chain=daisy_chain, read_args_from_stdin=read_args_from_stdin, print_ver=print_ver, use_manifest=use_manifest, preserve_acl=preserve_acl, canned_acl=canned_acl, skip_unsupported_objects=skip_unsupported_objects, test_callback_file=test_callback_file, dest_storage_class=dest_storage_class) return global_copy_helper_opts # pylint: disable=global-variable-undefined # pylint: disable=global-variable-not-assigned def GetCopyHelperOpts(): """Returns namedtuple holding CopyHelper options.""" global global_copy_helper_opts return global_copy_helper_opts def _SelectDownloadStrategy(dst_url): """Get download strategy based on the destination object. Args: dst_url: Destination StorageUrl. Returns: gsutil Cloud API DownloadStrategy. """ dst_is_special = False if dst_url.IsFileUrl(): # Check explicitly first because os.stat doesn't work on 'nul' in Windows. if dst_url.object_name == os.devnull: dst_is_special = True try: mode = os.stat(dst_url.object_name).st_mode if stat.S_ISCHR(mode): dst_is_special = True except OSError: pass if dst_is_special: return CloudApi.DownloadStrategy.ONE_SHOT else: return CloudApi.DownloadStrategy.RESUMABLE def InsistDstUrlNamesContainer(exp_dst_url, have_existing_dst_container, command_name): """Ensures the destination URL names a container. Acceptable containers include directory, bucket, bucket subdir, and non-existent bucket subdir. Args: exp_dst_url: Wildcard-expanded destination StorageUrl. have_existing_dst_container: bool indicator of whether exp_dst_url names a container (directory, bucket, or existing bucket subdir). command_name: Name of command making call. May not be the same as the calling class's self.command_name in the case of commands implemented atop other commands (like mv command). Raises: CommandException: if the URL being checked does not name a container. """ if ((exp_dst_url.IsFileUrl() and not exp_dst_url.IsDirectory()) or (exp_dst_url.IsCloudUrl() and exp_dst_url.IsBucket() and not have_existing_dst_container)): raise CommandException('Destination URL must name a directory, bucket, ' 'or bucket\nsubdirectory for the multiple ' 'source form of the %s command.' % command_name) def _ShouldTreatDstUrlAsBucketSubDir(have_multiple_srcs, dst_url, have_existing_dest_subdir, src_url_names_container, recursion_requested): """Checks whether dst_url should be treated as a bucket "sub-directory". The decision about whether something constitutes a bucket "sub-directory" depends on whether there are multiple sources in this request and whether there is an existing bucket subdirectory. For example, when running the command: gsutil cp file gs://bucket/abc if there's no existing gs://bucket/abc bucket subdirectory we should copy file to the object gs://bucket/abc. In contrast, if there's an existing gs://bucket/abc bucket subdirectory we should copy file to gs://bucket/abc/file. And regardless of whether gs://bucket/abc exists, when running the command: gsutil cp file1 file2 gs://bucket/abc we should copy file1 to gs://bucket/abc/file1 (and similarly for file2). Finally, for recursive copies, if the source is a container then we should copy to a container as the target. For example, when running the command: gsutil cp -r dir1 gs://bucket/dir2 we should copy the subtree of dir1 to gs://bucket/dir2. Note that we don't disallow naming a bucket "sub-directory" where there's already an object at that URL. For example it's legitimate (albeit confusing) to have an object called gs://bucket/dir and then run the command gsutil cp file1 file2 gs://bucket/dir Doing so will end up with objects gs://bucket/dir, gs://bucket/dir/file1, and gs://bucket/dir/file2. Args: have_multiple_srcs: Bool indicator of whether this is a multi-source operation. dst_url: StorageUrl to check. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. src_url_names_container: bool indicator of whether the source URL is a container. recursion_requested: True if a recursive operation has been requested. Returns: bool indicator. """ if have_existing_dest_subdir: return True if dst_url.IsCloudUrl(): return (have_multiple_srcs or (src_url_names_container and recursion_requested)) def _ShouldTreatDstUrlAsSingleton(src_url_names_container, have_multiple_srcs, have_existing_dest_subdir, dst_url, recursion_requested): """Checks that dst_url names a single file/object after wildcard expansion. It is possible that an object path might name a bucket sub-directory. Args: src_url_names_container: Bool indicator of whether the source for the operation is a container (bucket, bucket subdir, or directory). have_multiple_srcs: Bool indicator of whether this is a multi-source operation. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. dst_url: StorageUrl to check. recursion_requested: True if a recursive operation has been requested. Returns: bool indicator. """ if recursion_requested and src_url_names_container: return False if dst_url.IsFileUrl(): return not dst_url.IsDirectory() else: # dst_url.IsCloudUrl() return (not have_multiple_srcs and not have_existing_dest_subdir and dst_url.IsObject()) def _IsUrlValidParentDir(url): """Returns True if not FileUrl ending in relative path symbols. A URL is invalid if it is a FileUrl and the parent directory of the file is a relative path symbol. Unix will not allow a file itself to be named with a relative path symbol, but one can be the parent. Notably, "../obj" can lead to unexpected behavior at the copy destination. We examine the pre-recursion "url", which might point to "..", to see if the parent is valid. If the user does a recursive copy from a URL, it may not end up the final parent of the copied object. For example, see: "dir/nested_dir/obj". If you ran "cp -r dir gs://bucket" from the parent of "dir", then the "url" arg would be "dir", but "nested_dir" would be the parent of "obj". This actually doesn't matter since recursion won't add relative path symbols to the path. However, we still return if "url" is valid because there are cases where we need to copy every parent directory up to "dir" to prevent file name conflicts. Args: url: StorageUrl before recursion. Returns: Boolean indicating if the "url" is valid as a parent directory. """ if not url.IsFileUrl(): return True url_string_minus_trailing_delimiter = url.versionless_url_string.rstrip( url.delim) _, _, after_last_delimiter = (url_string_minus_trailing_delimiter.rpartition( url.delim)) return after_last_delimiter not in storage_url.RELATIVE_PATH_SYMBOLS and ( after_last_delimiter not in [ url.scheme + '://' + symbol for symbol in storage_url.RELATIVE_PATH_SYMBOLS ]) def ConstructDstUrl(src_url, exp_src_url, src_url_names_container, have_multiple_srcs, has_multiple_top_level_srcs, exp_dst_url, have_existing_dest_subdir, recursion_requested, preserve_posix=False): """Constructs the destination URL for a given exp_src_url/exp_dst_url pair. Uses context-dependent naming rules that mimic Linux cp and mv behavior. Args: src_url: Source StorageUrl to be copied. exp_src_url: Single StorageUrl from wildcard expansion of src_url. src_url_names_container: True if src_url names a container (including the case of a wildcard-named bucket subdir (like gs://bucket/abc, where gs://bucket/abc/* matched some objects). have_multiple_srcs: True if this is a multi-source request. This can be true if src_url wildcard-expanded to multiple URLs or if there were multiple source URLs in the request. has_multiple_top_level_srcs: Same as have_multiple_srcs but measured before recursion. exp_dst_url: the expanded StorageUrl requested for the cp destination. Final written path is constructed from this plus a context-dependent variant of src_url. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. recursion_requested: True if a recursive operation has been requested. preserve_posix: True if preservation of posix attributes has been requested. Returns: StorageUrl to use for copy. Raises: CommandException if destination object name not specified for source and source is a stream. """ if (exp_dst_url.IsFileUrl() and exp_dst_url.IsStream() and preserve_posix): raise CommandException('Cannot preserve POSIX attributes with a stream.') if _ShouldTreatDstUrlAsSingleton(src_url_names_container, have_multiple_srcs, have_existing_dest_subdir, exp_dst_url, recursion_requested): # We're copying one file or object to one file or object. return exp_dst_url if exp_src_url.IsFileUrl() and (exp_src_url.IsStream() or exp_src_url.IsFifo()): if have_existing_dest_subdir: type_text = 'stream' if exp_src_url.IsStream() else 'named pipe' raise CommandException('Destination object name needed when ' 'source is a %s' % type_text) return exp_dst_url if not recursion_requested and not have_multiple_srcs: # We're copying one file or object to a subdirectory. Append final comp # of exp_src_url to exp_dst_url. src_final_comp = exp_src_url.object_name.rpartition(src_url.delim)[-1] return StorageUrlFromString('%s%s%s' % (exp_dst_url.url_string.rstrip( exp_dst_url.delim), exp_dst_url.delim, src_final_comp)) # Else we're copying multiple sources to a directory, bucket, or a bucket # "sub-directory". # Ensure exp_dst_url ends in delim char if we're doing a multi-src copy or # a copy to a directory. (The check for copying to a directory needs # special-case handling so that the command: # gsutil cp gs://bucket/obj dir # will turn into file://dir/ instead of file://dir -- the latter would cause # the file "dirobj" to be created.) # Note: need to check have_multiple_srcs or src_url.names_container() # because src_url could be a bucket containing a single object, named # as gs://bucket. if ((have_multiple_srcs or src_url_names_container or (exp_dst_url.IsFileUrl() and exp_dst_url.IsDirectory())) and not exp_dst_url.url_string.endswith(exp_dst_url.delim)): exp_dst_url = StorageUrlFromString( '%s%s' % (exp_dst_url.url_string, exp_dst_url.delim)) src_url_is_valid_parent = _IsUrlValidParentDir(src_url) if not src_url_is_valid_parent and has_multiple_top_level_srcs: # To avoid top-level name conflicts, we need to copy the parent dir. # However, that cannot be done because the parent dir has an invalid name. raise InvalidUrlError( 'Presence of multiple top-level sources and invalid expanded URL' ' make file name conflicts possible for URL: {}'.format(src_url)) # Making naming behavior match how things work with local Linux cp and mv # operations depends on many factors, including whether the destination is a # container, and the plurality of the source(s). # 1. Recursively copying from directories, buckets, or bucket subdirs should # result in objects/files mirroring the source hierarchy. For example: # gsutil cp -r dir1/dir2 gs://bucket # should create the object gs://bucket/dir2/file2, assuming dir1/dir2 # contains file2). # # To be consistent with Linux cp behavior, there's one more wrinkle when # working with subdirs: The resulting object names depend on whether the # destination subdirectory exists. For example, if gs://bucket/subdir # exists, the command: # gsutil cp -r dir1/dir2 gs://bucket/subdir # should create objects named like gs://bucket/subdir/dir2/a/b/c. In # contrast, if gs://bucket/subdir does not exist, this same command # should create objects named like gs://bucket/subdir/a/b/c. # # If there are multiple top-level source items, preserve source parent # dirs. This is similar to when the destination dir already exists and # avoids conflicts such as "dir1/f.txt" and "dir2/f.txt" both getting # copied to "gs://bucket/f.txt". Linux normally errors on these conflicts, # but we cannot do that because we need to give users the ability to create # dirs as they copy to the cloud. # # Note: "mv" is similar to running "cp -r" followed by source deletion. # # 2. Copying individual files or objects to dirs, buckets or bucket subdirs # should result in objects/files named by the final source file name # component. Example: # gsutil cp dir1/*.txt gs://bucket # should create the objects gs://bucket/f1.txt and gs://bucket/f2.txt, # assuming dir1 contains f1.txt and f2.txt. # Ignore the "multiple top-level sources" rule if using double wildcard ** # because that treats all files as top-level, in which case the user doesn't # want to preserve directories. preserve_src_top_level_dirs = ('**' not in src_url.versionless_url_string and src_url_is_valid_parent and (has_multiple_top_level_srcs or have_existing_dest_subdir)) if preserve_src_top_level_dirs or (src_url_names_container and (exp_dst_url.IsCloudUrl() or exp_dst_url.IsDirectory())): # Case 1. Container copy to a destination other than a file. # Build dst_key_name from subpath of exp_src_url past # where src_url ends. For example, for src_url=gs://bucket/ and # exp_src_url=gs://bucket/src_subdir/obj, dst_key_name should be # src_subdir/obj. src_url_path_sans_final_dir = GetPathBeforeFinalDir(src_url, exp_src_url) dst_key_name = exp_src_url.versionless_url_string[ len(src_url_path_sans_final_dir):].lstrip(src_url.delim) if not preserve_src_top_level_dirs: # Only copy file name, not parent dir. dst_key_name = dst_key_name.partition(src_url.delim)[-1] else: # Case 2. dst_key_name = exp_src_url.object_name.rpartition(src_url.delim)[-1] if (exp_dst_url.IsFileUrl() or _ShouldTreatDstUrlAsBucketSubDir( have_multiple_srcs, exp_dst_url, have_existing_dest_subdir, src_url_names_container, recursion_requested)): if exp_dst_url.object_name and exp_dst_url.object_name.endswith( exp_dst_url.delim): dst_key_name = '%s%s%s' % (exp_dst_url.object_name.rstrip( exp_dst_url.delim), exp_dst_url.delim, dst_key_name) else: delim = exp_dst_url.delim if exp_dst_url.object_name else '' dst_key_name = '%s%s%s' % (exp_dst_url.object_name or '', delim, dst_key_name) new_exp_dst_url = exp_dst_url.Clone() new_exp_dst_url.object_name = dst_key_name.replace(src_url.delim, exp_dst_url.delim) return new_exp_dst_url def _CreateDigestsFromDigesters(digesters): digests = {} if digesters: for alg in digesters: digests[alg] = base64.b64encode( digesters[alg].digest()).rstrip(b'\n').decode('ascii') return digests def _CreateDigestsFromLocalFile(status_queue, algs, file_name, src_url, src_obj_metadata): """Creates a base64 CRC32C and/or MD5 digest from file_name. Args: status_queue: Queue for posting progress messages for UI/Analytics. algs: List of algorithms to compute. file_name: File to digest. src_url: StorageUrl for local object. Used to track progress. src_obj_metadata: Metadata of source object. Returns: Dict of algorithm name : base 64 encoded digest """ hash_dict = {} if 'md5' in algs: hash_dict['md5'] = GetMd5() if 'crc32c' in algs: hash_dict['crc32c'] = crcmod.predefined.Crc('crc-32c') with open(file_name, 'rb') as fp: CalculateHashesFromContents(fp, hash_dict, callback_processor=ProgressCallbackWithTimeout( src_obj_metadata.size, FileProgressCallbackHandler( status_queue, src_url=src_url, operation_name='Hashing').call)) digests = {} for alg_name, digest in six.iteritems(hash_dict): digests[alg_name] = Base64EncodeHash(digest.hexdigest()) return digests def _CheckCloudHashes(logger, src_url, dst_url, src_obj_metadata, dst_obj_metadata): """Validates integrity of two cloud objects copied via daisy-chain. Args: logger: for outputting log messages. src_url: CloudUrl for source cloud object. dst_url: CloudUrl for destination cloud object. src_obj_metadata: Cloud Object metadata for object being downloaded from. dst_obj_metadata: Cloud Object metadata for object being uploaded to. Raises: CommandException: if cloud digests don't match local digests. """ # See hack comment in _CheckHashes. # Sometimes (e.g. when kms is enabled for s3) the values we check below are # not actually content hashes. The early exit here provides users a workaround # for this case and any others we've missed. check_hashes_config = config.get('GSUtil', 'check_hashes', CHECK_HASH_IF_FAST_ELSE_FAIL) if check_hashes_config == CHECK_HASH_NEVER: return checked_one = False download_hashes = {} upload_hashes = {} if src_obj_metadata.md5Hash: src_md5hash = six.ensure_binary(src_obj_metadata.md5Hash) download_hashes['md5'] = src_md5hash if src_obj_metadata.crc32c: src_crc32c_hash = six.ensure_binary(src_obj_metadata.crc32c) download_hashes['crc32c'] = src_crc32c_hash if dst_obj_metadata.md5Hash: dst_md5hash = six.ensure_binary(dst_obj_metadata.md5Hash) upload_hashes['md5'] = dst_md5hash if dst_obj_metadata.crc32c: dst_crc32c_hash = six.ensure_binary(dst_obj_metadata.crc32c) upload_hashes['crc32c'] = dst_crc32c_hash for alg, upload_b64_digest in six.iteritems(upload_hashes): if alg not in download_hashes: continue download_b64_digest = download_hashes[alg] if six.PY3 and isinstance(download_b64_digest, str): download_b64_digest = download_b64_digest.encode('ascii') logger.debug('Comparing source vs destination %s-checksum for %s. (%s/%s)', alg, dst_url, download_b64_digest, upload_b64_digest) if download_b64_digest != upload_b64_digest: raise HashMismatchException( '%s signature for source object (%s) doesn\'t match ' 'destination object digest (%s). Object (%s) will be deleted.' % (alg, download_b64_digest, upload_b64_digest, dst_url)) checked_one = True if not checked_one: # One known way this can currently happen is when downloading objects larger # than 5 GiB from S3 (for which the etag is not an MD5). logger.warn( 'WARNING: Found no hashes to validate object downloaded from %s and ' 'uploaded to %s. Integrity cannot be assured without hashes.', src_url, dst_url) def _CheckHashes(logger, obj_url, obj_metadata, file_name, digests, is_upload=False): """Validates integrity by comparing cloud digest to local digest. Args: logger: for outputting log messages. obj_url: CloudUrl for cloud object. obj_metadata: Cloud Object being downloaded from or uploaded to. file_name: Local file name on disk being downloaded to or uploaded from (used only for logging). digests: Computed Digests for the object. is_upload: If true, comparing for an uploaded object (controls logging). Raises: CommandException: if cloud digests don't match local digests. """ # Hack below. # I cannot track down all of the code paths that get here, so I finally # gave up and opted to convert all of the hashes to str. I know that they # *should* be bytes, but the path of least resistance led to str. # Not a nice thing, but for now it makes tests pass... # Update: Since the _CheckCloudHashes function above needs to be changed # as well, I am going to make the executive decision that hashes are # bytes - here as well. It's what the hash and base64 PY3 libs return, # and should be the native format for these things. local_hashes = digests cloud_hashes = {} if obj_metadata.md5Hash: md5_b64_digest = six.ensure_binary(obj_metadata.md5Hash) cloud_hashes['md5'] = md5_b64_digest.rstrip(b'\n') if obj_metadata.crc32c: crc32c_b64_hash = six.ensure_binary(obj_metadata.crc32c) cloud_hashes['crc32c'] = crc32c_b64_hash.rstrip(b'\n') checked_one = False for alg in local_hashes: if alg not in cloud_hashes: continue local_b64_digest = six.ensure_binary(local_hashes[alg]) cloud_b64_digest = cloud_hashes[alg] logger.debug('Comparing local vs cloud %s-checksum for %s. (%s/%s)', alg, file_name, local_b64_digest, cloud_b64_digest) if local_b64_digest != cloud_b64_digest: raise HashMismatchException( '%s signature computed for local file (%s) doesn\'t match ' 'cloud-supplied digest (%s). %s (%s) will be deleted.' % (alg, local_b64_digest, cloud_b64_digest, 'Cloud object' if is_upload else 'Local file', obj_url if is_upload else file_name)) checked_one = True if not checked_one: if is_upload: logger.warn( 'WARNING: Found no hashes to validate object uploaded to %s. ' 'Integrity cannot be assured without hashes.', obj_url) else: # One known way this can currently happen is when downloading objects larger # than 5 GB from S3 (for which the etag is not an MD5). logger.warn( 'WARNING: Found no hashes to validate object downloaded to %s. ' 'Integrity cannot be assured without hashes.', file_name) def IsNoClobberServerException(e): """Checks to see if the server attempted to clobber a file. In this case we specified via a precondition that we didn't want the file clobbered. Args: e: The Exception that was generated by a failed copy operation Returns: bool indicator - True indicates that the server did attempt to clobber an existing file. """ return ((isinstance(e, PreconditionException)) or (isinstance(e, ResumableUploadException) and '412' in e.message)) def CheckForDirFileConflict(exp_src_url, dst_url): """Checks whether copying exp_src_url into dst_url is not possible. This happens if a directory exists in local file system where a file needs to go or vice versa. In that case we print an error message and exits. Example: if the file "./x" exists and you try to do: gsutil cp gs://mybucket/x/y . the request can't succeed because it requires a directory where the file x exists. Note that we don't enforce any corresponding restrictions for buckets, because the flat namespace semantics for buckets doesn't prohibit such cases the way hierarchical file systems do. For example, if a bucket contains an object called gs://bucket/dir and then you run the command: gsutil cp file1 file2 gs://bucket/dir you'll end up with objects gs://bucket/dir, gs://bucket/dir/file1, and gs://bucket/dir/file2. Args: exp_src_url: Expanded source StorageUrl. dst_url: Destination StorageUrl. Raises: CommandException: if errors encountered. """ if dst_url.IsCloudUrl(): # The problem can only happen for file destination URLs. return dst_path = dst_url.object_name final_dir = os.path.dirname(dst_path) if os.path.isfile(final_dir): raise CommandException('Cannot retrieve %s because a file exists ' 'where a directory needs to be created (%s).' % (exp_src_url.url_string, final_dir)) if os.path.isdir(dst_path): raise CommandException('Cannot retrieve %s because a directory exists ' '(%s) where the file needs to be created.' % (exp_src_url.url_string, dst_path)) def _PartitionFile(canned_acl, content_type, dst_bucket_url, file_size, fp, random_prefix, src_url, storage_class, tracker_file, tracker_file_lock, encryption_key_sha256=None, gzip_encoded=False): """Partitions a file into FilePart objects to be uploaded and later composed. These objects, when composed, will match the original file. This entails splitting the file into parts, naming and forming a destination URL for each part, and also providing the PerformParallelUploadFileToObjectArgs corresponding to each part. Args: canned_acl: The user-provided canned_acl, if applicable. content_type: content type for the component and final objects. dst_bucket_url: CloudUrl for the destination bucket. file_size: The size of fp, in bytes. fp: The file object to be partitioned. random_prefix: The randomly-generated prefix used to prevent collisions among the temporary component names. src_url: Source FileUrl from the original command. storage_class: storage class for the component and final objects. tracker_file: The path to the parallel composite upload tracker file. tracker_file_lock: The lock protecting access to the tracker file. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: dst_args: The destination URIs for the temporary component objects. """ parallel_composite_upload_component_size = HumanReadableToBytes( config.get('GSUtil', 'parallel_composite_upload_component_size', DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE)) (num_components, component_size) = _GetPartitionInfo( file_size, MAX_COMPOSE_ARITY, parallel_composite_upload_component_size) dst_args = {} # Arguments to create commands and pass to subprocesses. file_names = [] # Used for the 2-step process of forming dst_args. for i in range(num_components): # "Salt" the object name with something a user is very unlikely to have # used in an object name, then hash the extended name to make sure # we don't run into problems with name length. Using a deterministic # naming scheme for the temporary components allows users to take # advantage of resumable uploads for each component. encoded_name = six.ensure_binary(PARALLEL_UPLOAD_STATIC_SALT + fp.name) content_md5 = GetMd5() content_md5.update(encoded_name) digest = content_md5.hexdigest() temp_file_name = (random_prefix + PARALLEL_UPLOAD_TEMP_NAMESPACE + digest + '_' + str(i)) tmp_dst_url = dst_bucket_url.Clone() tmp_dst_url.object_name = temp_file_name if i < (num_components - 1): # Every component except possibly the last is the same size. file_part_length = component_size else: # The last component just gets all of the remaining bytes. file_part_length = (file_size - ((num_components - 1) * component_size)) offset = i * component_size func_args = PerformParallelUploadFileToObjectArgs( fp.name, offset, file_part_length, src_url, tmp_dst_url, canned_acl, content_type, storage_class, tracker_file, tracker_file_lock, encryption_key_sha256, gzip_encoded) file_names.append(temp_file_name) dst_args[temp_file_name] = func_args return dst_args def _GetComponentNumber(component): """Gets component number from component CloudUrl. Used during parallel composite upload. Args: component: CloudUrl representing component. Returns: component number """ return int(component.object_name[component.object_name.rfind('_') + 1:]) def _DoParallelCompositeUpload(fp, src_url, dst_url, dst_obj_metadata, canned_acl, file_size, preconditions, gsutil_api, command_obj, copy_exception_handler, logger, gzip_encoded=False): """Uploads a local file to a cloud object using parallel composite upload. The file is partitioned into parts, and then the parts are uploaded in parallel, composed to form the original destination object, and deleted. Args: fp: The file object to be uploaded. src_url: FileUrl representing the local file. dst_url: CloudUrl representing the destination file. dst_obj_metadata: apitools Object describing the destination object. canned_acl: The canned acl to apply to the object, if any. file_size: The size of the source file in bytes. preconditions: Cloud API Preconditions for the final object. gsutil_api: gsutil Cloud API instance to use. command_obj: Command object (for calling Apply). copy_exception_handler: Copy exception handler (for use in Apply). logger: logging.Logger for outputting log messages. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, crc32c, and size fields populated. """ start_time = time.time() dst_bucket_url = StorageUrlFromString(dst_url.bucket_url_string) api_selector = gsutil_api.GetApiSelector(provider=dst_url.scheme) encryption_keywrapper = GetEncryptionKeyWrapper(config) encryption_key_sha256 = (encryption_keywrapper.crypto_key_sha256 if encryption_keywrapper else None) # Determine which components, if any, have already been successfully # uploaded. tracker_file_name = GetTrackerFilePath(dst_url, TrackerFileType.PARALLEL_UPLOAD, api_selector, src_url) (existing_enc_key_sha256, existing_prefix, existing_components) = (ReadParallelUploadTrackerFile( tracker_file_name, logger)) # Ensure that the tracker data is still valid (encryption keys match) and # perform any necessary cleanup. (existing_prefix, existing_components) = ValidateParallelCompositeTrackerData( tracker_file_name, existing_enc_key_sha256, existing_prefix, existing_components, encryption_key_sha256, dst_bucket_url, command_obj, logger, _DeleteTempComponentObjectFn, _RmExceptionHandler) encryption_key_sha256 = (encryption_key_sha256.decode('ascii') if encryption_key_sha256 is not None else None) random_prefix = (existing_prefix if existing_prefix is not None else GenerateComponentObjectPrefix( encryption_key_sha256=encryption_key_sha256)) # Create (or overwrite) the tracker file for the upload. WriteParallelUploadTrackerFile(tracker_file_name, random_prefix, existing_components, encryption_key_sha256=encryption_key_sha256) # Protect the tracker file within calls to Apply. tracker_file_lock = parallelism_framework_util.CreateLock() # Dict to track component info so we may align FileMessage values # before and after the operation. components_info = {} # Get the set of all components that should be uploaded. dst_args = _PartitionFile(canned_acl, dst_obj_metadata.contentType, dst_bucket_url, file_size, fp, random_prefix, src_url, dst_obj_metadata.storageClass, tracker_file_name, tracker_file_lock, encryption_key_sha256=encryption_key_sha256, gzip_encoded=gzip_encoded) (components_to_upload, existing_components, existing_objects_to_delete) = (FilterExistingComponents( dst_args, existing_components, dst_bucket_url, gsutil_api)) # Assign a start message to each different component type for component in components_to_upload: components_info[component.dst_url.url_string] = ( FileMessage.COMPONENT_TO_UPLOAD, component.file_length) PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(component.src_url, component.dst_url, time.time(), size=component.file_length, finished=False, component_num=_GetComponentNumber(component.dst_url), message_type=FileMessage.COMPONENT_TO_UPLOAD)) for component in existing_components: component_str = component[0].versionless_url_string components_info[component_str] = (FileMessage.EXISTING_COMPONENT, component[1]) PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, component[0], time.time(), finished=False, size=component[1], component_num=_GetComponentNumber(component[0]), message_type=FileMessage.EXISTING_COMPONENT)) for component in existing_objects_to_delete: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, component, time.time(), finished=False, message_type=FileMessage.EXISTING_OBJECT_TO_DELETE)) # In parallel, copy all of the file parts that haven't already been # uploaded to temporary objects. cp_results = command_obj.Apply( _PerformParallelUploadFileToObject, components_to_upload, copy_exception_handler, ('op_failure_count', 'total_bytes_transferred'), arg_checker=gslib.command.DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SLICE, should_return_results=True) uploaded_components = [] for cp_result in cp_results: uploaded_components.append(cp_result[2]) components = uploaded_components + [i[0] for i in existing_components] if len(components) == len(dst_args): # Only try to compose if all of the components were uploaded successfully. # Sort the components so that they will be composed in the correct order. components = sorted(components, key=_GetComponentNumber) request_components = [] for component_url in components: src_obj_metadata = ( apitools_messages.ComposeRequest.SourceObjectsValueListEntry( name=component_url.object_name)) if component_url.HasGeneration(): src_obj_metadata.generation = long(component_url.generation) request_components.append(src_obj_metadata) composed_object = gsutil_api.ComposeObject( request_components, dst_obj_metadata, preconditions=preconditions, provider=dst_url.scheme, fields=['crc32c', 'generation', 'size'], encryption_tuple=encryption_keywrapper) try: # Make sure only to delete things that we know were successfully # uploaded (as opposed to all of the objects that we attempted to # create) so that we don't delete any preexisting objects, except for # those that were uploaded by a previous, failed run and have since # changed (but still have an old generation lying around). objects_to_delete = components + existing_objects_to_delete command_obj.Apply( _DeleteTempComponentObjectFn, objects_to_delete, _RmExceptionHandler, arg_checker=gslib.command.DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SLICE) # Assign an end message to each different component type for component in components: component_str = component.versionless_url_string try: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, component, time.time(), finished=True, component_num=_GetComponentNumber(component), size=components_info[component_str][1], message_type=components_info[component_str][0])) except: # pylint: disable=bare-except PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, component, time.time(), finished=True)) for component in existing_objects_to_delete: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, component, time.time(), finished=True, message_type=FileMessage.EXISTING_OBJECT_TO_DELETE)) except Exception: # pylint: disable=broad-except # If some of the delete calls fail, don't cause the whole command to # fail. The copy was successful iff the compose call succeeded, so # reduce this to a warning. logger.warn( 'Failed to delete some of the following temporary objects:\n' + '\n'.join(dst_args.keys())) finally: with tracker_file_lock: DeleteTrackerFile(tracker_file_name) else: # Some of the components failed to upload. In this case, we want to exit # without deleting the objects. raise CommandException( 'Some temporary components were not uploaded successfully. ' 'Please retry this upload.') elapsed_time = time.time() - start_time return elapsed_time, composed_object def _ShouldDoParallelCompositeUpload(logger, allow_splitting, src_url, dst_url, file_size, gsutil_api, canned_acl=None): """Determines whether parallel composite upload strategy should be used. Args: logger: for outputting log messages. allow_splitting: If false, then this function returns false. src_url: FileUrl corresponding to a local file. dst_url: CloudUrl corresponding to destination cloud object. file_size: The size of the source file, in bytes. gsutil_api: CloudApi that may be used to check if the destination bucket has any metadata attributes set that would discourage us from using parallel composite uploads. canned_acl: Canned ACL to apply to destination object, if any. Returns: True iff a parallel upload should be performed on the source file. """ global suggested_sliced_transfers, suggested_sliced_transfers_lock parallel_composite_upload_threshold = HumanReadableToBytes( config.get('GSUtil', 'parallel_composite_upload_threshold', DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD)) all_factors_but_size = ( allow_splitting # Don't split the pieces multiple times. and not src_url.IsStream() # We can't partition streams. and not src_url.IsFifo() # We can't partition fifos. and dst_url.scheme == 'gs' # Compose is only for gs. and not canned_acl) # TODO: Implement canned ACL support for compose. # Since parallel composite uploads are disabled by default, make user aware of # them. # TODO: Once compiled crcmod is being distributed by major Linux distributions # remove this check. if (all_factors_but_size and parallel_composite_upload_threshold == 0 and file_size >= PARALLEL_COMPOSITE_SUGGESTION_THRESHOLD): with suggested_sliced_transfers_lock: if not suggested_sliced_transfers.get('suggested'): logger.info('\n'.join( textwrap.wrap( '==> NOTE: You are uploading one or more large file(s), which ' 'would run significantly faster if you enable parallel composite ' 'uploads. This feature can be enabled by editing the ' '"parallel_composite_upload_threshold" value in your .boto ' 'configuration file. However, note that if you do this large files ' 'will be uploaded as ' '`composite objects <https://cloud.google.com/storage/docs/composite-objects>`_,' # pylint: disable=line-too-long 'which means that any user who downloads such objects will need to ' 'have a compiled crcmod installed (see "gsutil help crcmod"). This ' 'is because without a compiled crcmod, computing checksums on ' 'composite objects is so slow that gsutil disables downloads of ' 'composite objects.')) + '\n') suggested_sliced_transfers['suggested'] = True return (all_factors_but_size and parallel_composite_upload_threshold > 0 and file_size >= parallel_composite_upload_threshold) def ExpandUrlToSingleBlr(url_str, gsutil_api, project_id, treat_nonexistent_object_as_subdir=False, logger=None): """Expands wildcard if present in url_str. Args: url_str: String representation of requested url. gsutil_api: gsutil Cloud API instance to use. project_id: project ID to use (for iterators). treat_nonexistent_object_as_subdir: indicates if should treat a non-existent object as a subdir. logger: logging.Logger instance to use for output. If None, the root Logger will be used. Returns: (exp_url, have_existing_dst_container) where exp_url is a StorageUrl and have_existing_dst_container is a bool indicating whether exp_url names an existing directory, bucket, or bucket subdirectory. In the case where we match a subdirectory AND an object, the object is returned. Raises: CommandException: if url_str matched more than 1 URL. """ logger = logger or logging.Logger() # Handle wildcarded url case. if ContainsWildcard(url_str): blr_expansion = list( CreateWildcardIterator(url_str, gsutil_api, project_id=project_id, logger=logger)) if len(blr_expansion) != 1: raise CommandException('Destination (%s) must match exactly 1 URL' % url_str) blr = blr_expansion[0] # BLR is either an OBJECT, PREFIX, or BUCKET; the latter two represent # directories. return (StorageUrlFromString(blr.url_string), not blr.IsObject()) storage_url = StorageUrlFromString(url_str) # Handle non-wildcarded URL. if storage_url.IsFileUrl(): return (storage_url, storage_url.IsDirectory()) # At this point we have a cloud URL. if storage_url.IsBucket(): return (storage_url, True) # For object/prefix URLs, there are four cases that indicate the destination # is a cloud subdirectory; these are always considered to be an existing # container. Checking each case allows gsutil to provide Unix-like # destination folder semantics, but requires up to three HTTP calls, noted # below. # Case 1: If a placeholder object ending with '/' exists. if IsCloudSubdirPlaceholder(storage_url): return (storage_url, True) # Get version of object name without trailing slash for matching prefixes prefix = storage_url.object_name.rstrip('/') # HTTP call to make an eventually consistent check for a matching prefix, # _$folder$, or empty listing. list_iterator = gsutil_api.ListObjects(storage_url.bucket_name, prefix=prefix, delimiter='/', provider=storage_url.scheme, fields=['prefixes', 'items/name']) for obj_or_prefix in list_iterator: # To conserve HTTP calls for the common case, we make a single listing # that covers prefixes and object names. Listing object names covers the # _$folder$ case and the nonexistent-object-as-subdir case. However, if # there are many existing objects for which the target URL is an exact # prefix, this listing could be paginated and span multiple HTTP calls. # If this case becomes common, we could heurestically abort the # listing operation after the first page of results and just query for the # _$folder$ object directly using GetObjectMetadata. # TODO: currently the ListObjects iterator yields objects before prefixes, # because ls depends on this iteration order for proper display. We could # save up to 1ms in determining that a destination is a prefix if we had a # way to yield prefixes first, but this would require poking a major hole # through the abstraction to control this iteration order. if (obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.PREFIX and obj_or_prefix.data == prefix + '/'): # Case 2: If there is a matching prefix when listing the destination URL. return (storage_url, True) elif (obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT and obj_or_prefix.data.name == storage_url.object_name + '_$folder$'): # Case 3: If a placeholder object matching destination + _$folder$ # exists. return (storage_url, True) elif (obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT and obj_or_prefix.data.name == storage_url.object_name): # The object exists but it is not a container return (storage_url, False) # Case 4: If no objects/prefixes matched, and nonexistent objects should be # treated as subdirectories. return (storage_url, treat_nonexistent_object_as_subdir) def TriggerReauthForDestinationProviderIfNecessary(destination_url, gsutil_api, worker_count): """Makes a request to the destination API provider to trigger reauth. Addresses https://github.com/GoogleCloudPlatform/gsutil/issues/1639. If an API call occurs in a child process, the library that handles reauth will fail. We need to make at least one API call in the main process to allow a user to reauthorize. For cloud source URLs this already happens because the plurality of the source name expansion iterator is checked in the main thread. For cloud destination URLs, only some situations result in a similar API call. In these situations, this function exits without performing an API call. In others, this function performs an API call to trigger reauth. Args: destination_url (StorageUrl): The destination of the transfer. gsutil_api (CloudApiDelegator): API to use for the GetBucket call. worker_count (int): If greater than 1, assume that parallel execution is used. Technically, reauth challenges can be answered in the main process, but they may be triggered multiple times if multithreading is used. Returns: None, but performs an API call if necessary. """ # Only perform this check if the user has opted in. if not config.getbool( 'GSUtil', 'trigger_reauth_challenge_for_parallel_operations', False): return # Reauth is not necessary for non-cloud destinations. if not destination_url.IsCloudUrl(): return # Destination wildcards are expanded by an API call in the main process. if ContainsWildcard(destination_url.url_string): return # If gsutil executes sequentially, all calls will occur in the main process. if worker_count == 1: return try: # The specific API call is not important, but one must occur. gsutil_api.GetBucket( destination_url.bucket_name, fields=['location'], # Single field to limit XML API calls. provider=destination_url.scheme) except Exception as e: if isinstance(e, pyu2f.errors.PluginError): raise # Other exceptions can be ignored. The purpose was just to trigger # a reauth challenge. def FixWindowsNaming(src_url, dst_url): """Translates Windows pathnames to cloud pathnames. Rewrites the destination URL built by ConstructDstUrl(). Args: src_url: Source StorageUrl to be copied. dst_url: The destination StorageUrl built by ConstructDstUrl(). Returns: StorageUrl to use for copy. """ if (src_url.IsFileUrl() and src_url.delim == '\\' and dst_url.IsCloudUrl()): trans_url_str = re.sub(r'\\', '/', dst_url.url_string) dst_url = StorageUrlFromString(trans_url_str) return dst_url def SrcDstSame(src_url, dst_url): """Checks if src_url and dst_url represent the same object or file. We don't handle anything about hard or symbolic links. Args: src_url: Source StorageUrl. dst_url: Destination StorageUrl. Returns: Bool indicator. """ if src_url.IsFileUrl() and dst_url.IsFileUrl(): # Translate a/b/./c to a/b/c, so src=dst comparison below works. new_src_path = os.path.normpath(src_url.object_name) new_dst_path = os.path.normpath(dst_url.object_name) return new_src_path == new_dst_path else: return (src_url.url_string == dst_url.url_string and src_url.generation == dst_url.generation) def _LogCopyOperation(logger, src_url, dst_url, dst_obj_metadata): """Logs copy operation, including Content-Type if appropriate. Args: logger: logger instance to use for output. src_url: Source StorageUrl. dst_url: Destination StorageUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. """ if (dst_url.IsCloudUrl() and dst_obj_metadata and dst_obj_metadata.contentType): content_type_msg = ' [Content-Type=%s]' % dst_obj_metadata.contentType else: content_type_msg = '' if src_url.IsFileUrl() and (src_url.IsStream() or src_url.IsFifo()): src_text = '<STDIN>' if src_url.IsStream() else 'named pipe' logger.info('Copying from %s%s...', src_text, content_type_msg) else: logger.info('Copying %s%s...', src_url.url_string, content_type_msg) # pylint: disable=undefined-variable def _CopyObjToObjInTheCloud(src_url, src_obj_metadata, dst_url, dst_obj_metadata, preconditions, gsutil_api, decryption_key=None): """Performs copy-in-the cloud from specified src to dest object. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata for source object; must include etag and size. dst_url: Destination CloudUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API instance to use for the copy. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. """ decryption_keywrapper = CryptoKeyWrapperFromKey(decryption_key) encryption_keywrapper = GetEncryptionKeyWrapper(config) start_time = time.time() progress_callback = FileProgressCallbackHandler(gsutil_api.status_queue, src_url=src_url, dst_url=dst_url, operation_name='Copying').call if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call dst_obj = gsutil_api.CopyObject(src_obj_metadata, dst_obj_metadata, src_generation=src_url.generation, canned_acl=global_copy_helper_opts.canned_acl, preconditions=preconditions, progress_callback=progress_callback, provider=dst_url.scheme, fields=UPLOAD_RETURN_FIELDS, decryption_tuple=decryption_keywrapper, encryption_tuple=encryption_keywrapper) end_time = time.time() result_url = dst_url.Clone() result_url.generation = GenerationFromUrlAndString(result_url, dst_obj.generation) PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, end_time, message_type=FileMessage.FILE_CLOUD_COPY, size=src_obj_metadata.size, finished=True)) return (end_time - start_time, src_obj_metadata.size, result_url, dst_obj.md5Hash) def _SetContentTypeFromFile(src_url, dst_obj_metadata): """Detects and sets Content-Type if src_url names a local file. Args: src_url: Source StorageUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. """ # contentType == '' if user requested default type. if (dst_obj_metadata.contentType is None and src_url.IsFileUrl() and not src_url.IsStream() and not src_url.IsFifo()): # Only do content type recognition if src_url is a file. Object-to-object # copies with no -h Content-Type specified re-use the content type of the # source object. object_name = src_url.object_name content_type = None # Streams (denoted by '-') are expected to be 'application/octet-stream' # and 'file' would partially consume them. if object_name != '-': real_file_path = os.path.realpath(object_name) if config.getbool('GSUtil', 'use_magicfile', False) and not IS_WINDOWS: try: p = subprocess.Popen(['file', '-b', '--mime', real_file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() p.stdout.close() p.stderr.close() if p.returncode != 0 or error: raise CommandException( 'Encountered error running "file -b --mime %s" ' '(returncode=%d).\n%s' % (real_file_path, p.returncode, error)) # Parse output by removing line delimiter content_type = output.rstrip() content_type = six.ensure_str(content_type) except OSError as e: # 'file' executable may not always be present. raise CommandException( 'Encountered OSError running "file -b --mime %s"\n%s' % (real_file_path, e)) else: _, _, extension = real_file_path.rpartition('.') if extension in COMMON_EXTENSION_RULES: content_type = COMMON_EXTENSION_RULES[extension] else: content_type, _ = mimetypes.guess_type(real_file_path) if not content_type: content_type = DEFAULT_CONTENT_TYPE dst_obj_metadata.contentType = content_type # pylint: disable=undefined-variable def _UploadFileToObjectNonResumable(src_url, src_obj_filestream, src_obj_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, gzip_encoded=False): """Uploads the file using a non-resumable strategy. This function does not support component transfers. Args: src_url: Source StorageUrl to upload. src_obj_filestream: File pointer to uploadable bytes. src_obj_size (int or None): Size of the source object. dst_url: Destination StorageUrl for the upload. dst_obj_metadata: Metadata for the target object. preconditions: Preconditions for the upload, if any. gsutil_api: gsutil Cloud API instance to use for the upload. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, md5, and size fields populated. """ progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=src_url, dst_url=dst_url, operation_name='Uploading').call if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call start_time = time.time() encryption_keywrapper = GetEncryptionKeyWrapper(config) if src_url.IsStream() or src_url.IsFifo(): # TODO: gsutil-beta: Provide progress callbacks for streaming uploads. uploaded_object = gsutil_api.UploadObjectStreaming( src_obj_filestream, object_metadata=dst_obj_metadata, canned_acl=global_copy_helper_opts.canned_acl, preconditions=preconditions, progress_callback=progress_callback, encryption_tuple=encryption_keywrapper, provider=dst_url.scheme, fields=UPLOAD_RETURN_FIELDS, gzip_encoded=gzip_encoded) else: uploaded_object = gsutil_api.UploadObject( src_obj_filestream, object_metadata=dst_obj_metadata, canned_acl=global_copy_helper_opts.canned_acl, size=src_obj_size, preconditions=preconditions, progress_callback=progress_callback, encryption_tuple=encryption_keywrapper, provider=dst_url.scheme, fields=UPLOAD_RETURN_FIELDS, gzip_encoded=gzip_encoded) end_time = time.time() elapsed_time = end_time - start_time return elapsed_time, uploaded_object # pylint: disable=undefined-variable def _UploadFileToObjectResumable(src_url, src_obj_filestream, src_obj_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, is_component=False, gzip_encoded=False): """Uploads the file using a resumable strategy. Args: src_url: Source FileUrl to upload. Must not be a stream. src_obj_filestream: File pointer to uploadable bytes. src_obj_size (int or None): Size of the source object. dst_url: Destination StorageUrl for the upload. dst_obj_metadata: Metadata for the target object. preconditions: Preconditions for the upload, if any. gsutil_api: gsutil Cloud API instance to use for the upload. logger: for outputting log messages. is_component: indicates whether this is a single component or whole file. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, md5, and size fields populated. """ tracker_file_name = GetTrackerFilePath( dst_url, TrackerFileType.UPLOAD, gsutil_api.GetApiSelector(provider=dst_url.scheme)) encryption_keywrapper = GetEncryptionKeyWrapper(config) encryption_key_sha256 = ( encryption_keywrapper.crypto_key_sha256.decode('ascii') if encryption_keywrapper and encryption_keywrapper.crypto_key_sha256 else None) def _UploadTrackerCallback(serialization_data): """Creates a new tracker file for starting an upload from scratch. This function is called by the gsutil Cloud API implementation and the the serialization data is implementation-specific. Args: serialization_data: Serialization data used in resuming the upload. """ data = { ENCRYPTION_UPLOAD_TRACKER_ENTRY: encryption_key_sha256, SERIALIZATION_UPLOAD_TRACKER_ENTRY: str(serialization_data) } WriteJsonDataToTrackerFile(tracker_file_name, data) # This contains the upload URL, which will uniquely identify the # destination object. tracker_data = GetUploadTrackerData( tracker_file_name, logger, encryption_key_sha256=encryption_key_sha256) if tracker_data: logger.info('Resuming upload for %s', src_url.url_string) retryable = True component_num = _GetComponentNumber(dst_url) if is_component else None progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=src_url, component_num=component_num, dst_url=dst_url, operation_name='Uploading').call if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call start_time = time.time() num_startover_attempts = 0 # This loop causes us to retry when the resumable upload failed in a way that # requires starting over with a new upload ID. Retries within a single upload # ID within the current process are handled in # gsutil_api.UploadObjectResumable, and retries within a single upload ID # spanning processes happens if an exception occurs not caught below (which # will leave the tracker file in place, and cause the upload ID to be reused # the next time the user runs gsutil and attempts the same upload). while retryable: try: uploaded_object = gsutil_api.UploadObjectResumable( src_obj_filestream, object_metadata=dst_obj_metadata, canned_acl=global_copy_helper_opts.canned_acl, preconditions=preconditions, provider=dst_url.scheme, size=src_obj_size, serialization_data=tracker_data, encryption_tuple=encryption_keywrapper, fields=UPLOAD_RETURN_FIELDS, tracker_callback=_UploadTrackerCallback, progress_callback=progress_callback, gzip_encoded=gzip_encoded) retryable = False except ResumableUploadStartOverException as e: logger.info('Caught ResumableUploadStartOverException for upload of %s.' % src_url.url_string) # This can happen, for example, if the server sends a 410 response code. # In that case the current resumable upload ID can't be reused, so delete # the tracker file and try again up to max retries. num_startover_attempts += 1 retryable = (num_startover_attempts < GetNumRetries()) if not retryable: raise # If the server sends a 404 response code, then the upload should only # be restarted if it was the object (and not the bucket) that was missing. try: logger.info('Checking that bucket %s exists before retrying upload...' % dst_obj_metadata.bucket) gsutil_api.GetBucket(dst_obj_metadata.bucket, provider=dst_url.scheme) except AccessDeniedException: # Proceed with deleting the tracker file in the event that the bucket # exists, but the user does not have permission to view its metadata. pass except NotFoundException: raise finally: DeleteTrackerFile(tracker_file_name) logger.info('Deleted tracker file %s for resumable upload of %s before ' 'retrying.' % (tracker_file_name, src_url.url_string)) logger.info( 'Restarting upload of %s from scratch (retry #%d) after exception ' 'indicating we need to start over with a new resumable upload ID: %s' % (src_url.url_string, num_startover_attempts, e)) tracker_data = None src_obj_filestream.seek(0) # Reset the progress callback handler. component_num = _GetComponentNumber(dst_url) if is_component else None progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=src_url, component_num=component_num, dst_url=dst_url, operation_name='Uploading').call # Report the retryable error to the global status queue. PutToQueueWithTimeout( gsutil_api.status_queue, RetryableErrorMessage(e, time.time(), num_retries=num_startover_attempts)) time.sleep( min(random.random() * (2**num_startover_attempts), GetMaxRetryDelay())) except ResumableUploadAbortException: retryable = False raise finally: if not retryable: DeleteTrackerFile(tracker_file_name) end_time = time.time() elapsed_time = end_time - start_time return (elapsed_time, uploaded_object) def _SelectUploadCompressionStrategy(object_name, is_component=False, gzip_exts=False, gzip_encoded=False): """Selects how an upload should be compressed. This is a helper function for _UploadFileToObject. Args: object_name: The object name of the source FileUrl. is_component: indicates whether this is a single component or whole file. gzip_exts: List of file extensions to gzip prior to upload, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts for selecting which files will be encoded. Streaming files compressed is only supported on the JSON GCS API. Returns: A tuple: (If the file should be gzipped locally, if the file should be gzip transport encoded). """ zipped_file = False gzip_encoded_file = False fname_parts = object_name.split('.') # If gzip_encoded and is_component are marked as true, the file was already # filtered through the original gzip_exts filter and we must compress the # component via gzip transport encoding. if gzip_encoded and is_component: gzip_encoded_file = True elif (gzip_exts == GZIP_ALL_FILES or (gzip_exts and len(fname_parts) > 1 and fname_parts[-1] in gzip_exts)): zipped_file = not gzip_encoded gzip_encoded_file = gzip_encoded return zipped_file, gzip_encoded_file def _ApplyZippedUploadCompression(src_url, src_obj_filestream, src_obj_size, logger): """Compresses a to-be-uploaded local file to save bandwidth. This is a helper function for _UploadFileToObject. Args: src_url: Source FileUrl. src_obj_filestream: Read stream of the source file - will be consumed and closed. src_obj_size (int or None): Size of the source file. logger: for outputting log messages. Returns: StorageUrl path to compressed file, read stream of the compressed file, compressed file size. """ # TODO: Compress using a streaming model as opposed to all at once here. if src_obj_size is not None and src_obj_size >= MIN_SIZE_COMPUTE_LOGGING: logger.info('Compressing %s (to tmp)...', src_url) (gzip_fh, gzip_path) = tempfile.mkstemp() gzip_fp = None try: # Check for temp space. Assume the compressed object is at most 2x # the size of the object (normally should compress to smaller than # the object) if src_url.IsStream() or src_url.IsFifo(): # TODO: Support streaming gzip uploads. # https://github.com/GoogleCloudPlatform/gsutil/issues/364 raise CommandException( 'gzip compression is not currently supported on streaming uploads. ' 'Remove the compression flag or save the streamed output ' 'temporarily to a file before uploading.') if src_obj_size is not None and (CheckFreeSpace(gzip_path) < 2 * int(src_obj_size)): raise CommandException('Inadequate temp space available to compress ' '%s. See the CHANGING TEMP DIRECTORIES section ' 'of "gsutil help cp" for more info.' % src_url) compression_level = config.getint('GSUtil', 'gzip_compression_level', DEFAULT_GZIP_COMPRESSION_LEVEL) gzip_fp = gzip.open(gzip_path, 'wb', compresslevel=compression_level) data = src_obj_filestream.read(GZIP_CHUNK_SIZE) while data: gzip_fp.write(data) data = src_obj_filestream.read(GZIP_CHUNK_SIZE) finally: if gzip_fp: gzip_fp.close() os.close(gzip_fh) src_obj_filestream.close() gzip_size = os.path.getsize(gzip_path) compressed_filestream = open(gzip_path, 'rb') return StorageUrlFromString(gzip_path), compressed_filestream, gzip_size def _DelegateUploadFileToObject(upload_delegate, upload_url, upload_stream, zipped_file, gzip_encoded_file, parallel_composite_upload, logger): """Handles setup and tear down logic for uploads. This is a helper function for _UploadFileToObject. Args: upload_delegate: Function that handles uploading the file. upload_url: StorageURL path to the file. upload_stream: Read stream of the file being uploaded. This will be closed after the upload. zipped_file: Flag for if the file is locally compressed prior to calling this function. If true, the local temporary file is deleted after the upload. gzip_encoded_file: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. parallel_composite_upload: Set to true if this upload represents a top-level parallel composite upload (not an upload of a component). If true, resource locking is skipped. logger: For outputting log messages. Returns: The elapsed upload time, the uploaded object. """ elapsed_time = None uploaded_object = None try: # Parallel transport compressed uploads use a signifcant amount of memory. # The number of threads that may run concurrently are restricted as a # result. Parallel composite upload's don't actually upload data, but # instead fork for each component and calling _UploadFileToObject # individually. The parallel_composite_upload flag is false for the actual # upload invocation. if gzip_encoded_file and not parallel_composite_upload: with gslib.command.concurrent_compressed_upload_lock: elapsed_time, uploaded_object = upload_delegate() else: elapsed_time, uploaded_object = upload_delegate() finally: # In the zipped_file case, this is the gzip stream. When the gzip stream is # created, the original source stream is closed in # _ApplyZippedUploadCompression. This means that we do not have to # explicitly close the source stream here in the zipped_file case. upload_stream.close() if zipped_file: try: os.unlink(upload_url.object_name) # Windows sometimes complains the temp file is locked when you try to # delete it. except Exception: # pylint: disable=broad-except logger.warning( 'Could not delete %s. This can occur in Windows because the ' 'temporary file is still locked.', upload_url.object_name) return elapsed_time, uploaded_object def _UploadFileToObject(src_url, src_obj_filestream, src_obj_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, command_obj, copy_exception_handler, gzip_exts=None, allow_splitting=True, is_component=False, gzip_encoded=False): """Uploads a local file to an object. Args: src_url: Source FileUrl. src_obj_filestream: Read stream of the source file to be read and closed. src_obj_size (int or None): Size of the source file. dst_url: Destination CloudUrl. dst_obj_metadata: Metadata to be applied to the destination object. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API to use for the copy. logger: for outputting log messages. command_obj: command object for use in Apply in parallel composite uploads. copy_exception_handler: For handling copy exceptions during Apply. gzip_exts: List of file extensions to gzip prior to upload, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. allow_splitting: Whether to allow the file to be split into component pieces for an parallel composite upload. is_component: indicates whether this is a single component or whole file. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts for selecting which files will be encoded. Streaming files compressed is only supported on the JSON GCS API. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. """ if not dst_obj_metadata or not dst_obj_metadata.contentLanguage: content_language = config.get_value('GSUtil', 'content_language') if content_language: dst_obj_metadata.contentLanguage = content_language upload_url = src_url upload_stream = src_obj_filestream upload_size = src_obj_size zipped_file, gzip_encoded_file = _SelectUploadCompressionStrategy( src_url.object_name, is_component, gzip_exts, gzip_encoded) # The component's parent already printed this debug message. if gzip_encoded_file and not is_component: logger.debug('Using compressed transport encoding for %s.', src_url) elif zipped_file: upload_url, upload_stream, upload_size = _ApplyZippedUploadCompression( src_url, src_obj_filestream, src_obj_size, logger) dst_obj_metadata.contentEncoding = 'gzip' # If we're sending an object with gzip encoding, it's possible it also # has an incompressible content type. Google Cloud Storage will remove # the top layer of compression when serving the object, which would cause # the served content not to match the CRC32C/MD5 hashes stored and make # integrity checking impossible. Therefore we set cache control to # no-transform to ensure it is served in its original form. The caveat is # that to read this object, other clients must then support # accept-encoding:gzip. if not dst_obj_metadata.cacheControl: dst_obj_metadata.cacheControl = 'no-transform' elif 'no-transform' not in dst_obj_metadata.cacheControl.lower(): dst_obj_metadata.cacheControl += ',no-transform' if not is_component: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(upload_url, dst_url, time.time(), message_type=FileMessage.FILE_UPLOAD, size=upload_size, finished=False)) elapsed_time = None uploaded_object = None hash_algs = GetUploadHashAlgs() digesters = dict((alg, hash_algs[alg]()) for alg in hash_algs or {}) parallel_composite_upload = _ShouldDoParallelCompositeUpload( logger, allow_splitting, upload_url, dst_url, src_obj_size, gsutil_api, canned_acl=global_copy_helper_opts.canned_acl) non_resumable_upload = ((0 if upload_size is None else upload_size) < ResumableThreshold() or src_url.IsStream() or src_url.IsFifo()) if ((src_url.IsStream() or src_url.IsFifo()) and gsutil_api.GetApiSelector(provider=dst_url.scheme) == ApiSelector.JSON): orig_stream = upload_stream # Add limited seekable properties to the stream via buffering. upload_stream = ResumableStreamingJsonUploadWrapper( orig_stream, GetJsonResumableChunkSize()) if not parallel_composite_upload and len(hash_algs): # Parallel composite uploads calculate hashes per-component in subsequent # calls to this function, but the composition of the final object is a # cloud-only operation. wrapped_filestream = HashingFileUploadWrapper(upload_stream, digesters, hash_algs, upload_url, logger) else: wrapped_filestream = upload_stream def CallParallelCompositeUpload(): return _DoParallelCompositeUpload(upload_stream, upload_url, dst_url, dst_obj_metadata, global_copy_helper_opts.canned_acl, upload_size, preconditions, gsutil_api, command_obj, copy_exception_handler, logger, gzip_encoded=gzip_encoded_file) def CallNonResumableUpload(): return _UploadFileToObjectNonResumable(upload_url, wrapped_filestream, upload_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, gzip_encoded=gzip_encoded_file) def CallResumableUpload(): return _UploadFileToObjectResumable(upload_url, wrapped_filestream, upload_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, is_component=is_component, gzip_encoded=gzip_encoded_file) if parallel_composite_upload: delegate = CallParallelCompositeUpload elif non_resumable_upload: delegate = CallNonResumableUpload else: delegate = CallResumableUpload elapsed_time, uploaded_object = _DelegateUploadFileToObject( delegate, upload_url, upload_stream, zipped_file, gzip_encoded_file, parallel_composite_upload, logger) if not parallel_composite_upload: try: digests = _CreateDigestsFromDigesters(digesters) _CheckHashes(logger, dst_url, uploaded_object, src_url.object_name, digests, is_upload=True) except HashMismatchException: if _RENAME_ON_HASH_MISMATCH: corrupted_obj_metadata = apitools_messages.Object( name=dst_obj_metadata.name, bucket=dst_obj_metadata.bucket, etag=uploaded_object.etag) dst_obj_metadata.name = (dst_url.object_name + _RENAME_ON_HASH_MISMATCH_SUFFIX) gsutil_api.CopyObject(corrupted_obj_metadata, dst_obj_metadata, provider=dst_url.scheme) # If the digest doesn't match, delete the object. gsutil_api.DeleteObject(dst_url.bucket_name, dst_url.object_name, generation=uploaded_object.generation, provider=dst_url.scheme) raise result_url = dst_url.Clone() result_url.generation = uploaded_object.generation result_url.generation = GenerationFromUrlAndString(result_url, uploaded_object.generation) if not is_component: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(upload_url, dst_url, time.time(), message_type=FileMessage.FILE_UPLOAD, size=upload_size, finished=True)) return (elapsed_time, uploaded_object.size, result_url, uploaded_object.md5Hash) def _GetDownloadFile(dst_url, src_obj_metadata, logger): """Creates a new download file, and deletes the file that will be replaced. Names and creates a temporary file for this download. Also, if there is an existing file at the path where this file will be placed after the download is completed, that file will be deleted. Args: dst_url: Destination FileUrl. src_obj_metadata: Metadata from the source object. logger: for outputting log messages. Returns: (download_file_name, need_to_unzip) download_file_name: The name of the temporary file to which the object will be downloaded. need_to_unzip: If true, a temporary zip file was used and must be uncompressed as part of validation. """ dir_name = os.path.dirname(dst_url.object_name) if dir_name and not os.path.exists(dir_name): # Do dir creation in try block so can ignore case where dir already # exists. This is needed to avoid a race condition when running gsutil # -m cp. try: os.makedirs(dir_name) except OSError as e: if e.errno != errno.EEXIST: raise need_to_unzip = False # For gzipped objects download to a temp file and unzip. For the XML API, # this represents the result of a HEAD request. For the JSON API, this is # the stored encoding which the service may not respect. However, if the # server sends decompressed bytes for a file that is stored compressed # (double compressed case), there is no way we can validate the hash and # we will fail our hash check for the object. if ObjectIsGzipEncoded(src_obj_metadata): need_to_unzip = True download_file_name = temporary_file_util.GetTempZipFileName(dst_url) logger.info('Downloading to temp gzip filename %s', download_file_name) else: download_file_name = temporary_file_util.GetTempFileName(dst_url) # If a file exists at the permanent destination (where the file will be moved # after the download is completed), delete it here to reduce disk space # requirements. if os.path.exists(dst_url.object_name): os.unlink(dst_url.object_name) # Downloads open the temporary download file in r+b mode, which requires it # to already exist, so we create it here if it doesn't exist already. if not os.path.exists(download_file_name): fp = open(download_file_name, 'w') fp.close() return download_file_name, need_to_unzip def _ShouldDoSlicedDownload(download_strategy, src_obj_metadata, allow_splitting, logger): """Determines whether the sliced download strategy should be used. Args: download_strategy: CloudApi download strategy. src_obj_metadata: Metadata from the source object. allow_splitting: If false, then this function returns false. logger: logging.Logger for log message output. Returns: True iff a sliced download should be performed on the source file. """ sliced_object_download_threshold = HumanReadableToBytes( config.get('GSUtil', 'sliced_object_download_threshold', DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD)) max_components = config.getint('GSUtil', 'sliced_object_download_max_components', DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS) # Don't use sliced download if it will prevent us from performing an # integrity check. check_hashes_config = config.get('GSUtil', 'check_hashes', CHECK_HASH_IF_FAST_ELSE_FAIL) parallel_hashing = src_obj_metadata.crc32c and UsingCrcmodExtension() hashing_okay = parallel_hashing or check_hashes_config == CHECK_HASH_NEVER use_slice = (allow_splitting and download_strategy is not CloudApi.DownloadStrategy.ONE_SHOT and max_components > 1 and hashing_okay and sliced_object_download_threshold > 0 and src_obj_metadata.size >= sliced_object_download_threshold) if (not use_slice and src_obj_metadata.size >= PARALLEL_COMPOSITE_SUGGESTION_THRESHOLD and not UsingCrcmodExtension() and check_hashes_config != CHECK_HASH_NEVER): with suggested_sliced_transfers_lock: if not suggested_sliced_transfers.get('suggested'): logger.info('\n'.join( textwrap.wrap( '==> NOTE: You are downloading one or more large file(s), which ' 'would run significantly faster if you enabled sliced object ' 'downloads. This feature is enabled by default but requires that ' 'compiled crcmod be installed (see "gsutil help crcmod").')) + '\n') suggested_sliced_transfers['suggested'] = True return use_slice def _PerformSlicedDownloadObjectToFile(cls, args, thread_state=None): """Function argument to Apply for performing sliced downloads. Args: cls: Calling Command class. args: PerformSlicedDownloadObjectToFileArgs tuple describing the target. thread_state: gsutil Cloud API instance to use for the operation. Returns: PerformSlicedDownloadReturnValues named-tuple filled with: component_num: The component number for this download. crc32c: CRC32C hash value (integer) of the downloaded bytes. bytes_transferred: The number of bytes transferred, potentially less than the component size if the download was resumed. component_total_size: The number of bytes corresponding to the whole component size, potentially more than bytes_transferred if the download was resumed. """ gsutil_api = GetCloudApiInstance(cls, thread_state=thread_state) # Deserialize the picklable object metadata. src_obj_metadata = protojson.decode_message(apitools_messages.Object, args.src_obj_metadata_json) hash_algs = GetDownloadHashAlgs(cls.logger, consider_crc32c=src_obj_metadata.crc32c) digesters = dict((alg, hash_algs[alg]()) for alg in hash_algs or {}) (bytes_transferred, server_encoding) = (_DownloadObjectToFileResumable( args.src_url, src_obj_metadata, args.dst_url, args.download_file_name, gsutil_api, cls.logger, digesters, component_num=args.component_num, start_byte=args.start_byte, end_byte=args.end_byte, decryption_key=args.decryption_key)) crc32c_val = None if 'crc32c' in digesters: crc32c_val = digesters['crc32c'].crcValue return PerformSlicedDownloadReturnValues(args.component_num, crc32c_val, bytes_transferred, args.end_byte - args.start_byte + 1, server_encoding) def _MaintainSlicedDownloadTrackerFiles(src_obj_metadata, dst_url, download_file_name, logger, api_selector, num_components): """Maintains sliced download tracker files in order to permit resumability. Reads or creates a sliced download tracker file representing this object download. Upon an attempt at cross-process resumption, the contents of the sliced download tracker file are verified to make sure a resumption is possible and appropriate. In the case that a resumption should not be attempted, existing component tracker files are deleted (to prevent child processes from attempting resumption), and a new sliced download tracker file is created. Args: src_obj_metadata: Metadata from the source object. Must include etag and generation. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for the download. logger: for outputting log messages. api_selector: The Cloud API implementation used. num_components: The number of components to perform this download with. """ assert src_obj_metadata.etag tracker_file = None # Only can happen if the resumable threshold is set higher than the # parallel transfer threshold. if src_obj_metadata.size < ResumableThreshold(): return tracker_file_name = GetTrackerFilePath(dst_url, TrackerFileType.SLICED_DOWNLOAD, api_selector) fp = None # Check to see if we should attempt resuming the download. try: fp = open(download_file_name, 'rb') existing_file_size = GetFileSize(fp) # A parallel resumption should be attempted only if the destination file # size is exactly the same as the source size and the tracker file matches. if existing_file_size == src_obj_metadata.size: tracker_file = open(tracker_file_name, 'r') tracker_file_data = json.load(tracker_file) if (tracker_file_data['etag'] == src_obj_metadata.etag and tracker_file_data['generation'] == src_obj_metadata.generation and tracker_file_data['num_components'] == num_components): return else: tracker_file.close() logger.warn('Sliced download tracker file doesn\'t match for ' 'download of %s. Restarting download from scratch.' % dst_url.object_name) except (IOError, ValueError) as e: # Ignore non-existent file (happens first time a download # is attempted on an object), but warn user for other errors. if isinstance(e, ValueError) or e.errno != errno.ENOENT: logger.warn('Couldn\'t read sliced download tracker file (%s): %s. ' 'Restarting download from scratch.' % (tracker_file_name, str(e))) finally: if fp: fp.close() if tracker_file: tracker_file.close() # Delete component tracker files to guarantee download starts from scratch. DeleteDownloadTrackerFiles(dst_url, api_selector) # Create a new sliced download tracker file to represent this download. try: with open(tracker_file_name, 'w') as tracker_file: tracker_file_data = { 'etag': src_obj_metadata.etag, 'generation': src_obj_metadata.generation, 'num_components': num_components, } tracker_file.write(json.dumps(tracker_file_data)) except IOError as e: RaiseUnwritableTrackerFileException(tracker_file_name, e.strerror) class SlicedDownloadFileWrapper(object): """Wraps a file object to be used in GetObjectMedia for sliced downloads. In order to allow resumability, the file object used by each thread in a sliced object download should be wrapped using SlicedDownloadFileWrapper. Passing a SlicedDownloadFileWrapper object to GetObjectMedia will allow the download component tracker file for this component to be updated periodically, while the downloaded bytes are normally written to file. """ def __init__(self, fp, tracker_file_name, src_obj_metadata, start_byte, end_byte): """Initializes the SlicedDownloadFileWrapper. Args: fp: The already-open file object to be used for writing in GetObjectMedia. Data will be written to file starting at the current seek position. tracker_file_name: The name of the tracker file for this component. src_obj_metadata: Metadata from the source object. Must include etag and generation. start_byte: The first byte to be downloaded for this parallel component. end_byte: The last byte to be downloaded for this parallel component. """ self._orig_fp = fp self._tracker_file_name = tracker_file_name self._src_obj_metadata = src_obj_metadata self._last_tracker_file_byte = None self._start_byte = start_byte self._end_byte = end_byte @property def mode(self): """Returns the mode of the underlying file descriptor, or None.""" return getattr(self._orig_fp, 'mode', None) def write(self, data): # pylint: disable=invalid-name current_file_pos = self._orig_fp.tell() assert (self._start_byte <= current_file_pos and current_file_pos + len(data) <= self._end_byte + 1) text_util.write_to_fd(self._orig_fp, data) current_file_pos = self._orig_fp.tell() threshold = TRACKERFILE_UPDATE_THRESHOLD if (self._last_tracker_file_byte is None or current_file_pos - self._last_tracker_file_byte > threshold or current_file_pos == self._end_byte + 1): WriteDownloadComponentTrackerFile(self._tracker_file_name, self._src_obj_metadata, current_file_pos) self._last_tracker_file_byte = current_file_pos def seek(self, offset, whence=os.SEEK_SET): # pylint: disable=invalid-name if whence == os.SEEK_END: self._orig_fp.seek(offset + self._end_byte + 1) else: self._orig_fp.seek(offset, whence) assert self._start_byte <= self._orig_fp.tell() <= self._end_byte + 1 def tell(self): # pylint: disable=invalid-name return self._orig_fp.tell() def flush(self): # pylint: disable=invalid-name self._orig_fp.flush() def close(self): # pylint: disable=invalid-name if self._orig_fp: self._orig_fp.close() def _PartitionObject(src_url, src_obj_metadata, dst_url, download_file_name, decryption_key=None): """Partitions an object into components to be downloaded. Each component is a byte range of the object. The byte ranges of the returned components are mutually exclusive and collectively exhaustive. The byte ranges are inclusive at both end points. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object with non-pickleable fields removed. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for the download. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: components_to_download: A list of PerformSlicedDownloadObjectToFileArgs to be used in Apply for the sliced download. """ sliced_download_component_size = HumanReadableToBytes( config.get('GSUtil', 'sliced_object_download_component_size', DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE)) max_components = config.getint('GSUtil', 'sliced_object_download_max_components', DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS) num_components, component_size = _GetPartitionInfo( src_obj_metadata.size, max_components, sliced_download_component_size) components_to_download = [] component_lengths = [] for i in range(num_components): start_byte = i * component_size end_byte = min((i + 1) * (component_size) - 1, src_obj_metadata.size - 1) component_lengths.append(end_byte - start_byte + 1) # We need to serialize src_obj_metadata for pickling since it can # contain nested classes such as custom metadata. src_obj_metadata_json = protojson.encode_message(src_obj_metadata) components_to_download.append( PerformSlicedDownloadObjectToFileArgs(i, src_url, src_obj_metadata_json, dst_url, download_file_name, start_byte, end_byte, decryption_key)) return components_to_download, component_lengths def _DoSlicedDownload(src_url, src_obj_metadata, dst_url, download_file_name, command_obj, logger, copy_exception_handler, api_selector, decryption_key=None, status_queue=None): """Downloads a cloud object to a local file using sliced download. Byte ranges are decided for each thread/process, and then the parts are downloaded in parallel. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. command_obj: command object for use in Apply in parallel composite uploads. logger: for outputting log messages. copy_exception_handler: For handling copy exceptions during Apply. api_selector: The Cloud API implementation used. decryption_key: Base64-encoded decryption key for the source object, if any. status_queue: Queue for posting file messages for UI/Analytics. Returns: (bytes_transferred, crc32c) bytes_transferred: Number of bytes transferred from server this call. crc32c: a crc32c hash value (integer) for the downloaded bytes, or None if crc32c hashing wasn't performed. """ # CustomerEncryptionValue is a subclass and thus not pickleable for # multiprocessing, but at this point we already have the matching key, # so just discard the metadata. src_obj_metadata.customerEncryption = None components_to_download, component_lengths = _PartitionObject( src_url, src_obj_metadata, dst_url, download_file_name, decryption_key) num_components = len(components_to_download) _MaintainSlicedDownloadTrackerFiles(src_obj_metadata, dst_url, download_file_name, logger, api_selector, num_components) # Resize the download file so each child process can seek to its start byte. with open(download_file_name, 'r+b') as fp: fp.truncate(src_obj_metadata.size) # Assign a start FileMessage to each component for (i, component) in enumerate(components_to_download): size = component.end_byte - component.start_byte + 1 download_start_byte = GetDownloadStartByte(src_obj_metadata, dst_url, api_selector, component.start_byte, size, i) bytes_already_downloaded = download_start_byte - component.start_byte PutToQueueWithTimeout( status_queue, FileMessage(src_url, dst_url, time.time(), size=size, finished=False, component_num=i, message_type=FileMessage.COMPONENT_TO_DOWNLOAD, bytes_already_downloaded=bytes_already_downloaded)) cp_results = command_obj.Apply( _PerformSlicedDownloadObjectToFile, components_to_download, copy_exception_handler, arg_checker=gslib.command.DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SLICE, should_return_results=True) if len(cp_results) < num_components: raise CommandException( 'Some components of %s were not downloaded successfully. ' 'Please retry this download.' % dst_url.object_name) # Crc32c hashes have to be concatenated in the correct order. cp_results = sorted(cp_results, key=attrgetter('component_num')) crc32c = cp_results[0].crc32c if crc32c is not None: for i in range(1, num_components): crc32c = ConcatCrc32c(crc32c, cp_results[i].crc32c, component_lengths[i]) bytes_transferred = 0 expect_gzip = ObjectIsGzipEncoded(src_obj_metadata) # Assign an end FileMessage to each component for cp_result in cp_results: PutToQueueWithTimeout( status_queue, FileMessage(src_url, dst_url, time.time(), size=cp_result.component_total_size, finished=True, component_num=cp_result.component_num, message_type=FileMessage.COMPONENT_TO_DOWNLOAD)) bytes_transferred += cp_result.bytes_transferred server_gzip = (cp_result.server_encoding and cp_result.server_encoding.lower().endswith('gzip')) # If the server gzipped any components on the fly, we will have no chance of # properly reconstructing the file. if server_gzip and not expect_gzip: raise CommandException( 'Download of %s failed because the server sent back data with an ' 'unexpected encoding.' % dst_url.object_name) return bytes_transferred, crc32c def _DownloadObjectToFileResumable(src_url, src_obj_metadata, dst_url, download_file_name, gsutil_api, logger, digesters, component_num=None, start_byte=0, end_byte=None, decryption_key=None): """Downloads an object to a local file using the resumable strategy. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. gsutil_api: gsutil Cloud API instance to use for the download. logger: for outputting log messages. digesters: Digesters corresponding to the hash algorithms that will be used for validation. component_num: Which component of a sliced download this call is for, or None if this is not a sliced download. start_byte: The first byte of a byte range for a sliced download. end_byte: The last byte of a byte range for a sliced download. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (bytes_transferred, server_encoding) bytes_transferred: Number of bytes transferred from server this call. server_encoding: Content-encoding string if it was detected that the server sent encoded bytes during transfer, None otherwise. """ if end_byte is None: end_byte = src_obj_metadata.size - 1 download_size = end_byte - start_byte + 1 is_sliced = component_num is not None api_selector = gsutil_api.GetApiSelector(provider=src_url.scheme) server_encoding = None # Used for logging download_name = dst_url.object_name if is_sliced: download_name += ' component %d' % component_num fp = None try: fp = open(download_file_name, 'r+b') fp.seek(start_byte) api_selector = gsutil_api.GetApiSelector(provider=src_url.scheme) existing_file_size = GetFileSize(fp) tracker_file_name, download_start_byte = ReadOrCreateDownloadTrackerFile( src_obj_metadata, dst_url, logger, api_selector, start_byte, existing_file_size, component_num, ) if download_start_byte < start_byte or download_start_byte > end_byte + 1: DeleteTrackerFile(tracker_file_name) raise CommandException( 'Resumable download start point for %s is not in the correct byte ' 'range. Deleting tracker file, so if you re-try this download it ' 'will start from scratch' % download_name) download_complete = (download_start_byte == start_byte + download_size) resuming = (download_start_byte != start_byte) and not download_complete if resuming: logger.info('Resuming download for %s', download_name) elif download_complete: logger.info( 'Download already complete for %s, skipping download but ' 'will run integrity checks.', download_name) # This is used for resuming downloads, but also for passing the mediaLink # and size into the download for new downloads so that we can avoid # making an extra HTTP call. serialization_data = GetDownloadSerializationData( src_obj_metadata, progress=download_start_byte, user_project=gsutil_api.user_project) if resuming or download_complete: # Catch up our digester with the hash data. bytes_digested = 0 total_bytes_to_digest = download_start_byte - start_byte hash_callback = ProgressCallbackWithTimeout( total_bytes_to_digest, FileProgressCallbackHandler(gsutil_api.status_queue, component_num=component_num, src_url=src_url, dst_url=dst_url, operation_name='Hashing').call) while bytes_digested < total_bytes_to_digest: bytes_to_read = min(DEFAULT_FILE_BUFFER_SIZE, total_bytes_to_digest - bytes_digested) data = fp.read(bytes_to_read) bytes_digested += bytes_to_read for alg_name in digesters: digesters[alg_name].update(six.ensure_binary(data)) hash_callback.Progress(len(data)) elif not is_sliced: # Delete file contents and start entire object download from scratch. fp.truncate(0) existing_file_size = 0 progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, start_byte=start_byte, override_total_size=download_size, src_url=src_url, dst_url=dst_url, component_num=component_num, operation_name='Downloading').call if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call if is_sliced and src_obj_metadata.size >= ResumableThreshold(): fp = SlicedDownloadFileWrapper(fp, tracker_file_name, src_obj_metadata, start_byte, end_byte) compressed_encoding = ObjectIsGzipEncoded(src_obj_metadata) # TODO: With gzip encoding (which may occur on-the-fly and not be part of # the object's metadata), when we request a range to resume, it's possible # that the server will just resend the entire object, which means our # caught-up hash will be incorrect. We recalculate the hash on # the local file in the case of a failed gzip hash anyway, but it would # be better if we actively detected this case. if not download_complete: fp.seek(download_start_byte) server_encoding = gsutil_api.GetObjectMedia( src_url.bucket_name, src_url.object_name, fp, start_byte=download_start_byte, end_byte=end_byte, compressed_encoding=compressed_encoding, generation=src_url.generation, object_size=src_obj_metadata.size, download_strategy=CloudApi.DownloadStrategy.RESUMABLE, provider=src_url.scheme, serialization_data=serialization_data, digesters=digesters, progress_callback=progress_callback, decryption_tuple=CryptoKeyWrapperFromKey(decryption_key)) except ResumableDownloadException as e: logger.warning('Caught ResumableDownloadException (%s) for download of %s.', e.reason, download_name) raise finally: if fp: fp.close() bytes_transferred = end_byte - download_start_byte + 1 return bytes_transferred, server_encoding def _DownloadObjectToFileNonResumable(src_url, src_obj_metadata, dst_url, download_file_name, gsutil_api, digesters, decryption_key=None): """Downloads an object to a local file using the non-resumable strategy. This function does not support component transfers. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. gsutil_api: gsutil Cloud API instance to use for the download. digesters: Digesters corresponding to the hash algorithms that will be used for validation. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (bytes_transferred, server_encoding) bytes_transferred: Number of bytes transferred from server this call. server_encoding: Content-encoding string if it was detected that the server sent encoded bytes during transfer, None otherwise. """ fp = None try: fp = open(download_file_name, 'w') # This is used to pass the mediaLink and the size into the download so that # we can avoid making an extra HTTP call. serialization_data = GetDownloadSerializationData( src_obj_metadata, 0, user_project=gsutil_api.user_project) progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=src_url, dst_url=dst_url, operation_name='Downloading').call if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call server_encoding = gsutil_api.GetObjectMedia( src_url.bucket_name, src_url.object_name, fp, generation=src_url.generation, object_size=src_obj_metadata.size, download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, provider=src_url.scheme, serialization_data=serialization_data, digesters=digesters, progress_callback=progress_callback, decryption_tuple=CryptoKeyWrapperFromKey(decryption_key)) finally: if fp: fp.close() return src_obj_metadata.size, server_encoding def _DownloadObjectToFile(src_url, src_obj_metadata, dst_url, gsutil_api, logger, command_obj, copy_exception_handler, allow_splitting=True, decryption_key=None, is_rsync=False, preserve_posix=False, use_stet=False): """Downloads an object to a local file. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. gsutil_api: gsutil Cloud API instance to use for the download. logger: for outputting log messages. command_obj: command object for use in Apply in sliced downloads. copy_exception_handler: For handling copy exceptions during Apply. allow_splitting: Whether or not to allow sliced download. decryption_key: Base64-encoded decryption key for the source object, if any. is_rsync: Whether or not the caller is the rsync command. preserve_posix: Whether or not to preserve POSIX attributes. use_stet: Decrypt downloaded file with STET binary if available on system. Returns: (elapsed_time, bytes_transferred, dst_url, md5), where time elapsed excludes initial GET. Raises: FileConcurrencySkipError: if this download is already in progress. CommandException: if other errors encountered. """ global open_files_map, open_files_lock if dst_url.object_name.endswith(dst_url.delim): logger.warn('\n'.join( textwrap.wrap( 'Skipping attempt to download to filename ending with slash (%s). This ' 'typically happens when using gsutil to download from a subdirectory ' 'created by the Cloud Console (https://cloud.google.com/console)' % dst_url.object_name))) # The warning above is needed because errors might get ignored # for parallel processing. raise InvalidUrlError('Invalid destination path: %s' % dst_url.object_name) api_selector = gsutil_api.GetApiSelector(provider=src_url.scheme) download_strategy = _SelectDownloadStrategy(dst_url) sliced_download = _ShouldDoSlicedDownload(download_strategy, src_obj_metadata, allow_splitting, logger) download_file_name, need_to_unzip = _GetDownloadFile(dst_url, src_obj_metadata, logger) # Ensure another process/thread is not already writing to this file. with open_files_lock: if open_files_map.get(download_file_name, False): raise FileConcurrencySkipError open_files_map[download_file_name] = True # Set up hash digesters. consider_md5 = src_obj_metadata.md5Hash and not sliced_download hash_algs = GetDownloadHashAlgs(logger, consider_md5=consider_md5, consider_crc32c=src_obj_metadata.crc32c) digesters = dict((alg, hash_algs[alg]()) for alg in hash_algs or {}) # Tracks whether the server used a gzip encoding. server_encoding = None download_complete = (src_obj_metadata.size == 0) bytes_transferred = 0 start_time = time.time() if not download_complete: if sliced_download: (bytes_transferred, crc32c) = (_DoSlicedDownload(src_url, src_obj_metadata, dst_url, download_file_name, command_obj, logger, copy_exception_handler, api_selector, decryption_key=decryption_key, status_queue=gsutil_api.status_queue)) if 'crc32c' in digesters: digesters['crc32c'].crcValue = crc32c elif download_strategy is CloudApi.DownloadStrategy.ONE_SHOT: bytes_transferred, server_encoding = _DownloadObjectToFileNonResumable( src_url, src_obj_metadata, dst_url, download_file_name, gsutil_api, digesters, decryption_key=decryption_key, ) elif download_strategy is CloudApi.DownloadStrategy.RESUMABLE: bytes_transferred, server_encoding = _DownloadObjectToFileResumable( src_url, src_obj_metadata, dst_url, download_file_name, gsutil_api, logger, digesters, decryption_key=decryption_key, ) else: raise CommandException('Invalid download strategy %s chosen for' 'file %s' % (download_strategy, download_file_name)) end_time = time.time() server_gzip = server_encoding and server_encoding.lower().endswith('gzip') local_md5 = _ValidateAndCompleteDownload(logger, src_url, src_obj_metadata, dst_url, need_to_unzip, server_gzip, digesters, hash_algs, download_file_name, api_selector, bytes_transferred, gsutil_api, is_rsync=is_rsync, preserve_posix=preserve_posix, use_stet=use_stet) with open_files_lock: open_files_map.delete(download_file_name) PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, message_time=end_time, message_type=FileMessage.FILE_DOWNLOAD, size=src_obj_metadata.size, finished=True)) return (end_time - start_time, bytes_transferred, dst_url, local_md5) def _ValidateAndCompleteDownload(logger, src_url, src_obj_metadata, dst_url, need_to_unzip, server_gzip, digesters, hash_algs, temporary_file_name, api_selector, bytes_transferred, gsutil_api, is_rsync=False, preserve_posix=False, use_stet=False): """Validates and performs necessary operations on a downloaded file. Validates the integrity of the downloaded file using hash_algs. If the file was compressed (temporarily), the file will be decompressed. Then, if the integrity of the file was successfully validated, the file will be moved from its temporary download location to its permanent location on disk. Args: logger: For outputting log messages. src_url: StorageUrl for the source object. src_obj_metadata: Metadata for the source object, potentially containing hash values. dst_url: StorageUrl describing the destination file. need_to_unzip: If true, a temporary zip file was used and must be uncompressed as part of validation. server_gzip: If true, the server gzipped the bytes (regardless of whether the object metadata claimed it was gzipped). digesters: dict of {string, hash digester} that contains up-to-date digests computed during the download. If a digester for a particular algorithm is None, an up-to-date digest is not available and the hash must be recomputed from the local file. hash_algs: dict of {string, hash algorithm} that can be used if digesters don't have up-to-date digests. temporary_file_name: Temporary file name that was used for download. api_selector: The Cloud API implementation used (used tracker file naming). bytes_transferred: Number of bytes downloaded (used for logging). gsutil_api: Cloud API to use for service and status. is_rsync: Whether or not the caller is the rsync function. Used to determine if timeCreated should be used. preserve_posix: Whether or not to preserve the posix attributes. use_stet: If True, attempt to decrypt downloaded files with the STET binary if it's present on the system. Returns: An MD5 of the local file, if one was calculated as part of the integrity check. """ final_file_name = dst_url.object_name digesters_succeeded = True for alg in digesters: # If we get a digester with a None algorithm, the underlying # implementation failed to calculate a digest, so we will need to # calculate one from scratch. if not digesters[alg]: digesters_succeeded = False break if digesters_succeeded: local_hashes = _CreateDigestsFromDigesters(digesters) else: local_hashes = _CreateDigestsFromLocalFile(gsutil_api.status_queue, hash_algs, temporary_file_name, src_url, src_obj_metadata) digest_verified = True hash_invalid_exception = None try: _CheckHashes(logger, src_url, src_obj_metadata, final_file_name, local_hashes) DeleteDownloadTrackerFiles(dst_url, api_selector) except HashMismatchException as e: # If an non-gzipped object gets sent with gzip content encoding, the hash # we calculate will match the gzipped bytes, not the original object. Thus, # we'll need to calculate and check it after unzipping. if server_gzip: logger.debug('Hash did not match but server gzipped the content, will ' 'recalculate.') digest_verified = False elif api_selector == ApiSelector.XML: logger.debug( 'Hash did not match but server may have gzipped the content, will ' 'recalculate.') # Save off the exception in case this isn't a gzipped file. hash_invalid_exception = e digest_verified = False else: DeleteDownloadTrackerFiles(dst_url, api_selector) if _RENAME_ON_HASH_MISMATCH: os.rename(temporary_file_name, final_file_name + _RENAME_ON_HASH_MISMATCH_SUFFIX) else: os.unlink(temporary_file_name) raise if not (need_to_unzip or server_gzip): unzipped_temporary_file_name = temporary_file_name else: # This will not result in the same string as temporary_file_name b/c # GetTempFileName returns ".gstmp" and gzipped temp files have ".gztmp". unzipped_temporary_file_name = temporary_file_util.GetTempFileName(dst_url) # Log that we're uncompressing if the file is big enough that # decompressing would make it look like the transfer "stalled" at the end. if bytes_transferred > TEN_MIB: logger.info('Uncompressing temporarily gzipped file to %s...', final_file_name) gzip_fp = None try: # Downloaded temporarily gzipped file, unzip to file without '_.gztmp' # suffix. gzip_fp = gzip.open(temporary_file_name, 'rb') with open(unzipped_temporary_file_name, 'wb') as f_out: data = gzip_fp.read(GZIP_CHUNK_SIZE) while data: f_out.write(data) data = gzip_fp.read(GZIP_CHUNK_SIZE) except IOError as e: # In the XML case where we don't know if the file was gzipped, raise # the original hash exception if we find that it wasn't. if 'Not a gzipped file' in str(e) and hash_invalid_exception: # Linter improperly thinks we're raising None despite the above check. # pylint: disable=raising-bad-type raise hash_invalid_exception finally: if gzip_fp: gzip_fp.close() os.unlink(temporary_file_name) if not digest_verified: try: # Recalculate hashes on the unzipped local file. local_hashes = _CreateDigestsFromLocalFile(gsutil_api.status_queue, hash_algs, unzipped_temporary_file_name, src_url, src_obj_metadata) _CheckHashes(logger, src_url, src_obj_metadata, final_file_name, local_hashes) DeleteDownloadTrackerFiles(dst_url, api_selector) except HashMismatchException: DeleteDownloadTrackerFiles(dst_url, api_selector) if _RENAME_ON_HASH_MISMATCH: os.rename( unzipped_temporary_file_name, unzipped_temporary_file_name + _RENAME_ON_HASH_MISMATCH_SUFFIX) else: os.unlink(unzipped_temporary_file_name) raise if use_stet: # Decrypt data using STET binary. stet_util.decrypt_download(src_url, dst_url, unzipped_temporary_file_name, logger) os.rename(unzipped_temporary_file_name, final_file_name) ParseAndSetPOSIXAttributes(final_file_name, src_obj_metadata, is_rsync=is_rsync, preserve_posix=preserve_posix) if 'md5' in local_hashes: return local_hashes['md5'] def _CopyFileToFile(src_url, dst_url, status_queue=None, src_obj_metadata=None): """Copies a local file to a local file. Args: src_url: Source FileUrl. dst_url: Destination FileUrl. status_queue: Queue for posting file messages for UI/Analytics. src_obj_metadata: An apitools Object that may contain file size, or None. Returns: (elapsed_time, bytes_transferred, dst_url, md5=None). Raises: CommandException: if errors encountered. """ src_fp = GetStreamFromFileUrl(src_url) dir_name = os.path.dirname(dst_url.object_name) if dir_name: try: os.makedirs(dir_name) except OSError as e: if e.errno != errno.EEXIST: raise with open(dst_url.object_name, 'wb') as dst_fp: start_time = time.time() shutil.copyfileobj(src_fp, dst_fp) if not src_url.IsStream(): src_fp.close() # Explicitly close the src fp - necessary if it is a fifo. end_time = time.time() PutToQueueWithTimeout( status_queue, FileMessage(src_url, dst_url, end_time, message_type=FileMessage.FILE_LOCAL_COPY, size=src_obj_metadata.size if src_obj_metadata else None, finished=True)) return (end_time - start_time, os.path.getsize(dst_url.object_name), dst_url, None) def _DummyTrackerCallback(_): pass # pylint: disable=undefined-variable def _CopyObjToObjDaisyChainMode(src_url, src_obj_metadata, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, decryption_key=None): """Copies from src_url to dst_url in "daisy chain" mode. See -D OPTION documentation about what daisy chain mode is. Args: src_url: Source CloudUrl src_obj_metadata: Metadata from source object dst_url: Destination CloudUrl dst_obj_metadata: Object-specific metadata that should be overidden during the copy. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API to use for the copy. logger: For outputting log messages. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. """ # We don't attempt to preserve ACLs across providers because # GCS and S3 support different ACLs and disjoint principals. if (global_copy_helper_opts.preserve_acl and src_url.scheme != dst_url.scheme): raise NotImplementedError('Cross-provider cp -p not supported') if not global_copy_helper_opts.preserve_acl: dst_obj_metadata.acl = [] # Don't use callbacks for downloads on the daisy chain wrapper because # upload callbacks will output progress, but respect test hooks if present. progress_callback = None if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call compressed_encoding = ObjectIsGzipEncoded(src_obj_metadata) encryption_keywrapper = GetEncryptionKeyWrapper(config) start_time = time.time() upload_fp = DaisyChainWrapper(src_url, src_obj_metadata.size, gsutil_api, compressed_encoding=compressed_encoding, progress_callback=progress_callback, decryption_key=decryption_key) uploaded_object = None if src_obj_metadata.size == 0: # Resumable uploads of size 0 are not supported. uploaded_object = gsutil_api.UploadObject( upload_fp, object_metadata=dst_obj_metadata, canned_acl=global_copy_helper_opts.canned_acl, preconditions=preconditions, provider=dst_url.scheme, fields=UPLOAD_RETURN_FIELDS, size=src_obj_metadata.size, encryption_tuple=encryption_keywrapper) else: # TODO: Support process-break resumes. This will resume across connection # breaks and server errors, but the tracker callback is a no-op so this # won't resume across gsutil runs. # TODO: Test retries via test_callback_file. uploaded_object = gsutil_api.UploadObjectResumable( upload_fp, object_metadata=dst_obj_metadata, canned_acl=global_copy_helper_opts.canned_acl, preconditions=preconditions, provider=dst_url.scheme, fields=UPLOAD_RETURN_FIELDS, size=src_obj_metadata.size, progress_callback=FileProgressCallbackHandler( gsutil_api.status_queue, src_url=src_url, dst_url=dst_url, operation_name='Uploading').call, tracker_callback=_DummyTrackerCallback, encryption_tuple=encryption_keywrapper) end_time = time.time() try: _CheckCloudHashes(logger, src_url, dst_url, src_obj_metadata, uploaded_object) except HashMismatchException: if _RENAME_ON_HASH_MISMATCH: corrupted_obj_metadata = apitools_messages.Object( name=dst_obj_metadata.name, bucket=dst_obj_metadata.bucket, etag=uploaded_object.etag) dst_obj_metadata.name = (dst_url.object_name + _RENAME_ON_HASH_MISMATCH_SUFFIX) decryption_keywrapper = CryptoKeyWrapperFromKey(decryption_key) gsutil_api.CopyObject(corrupted_obj_metadata, dst_obj_metadata, provider=dst_url.scheme, decryption_tuple=decryption_keywrapper, encryption_tuple=encryption_keywrapper) # If the digest doesn't match, delete the object. gsutil_api.DeleteObject(dst_url.bucket_name, dst_url.object_name, generation=uploaded_object.generation, provider=dst_url.scheme) raise result_url = dst_url.Clone() result_url.generation = GenerationFromUrlAndString(result_url, uploaded_object.generation) PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, end_time, message_type=FileMessage.FILE_DAISY_COPY, size=src_obj_metadata.size, finished=True)) return (end_time - start_time, src_obj_metadata.size, result_url, uploaded_object.md5Hash) def GetSourceFieldsNeededForCopy(dst_is_cloud, skip_unsupported_objects, preserve_acl, is_rsync=False, preserve_posix=False, delete_source=False, file_size_will_change=False): """Determines the metadata fields needed for a copy operation. This function returns the fields we will need to successfully copy any cloud objects that might be iterated. By determining this prior to iteration, the cp command can request this metadata directly from the iterator's get/list calls, avoiding the need for a separate get metadata HTTP call for each iterated result. As a trade-off, filtering objects at the leaf nodes of the iteration (based on a remaining wildcard) is more expensive. This is because more metadata will be requested when object name is all that is required for filtering. The rsync command favors fast listing and comparison, and makes the opposite trade-off, optimizing for the low-delta case by making per-object get metadata HTTP call so that listing can return minimal metadata. It uses this function to determine what is needed for get metadata HTTP calls. Args: dst_is_cloud: if true, destination is a Cloud URL. skip_unsupported_objects: if true, get metadata for skipping unsupported object types. preserve_acl: if true, get object ACL. is_rsync: if true, the calling function is rsync. Determines if metadata is needed to verify download. preserve_posix: if true, retrieves POSIX attributes into user metadata. delete_source: if true, source object will be deleted after the copy (mv command). file_size_will_change: if true, do not try to record file size. Returns: List of necessary field metadata field names. """ src_obj_fields_set = set() if dst_is_cloud: # For cloud or daisy chain copy, we need every copyable field. # If we're not modifying or overriding any of the fields, we can get # away without retrieving the object metadata because the copy # operation can succeed with just the destination bucket and object # name. But if we are sending any metadata, the JSON API will expect a # complete object resource. Since we want metadata like the object size for # our own tracking, we just get all of the metadata here. src_obj_fields_set.update([ 'cacheControl', 'componentCount', 'contentDisposition', 'contentEncoding', 'contentLanguage', 'contentType', 'crc32c', 'customerEncryption', 'etag', 'generation', 'md5Hash', 'mediaLink', 'metadata', 'metageneration', 'storageClass', 'timeCreated', ]) # We only need the ACL if we're going to preserve it. if preserve_acl: src_obj_fields_set.update(['acl']) if not file_size_will_change: src_obj_fields_set.update(['size']) else: # Just get the fields needed to perform and validate the download. src_obj_fields_set.update([ 'crc32c', 'contentEncoding', 'contentType', 'customerEncryption', 'etag', 'mediaLink', 'md5Hash', 'size', 'generation', ]) if is_rsync: src_obj_fields_set.update(['metadata/%s' % MTIME_ATTR, 'timeCreated']) if preserve_posix: posix_fields = [ 'metadata/%s' % ATIME_ATTR, 'metadata/%s' % MTIME_ATTR, 'metadata/%s' % GID_ATTR, 'metadata/%s' % MODE_ATTR, 'metadata/%s' % UID_ATTR, ] src_obj_fields_set.update(posix_fields) if delete_source: src_obj_fields_set.update([ 'storageClass', 'timeCreated', ]) if skip_unsupported_objects: src_obj_fields_set.update(['storageClass']) return list(src_obj_fields_set) # Map of (lowercase) storage classes with early deletion charges to their # minimum lifetime in seconds. EARLY_DELETION_MINIMUM_LIFETIME = { 'nearline': 30 * SECONDS_PER_DAY, 'coldline': 90 * SECONDS_PER_DAY, 'archive': 365 * SECONDS_PER_DAY } def WarnIfMvEarlyDeletionChargeApplies(src_url, src_obj_metadata, logger): """Warns when deleting a gs:// object could incur an early deletion charge. This function inspects metadata for Google Cloud Storage objects that are subject to early deletion charges (such as Nearline), and warns when performing operations like mv that would delete them. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with necessary fields (per GetSourceFieldsNeededForCopy). logger: logging.Logger for outputting warning. """ if (src_url.scheme == 'gs' and src_obj_metadata and src_obj_metadata.timeCreated and src_obj_metadata.storageClass): object_storage_class = src_obj_metadata.storageClass.lower() early_deletion_cutoff_seconds = EARLY_DELETION_MINIMUM_LIFETIME.get( object_storage_class, None) if early_deletion_cutoff_seconds: minimum_delete_age = ( early_deletion_cutoff_seconds + ConvertDatetimeToPOSIX(src_obj_metadata.timeCreated)) if time.time() < minimum_delete_age: logger.warn( 'Warning: moving %s object %s may incur an early deletion ' 'charge, because the original object is less than %s ' 'days old according to the local system time.', object_storage_class, src_url.url_string, early_deletion_cutoff_seconds // SECONDS_PER_DAY) def MaybeSkipUnsupportedObject(src_url, src_obj_metadata): """Skips unsupported object types if requested. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with storageClass field (per GetSourceFieldsNeededForCopy). Raises: SkipGlacierError: if skipping a s3 Glacier object. """ if (src_url.scheme == 's3' and global_copy_helper_opts.skip_unsupported_objects and src_obj_metadata.storageClass == 'GLACIER'): raise SkipGlacierError() def GetDecryptionCSEK(src_url, src_obj_metadata): """Ensures a matching decryption key is available for the source object. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with optional customerEncryption field. Raises: EncryptionException if the object is encrypted and no matching key is found. Returns: Base64-encoded decryption key string if the object is encrypted and a matching key is found, or None if object is not encrypted. """ if src_obj_metadata.customerEncryption: decryption_key = FindMatchingCSEKInBotoConfig( src_obj_metadata.customerEncryption.keySha256, config) if not decryption_key: raise EncryptionException( 'Missing decryption key with SHA256 hash %s. No decryption key ' 'matches object %s' % (src_obj_metadata.customerEncryption.keySha256, src_url)) return decryption_key # pylint: disable=undefined-variable # pylint: disable=too-many-statements def PerformCopy(logger, src_url, dst_url, gsutil_api, command_obj, copy_exception_handler, src_obj_metadata=None, allow_splitting=True, headers=None, manifest=None, gzip_exts=None, is_rsync=False, preserve_posix=False, gzip_encoded=False, use_stet=False): """Performs copy from src_url to dst_url, handling various special cases. Args: logger: for outputting log messages. src_url: Source StorageUrl. dst_url: Destination StorageUrl. gsutil_api: gsutil Cloud API instance to use for the copy. command_obj: command object for use in Apply in parallel composite uploads and sliced object downloads. copy_exception_handler: for handling copy exceptions during Apply. src_obj_metadata: If source URL is a cloud object, source object metadata with all necessary fields (per GetSourceFieldsNeededForCopy). Required for cloud source URLs. If source URL is a file, an apitools Object that may contain file size, or None. allow_splitting: Whether to allow the file to be split into component pieces for an parallel composite upload or download. headers: optional headers to use for the copy operation. manifest: optional manifest for tracking copy operations. gzip_exts: List of file extensions to gzip, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. is_rsync: Whether or not the caller is the rsync command. preserve_posix: Whether or not to preserve posix attributes. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts. Streaming files compressed is only supported on the JSON GCS API. use_stet: If True, will perform STET encryption or decryption using the binary specified in the boto config or PATH. Returns: (elapsed_time, bytes_transferred, version-specific dst_url) excluding overhead like initial GET. Raises: ItemExistsError: if no clobber flag is specified and the destination object already exists. SkipUnsupportedObjectError: if skip_unsupported_objects flag is specified and the source is an unsupported type. CommandException: if other errors encountered. """ # TODO: Remove elapsed_time as it is currently unused by all callers. if headers: dst_obj_headers = headers.copy() else: dst_obj_headers = {} # Create a metadata instance for each destination object so metadata # such as content-type can be applied per-object. # Initialize metadata from any headers passed in via -h. dst_obj_metadata = ObjectMetadataFromHeaders(dst_obj_headers) if dst_url.IsCloudUrl() and dst_url.scheme == 'gs': preconditions = PreconditionsFromHeaders(dst_obj_headers) else: preconditions = Preconditions() src_obj_filestream = None decryption_key = None copy_in_the_cloud = False if src_url.IsCloudUrl(): if (dst_url.IsCloudUrl() and src_url.scheme == dst_url.scheme and not global_copy_helper_opts.daisy_chain): copy_in_the_cloud = True if global_copy_helper_opts.perform_mv: WarnIfMvEarlyDeletionChargeApplies(src_url, src_obj_metadata, logger) MaybeSkipUnsupportedObject(src_url, src_obj_metadata) decryption_key = GetDecryptionCSEK(src_url, src_obj_metadata) src_obj_size = src_obj_metadata.size dst_obj_metadata.contentType = src_obj_metadata.contentType if global_copy_helper_opts.preserve_acl and dst_url.IsCloudUrl(): if src_url.scheme == 'gs' and not src_obj_metadata.acl: raise CommandException( 'No OWNER permission found for object %s. OWNER permission is ' 'required for preserving ACLs.' % src_url) dst_obj_metadata.acl = src_obj_metadata.acl # Special case for S3-to-S3 copy URLs using # global_copy_helper_opts.preserve_acl. # dst_url will be verified in _CopyObjToObjDaisyChainMode if it # is not s3 (and thus differs from src_url). if src_url.scheme == 's3': acl_text = S3MarkerAclFromObjectMetadata(src_obj_metadata) if acl_text: AddS3MarkerAclToObjectMetadata(dst_obj_metadata, acl_text) else: # src_url.IsFileUrl() if use_stet: source_stream_url = stet_util.encrypt_upload(src_url, dst_url, logger) else: source_stream_url = src_url try: src_obj_filestream = GetStreamFromFileUrl(source_stream_url) except Exception as e: # pylint: disable=broad-except message = 'Error opening file "%s": %s.' % (src_url, str(e)) if command_obj.continue_on_error: command_obj.op_failure_count += 1 logger.error(message) return else: raise CommandException(message) if src_url.IsStream() or src_url.IsFifo(): src_obj_size = None elif src_obj_metadata and src_obj_metadata.size and not use_stet: # Iterator retrieved the file's size, no need to stat it again. # Unless STET changed the file size. src_obj_size = src_obj_metadata.size else: src_obj_size = os.path.getsize(source_stream_url.object_name) if global_copy_helper_opts.use_manifest: # Set the source size in the manifest. manifest.Set(src_url.url_string, 'size', src_obj_size) if (dst_url.scheme == 's3' and src_url != 's3' and src_obj_size is not None and # Can't compare int to None in py3 src_obj_size > S3_MAX_UPLOAD_SIZE): raise CommandException( '"%s" exceeds the maximum gsutil-supported size for an S3 upload. S3 ' 'objects greater than %s in size require multipart uploads, which ' 'gsutil does not support.' % (src_url, MakeHumanReadable(S3_MAX_UPLOAD_SIZE))) # On Windows, stdin is opened as text mode instead of binary which causes # problems when piping a binary file, so this switches it to binary mode. if IS_WINDOWS and src_url.IsFileUrl() and src_url.IsStream(): msvcrt.setmode(GetStreamFromFileUrl(src_url).fileno(), os.O_BINARY) if global_copy_helper_opts.no_clobber: # There are two checks to prevent clobbering: # 1) The first check is to see if the URL # already exists at the destination and prevent the upload/download # from happening. This is done by the exists() call. # 2) The second check is only relevant if we are writing to gs. We can # enforce that the server only writes the object if it doesn't exist # by specifying the header below. This check only happens at the # server after the complete file has been uploaded. We specify this # header to prevent a race condition where a destination file may # be created after the first check and before the file is fully # uploaded. # In order to save on unnecessary uploads/downloads we perform both # checks. However, this may come at the cost of additional HTTP calls. if preconditions.gen_match: raise ArgumentException('Specifying x-goog-if-generation-match is ' 'not supported with cp -n') else: preconditions.gen_match = 0 if dst_url.IsFileUrl() and os.path.exists(dst_url.object_name): raise ItemExistsError() elif dst_url.IsCloudUrl(): try: dst_object = gsutil_api.GetObjectMetadata(dst_url.bucket_name, dst_url.object_name, provider=dst_url.scheme) except NotFoundException: dst_object = None if dst_object: raise ItemExistsError() if dst_url.IsCloudUrl(): # Cloud storage API gets object and bucket name from metadata. dst_obj_metadata.name = dst_url.object_name dst_obj_metadata.bucket = dst_url.bucket_name if src_url.IsCloudUrl(): # Preserve relevant metadata from the source object if it's not already # provided from the headers. src_obj_metadata.name = src_url.object_name src_obj_metadata.bucket = src_url.bucket_name else: _SetContentTypeFromFile(src_url, dst_obj_metadata) # Only set KMS key name if destination provider is 'gs'. encryption_keywrapper = GetEncryptionKeyWrapper(config) if (encryption_keywrapper and encryption_keywrapper.crypto_type == CryptoKeyType.CMEK and dst_url.scheme == 'gs'): dst_obj_metadata.kmsKeyName = encryption_keywrapper.crypto_key if src_obj_metadata: # Note that CopyObjectMetadata only copies specific fields. We intentionally # do not copy storageClass, as the bucket's default storage class should be # used (when copying to a gs:// bucket) unless explicitly overridden. CopyObjectMetadata(src_obj_metadata, dst_obj_metadata, override=False) if global_copy_helper_opts.dest_storage_class: dst_obj_metadata.storageClass = global_copy_helper_opts.dest_storage_class if config.get('GSUtil', 'check_hashes') == CHECK_HASH_NEVER: # GCS server will perform MD5 validation if the md5 hash is present. # Remove md5_hash if check_hashes=never. dst_obj_metadata.md5Hash = None _LogCopyOperation(logger, src_url, dst_url, dst_obj_metadata) if src_url.IsCloudUrl(): if dst_url.IsFileUrl(): PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, time.time(), message_type=FileMessage.FILE_DOWNLOAD, size=src_obj_size, finished=False)) return _DownloadObjectToFile(src_url, src_obj_metadata, dst_url, gsutil_api, logger, command_obj, copy_exception_handler, allow_splitting=allow_splitting, decryption_key=decryption_key, is_rsync=is_rsync, preserve_posix=preserve_posix, use_stet=use_stet) elif copy_in_the_cloud: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, time.time(), message_type=FileMessage.FILE_CLOUD_COPY, size=src_obj_size, finished=False)) return _CopyObjToObjInTheCloud(src_url, src_obj_metadata, dst_url, dst_obj_metadata, preconditions, gsutil_api, decryption_key=decryption_key) else: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, time.time(), message_type=FileMessage.FILE_DAISY_COPY, size=src_obj_size, finished=False)) return _CopyObjToObjDaisyChainMode(src_url, src_obj_metadata, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, decryption_key=decryption_key) else: # src_url.IsFileUrl() if dst_url.IsCloudUrl(): # The FileMessage for this upload object is inside _UploadFileToObject(). # This is such because the function may alter src_url, which would prevent # us from correctly tracking the new url. uploaded_metadata = _UploadFileToObject(src_url, src_obj_filestream, src_obj_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, command_obj, copy_exception_handler, gzip_exts=gzip_exts, allow_splitting=allow_splitting, gzip_encoded=gzip_encoded) if use_stet: # Delete temporary file. os.unlink(src_obj_filestream.name) return uploaded_metadata else: # dst_url.IsFileUrl() PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, time.time(), message_type=FileMessage.FILE_LOCAL_COPY, size=src_obj_size, finished=False)) result = _CopyFileToFile(src_url, dst_url, status_queue=gsutil_api.status_queue, src_obj_metadata=src_obj_metadata) # Need to let _CopyFileToFile return before setting the POSIX attributes. if not src_url.IsStream() and not dst_url.IsStream(): ParseAndSetPOSIXAttributes(dst_url.object_name, src_obj_metadata, is_rsync=is_rsync, preserve_posix=preserve_posix) return result class Manifest(object): """Stores the manifest items for the CpCommand class.""" def __init__(self, path): # self.items contains a dictionary of rows self.items = {} self.manifest_filter = {} self.lock = parallelism_framework_util.CreateLock() self.manifest_path = os.path.expanduser(path) self._ParseManifest() self._CreateManifestFile() def _ParseManifest(self): """Load and parse a manifest file. This information will be used to skip any files that have a skip or OK status. """ try: if os.path.exists(self.manifest_path): # Note: we can't use io.open here or CSV reader will become upset # https://stackoverflow.com/a/18449496 with open(self.manifest_path, 'r') as f: first_row = True reader = csv.reader(f) for row in reader: if first_row: try: source_index = row.index('Source') result_index = row.index('Result') except ValueError: # No header and thus not a valid manifest file. raise CommandException('Missing headers in manifest file: %s' % self.manifest_path) first_row = False source = row[source_index] result = row[result_index] if result in ['OK', 'skip']: # We're always guaranteed to take the last result of a specific # source url. self.manifest_filter[source] = result except IOError: raise CommandException('Could not parse %s' % self.manifest_path) def WasSuccessful(self, src): """Returns whether the specified src url was marked as successful.""" return src in self.manifest_filter def _CreateManifestFile(self): """Opens the manifest file and assigns it to the file pointer.""" try: if ((not os.path.exists(self.manifest_path)) or (os.stat(self.manifest_path).st_size == 0)): # Add headers to the new file. if six.PY3: with open(self.manifest_path, 'w', newline='') as f: writer = csv.writer(f) writer.writerow([ 'Source', 'Destination', 'Start', 'End', 'Md5', 'UploadId', 'Source Size', 'Bytes Transferred', 'Result', 'Description', ]) else: with open(self.manifest_path, 'wb', 1) as f: writer = csv.writer(f) writer.writerow([ 'Source', 'Destination', 'Start', 'End', 'Md5', 'UploadId', 'Source Size', 'Bytes Transferred', 'Result', 'Description', ]) except IOError: raise CommandException('Could not create manifest file.') def Set(self, url, key, value): if value is None: # In case we don't have any information to set we bail out here. # This is so that we don't clobber existing information. # To zero information pass '' instead of None. return if url in self.items: self.items[url][key] = value else: self.items[url] = {key: value} def Initialize(self, source_url, destination_url): # Always use the source_url as the key for the item. This is unique. self.Set(source_url, 'source_uri', source_url) self.Set(source_url, 'destination_uri', destination_url) self.Set(source_url, 'start_time', datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None)) def SetResult(self, source_url, bytes_transferred, result, description=''): self.Set(source_url, 'bytes', bytes_transferred) self.Set(source_url, 'result', result) self.Set(source_url, 'description', description) self.Set(source_url, 'end_time', datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None)) self._WriteRowToManifestFile(source_url) self._RemoveItemFromManifest(source_url) def _WriteRowToManifestFile(self, url): """Writes a manifest entry to the manifest file for the url argument.""" row_item = self.items[url] data = [ row_item['source_uri'], row_item['destination_uri'], '%sZ' % row_item['start_time'].isoformat(), '%sZ' % row_item['end_time'].isoformat(), row_item['md5'] if 'md5' in row_item else '', row_item['upload_id'] if 'upload_id' in row_item else '', str(row_item['size']) if 'size' in row_item else '', str(row_item['bytes']) if 'bytes' in row_item else '', row_item['result'], row_item['description'], ] data = [six.ensure_str(value) for value in data] # Aquire a lock to prevent multiple threads writing to the same file at # the same time. This would cause a garbled mess in the manifest file. with self.lock: if IS_WINDOWS and six.PY3: f = open(self.manifest_path, 'a', 1, newline='') else: f = open(self.manifest_path, 'a', 1) # 1 == line buffered writer = csv.writer(f) writer.writerow(data) f.close() def _RemoveItemFromManifest(self, url): # Remove the item from the dictionary since we're done with it and # we don't want the dictionary to grow too large in memory for no good # reason. del self.items[url] class ItemExistsError(Exception): """Exception class for objects that are skipped because they already exist.""" pass class SkipUnsupportedObjectError(Exception): """Exception for objects skipped because they are an unsupported type.""" def __init__(self): super(SkipUnsupportedObjectError, self).__init__() self.unsupported_type = 'Unknown' class SkipGlacierError(SkipUnsupportedObjectError): """Exception for objects skipped because they are an unsupported type.""" def __init__(self): super(SkipGlacierError, self).__init__() self.unsupported_type = 'GLACIER' def GetPathBeforeFinalDir(url, exp_src_url): """Returns the path section before the final directory component of the URL. This handles cases for file system directories, bucket, and bucket subdirectories. Example: for gs://bucket/dir/ we'll return 'gs://bucket', and for file://dir we'll return file:// Args: url: StorageUrl representing a filesystem directory, cloud bucket or bucket subdir. exp_src_url: StorageUrl representing the fully expanded object to-be-copied; used for resolving cloud wildcards. Returns: String name of above-described path, sans final path separator. """ sep = url.delim if url.IsFileUrl(): past_scheme = url.url_string[len('file://'):] if past_scheme.find(sep) == -1: return 'file://' else: return 'file://%s' % past_scheme.rstrip(sep).rpartition(sep)[0] if url.IsBucket(): return '%s://' % url.scheme # Else it names a bucket subdir. path_sans_final_dir = url.url_string.rstrip(sep).rpartition(sep)[0] return ResolveWildcardsInPathBeforeFinalDir(path_sans_final_dir, exp_src_url) def ResolveWildcardsInPathBeforeFinalDir(src_url_path_sans_final_dir, exp_src_url): """Returns the path section for a bucket subdir with wildcards resolved. This handles cases for bucket subdirectories where the initial source URL contains a wildcard. In this case, src_url must be wildcard-expanded before calculating the final directory. Example: A bucket containing: gs://bucket/dir1/subdir/foo gs://bucket/dir2/subdir/foo and source URL gs://bucket/*/subdir and src_url_path_sans_final dir gs://bucket/* should yield final path gs://bucket/dir1 or gs://bucket/dir2 according to the expanded source URL. Args: src_url_path_sans_final_dir: URL string with wildcards representing a bucket subdir as computed from GetPathBeforeFinalDir. exp_src_url: CloudUrl representing the fully expanded object to-be-copied. Returns: String name of above-described path, sans final path separator. """ if not ContainsWildcard(src_url_path_sans_final_dir): return src_url_path_sans_final_dir # Parse the expanded source URL, replacing wildcarded # portions of the path with what they actually expanded to. wildcarded_src_obj_path = StorageUrlFromString( src_url_path_sans_final_dir).object_name.split('/') expanded_src_obj_path = exp_src_url.object_name.split('/') for path_segment_index in range(len(wildcarded_src_obj_path)): if ContainsWildcard(wildcarded_src_obj_path[path_segment_index]): # The expanded path is guaranteed to be have at least as many path # segments as the wildcarded path. wildcarded_src_obj_path[path_segment_index] = ( expanded_src_obj_path[path_segment_index]) resolved_src_path = '/'.join(wildcarded_src_obj_path) final_path_url = exp_src_url.Clone() final_path_url.object_name = resolved_src_path return final_path_url.url_string def _GetPartitionInfo(file_size, max_components, default_component_size): """Gets info about a file partition for parallel file/object transfers. Args: file_size: The number of bytes in the file to be partitioned. max_components: The maximum number of components that can be composed. default_component_size: The size of a component, assuming that max_components is infinite. Returns: The number of components in the partitioned file, and the size of each component (except the last, which will have a different size iff file_size != 0 (mod num_components)). """ # num_components = ceil(file_size / default_component_size) num_components = DivideAndCeil(file_size, default_component_size) # num_components must be in the range [2, max_components] num_components = max(min(num_components, max_components), 2) # component_size = ceil(file_size / num_components) component_size = DivideAndCeil(file_size, num_components) return (num_components, component_size) def _DeleteTempComponentObjectFn(cls, url_to_delete, thread_state=None): """Wrapper func to be used with command.Apply to delete temporary objects.""" gsutil_api = GetCloudApiInstance(cls, thread_state) try: gsutil_api.DeleteObject(url_to_delete.bucket_name, url_to_delete.object_name, generation=url_to_delete.generation, provider=url_to_delete.scheme) except NotFoundException: # The temporary object could already be gone if a retry was # issued at a lower layer but the original request succeeded. # Barring other errors, the top-level command should still report success, # so don't raise here. pass def FilterExistingComponents(dst_args, existing_components, bucket_url, gsutil_api): """Determines course of action for component objects. Given the list of all target objects based on partitioning the file and the list of objects that have already been uploaded successfully, this function determines which objects should be uploaded, which existing components are still valid, and which existing components should be deleted. Args: dst_args: The map of file_name -> PerformParallelUploadFileToObjectArgs calculated by partitioning the file. existing_components: A list of ObjectFromTracker objects that have been uploaded in the past. bucket_url: CloudUrl of the bucket in which the components exist. gsutil_api: gsutil Cloud API instance to use for retrieving object metadata. Returns: components_to_upload: List of components that need to be uploaded. uploaded_components: List of components that have already been uploaded and are still valid. Each element of the list contains the dst_url for the uploaded component and its size. existing_objects_to_delete: List of components that have already been uploaded, but are no longer valid and are in a versioned bucket, and therefore should be deleted. """ components_to_upload = [] existing_component_names = [ component.object_name for component in existing_components ] for component_name in dst_args: if component_name not in existing_component_names: components_to_upload.append(dst_args[component_name]) objects_already_chosen = [] # Don't reuse any temporary components whose MD5 doesn't match the current # MD5 of the corresponding part of the file. If the bucket is versioned, # also make sure that we delete the existing temporary version. existing_objects_to_delete = [] uploaded_components = [] for tracker_object in existing_components: if (tracker_object.object_name not in dst_args.keys() or tracker_object.object_name in objects_already_chosen): # This could happen if the component size has changed. This also serves # to handle object names that get duplicated in the tracker file due # to people doing things they shouldn't (e.g., overwriting an existing # temporary component in a versioned bucket). url = bucket_url.Clone() url.object_name = tracker_object.object_name url.generation = tracker_object.generation existing_objects_to_delete.append(url) continue dst_arg = dst_args[tracker_object.object_name] file_part = FilePart(dst_arg.filename, dst_arg.file_start, dst_arg.file_length) # TODO: calculate MD5's in parallel when possible. content_md5 = CalculateB64EncodedMd5FromContents(file_part) try: # Get the MD5 of the currently-existing component. dst_url = dst_arg.dst_url dst_metadata = gsutil_api.GetObjectMetadata( dst_url.bucket_name, dst_url.object_name, generation=dst_url.generation, provider=dst_url.scheme, fields=['customerEncryption', 'etag', 'md5Hash']) cloud_md5 = dst_metadata.md5Hash except Exception: # pylint: disable=broad-except # We don't actually care what went wrong - we couldn't retrieve the # object to check the MD5, so just upload it again. cloud_md5 = None if cloud_md5 != content_md5: components_to_upload.append(dst_arg) objects_already_chosen.append(tracker_object.object_name) if tracker_object.generation: # If the old object doesn't have a generation (i.e., it isn't in a # versioned bucket), then we will just overwrite it anyway. invalid_component_with_generation = dst_arg.dst_url.Clone() invalid_component_with_generation.generation = tracker_object.generation existing_objects_to_delete.append(invalid_component_with_generation) else: url = dst_arg.dst_url.Clone() url.generation = tracker_object.generation uploaded_components.append((url, dst_arg.file_length)) objects_already_chosen.append(tracker_object.object_name) if uploaded_components: logging.info('Found %d existing temporary components to reuse.', len(uploaded_components)) return (components_to_upload, uploaded_components, existing_objects_to_delete) utils/posix_util.py 0000644 00000045243 15025013616 0010464 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper file for POSIX methods.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from calendar import timegm import getpass import logging import os import re import time import six from gslib.exception import CommandException from gslib.tz_utc import UTC from gslib.utils.metadata_util import CreateCustomMetadata from gslib.utils.metadata_util import GetValueFromObjectCustomMetadata from gslib.utils.system_util import IS_WINDOWS from gslib.utils.unit_util import SECONDS_PER_DAY # pylint: disable=g-import-not-at-top if not IS_WINDOWS: import grp import pwd if six.PY3: long = int # Metadata attribute names for POSIX attributes. ATIME_ATTR = 'goog-reserved-file-atime' GID_ATTR = 'goog-reserved-posix-gid' MODE_ATTR = 'goog-reserved-posix-mode' MTIME_ATTR = 'goog-reserved-file-mtime' UID_ATTR = 'goog-reserved-posix-uid' # NA_TIME is a long value that takes the place of any invalid mtime. NA_TIME = -1 # NA_ID is a value that takes the place of any invalid POSIX UID or GID. NA_ID = -1 # NA_MODE is an octal value that takes the place of any invalid POSIX file mode. NA_MODE = -0o1 MODE_REGEX = re.compile('^[0-7]{3}$') # POSIX Permissions Bit-masks # User. U_R = 0o400 U_W = 0o200 U_X = 0o100 # Group. G_R = 0o040 G_W = 0o020 G_X = 0o010 # Other. O_R = 0o004 O_W = 0o002 O_X = 0o001 # The permissions newly created files get by default. SYSTEM_POSIX_MODE = None # A list of group IDs that the current user is a member of. USER_GROUPS = set() class POSIXAttributes(object): """Class to hold POSIX attributes for a file/object.""" def __init__(self, atime=NA_TIME, mtime=NA_TIME, uid=NA_ID, gid=NA_ID, mode=None): """Constructor for POSIXAttributes class which holds relevant data. Args: atime: The access time of the file/object. mtime: The modification time of the file/object. uid: The user ID that owns the file. gid: The group ID that the user is in. mode: An instance of POSIXMode. """ self.atime = atime self.mtime = mtime self.uid = uid self.gid = gid self.mode = POSIXMode(mode if mode else NA_MODE) class POSIXMode(object): def __init__(self, permissions): self.permissions = permissions def ConvertModeToBase8(mode): """Converts a base-10 mode integer from os.stat to base-8.""" # Strip out unnecessary bits in the mode. Mode is given as a base-10 # integer. It must be converted to base-8. return int(oct(mode)[-3:]) def DeserializeFileAttributesFromObjectMetadata(obj_metadata, url_str): """Parses the POSIX attributes from the supplied metadata. Args: obj_metadata: The metadata for an object. url_str: File/object path that provides context if a warning is thrown. Returns: A POSIXAttribute object with the retrieved values or a default value for any attribute that could not be found. """ posix_attrs = POSIXAttributes() # Parse atime. found, atime = GetValueFromObjectCustomMetadata(obj_metadata, ATIME_ATTR, NA_TIME) try: atime = long(atime) if found and atime <= NA_TIME: WarnNegativeAttribute('atime', url_str) atime = NA_TIME elif atime > long(time.time()) + SECONDS_PER_DAY: WarnFutureTimestamp('atime', url_str) atime = NA_TIME except ValueError: WarnInvalidValue('atime', url_str) atime = NA_TIME posix_attrs.atime = atime # Parse gid. DeserializeIDAttribute(obj_metadata, GID_ATTR, url_str, posix_attrs) # Parse uid. DeserializeIDAttribute(obj_metadata, UID_ATTR, url_str, posix_attrs) found, mode = GetValueFromObjectCustomMetadata(obj_metadata, MODE_ATTR, NA_MODE) if found and MODE_REGEX.match(mode): try: # Parse mode into a 3-digit base-8 number. posix_attrs.mode = POSIXMode(int(mode)) except ValueError: WarnInvalidValue('mode', url_str) return posix_attrs def SerializeFileAttributesToObjectMetadata(posix_attrs, custom_metadata, preserve_posix=False): """Takes a POSIXAttributes object and serializes it into custom metadata. Args: posix_attrs: A POSIXAttributes object. custom_metadata: A custom metadata object to serialize values into. preserve_posix: Whether or not to preserve POSIX attributes other than mtime. """ # mtime will always be needed in the object metadata for rsync. if posix_attrs.mtime != NA_TIME: CreateCustomMetadata(entries={MTIME_ATTR: posix_attrs.mtime}, custom_metadata=custom_metadata) # Only add other POSIX attributes if the preserve_posix flag is set. if preserve_posix: if posix_attrs.atime != NA_TIME: CreateCustomMetadata(entries={ATIME_ATTR: posix_attrs.atime}, custom_metadata=custom_metadata) if posix_attrs.uid != NA_ID: CreateCustomMetadata(entries={UID_ATTR: posix_attrs.uid}, custom_metadata=custom_metadata) if posix_attrs.gid != NA_ID: CreateCustomMetadata(entries={GID_ATTR: posix_attrs.gid}, custom_metadata=custom_metadata) if posix_attrs.mode.permissions != NA_MODE: CreateCustomMetadata(entries={MODE_ATTR: posix_attrs.mode.permissions}, custom_metadata=custom_metadata) def DeserializeIDAttribute(obj_metadata, attr, url_str, posix_attrs): """Parses the POSIX attributes from the supplied metadata into posix_attrs. Args: obj_metadata: The metadata for an object. attr: Either GID_ATTR or UID_ATTR. url_str: File/object path that provides context if a warning is thrown. posix_attrs: POSIXAttribute object. """ attr_name = attr.split('-')[-1] found, val = GetValueFromObjectCustomMetadata(obj_metadata, attr, NA_ID) try: val = int(val) if found and val <= NA_ID: WarnNegativeAttribute(attr_name, url_str) val = NA_ID except ValueError: WarnInvalidValue(attr_name, url_str) val = NA_ID setattr(posix_attrs, attr_name, val) def NeedsPOSIXAttributeUpdate(src_atime, dst_atime, src_mtime, dst_mtime, src_uid, dst_uid, src_gid, dst_gid, src_mode, dst_mode): """Checks whether an update for any POSIX attribute is needed. Args: src_atime: The source access time. dst_atime: The destination access time. src_mtime: The source modification time. dst_mtime: The destination modification time. src_uid: The source user ID. dst_uid: The destination user ID. src_gid: The source group ID. dst_gid: The destination group ID. src_mode: The source mode. dst_mode: The destination mode. Returns: A tuple containing a POSIXAttribute object and a boolean for whether an update was needed. """ posix_attrs = POSIXAttributes() has_src_atime = src_atime > NA_TIME has_dst_atime = dst_atime > NA_TIME has_src_mtime = src_mtime > NA_TIME has_dst_mtime = dst_mtime > NA_TIME has_src_uid = src_uid > NA_ID has_dst_uid = dst_uid > NA_ID has_src_gid = src_gid > NA_ID has_dst_gid = dst_gid > NA_ID has_src_mode = src_mode > NA_MODE has_dst_mode = dst_mode > NA_MODE if has_src_atime and not has_dst_atime: posix_attrs.atime = src_atime if has_src_mtime and not has_dst_mtime: posix_attrs.mtime = src_mtime if has_src_uid and not has_dst_uid: posix_attrs.uid = src_uid if has_src_gid and not has_dst_gid: posix_attrs.gid = src_gid if has_src_mode and not has_dst_mode: posix_attrs.mode.permissions = src_mode return posix_attrs, ((has_src_atime and not has_dst_atime) or (has_src_mtime and not has_dst_mtime) or (has_src_uid and not has_dst_uid) or (has_src_gid and not has_dst_gid) or (has_src_mode and not has_dst_mode)) def ValidatePOSIXMode(mode): """Validates whether the mode is valid. In order for the mode to be valid either the user, group, or other byte must be >= 4. Args: mode: The mode as a 3-digit, base-8 integer. Returns: True/False """ return MODE_REGEX.match(oct(mode)[-3:]) and (mode & U_R or mode & G_R or mode & O_R) def ValidateFilePermissionAccess(url_str, uid=NA_ID, gid=NA_ID, mode=NA_MODE): """Validates that the user has file access if uid, gid, and mode are applied. Args: url_str: The path to the object for which this is validating. uid: A POSIX user ID. gid: A POSIX group ID. mode: A 3-digit, number representing POSIX permissions, must be in base-8. Returns: A (bool, str) tuple, True if and only if it's safe to copy the file, and a string containing details for the error. """ # Windows doesn't use the POSIX system for file permissions, so all files will # validate. if IS_WINDOWS: return True, '' uid_present = uid > NA_ID gid_present = int(gid) > NA_ID mode_present = mode > NA_MODE # No need to perform validation if Posix attrs are not being preserved. if not (uid_present or gid_present or mode_present): return True, '' # The root user on non-Windows systems can access files regardless of their # permissions. if os.geteuid() == 0: return True, '' mode_valid = ValidatePOSIXMode(int(str(mode), 8)) if mode_present: if not mode_valid: return False, 'Mode for %s won\'t allow read access.' % url_str else: # Calculate the default mode if the mode doesn't exist. # Convert mode to a 3-digit, base-8 integer. mode = int(SYSTEM_POSIX_MODE) if uid_present: try: pwd.getpwuid(uid) except (KeyError, OverflowError): return (False, 'UID for %s doesn\'t exist on current system. uid: %d' % (url_str, uid)) if gid_present: try: grp.getgrgid(gid) except (KeyError, OverflowError): return (False, 'GID for %s doesn\'t exist on current system. gid: %d' % (url_str, gid)) # uid at this point must exist, but isn't necessarily the current user. # Likewise, gid must also exist at this point. uid_is_current_user = uid == os.getuid() # By this point uid and gid must exist on the system. However, the uid might # not match the current user's or the current user might not be a member of # the group identified by gid. In this case, the 'other' byte of the # permissions could provide sufficient access. mode = int(str(mode), 8) # Check that if the uid is not present and the gid and mode are, so that we # won't orphan the file. For example if the mode is set to 007, we can orphan # the file because the uid would default to the current user's ID and if the # current user wouldn't have read access or better, the file will be orphaned # even though they might otherwise have access through the gid or other bytes. if not uid_present and gid_present and mode_present and not bool(mode & U_R): return (False, 'Insufficient access with uid/gid/mode for %s, gid: %d, ' 'mode: %s' % (url_str, gid, oct(mode)[-3:])) if uid_is_current_user: valid = bool(mode & U_R) return (valid, '' if valid else 'Insufficient access with uid/gid/mode for %s, uid: %d, ' 'mode: %s' % (url_str, uid, oct(mode)[-3:])) elif int(gid) in USER_GROUPS: valid = bool(mode & G_R) return (valid, '' if valid else 'Insufficient access with uid/gid/mode for %s, gid: %d, ' 'mode: %s' % (url_str, gid, oct(mode)[-3:])) elif mode & O_R: return True, '' elif not uid_present and not gid_present and mode_valid: return True, '' return False, 'There was a problem validating %s.' % url_str def ParseAndSetPOSIXAttributes(path, obj_metadata, is_rsync=False, preserve_posix=False): """Parses POSIX attributes from obj_metadata and sets them. Attributes will only be set if they exist in custom metadata. This function should only be called after ValidateFilePermissionAccess has been called for the specific file/object so as not to orphan files. Args: path: The local filesystem path for the file. Valid metadata attributes will be set for the file located at path, some attributes will only be set if preserve_posix is set to True. obj_metadata: The metadata for the file/object. is_rsync: Whether or not the caller is the rsync command. Used to determine if timeCreated should be used. preserve_posix: Whether or not all POSIX attributes should be set. """ if obj_metadata is None: # This exception is meant for debugging purposes as it should never be # thrown unless there are unexpected code changes. raise CommandException('obj_metadata cannot be None for %s' % path) try: found_at, atime = GetValueFromObjectCustomMetadata(obj_metadata, ATIME_ATTR, default_value=NA_TIME) found_mt, mtime = GetValueFromObjectCustomMetadata(obj_metadata, MTIME_ATTR, default_value=NA_TIME) found_uid, uid = GetValueFromObjectCustomMetadata(obj_metadata, UID_ATTR, default_value=NA_ID) found_gid, gid = GetValueFromObjectCustomMetadata(obj_metadata, GID_ATTR, default_value=NA_ID) found_mode, mode = GetValueFromObjectCustomMetadata(obj_metadata, MODE_ATTR, default_value=NA_MODE) if found_mt: mtime = long(mtime) if not preserve_posix: atime_tmp = os.stat(path).st_atime os.utime(path, (atime_tmp, mtime)) return elif is_rsync: mtime = ConvertDatetimeToPOSIX(obj_metadata.timeCreated) os.utime(path, (mtime, mtime)) if not preserve_posix: return if found_at: atime = long(atime) if atime > NA_TIME and mtime > NA_TIME: # Set both atime and mtime. os.utime(path, (atime, mtime)) elif atime > NA_TIME and mtime <= NA_TIME: # atime is valid but mtime isn't. mtime_tmp = os.stat(path).st_mtime os.utime(path, (atime, mtime_tmp)) elif atime <= NA_TIME and mtime > NA_TIME: # mtime is valid but atime isn't atime_tmp = os.stat(path).st_atime os.utime(path, (atime_tmp, mtime)) if IS_WINDOWS: # Windows doesn't use POSIX uid/gid/mode unlike other systems. So there's # no point continuing. return # Only root can change ownership. if found_uid and os.geteuid() == 0: uid = int(uid) else: uid = NA_ID if found_gid: gid = int(gid) if uid > NA_ID and gid > NA_ID: # Set both uid and gid. os.chown(path, uid, gid) elif uid > NA_ID and gid <= NA_ID: # uid is valid but gid isn't. os.chown(path, uid, -1) elif uid <= NA_ID and gid > NA_ID: # gid is valid but uid isn't. os.chown(path, -1, gid) if found_mode: mode = int(str(mode), 8) os.chmod(path, mode) except ValueError: raise CommandException('Check POSIX attribute values for %s' % obj_metadata.name) def WarnNegativeAttribute(attr_name, url_str): """Logs if an attribute has a negative value. Args: attr_name: The name of the attribute to log. url_str: The path of the file for context. """ logging.getLogger().warn('%s has a negative %s in its metadata', url_str, attr_name) def WarnInvalidValue(attr_name, url_str): """Logs if an attribute has an invalid value. Args: attr_name: The name of the attribute to log. url_str: The path of the file for context. """ logging.getLogger().warn('%s has an invalid %s in its metadata', url_str, attr_name) def WarnFutureTimestamp(attr_name, url_str): """Logs if an attribute has an invalid value. Args: attr_name: The name of the attribute to log. url_str: The path of the file for context. """ logging.getLogger().warn( '%s has an %s more than 1 day from current system' ' time', url_str, attr_name) def ConvertDatetimeToPOSIX(dt): """Converts a datetime object to UTC and formats as POSIX. Sanitize the timestamp returned in dt, and put it in UTC format. For more information see the UTC class. Args: dt: A Python datetime object. Returns: A POSIX timestamp according to UTC. """ return long(timegm(dt.replace(tzinfo=UTC()).timetuple())) def InitializeDefaultMode(): """Records the default POSIX mode using os.umask.""" global SYSTEM_POSIX_MODE if IS_WINDOWS: # os.umask returns 0 on Windows. Below math works out to 666. SYSTEM_POSIX_MODE = '666' return # umask returns the permissions that should not be granted, so they must be # subtracted from the maximum set of permissions. max_permissions = 0o777 current_umask = os.umask(0o177) os.umask(current_umask) mode = max_permissions - current_umask # Files are not given execute privileges by default. Therefore we need to # subtract one from every odd permissions value. This is done via a bitmask. SYSTEM_POSIX_MODE = oct(mode & 0o666)[-3:] def InitializeUserGroups(): """Initializes the set of groups that the user is in. Should only be called if the flag for preserving POSIX attributes is set. """ global USER_GROUPS if IS_WINDOWS: return user_id = os.getuid() user_name = pwd.getpwuid(user_id).pw_name USER_GROUPS = set( # Primary group [pwd.getpwuid(user_id).pw_gid] + # Secondary groups [g.gr_gid for g in grp.getgrall() if user_name in g.gr_mem]) def InitializePreservePosixData(): """Initializes POSIX data. Run once at the beginning of a copy.""" InitializeDefaultMode() InitializeUserGroups() utils/cat_helper.py 0000644 00000014346 15025013616 0010373 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper for cat and cp streaming download.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import io import sys from boto import config from gslib.cloud_api import EncryptionException from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.storage_url import StorageUrlFromString from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.encryption_helper import FindMatchingCSEKInBotoConfig from gslib.utils.metadata_util import ObjectIsGzipEncoded from gslib.utils import text_util _CAT_BUCKET_LISTING_FIELDS = [ 'bucket', 'contentEncoding', 'crc32c', 'customerEncryption', 'generation', 'md5Hash', 'name', 'size', ] class CatHelper(object): """Provides methods for the "cat" command and associated functionality.""" def __init__(self, command_obj): """Initializes the helper object. Args: command_obj: gsutil command instance of calling command. """ self.command_obj = command_obj def _WriteBytesBufferedFileToFile(self, src_fd, dst_fd): """Copies contents of the source to the destination via buffered IO. Buffered reads are necessary in the case where you're reading from a source that produces more data than can fit into memory all at once. This method does not close either file when finished. Args: src_fd: The already-open source file to read from. dst_fd: The already-open destination file to write to. """ while True: buf = src_fd.read(io.DEFAULT_BUFFER_SIZE) if not buf: break text_util.write_to_fd(dst_fd, buf) def CatUrlStrings(self, url_strings, show_header=False, start_byte=0, end_byte=None, cat_out_fd=None): """Prints each of the url strings to stdout. Args: url_strings: String iterable. show_header: If true, print a header per file. start_byte: Starting byte of the file to print, used for constructing range requests. end_byte: Ending byte of the file to print; used for constructing range requests. If this is negative, the start_byte is ignored and and end range is sent over HTTP (such as range: bytes -9) cat_out_fd: File descriptor to which output should be written. Defaults to stdout if no file descriptor is supplied. Returns: 0 on success. Raises: CommandException if no URLs can be found. """ printed_one = False # This should refer to whatever sys.stdin refers to when this method is # run, not when this method is defined, so we do the initialization here # rather than define sys.stdin as the cat_out_fd parameter's default value. if cat_out_fd is None: cat_out_fd = sys.stdout # We manipulate the stdout so that all other data other than the Object # contents go to stderr. old_stdout = sys.stdout sys.stdout = sys.stderr try: if url_strings and url_strings[0] in ('-', 'file://-'): self._WriteBytesBufferedFileToFile(sys.stdin, cat_out_fd) else: for url_str in url_strings: did_some_work = False # TODO: Get only the needed fields here. for blr in self.command_obj.WildcardIterator(url_str).IterObjects( bucket_listing_fields=_CAT_BUCKET_LISTING_FIELDS): decryption_keywrapper = None if (blr.root_object and blr.root_object.customerEncryption and blr.root_object.customerEncryption.keySha256): decryption_key = FindMatchingCSEKInBotoConfig( blr.root_object.customerEncryption.keySha256, config) if not decryption_key: raise EncryptionException( 'Missing decryption key with SHA256 hash %s. No decryption ' 'key matches object %s' % (blr.root_object.customerEncryption.keySha256, blr.url_string)) decryption_keywrapper = CryptoKeyWrapperFromKey(decryption_key) did_some_work = True if show_header: if printed_one: print() print('==> %s <==' % blr) printed_one = True cat_object = blr.root_object # This if statement ensures nothing is outputted and no error # is thrown if the user enters an out of bounds range for the object. if 0 < getattr(cat_object, 'size', -1) <= start_byte: return 0 storage_url = StorageUrlFromString(blr.url_string) if storage_url.IsCloudUrl(): compressed_encoding = ObjectIsGzipEncoded(cat_object) self.command_obj.gsutil_api.GetObjectMedia( cat_object.bucket, cat_object.name, cat_out_fd, compressed_encoding=compressed_encoding, start_byte=start_byte, end_byte=end_byte, object_size=cat_object.size, generation=storage_url.generation, decryption_tuple=decryption_keywrapper, provider=storage_url.scheme) cat_out_fd.flush() else: with open(storage_url.object_name, 'rb') as f: self._WriteBytesBufferedFileToFile(f, cat_out_fd) if not did_some_work: raise CommandException(NO_URLS_MATCHED_TARGET % url_str) finally: sys.stdout = old_stdout return 0 utils/stet_util.py 0000644 00000007525 15025013616 0010302 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for Split Trust Encryption Tool integration.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import shutil from gslib import storage_url from gslib.utils import execution_util from gslib.utils import temporary_file_util from boto import config class StetSubcommandName(object): """Enum class for available STET subcommands.""" ENCRYPT = 'encrypt' DECRYPT = 'decrypt' def _get_stet_binary_from_path(): """Retrieves STET binary from path if available. Python 2 compatible.""" for path_directory in os.getenv('PATH').split(os.path.pathsep): binary_path = os.path.join(path_directory, 'stet') if os.path.exists(binary_path): return binary_path def _stet_transform(subcommand, blob_id, in_file_path, out_file_path, logger): """Runs a STET transform on a file. Encrypts for uploads. Decrypts for downloads. Automatically populates flags for the STET binary. Args: subcommand (StetSubcommandName): Subcommand to call on STET binary. blob_id (str): Cloud URL that binary uses for validation. in_file_path (str): File to be transformed source. out_file_path (str): Where to write result of transform. logger (logging.Logger): For logging STET binary output. Raises: KeyError: STET binary or config could not be found. """ binary_path = config.get('GSUtil', 'stet_binary_path', _get_stet_binary_from_path()) if not binary_path: raise KeyError('Could not find STET binary in boto config or PATH.') command_args = [os.path.expanduser(binary_path), subcommand] config_path = config.get('GSUtil', 'stet_config_path', None) if config_path: command_args.append('--config-file=' + os.path.expanduser(config_path)) command_args.extend(['--blob-id=' + blob_id, in_file_path, out_file_path]) _, stderr = execution_util.ExecuteExternalCommand(command_args) logger.debug(stderr) def encrypt_upload(source_url, destination_url, logger): """Encrypts a file with STET binary before upload. Args: source_url (StorageUrl): Copy source. destination_url (StorageUrl): Copy destination. logger (logging.Logger): For logging STET binary output. Returns: stet_temporary_file_url (StorageUrl): Path to STET-encrypted file. """ in_file = source_url.object_name out_file = temporary_file_util.GetStetTempFileName(source_url) blob_id = destination_url.url_string _stet_transform(StetSubcommandName.ENCRYPT, blob_id, in_file, out_file, logger) return storage_url.StorageUrlFromString(out_file) def decrypt_download(source_url, destination_url, temporary_file_name, logger): """STET-decrypts downloaded file. Args: source_url (StorageUrl): Copy source. destination_url (StorageUrl): Copy destination. temporary_file_name (str): Path to temporary file used for download. logger (logging.Logger): For logging STET binary output. """ in_file = temporary_file_name out_file = temporary_file_util.GetStetTempFileName(destination_url) blob_id = source_url.url_string _stet_transform(StetSubcommandName.DECRYPT, blob_id, in_file, out_file, logger) shutil.move(out_file, in_file) utils/iam_helper.py 0000644 00000025511 15025013616 0010366 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper module for the IAM command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import defaultdict from collections import namedtuple import six from apitools.base.protorpclite import protojson from gslib.exception import CommandException from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages TYPES = set([ 'user', 'deleted:user', 'serviceAccount', 'deleted:serviceAccount', 'group', 'deleted:group', 'domain', 'principal', 'principalSet', 'principalHierarchy' ]) DISCOURAGED_TYPES = set([ 'projectOwner', 'projectEditor', 'projectViewer', ]) DISCOURAGED_TYPES_MSG = ( 'Assigning roles (e.g. objectCreator, legacyBucketOwner) for project ' 'convenience groups is not supported by gsutil, as it goes against the ' 'principle of least privilege. Consider creating and using more granular ' 'groups with which to assign permissions. See ' 'https://cloud.google.com/iam/docs/using-iam-securely for more ' 'information. Assigning a role to a project group can be achieved by ' 'setting the IAM policy directly (see gsutil help iam for specifics).') PUBLIC_MEMBERS = set([ 'allUsers', 'allAuthenticatedUsers', ]) # These are convenience classes to handle returned results from # BindingStringToTuple. is_grant is a boolean specifying if the # bindings are to be granted or removed from a bucket / object. # bindings varies by type. # For BindingsTuple, it is a list of BindingsValueListEntry # For BindingsDictTuple it is a list of analogous dicts, e.g.: # { # 'members': ['member'], # 'role': 'role', # } BindingsTuple = namedtuple('BindingsTuple', ['is_grant', 'bindings']) BindingsDictTuple = namedtuple('BindingsDictTuple', ['is_grant', 'bindings']) # This is a special role value assigned to a specific member when all roles # assigned to the member should be dropped in the policy. A member:DROP_ALL # binding will be passed from BindingStringToTuple into PatchBindings. # This role will only ever appear on client-side (i.e. user-generated). It # will never be returned as a real role from an IAM get request. All roles # returned by PatchBindings are guaranteed to be "real" roles, i.e. not a # DROP_ALL role. DROP_ALL = '' def SerializeBindingsTuple(bindings_tuple): """Serializes the BindingsValueListEntry instances in a BindingsTuple. This is necessary when passing instances of BindingsTuple through Command.Apply, as apitools_messages classes are not by default pickleable. Args: bindings_tuple: A BindingsTuple instance to be serialized. Returns: A serialized BindingsTuple object. """ return (bindings_tuple.is_grant, [protojson.encode_message(t) for t in bindings_tuple.bindings]) def DeserializeBindingsTuple(serialized_bindings_tuple): (is_grant, bindings) = serialized_bindings_tuple return BindingsTuple(is_grant=is_grant, bindings=[ protojson.decode_message( apitools_messages.Policy.BindingsValueListEntry, t) for t in bindings ]) def BindingsMessageToUpdateDict(bindings): """Reformats policy bindings metadata. Args: bindings: A list of BindingsValueListEntry instances. Returns: A {role: set(members)} dictionary. """ tmp_bindings = defaultdict(set) for binding in bindings: tmp_bindings[binding.role].update(binding.members) return tmp_bindings def BindingsDictToUpdateDict(bindings): """Reformats policy bindings metadata. Args: bindings: List of dictionaries representing BindingsValueListEntry instances. e.g.: { "role": "some_role", "members": ["allAuthenticatedUsers", ...] } Returns: A {role: set(members)} dictionary. """ tmp_bindings = defaultdict(set) for binding in bindings: tmp_bindings[binding['role']].update(binding['members']) return tmp_bindings def IsEqualBindings(a, b): (granted, removed) = DiffBindings(a, b) return not granted.bindings and not removed.bindings def DiffBindings(old, new): """Computes the difference between two BindingsValueListEntry lists. Args: old: The original list of BindingValuesListEntry instances new: The updated list of BindingValuesListEntry instances Returns: A pair of BindingsTuple instances, one for roles granted between old and new, and one for roles removed between old and new. """ tmp_old = BindingsMessageToUpdateDict(old) tmp_new = BindingsMessageToUpdateDict(new) granted = BindingsMessageToUpdateDict([]) removed = BindingsMessageToUpdateDict([]) for (role, members) in six.iteritems(tmp_old): removed[role].update(members.difference(tmp_new[role])) for (role, members) in six.iteritems(tmp_new): granted[role].update(members.difference(tmp_old[role])) granted = [ apitools_messages.Policy.BindingsValueListEntry(role=r, members=list(m)) for (r, m) in six.iteritems(granted) if m ] removed = [ apitools_messages.Policy.BindingsValueListEntry(role=r, members=list(m)) for (r, m) in six.iteritems(removed) if m ] return (BindingsTuple(True, granted), BindingsTuple(False, removed)) def PatchBindings(base, diff, is_grant): """Patches a diff list of BindingsValueListEntry to the base. Will remove duplicate members for any given role on a grant operation. Args: base (dict): A dictionary returned by BindingsMessageToUpdateDict or BindingsDictToUpdateDict representing a resource's current IAM policy. diff (dict): A dictionary returned by BindingsMessageToUpdateDict or BindingsDictToUpdateDict representing the IAM policy bindings to add/remove from `base`. is_grant (bool): True if `diff` should be added to `base`, False if it should be removed from `base`. Returns: A {role: set(members)} dictionary created by applying `diff` to `base`. """ # Patch the diff into base if is_grant: for (role, members) in six.iteritems(diff): if not role: raise CommandException('Role must be specified for a grant request.') base[role].update(members) else: for role in base: base[role].difference_update(diff[role]) # Drop all members with the DROP_ALL role specifed from input. base[role].difference_update(diff[DROP_ALL]) return {role: members for role, members in six.iteritems(base) if members} def BindingStringToTuple(is_grant, input_str): """Parses an iam ch bind string to a list of binding tuples. Args: is_grant: If true, binding is to be appended to IAM policy; else, delete this binding from the policy. input_str: A string representing a member-role binding. e.g. user:foo@bar.com:objectAdmin user:foo@bar.com:objectAdmin,objectViewer user:foo@bar.com allUsers deleted:user:foo@bar.com?uid=123:objectAdmin,objectViewer deleted:serviceAccount:foo@bar.com?uid=123 Raises: CommandException in the case of invalid input. Returns: A BindingsDictTuple instance. """ if not input_str.count(':'): input_str += ':' # Allows user specified PUBLIC_MEMBERS, DISCOURAGED_TYPES, and TYPES to be # case insensitive. tokens = input_str.split(":") public_members = {s.lower(): s for s in PUBLIC_MEMBERS} types = {s.lower(): s for s in TYPES} discouraged_types = {s.lower(): s for s in DISCOURAGED_TYPES} possible_public_member_or_type = tokens[0].lower() possible_type = '%s:%s' % (tokens[0].lower(), tokens[1].lower()) if possible_public_member_or_type in public_members: tokens[0] = public_members[possible_public_member_or_type] elif possible_public_member_or_type in types: tokens[0] = types[possible_public_member_or_type] elif possible_public_member_or_type in discouraged_types: tokens[0] = discouraged_types[possible_public_member_or_type] elif possible_type in types: (tokens[0], tokens[1]) = types[possible_type].split(':') input_str = ":".join(tokens) # We can remove project convenience members, but not add them. removing_discouraged_type = not is_grant and tokens[0] in DISCOURAGED_TYPES if input_str.count(':') == 1: if '%s:%s' % (tokens[0], tokens[1]) in TYPES: raise CommandException('Incorrect public member type for binding %s' % input_str) elif tokens[0] in PUBLIC_MEMBERS: (member, roles) = tokens elif tokens[0] in TYPES or removing_discouraged_type: member = input_str roles = DROP_ALL else: raise CommandException('Incorrect public member type for binding %s' % input_str) elif input_str.count(':') == 2: if '%s:%s' % (tokens[0], tokens[1]) in TYPES: # case "deleted:user:foo@bar.com?uid=1234" member = input_str roles = DROP_ALL elif removing_discouraged_type: (member_type, project_id, roles) = tokens member = '%s:%s' % (member_type, project_id) else: (member_type, member_id, roles) = tokens _check_member_type(member_type, input_str) member = '%s:%s' % (member_type, member_id) elif input_str.count(':') == 3: # case "deleted:user:foo@bar.com?uid=1234:objectAdmin,objectViewer" (member_type_p1, member_type_p2, member_id, roles) = input_str.split(':') member_type = '%s:%s' % (member_type_p1, member_type_p2) _check_member_type(member_type, input_str) member = '%s:%s' % (member_type, member_id) else: raise CommandException('Invalid ch format %s' % input_str) if is_grant and not roles: raise CommandException('Must specify a role to grant.') roles = [ResolveRole(r) for r in roles.split(',')] bindings = [{'role': r, 'members': [member]} for r in set(roles)] return BindingsDictTuple(is_grant=is_grant, bindings=bindings) def _check_member_type(member_type, input_str): if member_type in DISCOURAGED_TYPES: raise CommandException(DISCOURAGED_TYPES_MSG) elif member_type not in TYPES: raise CommandException('Incorrect member type for binding %s' % input_str) def ResolveRole(role): if not role: return DROP_ALL if 'roles/' in role: return role return 'roles/storage.%s' % role utils/rsync_util.py 0000644 00000003535 15025013616 0010456 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods for rsync functionality.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class DiffAction(object): """Enum class representing possible actions to take for an rsync diff.""" COPY = 'copy' REMOVE = 'remove' MTIME_SRC_TO_DST = 'mtime_src_to_dst' POSIX_SRC_TO_DST = 'posix_src_to_dst' class RsyncDiffToApply(object): """Class that encapsulates info needed to apply diff for one object.""" def __init__(self, src_url_str, dst_url_str, src_posix_attrs, diff_action, copy_size): """Constructor. Args: src_url_str: (str or None) The source URL string, or None if diff_action is REMOVE. dst_url_str: (str) The destination URL string. src_posix_attrs: (posix_util.POSIXAttributes) The source POSIXAttributes. diff_action: (DiffAction) DiffAction to be applied. copy_size: (int or None) The amount of bytes to copy, or None if diff_action is REMOVE. """ self.src_url_str = src_url_str self.dst_url_str = dst_url_str self.src_posix_attrs = src_posix_attrs self.diff_action = diff_action self.copy_size = copy_size utils/execution_util.py 0000644 00000003465 15025013616 0011325 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for executing binaries.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import subprocess from gslib import exception def ExecuteExternalCommand(command_and_flags): """Runs external terminal command. Args: command_and_flags (List[str]): Ordered command and flag strings. Returns: (stdout (str|None), stderr (str|None)) from running command. Raises: OSError for any issues running the command. """ command_process = subprocess.Popen(command_and_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) command_stdout, command_stderr = command_process.communicate() # Python 3 outputs bytes from communicate() by default. if command_stdout is not None and not isinstance(command_stdout, str): command_stdout = command_stdout.decode() if command_stderr is not None and not isinstance(command_stderr, str): command_stderr = command_stderr.decode() if command_process.returncode != 0: raise exception.ExternalBinaryError(command_stderr) return command_stdout, command_stderr utils/arg_helper.py 0000644 00000002661 15025013616 0010372 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains helper for parsing command arguments and options.""" import getopt import sys from gslib.exception import CommandException def GetArgumentsAndOptions(): """Gets the list of arguments and options from the command input. Returns: The return value consists of two elements: the first is a list of (option, value) pairs; the second is the list of program arguments left after the option list was stripped (this is a trailing slice of the first argument). """ try: return getopt.getopt(sys.argv[1:], 'dDvo:?h:i:u:mq', [ 'debug', 'detailedDebug', 'version', 'option', 'help', 'header', 'impersonate-service-account=', 'multithreaded', 'quiet', 'testexceptiontraces', 'trace-token=', 'perf-trace-token=' ]) except getopt.GetoptError as e: raise CommandException(e.msg) utils/boto_util.py 0000644 00000064006 15025013616 0010263 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods that require importing boto. This module also imports httplib2 (as it is Boto's http transport and closely tied to some of Boto's core functionality) and oauth2client. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import pkgutil import tempfile import textwrap import six import boto from boto import config import boto.auth from boto.exception import NoAuthHandlerFound from boto.gs.connection import GSConnection from boto.provider import Provider from boto.pyami.config import BotoConfigLocations import gslib from gslib import context_config from gslib.exception import CommandException from gslib.utils import system_util from gslib.utils.constants import DEFAULT_GCS_JSON_API_VERSION from gslib.utils.constants import DEFAULT_GSUTIL_STATE_DIR from gslib.utils.constants import SSL_TIMEOUT_SEC from gslib.utils.constants import UTF8 from gslib.utils.unit_util import HumanReadableToBytes from gslib.utils.unit_util import ONE_MIB import httplib2 from oauth2client.client import HAS_CRYPTO if six.PY3: long = int # Globals in this module are set according to values in the boto config. BOTO_IS_SECURE = config.get('Boto', 'is_secure', True) CERTIFICATE_VALIDATION_ENABLED = config.get('Boto', 'https_validate_certificates', True) configured_certs_file = None # Single certs file for use across all processes. temp_certs_file = None # Temporary certs file for cleanup upon exit. def ConfigureCertsFile(): """Configures and returns the CA Certificates file. If one is already configured, use it. Otherwise, use the cert roots distributed with gsutil. Returns: string filename of the certs file to use. """ certs_file = boto.config.get('Boto', 'ca_certificates_file', None) # The 'system' keyword indicates to use the system installed certs. Some # Linux distributions patch the stack such that the Python SSL # infrastructure picks up the system installed certs by default, thus no # action necessary on our part if certs_file == 'system': return None if not certs_file: global configured_certs_file, temp_certs_file if not configured_certs_file: configured_certs_file = os.path.abspath( os.path.join(gslib.GSLIB_DIR, 'data', 'cacerts.txt')) if not os.path.exists(configured_certs_file): # If the file is not present on disk, this means the gslib module # doesn't actually exist on disk anywhere. This can happen if it's # being imported from a zip file. Unfortunately, we have to copy the # certs file to a local temp file on disk because the underlying SSL # socket requires it to be a filesystem path. certs_data = pkgutil.get_data('gslib', 'data/cacerts.txt') if not certs_data: raise CommandException('Certificates file not found. Please ' 'reinstall gsutil from scratch') certs_data = six.ensure_str(certs_data) fd, fname = tempfile.mkstemp(suffix='.txt', prefix='gsutil-cacerts') f = os.fdopen(fd, 'w') f.write(certs_data) f.close() temp_certs_file = fname configured_certs_file = temp_certs_file certs_file = configured_certs_file return certs_file def ConfigureNoOpAuthIfNeeded(): """Sets up no-op auth handler if no boto credentials are configured.""" if not HasConfiguredCredentials(): if (config.has_option('Credentials', 'gs_service_client_id') and not HAS_CRYPTO): if system_util.InvokedViaCloudSdk(): raise CommandException('\n'.join( textwrap.wrap( 'Your gsutil is configured with an OAuth2 service account, but ' 'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. ' 'Service account authentication requires one of these libraries; ' 'please reactivate your service account via the gcloud auth ' 'command and ensure any gcloud packages necessary for ' 'service accounts are present.'))) else: raise CommandException('\n'.join( textwrap.wrap( 'Your gsutil is configured with an OAuth2 service account, but ' 'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. ' 'Service account authentication requires one of these libraries; ' 'please install either of them to proceed, or configure a ' 'different type of credentials with "gsutil config".'))) else: # With no boto config file the user can still access publicly readable # buckets and objects. from gslib import no_op_auth_plugin # pylint: disable=unused-variable def GetCertsFile(): return configured_certs_file def GetCleanupFiles(): """Returns a list of temp files to delete (if possible) when program exits.""" return [temp_certs_file] if temp_certs_file else [] def GetConfigFilePaths(): """Returns a list of the path(s) to the boto config file(s) to be loaded.""" potential_config_paths = BotoConfigLocations # When Boto's pyami.config.Config class is initialized, it attempts to read # this file, transform it into valid Boto config syntax, and load credentials # from it, but it does not add this file to BotoConfigLocations. if 'AWS_CREDENTIAL_FILE' in os.environ: potential_config_paths.append(os.environ['AWS_CREDENTIAL_FILE']) # Provider-specific config files, like the one below, are checked for and # loaded when Boto's Provider class is initialized. These aren't listed in # BotoConfigLocations, but can still affect what credentials are loaded, so # we display them in our config list to make auth'n debugging easier. aws_cred_file = os.path.join(os.path.expanduser('~'), '.aws', 'credentials') if os.path.isfile(aws_cred_file): potential_config_paths.append(aws_cred_file) # Only return credential files which we have permission to read (and thus can # actually load credentials from). readable_config_paths = [] for path in potential_config_paths: try: with open(path, 'r'): readable_config_paths.append(path) except IOError: pass return readable_config_paths def GetFriendlyConfigFilePaths(): """Like GetConfigFilePaths but returns a not-found message if paths empty.""" readable_config_paths = GetConfigFilePaths() if len(readable_config_paths) == 0: readable_config_paths.append('No config found') return readable_config_paths def GetGsutilStateDir(): """Returns the location of the directory for gsutil state files. Certain operations, such as cross-process credential sharing and resumable transfer tracking, need a known location for state files which are created by gsutil as-needed. This location should only be used for storing data that is required to be in a static location. Returns: Path to directory for gsutil static state files. """ config_file_dir = config.get('GSUtil', 'state_dir', DEFAULT_GSUTIL_STATE_DIR) system_util.CreateDirIfNeeded(config_file_dir) return config_file_dir def GetCredentialStoreFilename(): # As of gsutil 4.29, this changed from 'credstore' to 'credstore2' because # of a change to the underlying credential storage format. return os.path.join(GetGsutilStateDir(), 'credstore2') def GetGceCredentialCacheFilename(): return os.path.join(GetGsutilStateDir(), 'gcecredcache') def GetGcsJsonApiVersion(): return config.get('GSUtil', 'json_api_version', DEFAULT_GCS_JSON_API_VERSION) # Resumable downloads and uploads make one HTTP call per chunk (and must be # in multiples of 256KiB). Overridable for testing. def GetJsonResumableChunkSize(): chunk_size = config.getint('GSUtil', 'json_resumable_chunk_size', long(1024 * 1024 * 100)) if chunk_size == 0: chunk_size = long(1024 * 256) elif chunk_size % long(1024 * 256) != 0: chunk_size += (long(1024 * 256) - (chunk_size % (long(1024 * 256)))) return chunk_size def GetLastCheckedForGsutilUpdateTimestampFile(): return os.path.join(GetGsutilStateDir(), '.last_software_update_check') def GetMaxConcurrentCompressedUploads(): """Gets the max concurrent transport compressed uploads allowed in parallel. Returns: The max number of concurrent transport compressed uploads allowed in parallel without exceeding the max_upload_compression_buffer_size. """ upload_chunk_size = GetJsonResumableChunkSize() # From apitools compression.py. compression_chunk_size = 16 * ONE_MIB total_upload_size = (upload_chunk_size + compression_chunk_size + 17 + 5 * (((compression_chunk_size - 1) / 16383) + 1)) max_concurrent_uploads = (GetMaxUploadCompressionBufferSize() / total_upload_size) if max_concurrent_uploads <= 0: max_concurrent_uploads = 1 return max_concurrent_uploads def GetMaxRetryDelay(): return config.getint('Boto', 'max_retry_delay', 32) def GetMaxUploadCompressionBufferSize(): """Get the max amount of memory compressed transport uploads may buffer.""" return HumanReadableToBytes( config.get('GSUtil', 'max_upload_compression_buffer_size', '2GiB')) def GetNewHttp(http_class=httplib2.Http, **kwargs): """Creates and returns a new httplib2.Http instance. Args: http_class: Optional custom Http class to use. **kwargs: Arguments to pass to http_class constructor. Returns: An initialized httplib2.Http instance. """ ##Get Proxy configuration from boto file, defaults are None, 0 and False proxy_host = config.get('Boto', 'proxy', None) #needed for if clause below boto_proxy_config = { 'proxy_host': proxy_host, 'proxy_type': config.get('Boto', 'proxy_type', 'http'), 'proxy_port': config.getint('Boto', 'proxy_port'), 'proxy_user': config.get('Boto', 'proxy_user', None), 'proxy_pass': config.get('Boto', 'proxy_pass', None), 'proxy_rdns': config.get('Boto', 'proxy_rdns', True if proxy_host else None) } #Use SetProxyInfo to convert boto config to httplib2.proxyinfo object proxy_info = SetProxyInfo(boto_proxy_config) # Some installers don't package a certs file with httplib2, so use the # one included with gsutil. kwargs['ca_certs'] = GetCertsFile() # Use a non-infinite SSL timeout to avoid hangs during network flakiness. kwargs['timeout'] = SSL_TIMEOUT_SEC http = http_class(proxy_info=proxy_info, **kwargs) http.disable_ssl_certificate_validation = (not config.getbool( 'Boto', 'https_validate_certificates')) global_context_config = context_config.get_context_config() if global_context_config and global_context_config.use_client_certificate: http.add_certificate(key=global_context_config.client_cert_path, cert=global_context_config.client_cert_path, domain='', password=global_context_config.client_cert_password) return http # Retry for 10 minutes with exponential backoff, which corresponds to # the maximum Downtime Period specified in the GCS SLA # (https://cloud.google.com/storage/sla) def GetNumRetries(): return config.getint('Boto', 'num_retries', 23) def GetTabCompletionLogFilename(): return os.path.join(GetGsutilStateDir(), 'tab-completion-logs') def GetTabCompletionCacheFilename(): tab_completion_dir = os.path.join(GetGsutilStateDir(), 'tab-completion') # Limit read permissions on the directory to owner for privacy. system_util.CreateDirIfNeeded(tab_completion_dir, mode=0o700) return os.path.join(tab_completion_dir, 'cache') def HasConfiguredCredentials(): """Determines if boto credential/config file exists.""" has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and config.has_option('Credentials', 'gs_secret_access_key')) has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and config.has_option('Credentials', 'aws_secret_access_key')) has_oauth_creds = (config.has_option('Credentials', 'gs_oauth2_refresh_token')) has_external_creds = (config.has_option('Credentials', 'gs_external_account_file')) has_external_account_authorized_user_creds = (config.has_option( 'Credentials', 'gs_external_account_authorized_user_file')) has_service_account_creds = ( HAS_CRYPTO and config.has_option('Credentials', 'gs_service_client_id') and config.has_option('Credentials', 'gs_service_key_file')) if (has_goog_creds or has_amzn_creds or has_oauth_creds or has_service_account_creds or has_external_creds or has_external_account_authorized_user_creds): return True valid_auth_handler = None try: valid_auth_handler = boto.auth.get_auth_handler(GSConnection.DefaultHost, config, Provider('google'), requested_capability=['s3']) # Exclude the no-op auth handler as indicating credentials are configured. # Note we can't use isinstance() here because the no-op module may not be # imported so we can't get a reference to the class type. if 'NoOpAuth' == getattr( getattr(valid_auth_handler, '__class__', None), '__name__', None): # yapf: disable valid_auth_handler = None except NoAuthHandlerFound: pass return valid_auth_handler def JsonResumableChunkSizeDefined(): chunk_size_defined = config.get('GSUtil', 'json_resumable_chunk_size', None) return chunk_size_defined is not None def MonkeyPatchBoto(): """Apply gsutil-specific patches to Boto. Here be dragons. Sorry. Note that this method should not be used as a replacement for contributing fixes to the upstream Boto library. However, the Boto library has historically not been consistent about release cadence, so upstream fixes may not be available immediately in a version which we can pin to. Also, some fixes may only be applicable to gsutil. In such cases, patches should be applied to the Boto library here (and removed if/when they are included in the upstream repository and included in an official new release that we pull in). This method should be invoked after all other Boto-related initialization has been completed. """ # We have to do all sorts of gross things here (dynamic imports, invalid names # to resolve symbols in copy/pasted methods, invalid spacing from copy/pasted # methods, etc.), so we just disable pylint warnings for this whole method. # yapf: disable # pylint: disable=all # This should have already been imported if this method was called in the # correct place, but we need to perform the import within this method for # this module to recognize it. We don't import this at module level because # its import timing is important and is managed elsewhere. import gcs_oauth2_boto_plugin # TODO(https://github.com/boto/boto/issues/3831): Remove this if the GitHub # issue is ever addressed. orig_get_plugin_method = boto.plugin.get_plugin def _PatchedGetPluginMethod(cls, requested_capability=None): handler_subclasses = orig_get_plugin_method( cls, requested_capability=requested_capability) # In Boto's logic, higher precedence handlers should be closer to the end # of the list. We always want to prefer OAuth2 credentials over HMAC # credentials if both are present, so we shuffle OAuth2 handlers to the end # of the handler list. xml_oauth2_handlers = ( gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2ServiceAccountAuth, gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2Auth) new_result = ( # We need to sort each of these to avoid inconsistent handler selection # when multiple credential types are configured in Python 3.5. See # https://issuetracker.google.com/issues/135709541 for specific logs. sorted( [r for r in handler_subclasses if r not in xml_oauth2_handlers], # Types aren't sortable, so we use their names: key=(lambda handler_t: handler_t.__name__), ) + # Now append XML handlers to the end (highest precedence) sorted( [r for r in handler_subclasses if r in xml_oauth2_handlers], key=(lambda handler_t: handler_t.__name__), )) return new_result boto.plugin.get_plugin = _PatchedGetPluginMethod ######################################################################### # TODO(boto>2.49.0): Remove this. # Fixes SSL issue where SNI was not being used for OpenSSL 1.1.1+ # https://github.com/boto/boto/pull/3843/files # Import modules and resolve symbols used by our patch method. import socket import ssl InvalidCertificateException = ( boto.https_connection.InvalidCertificateException) ValidateCertificateHostname = ( boto.https_connection.ValidateCertificateHostname) def _PatchedConnectMethod(self): # The lines below were copied directly from the Boto file, so we don't lint # or otherwise alter them. if hasattr(self, "timeout"): sock = socket.create_connection((self.host, self.port), self.timeout) else: sock = socket.create_connection((self.host, self.port)) msg = "wrapping ssl socket; " if self.ca_certs: msg += "CA certificate file=%s" % self.ca_certs else: msg += "using system provided SSL certs" boto.log.debug(msg) if hasattr(ssl, 'SSLContext') and getattr(ssl, 'HAS_SNI', False): # Use SSLContext so we can specify server_hostname for SNI # (Required for connections to storage.googleapis.com) context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = ssl.CERT_REQUIRED if self.ca_certs: context.load_verify_locations(self.ca_certs) if self.cert_file: context.load_cert_chain(self.cert_file, self.key_file) self.sock = context.wrap_socket(sock, server_hostname=self.host) # Add attributes only set in SSLSocket constructor without context: self.sock.keyfile = self.key_file self.sock.certfile = self.cert_file self.sock.cert_reqs = context.verify_mode self.sock.ssl_version = ssl.PROTOCOL_SSLv23 self.sock.ca_certs = self.ca_certs self.sock.ciphers = None else: self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_certs) cert = self.sock.getpeercert() hostname = self.host.split(':', 0)[0] if not ValidateCertificateHostname(cert, hostname): raise InvalidCertificateException( hostname, cert, 'remote hostname "%s" does not match ' 'certificate' % hostname) # End `_PatchedConnectMethod` declaration. boto.https_connection.CertValidatingHTTPSConnection.connect = ( _PatchedConnectMethod) def ProxyInfoFromEnvironmentVar(proxy_env_var): """Reads proxy info from the environment and converts to httplib2.ProxyInfo. Args: proxy_env_var: Environment variable string to read, such as http_proxy or https_proxy. Returns: httplib2.ProxyInfo constructed from the environment string. """ proxy_url = os.environ.get(proxy_env_var) if not proxy_url or not proxy_env_var.lower().startswith('http'): return httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, None, 0) proxy_protocol = proxy_env_var.lower().split('_')[0] if not proxy_url.lower().startswith('http'): # proxy_info_from_url requires a protocol, which is always http or https. proxy_url = proxy_protocol + '://' + proxy_url return httplib2.proxy_info_from_url(proxy_url, method=proxy_protocol) def ResumableThreshold(): return config.getint('GSUtil', 'resumable_threshold', 8 * ONE_MIB) def SetProxyInfo(boto_proxy_config): """Sets proxy info from boto and environment and converts to httplib2.ProxyInfo. Args: dict: Values read from the .boto file Returns: httplib2.ProxyInfo constructed from boto or environment variable string. """ #Defining proxy_type based on httplib2 library, accounting for None entry too. proxy_type_spec = {'socks4': 1, 'socks5': 2, 'http': 3, 'https': 3} #proxy_type defaults to 'http (3)' for backwards compatibility proxy_type = proxy_type_spec.get( boto_proxy_config.get('proxy_type').lower(), proxy_type_spec['http']) proxy_host = boto_proxy_config.get('proxy_host') proxy_port = boto_proxy_config.get('proxy_port') proxy_user = boto_proxy_config.get('proxy_user') proxy_pass = boto_proxy_config.get('proxy_pass') proxy_rdns = bool(boto_proxy_config.get('proxy_rdns')) #For proxy_info below, proxy_rdns fails for socks4 and socks5 so restricting use #to http only proxy_info = httplib2.ProxyInfo(proxy_host=proxy_host, proxy_type=proxy_type, proxy_port=proxy_port, proxy_user=proxy_user, proxy_pass=proxy_pass, proxy_rdns=proxy_rdns) #Added to force socks proxies not to use rdns if not (proxy_info.proxy_type == proxy_type_spec['http']): proxy_info.proxy_rdns = False if not (proxy_info.proxy_host and proxy_info.proxy_port): # Fall back to using the environment variable. Use only http proxies. for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: if proxy_env_var in os.environ and os.environ[proxy_env_var]: proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var) # Assume proxy_rnds is True if a proxy environment variable exists. if boto_proxy_config.get('proxy_rdns') == None: proxy_info.proxy_rdns = True break return proxy_info def UsingCrcmodExtension(): boto_opt = boto.config.get('GSUtil', 'test_assume_fast_crcmod', None) if boto_opt is not None: return boto_opt # Python 3 makes this attribute tough to access due to the way the top-level # crcmod package imports and (identically) names its crcmod module. The only # way to get it is "from crcmod.crcmod import _usingExtension". This is the # alternative form of that statement, but doesn't pollute this module's # namespace with a "_usingExtension" attribute. This also works in both Python # 2.7 and 3.5+. nested_crcmod = __import__( 'crcmod.crcmod', globals(), locals(), ['_usingExtension'], 0, ) return getattr(nested_crcmod, '_usingExtension', False) # TODO(boto-2.49.0): Remove when we pull in the next version of Boto. def _PatchedShouldRetryMethod(self, response, chunked_transfer=False): """Replaces boto.s3.key's should_retry() to handle KMS-encrypted objects.""" # We copy/pasted this from boto and slightly modified it; keep the old # formatting style. # yapf: disable provider = self.bucket.connection.provider if not chunked_transfer: if response.status in [500, 503]: # 500 & 503 can be plain retries. return True if response.getheader('location'): # If there's a redirect, plain retry. return True if 200 <= response.status <= 299: self.etag = response.getheader('etag') md5 = self.md5 if isinstance(md5, bytes): md5 = md5.decode(UTF8) # If you use customer-provided encryption keys, the ETag value that # Amazon S3 returns in the response will not be the MD5 of the # object. amz_server_side_encryption_customer_algorithm = response.getheader( 'x-amz-server-side-encryption-customer-algorithm', None) # The same is applicable for KMS-encrypted objects in gs buckets. goog_customer_managed_encryption = response.getheader( 'x-goog-encryption-kms-key-name', None) if (amz_server_side_encryption_customer_algorithm is None and goog_customer_managed_encryption is None): if self.etag != '"%s"' % md5: raise provider.storage_data_error( 'ETag from S3 did not match computed MD5. ' '%s vs. %s' % (self.etag, self.md5)) return True if response.status == 400: # The 400 must be trapped so the retry handler can check to # see if it was a timeout. # If ``RequestTimeout`` is present, we'll retry. Otherwise, bomb # out. body = response.read() err = provider.storage_response_error( response.status, response.reason, body ) if err.error_code in ['RequestTimeout']: raise boto.exception.PleaseRetryException( "Saw %s, retrying" % err.error_code, response=response ) return False def HasUserSpecifiedGsHost(): gs_host = boto.config.get('Credentials', 'gs_host', None) default_host = six.ensure_str(boto.gs.connection.GSConnection.DefaultHost) if gs_host is not None: return default_host == six.ensure_str(gs_host) return False def UsingGsHmac(): # NOTE: External credentials are omitted intentionally as HMAC takes priority # over the external credentials types. config = boto.config has_refresh_token = config.has_option('Credentials', 'gs_oauth2_refresh_token') has_service_account_credentials = ( config.has_option('Credentials', 'gs_service_client_id') and config.has_option('Credentials', 'gs_service_key_file')) has_hmac_credentials = ( config.has_option('Credentials', 'gs_access_key_id') and config.has_option('Credentials', 'gs_secret_access_key')) return (not has_refresh_token and not has_service_account_credentials and has_hmac_credentials) utils/metadata_util.py 0000644 00000010636 15025013616 0011100 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility methods for manipulating metadata of requests and resources.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import six from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages def AddAcceptEncodingGzipIfNeeded(headers_dict, compressed_encoding=False): if compressed_encoding: # If we send accept-encoding: gzip with a range request, the service # may respond with the whole object, which would be bad for resuming. # So only accept gzip encoding if the object we are downloading has # a gzip content encoding. # TODO: If we want to support compressive transcoding fully in the client, # condition on whether we are requesting the entire range of the object. # In this case, we can accept the first bytes of the object compressively # transcoded, but we must perform data integrity checking on bytes after # they are decompressed on-the-fly, and any connection break must be # resumed without compressive transcoding since we cannot specify an # offset. We would also need to ensure that hashes for downloaded data # from objects stored with content-encoding:gzip continue to be calculated # prior to our own on-the-fly decompression so they match the stored hashes. headers_dict['accept-encoding'] = 'gzip' def CreateCustomMetadata(entries=None, custom_metadata=None): """Creates a custom MetadataValue object. Inserts the key/value pairs in entries. Args: entries: (Dict[str, Any] or None) The dictionary containing key/value pairs to insert into metadata. Both the key and value must be able to be casted to a string type. custom_metadata (apitools_messages.Object.MetadataValue or None): A pre-existing custom metadata object to add to. If one is not provided, a new one will be constructed. Returns: An apitools_messages.Object.MetadataValue. """ if custom_metadata is None: custom_metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) if entries is None: entries = {} for key, value in six.iteritems(entries): custom_metadata.additionalProperties.append( apitools_messages.Object.MetadataValue.AdditionalProperty( key=str(key), value=str(value))) return custom_metadata def GetValueFromObjectCustomMetadata(obj_metadata, search_key, default_value=None): """Filters a specific element out of an object's custom metadata. Args: obj_metadata: (apitools_messages.Object) The metadata for an object. search_key: (str) The custom metadata key to search for. default_value: (Any) The default value to use for the key if it cannot be found. Returns: (Tuple(bool, Any)) A tuple indicating if the value could be found in metadata and a value corresponding to search_key (the value at the specified key in custom metadata, or the default value if the specified key does not exist in the custom metadata). """ try: value = next((attr.value for attr in obj_metadata.metadata.additionalProperties if attr.key == search_key), None) if value is None: return False, default_value return True, value except AttributeError: return False, default_value def IsCustomMetadataHeader(header): """Returns true if header (which must be lowercase) is a custom header.""" return header.startswith('x-goog-meta-') or header.startswith('x-amz-meta-') def ObjectIsGzipEncoded(obj_metadata): """Returns true if the apitools_messages.Object has gzip content-encoding.""" return (obj_metadata.contentEncoding and obj_metadata.contentEncoding.lower().endswith('gzip')) utils/retry_util.py 0000644 00000006165 15025013616 0010467 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods for handling request retries.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import time from apitools.base.py import http_wrapper from gslib import thread_message from gslib.utils import constants from retry_decorator import retry_decorator Retry = retry_decorator.retry # pylint: disable=invalid-name def LogAndHandleRetries(is_data_transfer=False, status_queue=None): """Higher-order function allowing retry handler to access global status queue. Args: is_data_transfer: If True, disable retries in apitools. status_queue: The global status queue. Returns: A retry function for retryable errors in apitools. """ def WarnAfterManyRetriesHandler(retry_args): """Exception handler for http failures in apitools. If the user has had to wait several seconds since their first request, print a progress message to the terminal to let them know we're still retrying, then perform the default retry logic and post a gslib.thread_message.RetryableErrorMessage to the global status queue. Args: retry_args: An apitools ExceptionRetryArgs tuple. """ if (retry_args.total_wait_sec is not None and retry_args.total_wait_sec >= constants.LONG_RETRY_WARN_SEC): logging.info('Retrying request, attempt #%d...', retry_args.num_retries) if status_queue: status_queue.put( thread_message.RetryableErrorMessage( retry_args.exc, time.time(), num_retries=retry_args.num_retries, total_wait_sec=retry_args.total_wait_sec)) http_wrapper.HandleExceptionsAndRebuildHttpConnections(retry_args) def RetriesInDataTransferHandler(retry_args): """Exception handler that disables retries in apitools data transfers. Post a gslib.thread_message.RetryableErrorMessage to the global status queue. We handle the actual retries within the download and upload functions. Args: retry_args: An apitools ExceptionRetryArgs tuple. """ if status_queue: status_queue.put( thread_message.RetryableErrorMessage( retry_args.exc, time.time(), num_retries=retry_args.num_retries, total_wait_sec=retry_args.total_wait_sec)) http_wrapper.RethrowExceptionHandler(retry_args) if is_data_transfer: return RetriesInDataTransferHandler return WarnAfterManyRetriesHandler utils/constants.py 0000644 00000011523 15025013616 0010273 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared, hard-coded constants. A constant should not be placed in this file if: - it requires complicated or conditional logic to initialize. - it requires importing any modules outside of the Python standard library. This helps reduce dependency graph complexity and the chance of cyclic deps. - it is only used in one file (in which case it should be defined within that module). - it semantically belongs somewhere else (e.g. 'BYTES_PER_KIB' would belong in unit_util.py). """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import sys import six from gslib.utils.unit_util import ONE_GIB from gslib.utils.unit_util import ONE_KIB from gslib.utils.unit_util import ONE_MIB # Readable descriptions for httplib2 logging levels. DEBUGLEVEL_DUMP_REQUESTS = 3 DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS = 4 DEFAULT_FILE_BUFFER_SIZE = 8 * ONE_KIB DEFAULT_GCS_JSON_API_VERSION = 'v1' DEFAULT_GSUTIL_STATE_DIR = os.path.expanduser(os.path.join('~', '.gsutil')) GSUTIL_PUB_TARBALL = 'gs://pub/gsutil.tar.gz' GSUTIL_PUB_TARBALL_PY2 = 'gs://pub/gsutil4.tar.gz' IAM_POLICY_VERSION = 3 IMPERSONATE_SERVICE_ACCOUNT = '' # Number of seconds to wait before printing a long retry warning message. LONG_RETRY_WARN_SEC = 10 # Compressed transport encoded uploads buffer chunks of compressed data. When # running many uploads in parallel, compression may consume more memory than # available. This restricts the number of compressed transport encoded uploads # running in parallel such that they don't consume more memory than set here. MAX_UPLOAD_COMPRESSION_BUFFER_SIZE = 2 * ONE_GIB # On Unix-like systems, we will set the maximum number of open files to avoid # hitting the limit imposed by the OS. This number was obtained experimentally. MIN_ACCEPTABLE_OPEN_FILES_LIMIT = 1000 # For files >= this size, output a message indicating that we're running an # operation on the file (like hashing or gzipping) so it does not appear to the # user that the command is hanging. # TODO: This should say the unit in the name. MIN_SIZE_COMPUTE_LOGGING = 100 * ONE_MIB # The way NO_MAX is used, what is really needed here is the maximum container # size in the Python C code, so using six.MAXSIZE which provides that portably. NO_MAX = six.MAXSIZE # Number of objects to request in listing calls. NUM_OBJECTS_PER_LIST_PAGE = 1000 RELEASE_NOTES_URL = 'https://pub.storage.googleapis.com/gsutil_ReleaseNotes.txt' REQUEST_REASON_ENV_VAR = 'CLOUDSDK_CORE_REQUEST_REASON' REQUEST_REASON_HEADER_KEY = 'x-goog-request-reason' RESUMABLE_THRESHOLD_MIB = 8 RESUMABLE_THRESHOLD_B = RESUMABLE_THRESHOLD_MIB * ONE_MIB # gsutil-specific GUIDs for marking special metadata for S3 compatibility. S3_ACL_MARKER_GUID = '3b89a6b5-b55a-4900-8c44-0b0a2f5eab43-s3-AclMarker' S3_DELETE_MARKER_GUID = 'eadeeee8-fa8c-49bb-8a7d-0362215932d8-s3-DeleteMarker' S3_MARKER_GUIDS = [S3_ACL_MARKER_GUID, S3_DELETE_MARKER_GUID] # By default, the timeout for SSL read errors is infinite. This could # cause gsutil to hang on network disconnect, so pick a more reasonable # timeout. SSL_TIMEOUT_SEC = 60 # Start with a progress callback every <X> KiB during uploads/downloads (JSON # API). Callback implementation should back off until it hits some maximum size # so that callbacks do not create huge amounts of log output. START_CALLBACK_PER_BYTES = 256 * ONE_KIB # Upload/download files in 8 KiB chunks over the HTTP connection. # TODO: This should say the unit in the name. if 'win32' in str(sys.platform).lower(): TRANSFER_BUFFER_SIZE = 64 * ONE_KIB else: TRANSFER_BUFFER_SIZE = 8 * ONE_KIB UTF8 = 'utf-8' WINDOWS_1252 = 'cp1252' # Default number of progress callbacks during transfer (XML API). XML_PROGRESS_CALLBACKS = 10 class Scopes(object): """Enum class for auth scopes, as unicode.""" CLOUD_PLATFORM = 'https://www.googleapis.com/auth/cloud-platform' CLOUD_PLATFORM_READ_ONLY = ( 'https://www.googleapis.com/auth/cloud-platform.read-only') FULL_CONTROL = 'https://www.googleapis.com/auth/devstorage.full_control' READ_ONLY = 'https://www.googleapis.com/auth/devstorage.read_only' READ_WRITE = 'https://www.googleapis.com/auth/devstorage.read_write' REAUTH = 'https://www.googleapis.com/auth/accounts.reauth' utils/user_agent_helper.py 0000644 00000004644 15025013616 0011760 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains helper for appending user agent information.""" import six import sys import gslib from gslib.utils import system_util from gslib.storage_url import StorageUrlFromString from gslib.exception import InvalidUrlError def GetUserAgent(args, metrics_off=True): """Using the command arguments return a suffix for the UserAgent string. Args: args: str[], parsed set of arguments entered in the CLI. metrics_off: boolean, whether the MetricsCollector is disabled. Returns: str, A string value that can be appended to an existing UserAgent. """ user_agent = ' gsutil/%s' % gslib.VERSION user_agent += ' (%s)' % sys.platform user_agent += ' analytics/%s' % ('disabled' if metrics_off else 'enabled') user_agent += ' interactive/%s' % system_util.IsRunningInteractively() if len(args) > 0: user_agent += ' command/%s' % args[0] if len(args) > 2: if args[0] in ['cp', 'mv', 'rsync']: # Any cp, mv or rsync commands that use daisy chain mode should be noted # as that represents a unique use case that may be better served by the # storage transfer service. try: src = StorageUrlFromString(six.ensure_text(args[-2])) dst = StorageUrlFromString(six.ensure_text(args[-1])) if src.IsCloudUrl() and dst.IsCloudUrl() and src.scheme != dst.scheme: user_agent += '-DaisyChain' except InvalidUrlError: pass elif args[0] == 'rewrite': if '-k' in args: # Rewrite encryption key. user_agent += '-k' if '-s' in args: # Rewrite storage class. user_agent += '-s' if system_util.InvokedViaCloudSdk(): user_agent += ' google-cloud-sdk' if system_util.CloudSdkVersion(): user_agent += '/%s' % system_util.CloudSdkVersion() return user_agent utils/cloud_api_helper.py 0000644 00000007734 15025013616 0011566 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for Cloud API implementations.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import re import six from gslib.cloud_api import ArgumentException from gslib.utils.text_util import AddQueryParamToUrl def GetCloudApiInstance(cls, thread_state=None): """Gets a gsutil Cloud API instance. Since Cloud API implementations are not guaranteed to be thread-safe, each thread needs its own instance. These instances are passed to each thread via the thread pool logic in command. Args: cls: Command class to be used for single-threaded case. thread_state: Per thread state from this thread containing a gsutil Cloud API instance. Returns: gsutil Cloud API instance. """ return thread_state or cls.gsutil_api def GetDownloadSerializationData(src_obj_metadata, progress=0, user_project=None): """Returns download serialization data. There are five entries: auto_transfer: JSON-specific field, always False. progress: How much of the download has already been completed. total_size: Total object size. url: Implementation-specific field used for saving a metadata get call. For JSON, this the download URL of the object. For XML, this is a pickled boto key. user_project: Project to be billed to, added as query param. Args: src_obj_metadata: Object to be downloaded. progress: See above. user_project: User project to add to query string. Returns: Serialization data for use with Cloud API GetObjectMedia. """ url = src_obj_metadata.mediaLink if user_project: url = AddQueryParamToUrl(url, 'userProject', user_project) if six.PY3: if isinstance(url, bytes): url = url.decode('ascii') serialization_dict = { 'auto_transfer': 'False', 'progress': progress, 'total_size': src_obj_metadata.size, 'url': url } return json.dumps(serialization_dict) def ListToGetFields(list_fields=None): """Removes 'items/' from the input fields and converts it to a set. Args: list_fields: Iterable fields usable in ListBuckets/ListObjects calls. Returns: Set of fields usable in GetBucket/GetObjectMetadata calls (None implies all fields should be returned). """ if list_fields: get_fields = set() for field in list_fields: if field in ('kind', 'nextPageToken', 'prefixes'): # These are not actually object / bucket metadata fields. # They are fields specific to listing, so we don't consider them. continue get_fields.add(re.sub(r'items/', '', field)) return get_fields def ValidateDstObjectMetadata(dst_obj_metadata): """Ensures dst_obj_metadata supplies the needed fields for copy and insert. Args: dst_obj_metadata: Metadata to validate. Raises: ArgumentException if metadata is invalid. """ if not dst_obj_metadata: raise ArgumentException( 'No object metadata supplied for destination object.') if not dst_obj_metadata.name: raise ArgumentException( 'Object metadata supplied for destination object had no object name.') if not dst_obj_metadata.bucket: raise ArgumentException( 'Object metadata supplied for destination object had no bucket name.') utils/acl_helper.py 0000644 00000031244 15025013616 0010357 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains helper objects for changing and deleting ACLs.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re from gslib.exception import CommandException from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages class ChangeType(object): USER = 'User' GROUP = 'Group' PROJECT = 'Project' class AclChange(object): """Represents a logical change to an access control list.""" public_scopes = ['AllAuthenticatedUsers', 'AllUsers'] id_scopes = ['UserById', 'GroupById'] email_scopes = ['UserByEmail', 'GroupByEmail'] domain_scopes = ['GroupByDomain'] project_scopes = ['Project'] scope_types = (public_scopes + id_scopes + email_scopes + domain_scopes + project_scopes) public_entity_all_users = 'allUsers' public_entity_all_auth_users = 'allAuthenticatedUsers' public_entity_types = (public_entity_all_users, public_entity_all_auth_users) project_entity_prefixes = ('project-editors-', 'project-owners-', 'project-viewers-') group_entity_prefix = 'group-' user_entity_prefix = 'user-' domain_entity_prefix = 'domain-' project_entity_prefix = 'project-' permission_shorthand_mapping = { 'R': 'READER', 'W': 'WRITER', 'FC': 'OWNER', 'O': 'OWNER', 'READ': 'READER', 'WRITE': 'WRITER', 'FULL_CONTROL': 'OWNER', } def __init__(self, acl_change_descriptor, scope_type): """Creates an AclChange object. Args: acl_change_descriptor: An acl change as described in the "ch" section of the "acl" command's help. scope_type: Either ChangeType.USER or ChangeType.GROUP or ChangeType.PROJECT, specifying the extent of the scope. """ self.identifier = '' self.raw_descriptor = acl_change_descriptor self._Parse(acl_change_descriptor, scope_type) self._Validate() def __str__(self): return 'AclChange<{0}|{1}|{2}>'.format(self.scope_type, self.perm, self.identifier) def _Parse(self, change_descriptor, scope_type): """Parses an ACL Change descriptor.""" def _ClassifyScopeIdentifier(text): re_map = { 'AllAuthenticatedUsers': r'^(AllAuthenticatedUsers|AllAuth)$', 'AllUsers': '^(AllUsers|All)$', 'Email': r'^.+@.+\..+$', 'Id': r'^[0-9A-Fa-f]{64}$', 'Domain': r'^[^@]+\.[^@]+$', 'Project': r'(owners|editors|viewers)\-.+$', } for type_string, regex in re_map.items(): if re.match(regex, text, re.IGNORECASE): return type_string if change_descriptor.count(':') != 1: raise CommandException( '{0} is an invalid change description.'.format(change_descriptor)) scope_string, perm_token = change_descriptor.split(':') perm_token = perm_token.upper() if perm_token in self.permission_shorthand_mapping: self.perm = self.permission_shorthand_mapping[perm_token] else: self.perm = perm_token scope_class = _ClassifyScopeIdentifier(scope_string) if scope_class == 'Domain': # This may produce an invalid UserByDomain scope, # which is good because then validate can complain. self.scope_type = '{0}ByDomain'.format(scope_type) self.identifier = scope_string elif scope_class in ('Email', 'Id'): self.scope_type = '{0}By{1}'.format(scope_type, scope_class) self.identifier = scope_string elif scope_class == 'AllAuthenticatedUsers': self.scope_type = 'AllAuthenticatedUsers' elif scope_class == 'AllUsers': self.scope_type = 'AllUsers' elif scope_class == 'Project': self.scope_type = 'Project' self.identifier = scope_string else: # This is just a fallback, so we set it to something # and the validate step has something to go on. self.scope_type = scope_string def _Validate(self): """Validates a parsed AclChange object.""" def _ThrowError(msg): raise CommandException('{0} is not a valid ACL change\n{1}'.format( self.raw_descriptor, msg)) if self.scope_type not in self.scope_types: _ThrowError('{0} is not a valid scope type'.format(self.scope_type)) if self.scope_type in self.public_scopes and self.identifier: _ThrowError('{0} requires no arguments'.format(self.scope_type)) if self.scope_type in self.id_scopes and not self.identifier: _ThrowError('{0} requires an id'.format(self.scope_type)) if self.scope_type in self.email_scopes and not self.identifier: _ThrowError('{0} requires an email address'.format(self.scope_type)) if self.scope_type in self.domain_scopes and not self.identifier: _ThrowError('{0} requires domain'.format(self.scope_type)) if self.perm not in self.permission_shorthand_mapping.values(): perms = ', '.join(set(self.permission_shorthand_mapping.values())) _ThrowError('Allowed permissions are {0}'.format(perms)) def _YieldMatchingEntries(self, current_acl): """Generator that yields entries that match the change descriptor. Args: current_acl: A list of apitools_messages.BucketAccessControls or ObjectAccessControls which will be searched for matching entries. Yields: An apitools_messages.BucketAccessControl or ObjectAccessControl. """ for entry in current_acl: if (self.scope_type in ('UserById', 'GroupById') and entry.entityId and self.identifier == entry.entityId): yield entry elif (self.scope_type in ('UserByEmail', 'GroupByEmail') and entry.email and self.identifier == entry.email): yield entry elif (self.scope_type == 'GroupByDomain' and entry.domain and self.identifier == entry.domain): yield entry elif (self.scope_type == 'Project' and entry.projectTeam and self.identifier == '%s-%s' % (entry.projectTeam.team, entry.projectTeam.projectNumber)): yield entry elif (self.scope_type == 'AllUsers' and entry.entity.lower() == self.public_entity_all_users.lower()): yield entry elif (self.scope_type == 'AllAuthenticatedUsers' and entry.entity.lower() == self.public_entity_all_auth_users.lower()): yield entry def GetEntity(self): """Gets an appropriate entity string for an ACL grant.""" if self.scope_type in ('UserById', 'UserByEmail'): return self.user_entity_prefix + self.identifier elif self.scope_type in ('GroupById', 'GroupByEmail'): return self.group_entity_prefix + self.identifier elif self.scope_type == 'Project': return self.project_entity_prefix + self.identifier elif self.scope_type == 'GroupByDomain': return self.domain_entity_prefix + self.identifier elif self.scope_type == 'AllAuthenticatedUsers': return self.public_entity_all_auth_users elif self.scope_type == 'AllUsers': return self.public_entity_all_users else: raise CommandException('Add entry to ACL got unexpected scope type %s.' % self.scope_type) def _AddEntry(self, current_acl, entry_class): """Adds an entry to current_acl.""" entity = self.GetEntity() entry = entry_class(role=self.perm, entity=entity) if self.scope_type in ('UserById', 'GroupById', 'Project'): entry.entityId = self.identifier elif self.scope_type in ('UserByEmail', 'GroupByEmail'): entry.email = self.identifier elif self.scope_type == 'GroupByDomain': entry.domain = self.identifier current_acl.append(entry) def _GetEntriesClass(self, current_acl): # Entries will share the same class, so just return the first one. for acl_entry in current_acl: return acl_entry.__class__ # It's possible that a default object ACL is empty, so if we have # an empty list, assume it is an object ACL. return apitools_messages.ObjectAccessControl().__class__ def Execute(self, storage_url, current_acl, command_name, logger): """Executes the described change on an ACL. Args: storage_url: StorageUrl representing the object to change. current_acl: A list of ObjectAccessControls or BucketAccessControls to permute. command_name: String name of comamnd being run (e.g., 'acl'). logger: An instance of logging.Logger. Returns: The number of changes that were made. """ logger.debug('Executing %s %s on %s', command_name, self.raw_descriptor, storage_url) if self.perm == 'WRITER': if command_name == 'acl' and storage_url.IsObject(): logger.warning('Skipping %s on %s, as WRITER does not apply to objects', self.raw_descriptor, storage_url) return 0 elif command_name == 'defacl': raise CommandException('WRITER cannot be set as a default object ACL ' 'because WRITER does not apply to objects') entry_class = self._GetEntriesClass(current_acl) matching_entries = list(self._YieldMatchingEntries(current_acl)) change_count = 0 if matching_entries: for entry in matching_entries: if entry.role != self.perm: entry.role = self.perm change_count += 1 else: self._AddEntry(current_acl, entry_class) change_count = 1 logger.debug('New Acl:\n%s', str(current_acl)) return change_count class AclDel(object): """Represents a logical change from an access control list.""" scope_regexes = { r'All(Users)?$': 'AllUsers', r'AllAuth(enticatedUsers)?$': 'AllAuthenticatedUsers', } def __init__(self, identifier): self.raw_descriptor = '-d {0}'.format(identifier) self.identifier = identifier for regex, scope in self.scope_regexes.items(): if re.match(regex, self.identifier, re.IGNORECASE): self.identifier = scope self.scope_type = 'Any' self.perm = 'NONE' def _YieldMatchingEntries(self, current_acl): """Generator that yields entries that match the change descriptor. Args: current_acl: An instance of apitools_messages.BucketAccessControls or ObjectAccessControls which will be searched for matching entries. Yields: An apitools_messages.BucketAccessControl or ObjectAccessControl. """ for entry in current_acl: if entry.entityId and self.identifier.lower() == entry.entityId.lower(): yield entry elif entry.email and self.identifier.lower() == entry.email.lower(): yield entry elif entry.domain and self.identifier.lower() == entry.domain.lower(): yield entry elif (entry.projectTeam and self.identifier.lower() == '%s-%s'.lower() % (entry.projectTeam.team, entry.projectTeam.projectNumber)): yield entry elif entry.entity.lower() == 'allusers' and self.identifier == 'AllUsers': yield entry elif (entry.entity.lower() == 'allauthenticatedusers' and self.identifier == 'AllAuthenticatedUsers'): yield entry def Execute(self, storage_url, current_acl, command_name, logger): logger.debug('Executing %s %s on %s', command_name, self.raw_descriptor, storage_url) matching_entries = list(self._YieldMatchingEntries(current_acl)) for entry in matching_entries: current_acl.remove(entry) logger.debug('New Acl:\n%s', str(current_acl)) return len(matching_entries) def translate_sub_opts_for_shim(sub_opts): translated_sub_opts = [] scope_type_from_flag = { '-g': ChangeType.GROUP, '-p': ChangeType.PROJECT, '-u': ChangeType.USER, } for (flag, value) in sub_opts: if flag in scope_type_from_flag: change = AclChange(value, scope_type=scope_type_from_flag[flag]) new_value = 'entity={},role={}'.format(change.GetEntity(), change.perm) translated_sub_opts.append((flag, new_value)) elif flag == '-d': change = AclDel(value) translated_sub_opts.append(('-d', change.identifier)) else: translated_sub_opts.append((flag, value)) return translated_sub_opts utils/wrapped_credentials.py 0000644 00000015524 15025013616 0012303 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2022 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Classes and functions to allow google.auth credentials to be used within oauth2client. In particular, the External Account credentials don't have an equivalent in oauth2client, so we create helper methods to allow variants of this particular class to be used in oauth2client workflows. """ import copy import datetime import io import json from google.auth import aws from google.auth import credentials from google.auth import exceptions from google.auth import external_account from google.auth import external_account_authorized_user from google.auth import identity_pool from google.auth import pluggable from google.auth.transport import requests from gslib.utils import constants import oauth2client DEFAULT_SCOPES = [ constants.Scopes.CLOUD_PLATFORM, constants.Scopes.CLOUD_PLATFORM_READ_ONLY, constants.Scopes.FULL_CONTROL, constants.Scopes.READ_ONLY, constants.Scopes.READ_WRITE, ] class WrappedCredentials(oauth2client.client.OAuth2Credentials): """A utility class to use Google Auth credentials in place of oauth2client credentials. """ NON_SERIALIZED_MEMBERS = frozenset( list(oauth2client.client.OAuth2Credentials.NON_SERIALIZED_MEMBERS) + ['_base']) def __init__(self, base_creds): """Initializes oauth2client credentials based on underlying Google Auth credentials. Args: external_account_creds: subclass of google.auth.external_account.Credentials """ self._base = base_creds if isinstance(base_creds, external_account.Credentials): client_id = self._base._audience client_secret = None refresh_token = None elif isinstance(base_creds, external_account_authorized_user.Credentials): client_id = self._base.client_id client_secret = self._base.client_secret refresh_token = self._base.refresh_token else: raise TypeError("Invalid Credentials") super(WrappedCredentials, self).__init__(access_token=self._base.token, client_id=client_id, client_secret=client_secret, refresh_token=refresh_token, token_expiry=self._base.expiry, token_uri=None, user_agent=None) def _do_refresh_request(self, http): self._base.refresh(requests.Request()) if self.store is not None: self.store.locked_put(self) @property def access_token(self): return self._base.token @access_token.setter def access_token(self, value): self._base.token = value @property def token_expiry(self): return self._base.expiry @token_expiry.setter def token_expiry(self, value): self._base.expiry = value def to_json(self): """Utility function that creates JSON repr. of a Credentials object. Returns: string, a JSON representation of this instance, suitable to pass to from_json(). """ serialized_data = super().to_json() deserialized_data = json.loads(serialized_data) deserialized_data['_base'] = copy.copy(self._base.info) deserialized_data['access_token'] = self._base.token deserialized_data['token_expiry'] = _parse_expiry(self.token_expiry) return json.dumps(deserialized_data) @classmethod def for_external_account(cls, filename): creds = _get_external_account_credentials_from_file(filename) return cls(creds) @classmethod def for_external_account_authorized_user(cls, filename): creds = _get_external_account_authorized_user_credentials_from_file( filename) return cls(creds) @classmethod def from_json(cls, json_data): """Instantiate a Credentials object from a JSON description of it. The JSON should have been produced by calling .to_json() on the object. Args: data: dict, A deserialized JSON object. Returns: An instance of a Credentials subclass. """ data = json.loads(json_data) # Rebuild the credentials. base = data.get("_base") # Init base cred. base_creds = None if base.get('type') == 'external_account': base_creds = _get_external_account_credentials_from_info(base) elif base.get('type') == 'external_account_authorized_user': base_creds = _get_external_account_authorized_user_credentials_from_info( base) creds = cls(base_creds) # Inject token and expiry. creds.access_token = data.get("access_token") if (data.get('token_expiry') and not isinstance(data['token_expiry'], datetime.datetime)): try: data['token_expiry'] = datetime.datetime.strptime( data['token_expiry'], oauth2client.client.EXPIRY_FORMAT) except ValueError: data['token_expiry'] = None creds.token_expiry = data.get("token_expiry") return creds def _get_external_account_credentials_from_info(info): if info.get( 'subject_token_type') == 'urn:ietf:params:aws:token-type:aws4_request': # Configuration corresponds to an AWS credentials. return aws.Credentials.from_info(info, scopes=DEFAULT_SCOPES) elif (info.get('credential_source') is not None and info.get('credential_source').get('executable') is not None): # Configuration corresponds to pluggable credentials. return pluggable.Credentials.from_info(info, scopes=DEFAULT_SCOPES) else: # Configuration corresponds to an identity pool credentials. return identity_pool.Credentials.from_info(info, scopes=DEFAULT_SCOPES) def _get_external_account_credentials_from_file(filename): with io.open(filename, "r", encoding="utf-8") as json_file: data = json.load(json_file) return _get_external_account_credentials_from_info(data) def _get_external_account_authorized_user_credentials_from_info(info): return external_account_authorized_user.Credentials.from_info(info) def _get_external_account_authorized_user_credentials_from_file(filename): with io.open(filename, "r", encoding="utf-8") as json_file: data = json.load(json_file) return _get_external_account_authorized_user_credentials_from_info(data) def _parse_expiry(expiry): if expiry and isinstance(expiry, datetime.datetime): return expiry.strftime(oauth2client.client.EXPIRY_FORMAT) else: return None utils/signurl_helper.py 0000644 00000012534 15025013616 0011304 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility functions for signurl command.""" import base64 from datetime import datetime from datetime import timezone import hashlib from gslib.utils.constants import UTF8 import six from six.moves import urllib _CANONICAL_REQUEST_FORMAT = ('{method}\n{resource}\n{query_string}\n{headers}' '\n{signed_headers}\n{hashed_payload}') _SIGNING_ALGO = 'GOOG4-RSA-SHA256' _STRING_TO_SIGN_FORMAT = ('{signing_algo}\n{request_time}\n{credential_scope}' '\n{hashed_request}') _SIGNED_URL_FORMAT = ('https://{host}/{path}?x-goog-signature={sig}&' '{query_string}') _UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD' def _NowUTC(): return datetime.now(tz=timezone.utc).replace(tzinfo=None) def CreatePayload(client_id, method, duration, path, generation, logger, region, signed_headers, billing_project=None, string_to_sign_debug=False): """Create a string that needs to be signed. Args: client_id: Client ID signing this URL. method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. path: String path to the bucket of object for signing, in the form 'bucket' or 'bucket/object'. generation: If not None, specifies a version of an object for signing. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. signed_headers: Dict containing the header info like host content-type etc. billing_project: Specify a user project to be billed for the request. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. Returns: A tuple where the 1st element is the string to sign. The second element is the query string. """ signing_time = _NowUTC() canonical_day = signing_time.strftime('%Y%m%d') canonical_time = signing_time.strftime('%Y%m%dT%H%M%SZ') canonical_scope = '{date}/{region}/storage/goog4_request'.format( date=canonical_day, region=region) signed_query_params = { 'x-goog-algorithm': _SIGNING_ALGO, 'x-goog-credential': client_id + '/' + canonical_scope, 'x-goog-date': canonical_time, 'x-goog-signedheaders': ';'.join(sorted(signed_headers.keys())), 'x-goog-expires': '%d' % duration.total_seconds() } if (billing_project is not None): signed_query_params['userProject'] = billing_project if generation is not None: signed_query_params['generation'] = generation canonical_resource = '/{}'.format(path) canonical_query_string = '&'.join([ '{}={}'.format(param, urllib.parse.quote_plus(signed_query_params[param])) for param in sorted(signed_query_params.keys()) ]) canonical_headers = '\n'.join([ '{}:{}'.format(header.lower(), signed_headers[header]) for header in sorted(signed_headers.keys()) ]) + '\n' canonical_signed_headers = ';'.join(sorted(signed_headers.keys())) canonical_request = _CANONICAL_REQUEST_FORMAT.format( method=method, resource=canonical_resource, query_string=canonical_query_string, headers=canonical_headers, signed_headers=canonical_signed_headers, hashed_payload=_UNSIGNED_PAYLOAD) if six.PY3: canonical_request = canonical_request.encode(UTF8) canonical_request_hasher = hashlib.sha256() canonical_request_hasher.update(canonical_request) hashed_canonical_request = base64.b16encode( canonical_request_hasher.digest()).lower().decode(UTF8) string_to_sign = _STRING_TO_SIGN_FORMAT.format( signing_algo=_SIGNING_ALGO, request_time=canonical_time, credential_scope=canonical_scope, hashed_request=hashed_canonical_request) if string_to_sign_debug and logger: logger.debug( 'Canonical request (ignore opening/closing brackets): [[[%s]]]' % canonical_request) logger.debug('String to sign (ignore opening/closing brackets): [[[%s]]]' % string_to_sign) return string_to_sign, canonical_query_string def GetFinalUrl(raw_signature, host, path, canonical_query_string): """Get the final signed url.""" signature = base64.b16encode(raw_signature).lower().decode() return _SIGNED_URL_FORMAT.format(host=host, path=path, sig=signature, query_string=canonical_query_string) def to_bytes(value, encoding=UTF8): if isinstance(value, bytes): return value return value.encode(encoding) utils/hashing_helper.py 0000644 00000044510 15025013616 0011241 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for hashing functionality.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import base64 import binascii import hashlib import os import six from boto import config import crcmod from gslib.exception import CommandException from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.constants import DEFAULT_FILE_BUFFER_SIZE from gslib.utils.constants import MIN_SIZE_COMPUTE_LOGGING from gslib.utils.constants import TRANSFER_BUFFER_SIZE from gslib.utils.constants import UTF8 SLOW_CRCMOD_WARNING = """ WARNING: You have requested checksumming but your crcmod installation isn't using the module's C extension, so checksumming will run very slowly. For help installing the extension, please see "gsutil help crcmod". """ SLOW_CRCMOD_RSYNC_WARNING = """ WARNING: gsutil rsync uses hashes when modification time is not available at both the source and destination. Your crcmod installation isn't using the module's C extension, so checksumming will run very slowly. If this is your first rsync since updating gsutil, this rsync can take significantly longer than usual. For help installing the extension, please see "gsutil help crcmod". """ _SLOW_CRCMOD_DOWNLOAD_WARNING = """ WARNING: Downloading this composite object requires integrity checking with CRC32c, but your crcmod installation isn't using the module's C extension, so the hash computation will likely throttle download performance. For help installing the extension, please see "gsutil help crcmod". To disable slow integrity checking, see the "check_hashes" option in your boto config file. """ _SLOW_CRC_EXCEPTION_TEXT = """ Downloading this composite object requires integrity checking with CRC32c, but your crcmod installation isn't using the module's C extension, so the hash computation will likely throttle download performance. For help installing the extension, please see "gsutil help crcmod". To download regardless of crcmod performance or to skip slow integrity checks, see the "check_hashes" option in your boto config file. NOTE: It is strongly recommended that you not disable integrity checks. Doing so could allow data corruption to go undetected during uploading/downloading.""" _NO_HASH_CHECK_WARNING = """ WARNING: This download will not be validated since your crcmod installation doesn't use the module's C extension, so the hash computation would likely throttle download performance. For help in installing the extension, please see "gsutil help crcmod". To force integrity checking, see the "check_hashes" option in your boto config file. """ # Configuration values for hashing. CHECK_HASH_IF_FAST_ELSE_FAIL = 'if_fast_else_fail' CHECK_HASH_IF_FAST_ELSE_SKIP = 'if_fast_else_skip' CHECK_HASH_ALWAYS = 'always' CHECK_HASH_NEVER = 'never' # Table storing polynomial values of x^(2^k) mod CASTAGNOLI_POLY for all k < 31, # where x^(2^k) and CASTAGNOLI_POLY are both considered polynomials. This is # sufficient since x^(2^31) mod CASTAGNOLI_POLY = x. X_POW_2K_TABLE = [ 2, 4, 16, 256, 65536, 517762881, 984302966, 408362264, 1503875210, 2862076957, 3884826397, 1324787473, 621200174, 1758783527, 1416537776, 1180494764, 648569364, 2521473789, 994858823, 1728245375, 3498467999, 4059169852, 3345064394, 2828422810, 2429203150, 3336788029, 860151998, 2102628683, 1033187991, 4243778976, 1123580069 ] # Castagnoli polynomial and its degree. CASTAGNOLI_POLY = 4812730177 DEGREE = 32 def ConcatCrc32c(crc_a, crc_b, num_bytes_in_b): """Computes CRC32C for concat(A, B) given crc(A), crc(B) and len(B). An explanation of the algorithm can be found at crcutil.googlecode.com/files/crc-doc.1.0.pdf. Args: crc_a: A 32-bit integer representing crc(A) with least-significant coefficient first. crc_b: Same as crc_a. num_bytes_in_b: Length of B in bytes. Returns: CRC32C for concat(A, B) """ if not num_bytes_in_b: return crc_a return _ExtendByZeros(crc_a, 8 * num_bytes_in_b) ^ crc_b def _CrcMultiply(p, q): """Multiplies two polynomials together modulo CASTAGNOLI_POLY. Args: p: The first polynomial. q: The second polynomial. Returns: Result of the multiplication. """ result = 0 top_bit = 1 << DEGREE for _ in range(DEGREE): if p & 1: result ^= q q <<= 1 if q & top_bit: q ^= CASTAGNOLI_POLY p >>= 1 return result def _ExtendByZeros(crc, num_bits): """Given crc representing polynomial P(x), compute P(x)*x^num_bits. Args: crc: crc respresenting polynomial P(x). num_bits: number of bits in crc. Returns: P(x)*x^num_bits """ def _ReverseBits32(crc): return int('{0:032b}'.format(crc, width=32)[::-1], 2) crc = _ReverseBits32(crc) i = 0 while num_bits != 0: if num_bits & 1: crc = _CrcMultiply(crc, X_POW_2K_TABLE[i % len(X_POW_2K_TABLE)]) i += 1 num_bits >>= 1 crc = _ReverseBits32(crc) return crc def _CalculateHashFromContents(fp, hash_alg): """Calculates a base64 digest of the contents of a seekable stream. This function resets the file pointer to position 0. Args: fp: An already-open file object. hash_alg: Instance of hashing class initialized to start state. Returns: Hash of the stream in hex string format. """ hash_dict = {'placeholder': hash_alg} fp.seek(0) CalculateHashesFromContents(fp, hash_dict) fp.seek(0) return hash_dict['placeholder'].hexdigest() def CalculateHashesFromContents(fp, hash_dict, callback_processor=None): """Calculates hashes of the contents of a file. Args: fp: An already-open file object (stream will be consumed). hash_dict: Dict of (string alg_name: initialized hashing class) Hashing class will be populated with digests upon return. callback_processor: Optional callback processing class that implements Progress(integer amount of bytes processed). """ while True: data = fp.read(DEFAULT_FILE_BUFFER_SIZE) if not data: break if six.PY3: if isinstance(data, str): data = data.encode(UTF8) for hash_alg in six.itervalues(hash_dict): hash_alg.update(data) if callback_processor: callback_processor.Progress(len(data)) def CalculateB64EncodedCrc32cFromContents(fp): """Calculates a base64 CRC32c checksum of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. Returns: CRC32c checksum of the file in base64 format. """ return _CalculateB64EncodedHashFromContents(fp, crcmod.predefined.Crc('crc-32c')) def CalculateB64EncodedMd5FromContents(fp): """Calculates a base64 MD5 digest of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. Returns: MD5 digest of the file in base64 format. """ return _CalculateB64EncodedHashFromContents(fp, GetMd5()) def CalculateMd5FromContents(fp): """Calculates a base64 MD5 digest of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. Returns: MD5 digest of the file in hex format. """ return _CalculateHashFromContents(fp, GetMd5()) def Base64EncodeHash(digest_value): """Returns the base64-encoded version of the input hex digest value.""" encoded_bytes = base64.b64encode(binascii.unhexlify(digest_value)) return encoded_bytes.rstrip(b'\n').decode(UTF8) def Base64ToHexHash(base64_hash): """Returns the hex digest value of the input base64-encoded hash. Args: base64_hash: Base64-encoded hash, which may contain newlines and single or double quotes. Returns: Hex digest of the input argument. """ decoded_bytes = base64.b64decode(base64_hash.strip('\n"\'').encode(UTF8)) return binascii.hexlify(decoded_bytes) def _CalculateB64EncodedHashFromContents(fp, hash_alg): """Calculates a base64 digest of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. hash_alg: Instance of hashing class initialized to start state. Returns: Hash of the stream in base64 format. """ return Base64EncodeHash(_CalculateHashFromContents(fp, hash_alg)) def GetUploadHashAlgs(): """Returns a dict of hash algorithms for validating an uploaded object. This is for use only with single object uploads, not compose operations such as those used by parallel composite uploads (though it can be used to validate the individual components). Returns: dict of (algorithm_name: hash_algorithm) """ check_hashes_config = config.get('GSUtil', 'check_hashes', CHECK_HASH_IF_FAST_ELSE_FAIL) if check_hashes_config == 'never': return {} return {'md5': GetMd5} def GetDownloadHashAlgs(logger, consider_md5=False, consider_crc32c=False): """Returns a dict of hash algorithms for validating an object. Args: logger: logging.Logger for outputting log messages. consider_md5: If True, consider using a md5 hash. consider_crc32c: If True, consider using a crc32c hash. Returns: Dict of (string, hash algorithm). Raises: CommandException if hash algorithms satisfying the boto config file cannot be returned. """ check_hashes_config = config.get('GSUtil', 'check_hashes', CHECK_HASH_IF_FAST_ELSE_FAIL) if check_hashes_config == CHECK_HASH_NEVER: return {} hash_algs = {} if consider_md5: hash_algs['md5'] = GetMd5 elif consider_crc32c: # If the cloud provider supplies a CRC, we'll compute a checksum to # validate if we're using a native crcmod installation and MD5 isn't # offered as an alternative. if UsingCrcmodExtension(): hash_algs['crc32c'] = lambda: crcmod.predefined.Crc('crc-32c') elif not hash_algs: if check_hashes_config == CHECK_HASH_IF_FAST_ELSE_FAIL: raise CommandException(_SLOW_CRC_EXCEPTION_TEXT) elif check_hashes_config == CHECK_HASH_IF_FAST_ELSE_SKIP: logger.warn(_NO_HASH_CHECK_WARNING) elif check_hashes_config == CHECK_HASH_ALWAYS: logger.warn(_SLOW_CRCMOD_DOWNLOAD_WARNING) hash_algs['crc32c'] = lambda: crcmod.predefined.Crc('crc-32c') else: raise CommandException( 'Your boto config \'check_hashes\' option is misconfigured.') return hash_algs def GetMd5(byte_string=b''): """Returns md5 object, avoiding incorrect FIPS error on Red Hat systems. Examples: GetMd5(b'abc') GetMd5(bytes('abc', encoding='utf-8')) Args: byte_string (bytes): String in bytes form to hash. Don't include for empty hash object, since md5(b'').digest() == md5().digest(). Returns: md5 hash object. """ try: return hashlib.md5(byte_string) except ValueError: # On Red Hat-based platforms, may catch a FIPS error. # "usedforsecurity" flag only available on Red Hat systems or Python 3.9+. # pylint:disable=unexpected-keyword-arg return hashlib.md5(byte_string, usedforsecurity=False) # pylint:enable=unexpected-keyword-arg class HashingFileUploadWrapper(object): """Wraps an input stream in a hash digester and exposes a stream interface. This class provides integrity checking during file uploads via the following properties: Calls to read will appropriately update digesters with all bytes read. Calls to seek (assuming it is supported by the wrapped stream) using os.SEEK_SET will catch up / reset the digesters to the specified position. If seek is called with a different os.SEEK mode, the caller must return to the original position using os.SEEK_SET before further reads. Calls to seek are fast if the desired position is equal to the position at the beginning of the last read call (we only need to re-hash bytes from that point on). """ def __init__(self, stream, digesters, hash_algs, src_url, logger): """Initializes the wrapper. Args: stream: Input stream. digesters: dict of {string: hash digester} containing digesters, where string is the name of the hash algorithm. hash_algs: dict of {string: hash algorithm} for resetting and recalculating digesters. String is the name of the hash algorithm. src_url: Source FileUrl that is being copied. logger: For outputting log messages. """ if not digesters: raise CommandException('HashingFileUploadWrapper used with no digesters.') elif not hash_algs: raise CommandException('HashingFileUploadWrapper used with no hash_algs.') self._orig_fp = stream self._digesters = digesters self._src_url = src_url self._logger = logger self._seek_away = None self._digesters_previous = {} for alg in self._digesters: self._digesters_previous[alg] = self._digesters[alg].copy() self._digesters_previous_mark = 0 self._digesters_current_mark = 0 self._hash_algs = hash_algs @property def mode(self): """Returns the mode of the underlying file descriptor, or None.""" return getattr(self._orig_fp, 'mode', None) def read(self, size=-1): # pylint: disable=invalid-name """"Reads from the wrapped file pointer and calculates hash digests. Args: size: The amount of bytes to read. If ommited or negative, the entire contents of the file will be read, hashed, and returned. Returns: Bytes from the wrapped stream. Raises: CommandException if the position of the wrapped stream is unknown. """ if self._seek_away is not None: raise CommandException('Read called on hashing file pointer in an ' 'unknown position; cannot correctly compute ' 'digest.') data = self._orig_fp.read(size) if isinstance(data, six.text_type): data = data.encode(UTF8) self._digesters_previous_mark = self._digesters_current_mark for alg in self._digesters: self._digesters_previous[alg] = self._digesters[alg].copy() self._digesters[alg].update(data) self._digesters_current_mark += len(data) return data def tell(self): # pylint: disable=invalid-name """Returns the current stream position.""" return self._orig_fp.tell() def seekable(self): # pylint: disable=invalid-name """Returns true if the stream is seekable.""" return self._orig_fp.seekable() def seek(self, offset, whence=os.SEEK_SET): # pylint: disable=invalid-name """Seeks in the wrapped file pointer and catches up hash digests. Args: offset: The offset to seek to. whence: os.SEEK_CUR, or SEEK_END, SEEK_SET. Returns: Return value from the wrapped stream's seek call. """ if whence != os.SEEK_SET: # We do not catch up hashes for non-absolute seeks, and rely on the # caller to seek to an absolute position before reading. self._seek_away = self._orig_fp.tell() else: # Hashes will be correct and it's safe to call read(). self._seek_away = None if offset < self._digesters_previous_mark: # This is earlier than our earliest saved digest, so we need to # reset the digesters and scan from the beginning. for alg in self._digesters: self._digesters[alg] = self._hash_algs[alg]() self._digesters_current_mark = 0 self._orig_fp.seek(0) self._CatchUp(offset) elif offset == self._digesters_previous_mark: # Just load the saved digests. self._digesters_current_mark = self._digesters_previous_mark for alg in self._digesters: self._digesters[alg] = self._digesters_previous[alg] elif offset < self._digesters_current_mark: # Reset the position to our previous digest and scan forward. self._digesters_current_mark = self._digesters_previous_mark for alg in self._digesters: self._digesters[alg] = self._digesters_previous[alg] self._orig_fp.seek(self._digesters_previous_mark) self._CatchUp(offset - self._digesters_previous_mark) else: # Scan forward from our current digest and position. self._orig_fp.seek(self._digesters_current_mark) self._CatchUp(offset - self._digesters_current_mark) return self._orig_fp.seek(offset, whence) def _CatchUp(self, bytes_to_read): """Catches up hashes, but does not return data and uses little memory. Before calling this function, digesters_current_mark should be updated to the current location of the original stream and the self._digesters should be current to that point (but no further). Args: bytes_to_read: Number of bytes to catch up from the original stream. """ if self._orig_fp.tell() != self._digesters_current_mark: raise CommandException( 'Invalid mark when catching up hashes. Stream position %s, hash ' 'position %s' % (self._orig_fp.tell(), self._digesters_current_mark)) for alg in self._digesters: if bytes_to_read >= MIN_SIZE_COMPUTE_LOGGING: self._logger.debug('Catching up %s for %s...', alg, self._src_url.url_string) self._digesters_previous[alg] = self._digesters[alg].copy() self._digesters_previous_mark = self._digesters_current_mark bytes_remaining = bytes_to_read bytes_this_round = min(bytes_remaining, TRANSFER_BUFFER_SIZE) while bytes_this_round: data = self._orig_fp.read(bytes_this_round) if isinstance(data, six.text_type): data = data.encode(UTF8) bytes_remaining -= bytes_this_round for alg in self._digesters: self._digesters[alg].update(data) bytes_this_round = min(bytes_remaining, TRANSFER_BUFFER_SIZE) self._digesters_current_mark += bytes_to_read utils/text_util.py 0000644 00000031767 15025013616 0010314 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods for manipulating text.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import binascii import codecs import os import sys import io import re import locale import collections import random import six import string from six.moves import urllib from six.moves import range from gslib.exception import CommandException from gslib.lazy_wrapper import LazyWrapper from gslib.utils.constants import UTF8 from gslib.utils.constants import WINDOWS_1252 from gslib.utils.system_util import IS_CP1252 if six.PY3: long = int STORAGE_CLASS_SHORTHAND_TO_FULL_NAME = { # Values should remain uppercase, as required by non-gs providers. 'CL': 'COLDLINE', 'DRA': 'DURABLE_REDUCED_AVAILABILITY', 'NL': 'NEARLINE', 'S': 'STANDARD', 'STD': 'STANDARD', 'A': 'ARCHIVE', } VERSION_MATCHER = LazyWrapper( lambda: re.compile(r'^(?P<maj>\d+)(\.(?P<min>\d+)(?P<suffix>.*))?')) def AddQueryParamToUrl(url_str, param_name, param_value): """Adds a query parameter to a URL string. Appends a query parameter to the query string portion of a url. If a parameter with the given name was already present, it is not removed; the new name/value pair will be appended to the end of the query string. It is assumed that all arguments will be of type `str` (either ASCII or UTF-8 encoded) or `unicode`. Note that this method performs no URL-encoding. It is the caller's responsibility to ensure proper URL encoding of the entire URL; i.e. if the URL is already URL-encoded, you should pass in URL-encoded values for param_name and param_value. If the URL is not URL-encoded, you should not pass in URL-encoded parameters; instead, you could perform URL-encoding using the URL string returned from this function. Args: url_str: (str or unicode) String representing the URL. param_name: (str or unicode) String key of the query parameter. param_value: (str or unicode) String value of the query parameter. Returns: (str or unicode) A string representing the modified url, of type `unicode` if the url_str argument was a `unicode`, otherwise a `str` encoded in UTF-8. """ scheme, netloc, path, query_str, fragment = urllib.parse.urlsplit(url_str) query_params = urllib.parse.parse_qsl(query_str, keep_blank_values=True) query_params.append((param_name, param_value)) new_query_str = '&'.join(['%s=%s' % (k, v) for (k, v) in query_params]) new_url = urllib.parse.urlunsplit( (scheme, netloc, path, new_query_str, fragment)) return new_url def CompareVersions(first, second): """Compares the first and second gsutil version strings. For example, 3.33 > 3.7, and 4.1 is a greater major version than 3.33. Does not handle multiple periods (e.g. 3.3.4) or complicated suffixes (e.g., 3.3RC4 vs. 3.3RC5). A version string with a suffix is treated as less than its non-suffix counterpart (e.g. 3.32 > 3.32pre). Args: first: First gsutil version string. second: Second gsutil version string. Returns: (g, m): g is True if first known to be greater than second, else False. m is True if first known to be greater by at least 1 major version, else False. """ m1 = VERSION_MATCHER().match(str(first)) m2 = VERSION_MATCHER().match(str(second)) # If passed strings we don't know how to handle, be conservative. if not m1 or not m2: return (False, False) major_ver1 = int(m1.group('maj')) minor_ver1 = int(m1.group('min')) if m1.group('min') else 0 suffix_ver1 = m1.group('suffix') major_ver2 = int(m2.group('maj')) minor_ver2 = int(m2.group('min')) if m2.group('min') else 0 suffix_ver2 = m2.group('suffix') if major_ver1 > major_ver2: return (True, True) elif major_ver1 == major_ver2: if minor_ver1 > minor_ver2: return (True, False) elif minor_ver1 == minor_ver2: return (bool(suffix_ver2) and not suffix_ver1, False) return (False, False) def ConvertRecursiveToFlatWildcard(url_strs): """A generator that adds '**' to each url string in url_strs.""" for url_str in url_strs: yield '%s**' % url_str def DecodeLongAsString(long_to_convert): """Decodes an encoded python long into an ASCII string. This is used for modeling S3 version_id's as apitools generation. Args: long_to_convert: long to convert to ASCII string. If this is already a string, it is simply returned. Returns: String decoded from the input long. """ unhexed = binascii.unhexlify(hex(long_to_convert)[2:].rstrip('L')) return six.ensure_str(unhexed) def EncodeStringAsLong(string_to_convert): """Encodes an ASCII string as a python long. This is used for modeling S3 version_id's as apitools generation. Because python longs can be arbitrarily large, this works. Args: string_to_convert: ASCII string to convert to a long. Returns: Long that represents the input string. """ hex_bytestr = codecs.encode(six.ensure_binary(string_to_convert), 'hex_codec') # Note that `long`/`int` accepts either `bytes` or `unicode` as the # first arg in both py2 and py3: return long(hex_bytestr, 16) def FixWindowsEncodingIfNeeded(input_str): """Attempts to detect Windows CP1252 encoding and convert to UTF8. Windows doesn't provide a way to set UTF-8 for string encodings; you can set the system locale (see http://windows.microsoft.com/en-us/windows/change-system-locale#1TC=windows-7) but that takes you to a "Change system locale" dropdown that just lists languages (e.g., "English (United States)". Instead, we're forced to check if a encoding as UTF8 raises an exception and if so, try converting from CP1252 to Unicode. Args: input_str: (str or bytes) The input string. Returns: (unicode) The converted string or the original, if conversion wasn't needed. """ if IS_CP1252: return six.ensure_text(input_str, WINDOWS_1252) else: return six.ensure_text(input_str, UTF8) def GetPrintableExceptionString(exc): """Returns a short Unicode string describing the exception.""" return six.text_type(exc).encode(UTF8) or six.text_type(exc.__class__) def InsistAscii(string, message): """Ensures that the string passed in consists of only ASCII values. Args: string: Union[str, unicode, bytes] Text that will be checked for ASCII values. message: Union[str, unicode, bytes] Error message, passed into the exception, in the event that the check on `string` fails. Returns: None Raises: CommandException """ if not all(ord(c) < 128 for c in string): raise CommandException(message) def InsistAsciiHeader(header): """Checks for ASCII-only characters in `header`. Also constructs an error message using `header` if the check fails. Args: header: Union[str, binary, unicode] Text being checked for ASCII values. Returns: None """ InsistAscii(header, 'Invalid non-ASCII header (%s).' % header) def InsistAsciiHeaderValue(header, value): """Checks for ASCII-only characters in `value`. Also constructs an error message using `header` and `value` if the check fails. Args: header: Header name, only used in error message in case of an exception. value: Union[str, binary, unicode] Text being checked for ASCII values. Returns: None """ InsistAscii( value, 'Invalid non-ASCII value (%s) was provided for header %s.\nOnly ASCII ' 'characters are allowed in headers other than x-goog-meta- and ' 'x-amz-meta- headers' % (repr(value), header)) def InsistOnOrOff(value, message): """Ensures that the value passed in consists of only "on" or "off" Args: value: (unicode) Unicode string that will be checked for correct text. message: Union[str, unicode, bytes] Error message passed into the exception in the event that the check on value fails. Returns: None Raises: CommandException """ if value != 'on' and value != 'off': raise CommandException(message) def NormalizeStorageClass(sc): """Returns a normalized form of the given storage class name. Converts the given string to uppercase and expands valid abbreviations to full storage class names (e.g 'std' would return 'STANDARD'). Note that this method does not check if the given storage class is valid. Args: sc: (str) String representing the storage class's full name or abbreviation. Returns: (str) A string representing the full name of the given storage class. """ # Use uppercase; storage class argument for the S3 API must be uppercase, # and it's case-insensitive for GS APIs. sc = sc.upper() if sc in STORAGE_CLASS_SHORTHAND_TO_FULL_NAME: sc = STORAGE_CLASS_SHORTHAND_TO_FULL_NAME[sc] return sc def PrintableStr(input_val): """Return an UTF8-encoded string type, or None if `input_val` is None. Args: input_val: (unicode, str, or None) A string-like object or None. This method simply calls encode() on `input_val` if it is not None; if `input_val` is not of type "unicode", this will implicitly call decode() with the default encoding for strings (for Python 2, this is ASCII), then call encode(). Returns: (str) A UTF-8 encoded string, or None. """ return input_val def print_to_fd(*objects, **kwargs): """A Python 2/3 compatible analogue to the print function. This function writes text to a file descriptor as the builtin print function would, favoring unicode encoding. Aguments and return values are the same as documented in the Python 2 print function. """ def _get_args(**kwargs): """Validates keyword arguments that would be used in Print Valid keyword arguments, mirroring print(), are 'sep', 'end', and 'file'. These must be of types string, string, and file / file interface respectively. Returns the above kwargs of the above types. """ expected_keywords = collections.OrderedDict([('sep', ' '), ('end', '\n'), ('file', sys.stdout)]) for key, value in kwargs.items(): if key not in expected_keywords: error_msg = ('{} is not a valid keyword argument. ' 'Please use one of: {}') raise KeyError(error_msg.format(key, ' '.join(expected_keywords.keys()))) else: expected_keywords[key] = value return expected_keywords.values() def _get_byte_strings(*objects): """Gets a `bytes` string for each item in a list of printable objects.""" byte_objects = [] for item in objects: if not isinstance(item, (six.binary_type, six.text_type)): # If the item wasn't bytes or unicode, its __str__ method # should return one of those types. item = str(item) if isinstance(item, six.binary_type): byte_objects.append(item) else: # The item should be unicode. If it's not, ensure_binary() # will throw a TypeError. byte_objects.append(six.ensure_binary(item)) return byte_objects sep, end, file = _get_args(**kwargs) sep = six.ensure_binary(sep) end = six.ensure_binary(end) data = _get_byte_strings(*objects) data = sep.join(data) data += end write_to_fd(file, data) def write_to_fd(fd, data): """Write given data to given file descriptor, doing any conversions needed""" if six.PY2: fd.write(data) return # PY3 logic: if isinstance(data, bytes): if (hasattr(fd, 'mode') and 'b' in fd.mode) or isinstance(fd, io.BytesIO): fd.write(data) elif hasattr(fd, 'buffer'): fd.buffer.write(data) else: fd.write(six.ensure_text(data)) elif 'b' in fd.mode: fd.write(six.ensure_binary(data)) else: fd.write(data) def RemoveCRLFFromString(input_str): r"""Returns the input string with all \n and \r removed.""" return re.sub(r'[\r\n]', '', input_str) def get_random_ascii_chars(size, seed=0): """Generates binary string representation of a list of ASCII characters. Args: size: Integer quantity of characters to generate. seed: A seed may be specified for deterministic behavior. Int 0 is used as the default value. Returns: Binary encoded string representation of a list of characters of length equal to size argument. """ random.seed(seed) contents = str([random.choice(string.ascii_letters) for _ in range(size)]) contents = six.ensure_binary(contents) random.seed() # Reset the seed for any other tests. return contents utils/translation_helper.py 0000644 00000114645 15025013616 0012165 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility module for translating XML API objects to/from JSON objects.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import datetime import json import re import textwrap import xml.etree.ElementTree from xml.etree.ElementTree import ParseError as XmlParseError import six from apitools.base.protorpclite.util import decode_datetime from apitools.base.py import encoding import boto from boto.gs.acl import ACL from boto.gs.acl import ALL_AUTHENTICATED_USERS from boto.gs.acl import ALL_USERS from boto.gs.acl import Entries from boto.gs.acl import Entry from boto.gs.acl import GROUP_BY_DOMAIN from boto.gs.acl import GROUP_BY_EMAIL from boto.gs.acl import GROUP_BY_ID from boto.gs.acl import USER_BY_EMAIL from boto.gs.acl import USER_BY_ID from boto.s3.tagging import Tags from boto.s3.tagging import TagSet from gslib.cloud_api import ArgumentException from gslib.cloud_api import BucketNotFoundException from gslib.cloud_api import NotFoundException from gslib.cloud_api import Preconditions from gslib.exception import CommandException from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import S3_ACL_MARKER_GUID from gslib.utils.constants import S3_MARKER_GUIDS if six.PY3: long = int CACHE_CONTROL_REGEX = re.compile(r'^cache-control', re.I) CONTENT_DISPOSITION_REGEX = re.compile(r'^content-disposition', re.I) CONTENT_ENCODING_REGEX = re.compile(r'^content-encoding', re.I) CONTENT_LANGUAGE_REGEX = re.compile(r'^content-language', re.I) CONTENT_MD5_REGEX = re.compile(r'^content-md5', re.I) CONTENT_TYPE_REGEX = re.compile(r'^content-type', re.I) CUSTOM_TIME_REGEX = re.compile(r'^custom-time', re.I) GOOG_API_VERSION_REGEX = re.compile(r'^x-goog-api-version', re.I) GOOG_GENERATION_MATCH_REGEX = re.compile(r'^x-goog-if-generation-match', re.I) GOOG_METAGENERATION_MATCH_REGEX = re.compile(r'^x-goog-if-metageneration-match', re.I) CUSTOM_GOOG_METADATA_REGEX = re.compile(r'^x-goog-meta-(?P<header_key>.*)', re.I) CUSTOM_AMZ_METADATA_REGEX = re.compile(r'^x-amz-meta-(?P<header_key>.*)', re.I) CUSTOM_AMZ_HEADER_REGEX = re.compile(r'^x-amz-(?P<header_key>.*)', re.I) metadata_header_regexes = frozenset({ CACHE_CONTROL_REGEX, CONTENT_DISPOSITION_REGEX, CONTENT_ENCODING_REGEX, CONTENT_LANGUAGE_REGEX, CONTENT_MD5_REGEX, CONTENT_TYPE_REGEX, CUSTOM_TIME_REGEX, GOOG_API_VERSION_REGEX, GOOG_GENERATION_MATCH_REGEX, GOOG_METAGENERATION_MATCH_REGEX, CUSTOM_GOOG_METADATA_REGEX, CUSTOM_AMZ_METADATA_REGEX, }) # This distinguishes S3 custom headers from S3 metadata on objects. S3_HEADER_PREFIX = 'custom-amz-header' DEFAULT_CONTENT_TYPE = 'application/octet-stream' # Because CORS is just a list in apitools, we need special handling or blank # CORS lists will get sent with other configuration commands such as lifecycle, # which would cause CORS configuration to be unintentionally removed. # Protorpc defaults list values to an empty list, and won't allow us to set the # value to None like other configuration fields, so there is no way to # distinguish the default value from when we actually want to remove the CORS # configuration. To work around this, we create a dummy CORS entry that # signifies that we should nullify the CORS configuration. # A value of [] means don't modify the CORS configuration. # A value of REMOVE_CORS_CONFIG means remove the CORS configuration. REMOVE_CORS_CONFIG = [ apitools_messages.Bucket.CorsValueListEntry(maxAgeSeconds=-1, method=['REMOVE_CORS_CONFIG']) ] # Similar to CORS above, we need a sentinel value allowing us to specify # when a default object ACL should be private (containing no entries). # A defaultObjectAcl value of [] means don't modify the default object ACL. # A value of [PRIVATE_DEFAULT_OBJ_ACL] means create an empty/private default # object ACL. PRIVATE_DEFAULT_OBJ_ACL = apitools_messages.ObjectAccessControl( id='PRIVATE_DEFAULT_OBJ_ACL') def GetNonMetadataHeaders(headers): arbitrary_headers = {} for header, value in headers.items(): if not any(regex.match(header) for regex in metadata_header_regexes): arbitrary_headers[header] = value return arbitrary_headers def ObjectMetadataFromHeaders(headers): """Creates object metadata according to the provided headers. gsutil -h allows specifiying various headers (originally intended to be passed to boto in gsutil v3). For the JSON API to be compatible with this option, we need to parse these headers into gsutil_api Object fields. Args: headers: Dict of headers passed via gsutil -h Raises: ArgumentException if an invalid header is encountered. Returns: apitools Object with relevant fields populated from headers. """ obj_metadata = apitools_messages.Object() for header, value in headers.items(): if CACHE_CONTROL_REGEX.match(header): obj_metadata.cacheControl = value.strip() elif CONTENT_DISPOSITION_REGEX.match(header): obj_metadata.contentDisposition = value.strip() elif CONTENT_ENCODING_REGEX.match(header): obj_metadata.contentEncoding = value.strip() elif CONTENT_MD5_REGEX.match(header): obj_metadata.md5Hash = value.strip() elif CONTENT_LANGUAGE_REGEX.match(header): obj_metadata.contentLanguage = value.strip() elif CONTENT_TYPE_REGEX.match(header): if not value: obj_metadata.contentType = DEFAULT_CONTENT_TYPE else: obj_metadata.contentType = value.strip() elif CUSTOM_TIME_REGEX.match(header): obj_metadata.customTime = decode_datetime(value.strip()) elif GOOG_API_VERSION_REGEX.match(header): # API version is only relevant for XML, ignore and rely on the XML API # to add the appropriate version. continue elif GOOG_GENERATION_MATCH_REGEX.match(header): # Preconditions are handled elsewhere, but allow these headers through. continue elif GOOG_METAGENERATION_MATCH_REGEX.match(header): # Preconditions are handled elsewhere, but allow these headers through. continue else: custom_goog_metadata_match = CUSTOM_GOOG_METADATA_REGEX.match(header) custom_amz_metadata_match = CUSTOM_AMZ_METADATA_REGEX.match(header) custom_amz_header_match = CUSTOM_AMZ_HEADER_REGEX.match(header) header_key = None if custom_goog_metadata_match: header_key = custom_goog_metadata_match.group('header_key') elif custom_amz_metadata_match: header_key = custom_amz_metadata_match.group('header_key') elif custom_amz_header_match: # If we got here we are guaranteed by the prior statement that this is # not an x-amz-meta- header. header_key = (S3_HEADER_PREFIX + custom_amz_header_match.group('header_key')) if header_key: if header_key.lower() == 'x-goog-content-language': # Work around content-language being inserted into custom metadata. continue if not obj_metadata.metadata: obj_metadata.metadata = apitools_messages.Object.MetadataValue() if not obj_metadata.metadata.additionalProperties: obj_metadata.metadata.additionalProperties = [] obj_metadata.metadata.additionalProperties.append( apitools_messages.Object.MetadataValue.AdditionalProperty( key=header_key, value=value)) return obj_metadata def HeadersFromObjectMetadata(dst_obj_metadata, provider): """Creates a header dictionary based on existing object metadata. Args: dst_obj_metadata: Object metadata to create the headers from. provider: Provider string ('gs' or 's3'). Returns: Headers dictionary. """ headers = {} if not dst_obj_metadata: return # Metadata values of '' mean suppress/remove this header. if dst_obj_metadata.cacheControl is not None: if not dst_obj_metadata.cacheControl: headers['cache-control'] = None else: headers['cache-control'] = dst_obj_metadata.cacheControl.strip() if dst_obj_metadata.contentDisposition: if not dst_obj_metadata.contentDisposition: headers['content-disposition'] = None else: headers['content-disposition'] = ( dst_obj_metadata.contentDisposition.strip()) if dst_obj_metadata.contentEncoding: if not dst_obj_metadata.contentEncoding: headers['content-encoding'] = None else: headers['content-encoding'] = dst_obj_metadata.contentEncoding.strip() if dst_obj_metadata.contentLanguage: if not dst_obj_metadata.contentLanguage: headers['content-language'] = None else: headers['content-language'] = dst_obj_metadata.contentLanguage.strip() if dst_obj_metadata.md5Hash: if not dst_obj_metadata.md5Hash: headers['Content-MD5'] = None else: headers['Content-MD5'] = dst_obj_metadata.md5Hash.strip() if dst_obj_metadata.contentType is not None: if not dst_obj_metadata.contentType: headers['content-type'] = None else: headers['content-type'] = dst_obj_metadata.contentType.strip() if dst_obj_metadata.customTime is not None: if not dst_obj_metadata.customTime: headers['custom-time'] = None else: headers['custom-time'] = dst_obj_metadata.customTime.strip() if dst_obj_metadata.storageClass: header_name = 'storage-class' if provider == 'gs': header_name = 'x-goog-' + header_name elif provider == 's3': header_name = 'x-amz-' + header_name else: raise ArgumentException('Invalid provider specified: %s' % provider) headers[header_name] = dst_obj_metadata.storageClass.strip() if (dst_obj_metadata.metadata and dst_obj_metadata.metadata.additionalProperties): for additional_property in dst_obj_metadata.metadata.additionalProperties: # Work around content-language being inserted into custom metadata by # the XML API. if additional_property.key == 'content-language': continue # Don't translate special metadata markers. if additional_property.key in S3_MARKER_GUIDS: continue if provider == 'gs': header_name = 'x-goog-meta-' + additional_property.key elif provider == 's3': if additional_property.key.startswith(S3_HEADER_PREFIX): header_name = ('x-amz-' + additional_property.key[len(S3_HEADER_PREFIX):]) else: header_name = 'x-amz-meta-' + additional_property.key else: raise ArgumentException('Invalid provider specified: %s' % provider) if (additional_property.value is not None and not additional_property.value): headers[header_name] = None else: headers[header_name] = additional_property.value return headers def CopyObjectMetadata(src_obj_metadata, dst_obj_metadata, override=False): """Copies metadata from src_obj_metadata to dst_obj_metadata. Args: src_obj_metadata: Metadata from source object. dst_obj_metadata: Initialized metadata for destination object. override: If true, will overwrite metadata in destination object. If false, only writes metadata for values that don't already exist. """ if override or not dst_obj_metadata.cacheControl: dst_obj_metadata.cacheControl = src_obj_metadata.cacheControl if override or not dst_obj_metadata.contentDisposition: dst_obj_metadata.contentDisposition = src_obj_metadata.contentDisposition if override or not dst_obj_metadata.contentEncoding: dst_obj_metadata.contentEncoding = src_obj_metadata.contentEncoding if override or not dst_obj_metadata.contentLanguage: dst_obj_metadata.contentLanguage = src_obj_metadata.contentLanguage if override or not dst_obj_metadata.contentType: dst_obj_metadata.contentType = src_obj_metadata.contentType if override or not dst_obj_metadata.customTime: dst_obj_metadata.customTime = src_obj_metadata.customTime if override or not dst_obj_metadata.md5Hash: dst_obj_metadata.md5Hash = src_obj_metadata.md5Hash CopyCustomMetadata(src_obj_metadata, dst_obj_metadata, override=override) def CopyCustomMetadata(src_obj_metadata, dst_obj_metadata, override=False): """Copies custom metadata from src_obj_metadata to dst_obj_metadata. Args: src_obj_metadata: Metadata from source object. dst_obj_metadata: Initialized metadata for destination object. override: If true, will overwrite metadata in destination object. If false, only writes metadata for values that don't already exist. """ # TODO: Apitools should ideally treat metadata like a real dictionary instead # of a list of key/value pairs (with an O(N^2) lookup). In practice the # number of values is typically small enough not to matter. # Work around this by creating our own dictionary. if (src_obj_metadata.metadata and src_obj_metadata.metadata.additionalProperties): if not dst_obj_metadata.metadata: dst_obj_metadata.metadata = apitools_messages.Object.MetadataValue() if not dst_obj_metadata.metadata.additionalProperties: dst_obj_metadata.metadata.additionalProperties = [] dst_metadata_dict = {} for dst_prop in dst_obj_metadata.metadata.additionalProperties: dst_metadata_dict[dst_prop.key] = dst_prop.value for src_prop in src_obj_metadata.metadata.additionalProperties: if src_prop.key in dst_metadata_dict: if override: # Metadata values of '' mean suppress/remove this header. if src_prop.value is not None and not src_prop.value: dst_metadata_dict[src_prop.key] = None else: dst_metadata_dict[src_prop.key] = src_prop.value elif src_prop.value != '': # pylint: disable=g-explicit-bool-comparison # Don't propagate '' value since that means to remove the header. dst_metadata_dict[src_prop.key] = src_prop.value # Rewrite the list with our updated dict. dst_obj_metadata.metadata.additionalProperties = [] for k, v in six.iteritems(dst_metadata_dict): dst_obj_metadata.metadata.additionalProperties.append( apitools_messages.Object.MetadataValue.AdditionalProperty(key=k, value=v)) def PreconditionsFromHeaders(headers): """Creates bucket or object preconditions acccording to the provided headers. Args: headers: Dict of headers passed via gsutil -h Returns: gsutil Cloud API Preconditions object fields populated from headers, or None if no precondition headers are present. """ return_preconditions = Preconditions() try: for header, value in headers.items(): if GOOG_GENERATION_MATCH_REGEX.match(header): return_preconditions.gen_match = long(value) if GOOG_METAGENERATION_MATCH_REGEX.match(header): return_preconditions.meta_gen_match = long(value) except ValueError as _: raise ArgumentException('Invalid precondition header specified. ' 'x-goog-if-generation-match and ' 'x-goog-if-metageneration match must be specified ' 'with a positive integer value.') return return_preconditions def CreateNotFoundExceptionForObjectWrite(dst_provider, dst_bucket_name, src_provider=None, src_bucket_name=None, src_object_name=None, src_generation=None): """Creates a NotFoundException for an object upload or copy. This is necessary because 404s don't necessarily specify which resource does not exist. Args: dst_provider: String abbreviation of destination provider, e.g., 'gs'. dst_bucket_name: Destination bucket name for the write operation. src_provider: String abbreviation of source provider, i.e. 'gs', if any. src_bucket_name: Source bucket name, if any (for the copy case). src_object_name: Source object name, if any (for the copy case). src_generation: Source object generation, if any (for the copy case). Returns: NotFoundException with appropriate message. """ dst_url_string = '%s://%s' % (dst_provider, dst_bucket_name) if src_bucket_name and src_object_name: src_url_string = '%s://%s/%s' % (src_provider, src_bucket_name, src_object_name) if src_generation: src_url_string += '#%s' % str(src_generation) return NotFoundException( 'The source object %s or the destination bucket %s does not exist.' % (src_url_string, dst_url_string)) return NotFoundException( 'The destination bucket %s does not exist or the write to the ' 'destination must be restarted' % dst_url_string) def CreateBucketNotFoundException(code, provider, bucket_name): return BucketNotFoundException('%s://%s bucket does not exist.' % (provider, bucket_name), bucket_name, status=code) def CreateObjectNotFoundException(code, provider, bucket_name, object_name, generation=None): uri_string = '%s://%s/%s' % (provider, bucket_name, object_name) if generation: uri_string += '#%s' % str(generation) return NotFoundException('%s does not exist.' % uri_string, status=code) def CheckForXmlConfigurationAndRaise(config_type_string, json_txt): """Checks a JSON parse exception for provided XML configuration.""" try: xml.etree.ElementTree.fromstring(str(json_txt)) raise ArgumentException('\n'.join( textwrap.wrap( 'XML {0} data provided; Google Cloud Storage {0} configuration ' 'now uses JSON format. To convert your {0}, set the desired XML ' 'ACL using \'gsutil {1} set ...\' with gsutil version 3.x. Then ' 'use \'gsutil {1} get ...\' with gsutil version 4 or greater to ' 'get the corresponding JSON {0}.'.format( config_type_string, config_type_string.lower())))) except XmlParseError: pass raise ArgumentException('JSON %s data could not be loaded ' 'from: %s' % (config_type_string, json_txt)) class LifecycleTranslation(object): """Functions for converting between various lifecycle formats. This class handles conversation to and from Boto Cors objects, JSON text, and apitools Message objects. """ @classmethod def BotoLifecycleFromMessage(cls, lifecycle_message): """Translates an apitools message to a boto lifecycle object.""" boto_lifecycle = boto.gs.lifecycle.LifecycleConfig() if lifecycle_message: for rule_message in lifecycle_message.rule: boto_rule = boto.gs.lifecycle.Rule() if rule_message.action and rule_message.action.type: if rule_message.action.type.lower() == 'delete': boto_rule.action = boto.gs.lifecycle.DELETE elif rule_message.action.type.lower() == 'setstorageclass': boto_rule.action = boto.gs.lifecycle.SET_STORAGE_CLASS boto_rule.action_text = rule_message.action.storageClass if rule_message.condition: if rule_message.condition.age is not None: boto_rule.conditions[boto.gs.lifecycle.AGE] = (str( rule_message.condition.age)) if rule_message.condition.createdBefore: boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE] = (str( rule_message.condition.createdBefore)) if rule_message.condition.isLive is not None: boto_rule.conditions[boto.gs.lifecycle.IS_LIVE] = ( # Note that the GCS XML API only accepts "false" or "true" # in all lower case. str(rule_message.condition.isLive).lower()) if rule_message.condition.matchesStorageClass: boto_rule.conditions[boto.gs.lifecycle.MATCHES_STORAGE_CLASS] = [ str(sc) for sc in rule_message.condition.matchesStorageClass ] if rule_message.condition.numNewerVersions is not None: boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS] = (str( rule_message.condition.numNewerVersions)) boto_lifecycle.append(boto_rule) return boto_lifecycle @classmethod def BotoLifecycleToMessage(cls, boto_lifecycle): """Translates a boto lifecycle object to an apitools message.""" lifecycle_message = None if boto_lifecycle: lifecycle_message = apitools_messages.Bucket.LifecycleValue() for boto_rule in boto_lifecycle: lifecycle_rule = ( apitools_messages.Bucket.LifecycleValue.RuleValueListEntry()) lifecycle_rule.condition = (apitools_messages.Bucket.LifecycleValue. RuleValueListEntry.ConditionValue()) if boto_rule.action: if boto_rule.action == boto.gs.lifecycle.DELETE: lifecycle_rule.action = (apitools_messages.Bucket.LifecycleValue. RuleValueListEntry.ActionValue( type='Delete')) elif boto_rule.action == boto.gs.lifecycle.SET_STORAGE_CLASS: lifecycle_rule.action = (apitools_messages.Bucket.LifecycleValue. RuleValueListEntry.ActionValue( type='SetStorageClass', storageClass=boto_rule.action_text)) if boto.gs.lifecycle.AGE in boto_rule.conditions: lifecycle_rule.condition.age = int( boto_rule.conditions[boto.gs.lifecycle.AGE]) if boto.gs.lifecycle.CREATED_BEFORE in boto_rule.conditions: lifecycle_rule.condition.createdBefore = ( LifecycleTranslation.TranslateBotoLifecycleTimestamp( boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE])) if boto.gs.lifecycle.IS_LIVE in boto_rule.conditions: boto_is_live_str = ( boto_rule.conditions[boto.gs.lifecycle.IS_LIVE].lower()) if boto_is_live_str == 'true': lifecycle_rule.condition.isLive = True elif boto_is_live_str == 'false': lifecycle_rule.condition.isLive = False else: raise CommandException( 'Got an invalid Boto value for IsLive condition ("%s"), ' 'expected "true" or "false".' % boto_rule.conditions[boto.gs.lifecycle.IS_LIVE]) if boto.gs.lifecycle.MATCHES_STORAGE_CLASS in boto_rule.conditions: for storage_class in ( boto_rule.conditions[boto.gs.lifecycle.MATCHES_STORAGE_CLASS]): lifecycle_rule.condition.matchesStorageClass.append(storage_class) if boto.gs.lifecycle.NUM_NEWER_VERSIONS in boto_rule.conditions: lifecycle_rule.condition.numNewerVersions = int( boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS]) lifecycle_message.rule.append(lifecycle_rule) return lifecycle_message @classmethod def JsonLifecycleFromMessage(cls, lifecycle_message): """Translates an apitools message to lifecycle JSON.""" return str(encoding.MessageToJson(lifecycle_message)) + '\n' @classmethod def JsonLifecycleToMessage(cls, json_txt): """Translates lifecycle JSON to an apitools message.""" try: deserialized_lifecycle = json.loads(json_txt) # If lifecycle JSON is the in the following format # {'lifecycle': {'rule': ... then strip out the 'lifecycle' key # and reduce it to the following format # {'rule': ... if 'lifecycle' in deserialized_lifecycle: deserialized_lifecycle = deserialized_lifecycle['lifecycle'] lifecycle = encoding.DictToMessage( deserialized_lifecycle or {}, apitools_messages.Bucket.LifecycleValue) return lifecycle except ValueError: CheckForXmlConfigurationAndRaise('lifecycle', json_txt) @classmethod def TranslateBotoLifecycleTimestamp(cls, lifecycle_datetime): """Parses the timestamp from the boto lifecycle into a datetime object.""" return datetime.datetime.strptime(lifecycle_datetime, '%Y-%m-%d').date() class CorsTranslation(object): """Functions for converting between various CORS formats. This class handles conversation to and from Boto Cors objects, JSON text, and apitools Message objects. """ @classmethod def BotoCorsFromMessage(cls, cors_message): """Translates an apitools message to a boto Cors object.""" cors = boto.gs.cors.Cors() cors.cors = [] for collection_message in cors_message: collection_elements = [] if collection_message.maxAgeSeconds: collection_elements.append( (boto.gs.cors.MAXAGESEC, str(collection_message.maxAgeSeconds))) if collection_message.method: method_elements = [] for method in collection_message.method: method_elements.append((boto.gs.cors.METHOD, method)) collection_elements.append((boto.gs.cors.METHODS, method_elements)) if collection_message.origin: origin_elements = [] for origin in collection_message.origin: origin_elements.append((boto.gs.cors.ORIGIN, origin)) collection_elements.append((boto.gs.cors.ORIGINS, origin_elements)) if collection_message.responseHeader: header_elements = [] for header in collection_message.responseHeader: header_elements.append((boto.gs.cors.HEADER, header)) collection_elements.append((boto.gs.cors.HEADERS, header_elements)) cors.cors.append(collection_elements) return cors @classmethod def BotoCorsToMessage(cls, boto_cors): """Translates a boto Cors object to an apitools message.""" message_cors = [] if boto_cors.cors: for cors_collection in boto_cors.cors: if cors_collection: collection_message = apitools_messages.Bucket.CorsValueListEntry() for element_tuple in cors_collection: if element_tuple[0] == boto.gs.cors.MAXAGESEC: collection_message.maxAgeSeconds = int(element_tuple[1]) if element_tuple[0] == boto.gs.cors.METHODS: for method_tuple in element_tuple[1]: collection_message.method.append(method_tuple[1]) if element_tuple[0] == boto.gs.cors.ORIGINS: for origin_tuple in element_tuple[1]: collection_message.origin.append(origin_tuple[1]) if element_tuple[0] == boto.gs.cors.HEADERS: for header_tuple in element_tuple[1]: collection_message.responseHeader.append(header_tuple[1]) message_cors.append(collection_message) return message_cors @classmethod def JsonCorsToMessageEntries(cls, json_cors): """Translates CORS JSON to an apitools message. Args: json_cors: JSON string representing CORS configuration. Raises: ArgumentException on invalid CORS JSON data. Returns: List of apitools Bucket.CorsValueListEntry. An empty list represents no CORS configuration. """ deserialized_cors = None try: deserialized_cors = json.loads(json_cors) except ValueError: CheckForXmlConfigurationAndRaise('CORS', json_cors) if not isinstance(deserialized_cors, list): raise ArgumentException( 'CORS JSON should be formatted as a list containing one or more JSON ' 'objects.\nSee "gsutil help cors".') cors = [] for cors_entry in deserialized_cors: cors.append( encoding.DictToMessage(cors_entry, apitools_messages.Bucket.CorsValueListEntry)) return cors @classmethod def MessageEntriesToJson(cls, cors_message): """Translates an apitools message to CORS JSON.""" json_text = '' # Because CORS is a MessageField, serialize/deserialize as JSON list. json_text += '[' printed_one = False for cors_entry in cors_message: if printed_one: json_text += ',' else: printed_one = True json_text += encoding.MessageToJson(cors_entry) json_text += ']\n' return json_text def S3MarkerAclFromObjectMetadata(object_metadata): """Retrieves GUID-marked S3 ACL from object metadata, if present. Args: object_metadata: Object metadata to check. Returns: S3 ACL text, if present, None otherwise. """ if (object_metadata and object_metadata.metadata and object_metadata.metadata.additionalProperties): for prop in object_metadata.metadata.additionalProperties: if prop.key == S3_ACL_MARKER_GUID: return prop.value def AddS3MarkerAclToObjectMetadata(object_metadata, acl_text): """Adds a GUID-marked S3 ACL to the object metadata. Args: object_metadata: Object metadata to add the acl to. acl_text: S3 ACL text to add. """ if not object_metadata.metadata: object_metadata.metadata = apitools_messages.Object.MetadataValue() if not object_metadata.metadata.additionalProperties: object_metadata.metadata.additionalProperties = [] object_metadata.metadata.additionalProperties.append( apitools_messages.Object.MetadataValue.AdditionalProperty( key=S3_ACL_MARKER_GUID, value=acl_text)) def UnaryDictToXml(message): """Generates XML representation of a nested dict. This dict contains exactly one top-level entry and an arbitrary number of 2nd-level entries, e.g. capturing a WebsiteConfiguration message. Args: message: The dict encoding the message. Returns: XML string representation of the input dict. Raises: Exception: if dict contains more than one top-level entry. """ if len(message) != 1: raise Exception('Expected dict of size 1, got size %d' % len(message)) name, content = message.items()[0] element_type = xml.etree.ElementTree.Element(name) for element_property, value in sorted(content.items()): node = xml.etree.ElementTree.SubElement(element_type, element_property) node.text = value return xml.etree.ElementTree.tostring(element_type) class LabelTranslation(object): """Functions for converting between various Label(JSON)/Tags(XML) formats. This class handles conversion to and from Boto Tags objects, JSON text, and apitools LabelsValue message objects. """ @classmethod def BotoTagsToMessage(cls, tags): label_dict = {} for tag_set in tags: label_dict.update(dict((i.key, i.value) for i in tag_set)) return cls.DictToMessage(label_dict) if label_dict else None @classmethod def BotoTagsFromMessage(cls, message): label_dict = json.loads(cls.JsonFromMessage(message)) tag_set = TagSet() for key, value in six.iteritems(label_dict): if value: # Skip values which may be set to None. tag_set.add_tag(key, value) tags = Tags() tags.add_tag_set(tag_set) return tags @classmethod def JsonFromMessage(cls, message, pretty_print=False): json_str = encoding.MessageToJson(message) if pretty_print: return json.dumps(json.loads(json_str), sort_keys=True, indent=2, separators=(',', ': ')) return json_str @classmethod def DictToMessage(cls, label_dict): return encoding.DictToMessage(label_dict, apitools_messages.Bucket.LabelsValue) class AclTranslation(object): """Functions for converting between various ACL formats. This class handles conversion to and from Boto ACL objects, JSON text, and apitools Message objects. """ JSON_TO_XML_ROLES = { 'READER': 'READ', 'WRITER': 'WRITE', 'OWNER': 'FULL_CONTROL', } XML_TO_JSON_ROLES = { 'READ': 'READER', 'WRITE': 'WRITER', 'FULL_CONTROL': 'OWNER', } @classmethod def BotoAclFromJson(cls, acl_json): acl = ACL() acl.parent = None acl.entries = cls.BotoEntriesFromJson(acl_json, acl) return acl @classmethod # acl_message is a list of messages, either object or bucketaccesscontrol def BotoAclFromMessage(cls, acl_message): acl_dicts = [] for message in acl_message: if message == PRIVATE_DEFAULT_OBJ_ACL: # Sentinel value indicating acl_dicts should be an empty list to create # a private (no entries) default object ACL. break acl_dicts.append(encoding.MessageToDict(message)) return cls.BotoAclFromJson(acl_dicts) @classmethod def BotoAclToJson(cls, acl): if hasattr(acl, 'entries'): return cls.BotoEntriesToJson(acl.entries) return [] @classmethod def BotoObjectAclToMessage(cls, acl): for entry in cls.BotoAclToJson(acl): message = encoding.DictToMessage(entry, apitools_messages.ObjectAccessControl) message.kind = 'storage#objectAccessControl' yield message @classmethod def BotoBucketAclToMessage(cls, acl): for entry in cls.BotoAclToJson(acl): message = encoding.DictToMessage(entry, apitools_messages.BucketAccessControl) message.kind = 'storage#bucketAccessControl' yield message @classmethod def BotoEntriesFromJson(cls, acl_json, parent): entries = Entries(parent) entries.parent = parent entries.entry_list = [ cls.BotoEntryFromJson(entry_json) for entry_json in acl_json ] return entries @classmethod def BotoEntriesToJson(cls, entries): return [cls.BotoEntryToJson(entry) for entry in entries.entry_list] @classmethod def BotoEntryFromJson(cls, entry_json): """Converts a JSON entry into a Boto ACL entry.""" entity = entry_json['entity'] permission = cls.JSON_TO_XML_ROLES[entry_json['role']] if entity.lower() == ALL_USERS.lower(): return Entry(type=ALL_USERS, permission=permission) elif entity.lower() == ALL_AUTHENTICATED_USERS.lower(): return Entry(type=ALL_AUTHENTICATED_USERS, permission=permission) elif entity.startswith('project'): raise CommandException('XML API does not support project scopes, ' 'cannot translate ACL.') elif 'email' in entry_json: if entity.startswith('user'): scope_type = USER_BY_EMAIL elif entity.startswith('group'): scope_type = GROUP_BY_EMAIL return Entry(type=scope_type, email_address=entry_json['email'], permission=permission) elif 'entityId' in entry_json: if entity.startswith('user'): scope_type = USER_BY_ID elif entity.startswith('group'): scope_type = GROUP_BY_ID return Entry(type=scope_type, id=entry_json['entityId'], permission=permission) elif 'domain' in entry_json: if entity.startswith('domain'): scope_type = GROUP_BY_DOMAIN return Entry(type=scope_type, domain=entry_json['domain'], permission=permission) raise CommandException('Failed to translate JSON ACL to XML.') @classmethod def BotoEntryToJson(cls, entry): """Converts a Boto ACL entry to a valid JSON dictionary.""" acl_entry_json = {} # JSON API documentation uses camel case. scope_type_lower = entry.scope.type.lower() if scope_type_lower == ALL_USERS.lower(): acl_entry_json['entity'] = 'allUsers' elif scope_type_lower == ALL_AUTHENTICATED_USERS.lower(): acl_entry_json['entity'] = 'allAuthenticatedUsers' elif scope_type_lower == USER_BY_EMAIL.lower(): acl_entry_json['entity'] = 'user-%s' % entry.scope.email_address acl_entry_json['email'] = entry.scope.email_address elif scope_type_lower == USER_BY_ID.lower(): acl_entry_json['entity'] = 'user-%s' % entry.scope.id acl_entry_json['entityId'] = entry.scope.id elif scope_type_lower == GROUP_BY_EMAIL.lower(): acl_entry_json['entity'] = 'group-%s' % entry.scope.email_address acl_entry_json['email'] = entry.scope.email_address elif scope_type_lower == GROUP_BY_ID.lower(): acl_entry_json['entity'] = 'group-%s' % entry.scope.id acl_entry_json['entityId'] = entry.scope.id elif scope_type_lower == GROUP_BY_DOMAIN.lower(): acl_entry_json['entity'] = 'domain-%s' % entry.scope.domain acl_entry_json['domain'] = entry.scope.domain else: raise ArgumentException('ACL contains invalid scope type: %s' % scope_type_lower) acl_entry_json['role'] = cls.XML_TO_JSON_ROLES[entry.permission] return acl_entry_json @classmethod def JsonToMessage(cls, json_data, message_type): """Converts the input JSON data into list of Object/BucketAccessControls. Args: json_data: String of JSON to convert. message_type: Which type of access control entries to return, either ObjectAccessControl or BucketAccessControl. Raises: ArgumentException on invalid JSON data. Returns: List of ObjectAccessControl or BucketAccessControl elements. """ try: deserialized_acl = json.loads(json_data) acl = [] for acl_entry in deserialized_acl: acl.append(encoding.DictToMessage(acl_entry, message_type)) return acl except ValueError: CheckForXmlConfigurationAndRaise('ACL', json_data) @classmethod def JsonFromMessage(cls, acl): """Strips unnecessary fields from an ACL message and returns valid JSON. Args: acl: iterable ObjectAccessControl or BucketAccessControl Returns: ACL JSON string. """ serializable_acl = [] if acl is not None: for acl_entry in acl: if acl_entry.kind == 'storage#objectAccessControl': acl_entry.object = None acl_entry.generation = None acl_entry.kind = None acl_entry.bucket = None acl_entry.id = None acl_entry.selfLink = None acl_entry.etag = None serializable_acl.append(encoding.MessageToDict(acl_entry)) return json.dumps(serializable_acl, sort_keys=True, indent=2, separators=(',', ': ')) utils/shim_util.py 0000644 00000066373 15025013616 0010271 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2021 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper for shim used to translate gsutil command to gcloud storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import enum import os import re import subprocess from boto import config from gslib import exception from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.utils import boto_util from gslib.utils import constants from gslib.utils import system_util class HIDDEN_SHIM_MODE(enum.Enum): NO_FALLBACK = 'no_fallback' DRY_RUN = 'dry_run' NONE = 'none' class RepeatFlagType(enum.Enum): LIST = 0 DICT = 1 DECRYPTION_KEY_REGEX = re.compile(r'^decryption_key([1-9]$|[1-9][0-9]$|100$)') # Required for headers translation and boto config translation. COMMANDS_SUPPORTING_ALL_HEADERS = frozenset(['cp', 'mv', 'rsync', 'setmeta']) ENCRYPTION_SUPPORTED_COMMANDS = COMMANDS_SUPPORTING_ALL_HEADERS | frozenset( ['ls', 'rewrite', 'stat', 'cat', 'compose']) PRECONDITONS_ONLY_SUPPORTED_COMMANDS = frozenset( ['compose', 'rewrite', 'rm', 'retention']) DATA_TRANSFER_HEADERS = frozenset([ 'cache-control', 'content-disposition', 'content-encoding', 'content-md5', 'content-language', 'content-type', 'custom-time', ]) PRECONDITIONS_HEADERS = frozenset([ 'x-goog-generation-match', 'x-goog-if-generation-match', 'x-goog-metageneration-match', 'x-goog-if-metageneration-match' ]) # The format for _BOTO_CONFIG_MAP is as follows: # { # 'Boto section name': { # 'boto field name': 'correspnding env variable name in Cloud SDK' # } # } _BOTO_CONFIG_MAP = { 'Credentials': { 'aws_access_key_id': 'AWS_ACCESS_KEY_ID', 'aws_secret_access_key': 'AWS_SECRET_ACCESS_KEY', 'gs_access_key_id': 'CLOUDSDK_STORAGE_GS_XML_ACCESS_KEY_ID', 'gs_secret_access_key': 'CLOUDSDK_STORAGE_GS_XML_SECRET_ACCESS_KEY', 'use_client_certificate': 'CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE', }, 'Boto': { 'proxy': 'CLOUDSDK_PROXY_ADDRESS', 'proxy_type': 'CLOUDSDK_PROXY_TYPE', 'proxy_port': 'CLOUDSDK_PROXY_PORT', 'proxy_user': 'CLOUDSDK_PROXY_USERNAME', 'proxy_pass': 'CLOUDSDK_PROXY_PASSWORD', 'proxy_rdns': 'CLOUDSDK_PROXY_RDNS', 'http_socket_timeout': 'CLOUDSDK_CORE_HTTP_TIMEOUT', 'ca_certificates_file': 'CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILE', 'max_retry_delay': 'CLOUDSDK_STORAGE_BASE_RETRY_DELAY', 'num_retries': 'CLOUDSDK_STORAGE_MAX_RETRIES', }, 'GSUtil': { 'check_hashes': 'CLOUDSDK_STORAGE_CHECK_HASHES', 'default_project_id': 'CLOUDSDK_CORE_PROJECT', 'disable_analytics_prompt': 'CLOUDSDK_CORE_DISABLE_USAGE_REPORTING', 'use_magicfile': 'CLOUDSDK_STORAGE_USE_MAGICFILE', 'parallel_composite_upload_threshold': 'CLOUDSDK_STORAGE_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD', 'resumable_threshold': 'CLOUDSDK_STORAGE_RESUMABLE_THRESHOLD', 'rsync_buffer_lines': 'CLOUDSDK_STORAGE_RSYNC_LIST_CHUNK_SIZE', }, 'OAuth2': { 'client_id': 'CLOUDSDK_AUTH_CLIENT_ID', 'client_secret': 'CLOUDSDK_AUTH_CLIENT_SECRET', 'provider_authorization_uri': 'CLOUDSDK_AUTH_AUTH_HOST', 'provider_token_uri': 'CLOUDSDK_AUTH_TOKEN_HOST', }, } _REQUIRED_BOTO_CONFIG_NOT_YET_SUPPORTED = frozenset( # TOTO(b/214245419) Remove this once STET is support and add equivalent # mapping above. ['stet_binary_path', 'stet_config_path']) def get_flag_from_header(raw_header_key, header_value, unset=False): """Returns the gcloud storage flag for the given gsutil header. Args: raw_header_key: The header key. header_value: The header value unset: If True, the equivalent clear/remove flag is returned instead of the setter flag. This only applies to setmeta. Returns: A string representing the equivalent gcloud storage flag and value, if translation is possible, else returns None. Examples: >> get_flag_from_header('Cache-Control', 'val') --cache-control=val >> get_flag_from_header('x-goog-meta-foo', 'val') --update-custom-metadata=foo=val >> get_flag_from_header('x-goog-meta-foo', 'val', unset=True) --remove-custom-metadata=foo """ lowercase_header_key = raw_header_key.lower() if lowercase_header_key in PRECONDITIONS_HEADERS: providerless_flag = raw_header_key[len('x-goog-'):] if not providerless_flag.startswith('if-'): flag_name = 'if-' + providerless_flag else: flag_name = providerless_flag elif lowercase_header_key in DATA_TRANSFER_HEADERS: flag_name = lowercase_header_key else: flag_name = None if flag_name is not None: if unset: if lowercase_header_key in PRECONDITIONS_HEADERS or lowercase_header_key == 'content-md5': # Precondition headers and content-md5 cannot be cleared. return None else: return '--clear-' + flag_name return '--{}={}'.format(flag_name, header_value) for header_prefix in ('x-goog-meta-', 'x-amz-meta-'): if lowercase_header_key.startswith(header_prefix): # Preserving raw header keys to avoid forcing lowercase on custom data. metadata_key = raw_header_key[len(header_prefix):] if unset: return '--remove-custom-metadata=' + metadata_key else: return '--update-custom-metadata={}={}'.format(metadata_key, header_value) return None def get_format_flag_caret(): if system_util.IS_WINDOWS: return '^^^^' return '^' def get_format_flag_newline(): if system_util.IS_WINDOWS: return '^\\^n' return '\n' class GcloudStorageFlag(object): def __init__(self, gcloud_flag, repeat_type=None, supports_output_translation=False): """Initializes GcloudStorageFlag. Args: gcloud_flag (str|dict): The name of the gcloud flag or a dictionary for when the gcloud flag depends on a gsutil value. gsutil "--pap off" -> gcloud "--no-public-access-prevention" repeat_type (RepeatFlagType|None): Gsutil sometimes handles list and dictionary inputs by accepting a flag multiple times. support_output_translation (bool): If True, this flag in gcloud storage supports printing gsutil formatted output. """ self.gcloud_flag = gcloud_flag self.repeat_type = repeat_type self.supports_output_translation = supports_output_translation class GcloudStorageMap(object): """Mapping to translate gsutil command to its gcloud storage equivalent.""" def __init__(self, gcloud_command, flag_map, supports_output_translation=False): """Intalizes GcloudStorageMap. Args: gcloud_command (dict|str): The corresponding name of the command to be called in gcloud. If this command supports sub-commands, then this field must be a dict of sub-command-name:GcloudStorageMap pairs. flag_map (dict): A dict of str to GcloudStorageFlag. Mapping of gsutil flags to their equivalent gcloud storage flag names. supports_output_translation (bool): Indicates if the corresponding gcloud storage command supports the printing gsutil formatted output. """ self.gcloud_command = gcloud_command self.flag_map = flag_map self.supports_output_translation = supports_output_translation def _get_gcloud_binary_path(cloudsdk_root): return os.path.join(cloudsdk_root, 'bin', 'gcloud.cmd' if system_util.IS_WINDOWS else 'gcloud') def _get_validated_gcloud_binary_path(): # GCLOUD_BINARY_PATH is used for testing purpose only. # It helps to run the parity_check.py script directly without having # to build gcloud. gcloud_binary_path = os.environ.get('GCLOUD_BINARY_PATH') if gcloud_binary_path: return gcloud_binary_path cloudsdk_root = os.environ.get('CLOUDSDK_ROOT_DIR') if cloudsdk_root is None: raise exception.GcloudStorageTranslationError( 'Requested to use "gcloud storage" but the gcloud binary path cannot' ' be found. This might happen if you attempt to use gsutil that was' ' not installed via Cloud SDK. You can manually set the' ' `CLOUDSDK_ROOT_DIR` environment variable to point to the' ' google-cloud-sdk installation directory to resolve the issue.' ' Alternatively, you can set `use_gcloud_storage=False` to disable' ' running the command using gcloud storage.') return _get_gcloud_binary_path(cloudsdk_root) def _get_gcs_json_endpoint_from_boto_config(config): gs_json_host = config.get('Credentials', 'gs_json_host') if gs_json_host: gs_json_port = config.get('Credentials', 'gs_json_port') port = ':' + gs_json_port if gs_json_port else '' json_api_version = config.get('Credentials', 'json_api_version', 'v1') return 'https://{}{}/storage/{}'.format(gs_json_host, port, json_api_version) return None def _get_s3_endpoint_from_boto_config(config): s3_host = config.get('Credentials', 's3_host') if s3_host: s3_port = config.get('Credentials', 's3_port') port = ':' + s3_port if s3_port else '' return 'https://{}{}'.format(s3_host, port) return None def _convert_args_to_gcloud_values(args, gcloud_storage_map): gcloud_args = [] repeat_flag_data = collections.defaultdict(list) i = 0 while i < len(args): if args[i] not in gcloud_storage_map.flag_map: # Add raw value (positional args and flag values for non-repeated flags). gcloud_args.append(args[i]) i += 1 continue gcloud_flag_object = gcloud_storage_map.flag_map[args[i]] if not gcloud_flag_object: # Flag asked to be skipped over. i += 1 elif gcloud_flag_object.repeat_type: # Capture "v1" and "v2" in ["-k", "v1", "-k", "v2"]. repeat_flag_data[gcloud_flag_object].append(args[i + 1]) i += 2 elif isinstance(gcloud_flag_object.gcloud_flag, str): # Simple translation. # gsutil: "-x" -> gcloud: "-y" gcloud_args.append(gcloud_flag_object.gcloud_flag) i += 1 else: # isinstance(gcloud_flag_object.gcloud_flag, dict) # gsutil: "--pap on" -> gcloud: "--pap" # gsutil: "--pap off" -> gcloud: "--no-pap" if args[i + 1] not in gcloud_flag_object.gcloud_flag: raise ValueError( 'Flag value not in translation map for "{}": {}'.format( args[i], args[i + 1])) translated_flag_and_value = gcloud_flag_object.gcloud_flag[args[i + 1]] if translated_flag_and_value: gcloud_args.append(translated_flag_and_value) i += 2 for gcloud_flag_object, values in repeat_flag_data.items(): if gcloud_flag_object.repeat_type is RepeatFlagType.LIST: # gsutil: "-k v1 -k v2" -> gcloud: "-k=v1,v2" condensed_flag_values = ','.join(values) elif gcloud_flag_object.repeat_type is RepeatFlagType.DICT: # gcloud: "-d k1:v1 -d k2:v2" -> gcloud: "-d=k1=v1,k2=v2" condensed_flag_values = ','.join( ['{}={}'.format(*s.split(':', 1)) for s in values]) else: raise ValueError('Shim cannot handle repeat flag type: {}'.format( repeat_flag_data.repeat_type)) gcloud_args.append('{}={}'.format(gcloud_flag_object.gcloud_flag, condensed_flag_values)) return gcloud_args class GcloudStorageCommandMixin(object): """Provides gcloud storage translation functionality. The command.Command class must inherit this class in order to support converting the gsutil command to it's gcloud storage equivalent. """ # Mapping for translating gsutil command to gcloud storage. gcloud_storage_map = None def __init__(self): self._translated_gcloud_storage_command = None self._translated_env_variables = None self._gcloud_has_active_account = None def gcloud_has_active_account(self): """Returns True if gcloud has an active account configured.""" gcloud_path = _get_validated_gcloud_binary_path() if self._gcloud_has_active_account is None: process = subprocess.run([gcloud_path, 'config', 'get', 'account'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8') if process.returncode: raise exception.GcloudStorageTranslationError( "Error occurred while trying to retrieve gcloud's active account." " Error: {}".format(process.stderr)) # stdout will have the name of the account if active account is available. # Empty string indicates no active account available. self._gcloud_has_active_account = process.stdout.strip() != '' self.logger.debug('Result for "gcloud config get account" command:\n' ' STDOUT: {}.\n STDERR: {}'.format( process.stdout, process.stderr)) return self._gcloud_has_active_account def _get_gcloud_storage_args(self, sub_opts, gsutil_args, gcloud_storage_map): if gcloud_storage_map is None: raise exception.GcloudStorageTranslationError( 'Command "{}" cannot be translated to gcloud storage because the' ' translation mapping is missing.'.format(self.command_name)) args = [] if isinstance(gcloud_storage_map.gcloud_command, list): args.extend(gcloud_storage_map.gcloud_command) elif isinstance(gcloud_storage_map.gcloud_command, dict): # If a command has sub-commands, e.g gsutil pap set, gsutil pap get. # All the flags mapping must be present in the subcommand's map # because gsutil does not have command specific flags # if sub-commands are present. if gcloud_storage_map.flag_map: raise ValueError( 'Flags mapping should not be present at the top-level command if ' 'a sub-command is used. Command: {}.'.format(self.command_name)) sub_command = gsutil_args[0] sub_opts, parsed_args = self.ParseSubOpts( args=gsutil_args[1:], should_update_sub_opts_and_args=False) return self._get_gcloud_storage_args( sub_opts, parsed_args, gcloud_storage_map.gcloud_command.get(sub_command)) else: raise ValueError('Incorrect mapping found for "{}" command'.format( self.command_name)) if sub_opts: for option, value in sub_opts: if option not in gcloud_storage_map.flag_map: raise exception.GcloudStorageTranslationError( 'Command option "{}" cannot be translated to' ' gcloud storage'.format(option)) else: args.append(option) if value != '': # Empty string represents that the user did not passed in a value # for the flag. args.append(value) return _convert_args_to_gcloud_values(args + gsutil_args, gcloud_storage_map) def _translate_top_level_flags(self): """Translates gsutil's top level flags. Gsutil specifies the headers (-h) and boto config (-o) as top level flags as well, but we handle those separately. Returns: A tuple. The first item is a list of top level flags that can be appended to the gcloud storage command. The second item is a dict of environment variables that can be set for the gcloud storage command execution. """ top_level_flags = [] env_variables = { 'CLOUDSDK_METRICS_ENVIRONMENT': 'gsutil_shim', 'CLOUDSDK_STORAGE_RUN_BY_GSUTIL_SHIM': 'True' } if self.debug >= 3: top_level_flags.extend(['--verbosity', 'debug']) if self.debug == 4: top_level_flags.append('--log-http') if self.quiet_mode: top_level_flags.append('--no-user-output-enabled') if self.user_project: top_level_flags.append('--billing-project=' + self.user_project) if self.trace_token: top_level_flags.append('--trace-token=' + self.trace_token) if constants.IMPERSONATE_SERVICE_ACCOUNT: top_level_flags.append('--impersonate-service-account=' + constants.IMPERSONATE_SERVICE_ACCOUNT) # TODO(b/208294509) Add --perf-trace-token translation. should_use_rsync_override = self.command_name == 'rsync' and not ( config.get('GSUtil', 'parallel_process_count') == '1' and config.get('GSUtil', 'thread_process_count') == '1') if not (self.parallel_operations or should_use_rsync_override): # TODO(b/208301084) Set the --sequential flag instead. env_variables['CLOUDSDK_STORAGE_THREAD_COUNT'] = '1' env_variables['CLOUDSDK_STORAGE_PROCESS_COUNT'] = '1' return top_level_flags, env_variables def _translate_headers(self, headers=None, unset=False): """Translates gsutil headers to equivalent gcloud storage flags. Args: headers (dict|None): If absent, extracts headers from command instance. unset (bool): Yield metadata clear flags instead of setter flags. Returns: List[str]: Translated flags for gcloud. Raises: GcloudStorageTranslationError: Could not translate flag. """ flags = [] # Accept custom headers or extract headers dict from Command class. headers_to_translate = headers if headers is not None else self.headers additional_headers = [] for raw_header_key, header_value in headers_to_translate.items(): lowercase_header_key = raw_header_key.lower() if lowercase_header_key == 'x-goog-api-version': # Gsutil adds this header. We don't have to translate it for gcloud. continue flag = get_flag_from_header(raw_header_key, header_value, unset=unset) if self.command_name in COMMANDS_SUPPORTING_ALL_HEADERS: if flag: flags.append(flag) elif (self.command_name in PRECONDITONS_ONLY_SUPPORTED_COMMANDS and lowercase_header_key in PRECONDITIONS_HEADERS): flags.append(flag) if not flag: self.logger.warn('Header {}:{} cannot be translated to a gcloud storage' ' equivalent flag. It is being treated as an arbitrary' ' request header.'.format(raw_header_key, header_value)) additional_headers.append('{}={}'.format(raw_header_key, header_value)) if additional_headers: flags.append('--additional-headers=' + ','.join(additional_headers)) return flags def _translate_boto_config(self): """Translates boto config options to gcloud storage properties. Returns: A tuple where first element is a list of flags and the second element is a dict representing the env variables that can be set to set the gcloud storage properties. """ flags = [] env_vars = {} # Handle gs_json_host and gs_json_port. gcs_json_endpoint = _get_gcs_json_endpoint_from_boto_config(config) if gcs_json_endpoint: env_vars['CLOUDSDK_API_ENDPOINT_OVERRIDES_STORAGE'] = gcs_json_endpoint # Handle s3_host and s3_port. s3_endpoint = _get_s3_endpoint_from_boto_config(config) if s3_endpoint: env_vars['CLOUDSDK_STORAGE_S3_ENDPOINT_URL'] = s3_endpoint decryption_keys = [] for section_name, section in config.items(): for key, value in section.items(): if (key == 'encryption_key' and self.command_name in ENCRYPTION_SUPPORTED_COMMANDS): flags.append('--encryption-key=' + value) # Boto config can have decryption keys in the form of # decryption_key1..100. elif (DECRYPTION_KEY_REGEX.match(key) and self.command_name in ENCRYPTION_SUPPORTED_COMMANDS): decryption_keys.append(value) elif (key == 'content_language' and self.command_name in COMMANDS_SUPPORTING_ALL_HEADERS): flags.append('--content-language=' + value) elif key in _REQUIRED_BOTO_CONFIG_NOT_YET_SUPPORTED: self.logger.error('The boto config field {}:{} cannot be translated' ' to gcloud storage equivalent.'.format( section_name, key)) elif key == 'https_validate_certificates' and not value: env_vars['CLOUDSDK_AUTH_DISABLE_SSL_VALIDATION'] = True # Skip mapping GS HMAC auth keys if gsutil wouldn't use them. elif (key in ('gs_access_key_id', 'gs_secret_access_key') and not boto_util.UsingGsHmac()): self.logger.debug('The boto config field {}:{} skipped translation' ' to the gcloud storage equivalent as it would' ' have been unused in gsutil.'.format( section_name, key)) else: env_var = _BOTO_CONFIG_MAP.get(section_name, {}).get(key, None) if env_var is not None: env_vars[env_var] = value if decryption_keys: flags.append('--decryption-keys=' + ','.join(decryption_keys)) return flags, env_vars def get_gcloud_storage_args(self, gcloud_storage_map=None): """Translates the gsutil command flags to gcloud storage flags. It uses the command_spec.gcloud_storage_map field that provides the translation mapping for all the flags. Args: gcloud_storage_map (GcloudStorageMap|None): Command surface may pass a custom translation map instead of using the default class constant. Useful for when translations change based on conditional logic. Returns: A list of all the options and arguments that can be used with the equivalent gcloud storage command. Raises: GcloudStorageTranslationError: If a flag or command cannot be translated. ValueError: If there is any issue with the mapping provided by GcloudStorageMap. """ return self._get_gcloud_storage_args( self.sub_opts, self.args, gcloud_storage_map or self.gcloud_storage_map) def _print_gcloud_storage_command_info(self, gcloud_command, env_variables, dry_run=False): logger_func = self.logger.info if dry_run else self.logger.debug logger_func('Gcloud Storage Command: {}'.format(' '.join(gcloud_command))) if env_variables: logger_func('Environment variables for Gcloud Storage:') for k, v in env_variables.items(): logger_func('%s=%s', k, v) def _get_full_gcloud_storage_execution_information(self, args): top_level_flags, env_variables = self._translate_top_level_flags() header_flags = self._translate_headers() flags_from_boto, env_vars_from_boto = self._translate_boto_config() env_variables.update(env_vars_from_boto) gcloud_binary_path = _get_validated_gcloud_binary_path() gcloud_storage_command = ([gcloud_binary_path] + args + top_level_flags + header_flags + flags_from_boto) return env_variables, gcloud_storage_command def translate_to_gcloud_storage_if_requested(self): """Translates the gsutil command to gcloud storage equivalent. The translated commands get stored at self._translated_gcloud_storage_command. This command also translate the boto config, which gets stored as a dict at self._translated_env_variables Returns: True if the command was successfully translated, else False. """ if self.command_name == 'version' or self.command_name == 'test': # Running any command in debug mode will lead to calling gsutil version # command. We don't want to translate the version command as this # should always reflect the version that gsutil is using. # We don't want to run any translation for the "test" command. return False use_gcloud_storage = config.getbool('GSUtil', 'use_gcloud_storage', False) try: hidden_shim_mode = HIDDEN_SHIM_MODE( config.get('GSUtil', 'hidden_shim_mode', 'none')) except ValueError: raise exception.CommandException( 'Invalid option specified for' ' GSUtil:hidden_shim_mode config setting. Should be one of: {}'. format(' | '.join([x.value for x in HIDDEN_SHIM_MODE]))) if use_gcloud_storage: try: env_variables, gcloud_storage_command = self._get_full_gcloud_storage_execution_information( self.get_gcloud_storage_args()) if not self.gcloud_has_active_account(): # Allow running gcloud with anonymous credentials. env_variables['CLOUDSDK_AUTH_DISABLE_CREDENTIALS'] = 'True' if hidden_shim_mode == HIDDEN_SHIM_MODE.DRY_RUN: self._print_gcloud_storage_command_info(gcloud_storage_command, env_variables, dry_run=True) elif not os.environ.get('CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL'): raise exception.GcloudStorageTranslationError( 'Requested to use "gcloud storage" but gsutil is not using the' ' same credentials as gcloud.' ' You can make gsutil use the same credentials by running:\n' '{} config set pass_credentials_to_gsutil True'.format( _get_validated_gcloud_binary_path())) elif (boto_util.UsingGsHmac() and ApiSelector.XML not in self.command_spec.gs_api_support): raise CommandException( 'Requested to use "gcloud storage" with Cloud Storage XML API' ' HMAC credentials but the "{}" command can only be used' ' with the Cloud Storage JSON API.'.format(self.command_name)) else: self._print_gcloud_storage_command_info(gcloud_storage_command, env_variables) self._translated_gcloud_storage_command = gcloud_storage_command self._translated_env_variables = env_variables return True except exception.GcloudStorageTranslationError as e: # Raise error if no_fallback mode has been requested. This mode # should only be used for debuggling and testing purposes. if hidden_shim_mode == HIDDEN_SHIM_MODE.NO_FALLBACK: raise exception.CommandException(e) # For all other cases, we want to run gsutil. self.logger.error( 'Cannot translate gsutil command to gcloud storage.' ' Going to run gsutil command. Error: %s', e) return False def _get_shim_command_environment_variables(self): subprocess_envs = os.environ.copy() subprocess_envs.update(self._translated_env_variables) return subprocess_envs def run_gcloud_storage(self): process = subprocess.run(self._translated_gcloud_storage_command, env=self._get_shim_command_environment_variables()) return process.returncode utils/system_util.py 0000644 00000023465 15025013616 0010650 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods for interacting with the host system. The methods in this module should be limited to obtaining system information and simple file operations (disk info, retrieving metadata about existing files, creating directories, fetching environment variables, etc.). """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import errno import locale import os import struct import sys import six from gslib.utils.constants import WINDOWS_1252 _DEFAULT_NUM_TERM_LINES = 25 PLATFORM = str(sys.platform).lower() # Detect platform types. IS_WINDOWS = 'win32' in PLATFORM IS_CYGWIN = 'cygwin' in PLATFORM IS_LINUX = 'linux' in PLATFORM IS_OSX = 'darwin' in PLATFORM # pylint: disable=g-import-not-at-top if IS_WINDOWS: from ctypes import c_int from ctypes import c_uint64 from ctypes import c_char_p from ctypes import c_wchar_p from ctypes import windll from ctypes import POINTER from ctypes import WINFUNCTYPE from ctypes import WinError IS_CP1252 = locale.getdefaultlocale()[1] == WINDOWS_1252 else: IS_CP1252 = False def CheckFreeSpace(path): """Return path/drive free space (in bytes).""" if IS_WINDOWS: try: # pylint: disable=invalid-name get_disk_free_space_ex = WINFUNCTYPE(c_int, c_wchar_p, POINTER(c_uint64), POINTER(c_uint64), POINTER(c_uint64)) get_disk_free_space_ex = get_disk_free_space_ex( ('GetDiskFreeSpaceExW', windll.kernel32), ( (1, 'lpszPathName'), (2, 'lpFreeUserSpace'), (2, 'lpTotalSpace'), (2, 'lpFreeSpace'), )) except AttributeError: get_disk_free_space_ex = WINFUNCTYPE(c_int, c_char_p, POINTER(c_uint64), POINTER(c_uint64), POINTER(c_uint64)) get_disk_free_space_ex = get_disk_free_space_ex( ('GetDiskFreeSpaceExA', windll.kernel32), ( (1, 'lpszPathName'), (2, 'lpFreeUserSpace'), (2, 'lpTotalSpace'), (2, 'lpFreeSpace'), )) def GetDiskFreeSpaceExErrCheck(result, unused_func, args): if not result: raise WinError() return args[1].value get_disk_free_space_ex.errcheck = GetDiskFreeSpaceExErrCheck return get_disk_free_space_ex(os.getenv('SystemDrive')) else: (_, f_frsize, _, _, f_bavail, _, _, _, _, _) = os.statvfs(path) return f_frsize * f_bavail def CloudSdkCredPassingEnabled(): return os.environ.get('CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL') == '1' def CloudSdkVersion(): return os.environ.get('CLOUDSDK_VERSION', '') def CreateDirIfNeeded(dir_path, mode=0o777): """Creates a directory, suppressing already-exists errors.""" if not os.path.exists(dir_path): try: # Unfortunately, even though we catch and ignore EEXIST, this call will # output a (needless) error message (no way to avoid that in Python). os.makedirs(dir_path, mode) # Ignore 'already exists' in case user tried to start up several # resumable uploads concurrently from a machine where no tracker dir had # yet been created. except OSError as e: if e.errno != errno.EEXIST and e.errno != errno.EISDIR: raise def GetDiskCounters(): """Retrieves disk I/O statistics for all disks. Adapted from the psutil module's psutil._pslinux.disk_io_counters: http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py Originally distributed under under a BSD license. Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola. Returns: A dictionary containing disk names mapped to the disk counters from /disk/diskstats. """ # iostat documentation states that sectors are equivalent with blocks and # have a size of 512 bytes since 2.4 kernels. This value is needed to # calculate the amount of disk I/O in bytes. sector_size = 512 partitions = [] with open('/proc/partitions', 'r') as f: lines = f.readlines()[2:] for line in lines: _, _, _, name = line.split() if name[-1].isdigit(): partitions.append(name) retdict = {} with open('/proc/diskstats', 'r') as f: for line in f: values = line.split()[:11] _, _, name, reads, _, rbytes, rtime, writes, _, wbytes, wtime = values if name in partitions: rbytes = int(rbytes) * sector_size wbytes = int(wbytes) * sector_size reads = int(reads) writes = int(writes) rtime = int(rtime) wtime = int(wtime) retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime) return retdict def GetFileSize(fp, position_to_eof=False): """Returns size of file, optionally leaving fp positioned at EOF.""" if not position_to_eof: cur_pos = fp.tell() fp.seek(0, os.SEEK_END) cur_file_size = fp.tell() if not position_to_eof: fp.seek(cur_pos) return cur_file_size def GetGsutilClientIdAndSecret(): """Returns a tuple of the gsutil OAuth2 client ID and secret. Google OAuth2 clients always have a secret, even if the client is an installed application/utility such as gsutil. Of course, in such cases the "secret" is actually publicly known; security depends entirely on the secrecy of refresh tokens, which effectively become bearer tokens. Returns: (str, str) A 2-tuple of (client ID, secret). """ if InvokedViaCloudSdk() and CloudSdkCredPassingEnabled(): # Cloud SDK installs have a separate client ID / secret. return ( '32555940559.apps.googleusercontent.com', # Cloud SDK client ID 'ZmssLNjJy2998hD4CTg2ejr2') # Cloud SDK secret return ( '909320924072.apps.googleusercontent.com', # gsutil client ID 'p3RlpR10xMFh9ZXBS/ZNLYUu') # gsutil secret def GetStreamFromFileUrl(storage_url, mode='rb'): if storage_url.IsStream(): return sys.stdin if six.PY2 else sys.stdin.buffer else: return open(storage_url.object_name, mode) def GetTermLines(): """Returns number of terminal lines.""" # fcntl isn't supported in Windows. try: import fcntl # pylint: disable=g-import-not-at-top import termios # pylint: disable=g-import-not-at-top except ImportError: return _DEFAULT_NUM_TERM_LINES def ioctl_GWINSZ(fd): # pylint: disable=invalid-name try: return struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))[0] except: # pylint: disable=bare-except return 0 # Failure (so will retry on different file descriptor below). # Try to find a valid number of lines from termio for stdin, stdout, # or stderr, in that order. ioc = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) if not ioc: try: fd = os.open(os.ctermid(), os.O_RDONLY) ioc = ioctl_GWINSZ(fd) os.close(fd) except: # pylint: disable=bare-except pass if not ioc: ioc = os.environ.get('LINES', _DEFAULT_NUM_TERM_LINES) return int(ioc) def InvokedViaCloudSdk(): return os.environ.get('CLOUDSDK_WRAPPER') == '1' def IsRunningInCiEnvironment(): """Returns True if running in a CI environment, e.g. GitHub CI.""" # https://docs.github.com/en/actions/reference/environment-variables on_github_ci = 'CI' in os.environ on_kokoro = 'KOKORO_ROOT' in os.environ return on_github_ci or on_kokoro def IsRunningInteractively(): """Returns True if currently running interactively on a TTY.""" return sys.stdout.isatty() and sys.stderr.isatty() and sys.stdin.isatty() def MonkeyPatchHttp(): ver = sys.version_info # Checking for and applying monkeypatch for Python versions: # 3.0 - 3.6.6, 3.7.0 if ver.major == 3: if (ver.minor < 6 or (ver.minor == 6 and ver.micro < 7) or (ver.minor == 7 and ver.micro == 0)): _MonkeyPatchHttpForPython_3x() def _MonkeyPatchHttpForPython_3x(): # We generally have to do all sorts of gross things when applying runtime # patches (dynamic imports, invalid names to resolve symbols in copy/pasted # methods, invalid spacing from copy/pasted methods, etc.), so we just disable # pylint warnings for this whole method. # pylint: disable=all # This fixes https://bugs.python.org/issue33365. A fix was applied in # https://github.com/python/cpython/commit/936f03e7fafc28fd6fdfba11d162c776b89c0167 # but to apply that at runtime would mean patching the entire begin() method. # Rather, we just override begin() to call its old self, followed by printing # the HTTP headers afterward. This prevents us from overriding more behavior # than we have to. import http old_begin = http.client.HTTPResponse.begin def PatchedBegin(self): old_begin(self) if self.debuglevel > 0: for hdr, val in self.headers.items(): print("header:", hdr + ":", val) http.client.HTTPResponse.begin = PatchedBegin def StdinIterator(): """A generator function that returns lines from stdin.""" for line in sys.stdin: # Strip CRLF. yield line.rstrip() class StdinIteratorCls(six.Iterator): """An iterator that returns lines from stdin. This is needed because Python 3 balks at pickling the generator version above. """ def __iter__(self): return self def __next__(self): line = sys.stdin.readline() if not line: raise StopIteration() return line.rstrip() utils/encryption_helper.py 0000644 00000016001 15025013616 0012004 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for dealing with encryption keys used with cloud APIs.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import base64 import binascii from hashlib import sha256 import re import sys import six from gslib.exception import CommandException from gslib.lazy_wrapper import LazyWrapper MAX_DECRYPTION_KEYS = 100 VALID_CMEK_RE = LazyWrapper( lambda: re.compile('projects/([^/]+)/' 'locations/([a-zA-Z0-9_-]{1,63})/' 'keyRings/([a-zA-Z0-9_-]{1,63})/' 'cryptoKeys/([a-zA-Z0-9_-]{1,63})$')) class CryptoKeyType(object): """Enum of valid types of encryption keys used with cloud API requests.""" CSEK = 'CSEK' CMEK = 'CMEK' class CryptoKeyWrapper(object): """Class describing a crypto key used with cloud API requests. This class should be instantiated via the `CryptoKeyWrapperFromKey` method. """ def __init__(self, crypto_key): """Initialize the CryptoKeyWrapper. Args: crypto_key: Base64-encoded string of a CSEK, or the name of a Cloud KMS CMEK. Raises: CommandException: The specified crypto key was neither a CMEK key name nor a valid base64-encoded string of a CSEK. """ self.crypto_key = crypto_key # Base64-encoded CSEKs always have a length of 44 characters, whereas # fully-qualified CMEK names are guaranteed to be longer than 45 characters. if len(crypto_key) == 44: self.crypto_type = CryptoKeyType.CSEK self.crypto_alg = 'AES256' # Only currently supported algorithm for CSEK. try: self.crypto_key_sha256 = Base64Sha256FromBase64EncryptionKey(crypto_key) except: raise CommandException( 'Configured encryption_key or decryption_key looked like a CSEK, ' 'but it was not a valid 44-character base64 string. Please ' 'double-check your configuration and ensure the key is correct.') else: # CMEK try: ValidateCMEK(crypto_key) except CommandException as e: raise CommandException( 'Configured encryption_key or decryption_key looked like a CMEK, ' 'but the key failed validation:\n%s' % e.reason) self.crypto_type = CryptoKeyType.CMEK self.crypto_alg = None self.crypto_key_sha256 = None def CryptoKeyWrapperFromKey(crypto_key): """Returns a CryptoKeyWrapper for crypto_key, or None for no key.""" return CryptoKeyWrapper(crypto_key) if crypto_key else None def FindMatchingCSEKInBotoConfig(key_sha256, boto_config): """Searches boto_config for a CSEK with the given base64-encoded SHA256 hash. Args: key_sha256: (str) Base64-encoded SHA256 hash of the AES256 encryption key. boto_config: (boto.pyami.config.Config) The boto config in which to check for a matching encryption key. Returns: (str) Base64-encoded encryption key string if a match is found, None otherwise. """ if six.PY3: if not isinstance(key_sha256, bytes): key_sha256 = key_sha256.encode('ascii') keywrapper = CryptoKeyWrapperFromKey( boto_config.get('GSUtil', 'encryption_key', None)) if (keywrapper is not None and keywrapper.crypto_type == CryptoKeyType.CSEK and keywrapper.crypto_key_sha256 == key_sha256): return keywrapper.crypto_key for i in range(MAX_DECRYPTION_KEYS): key_number = i + 1 keywrapper = CryptoKeyWrapperFromKey( boto_config.get('GSUtil', 'decryption_key%s' % str(key_number), None)) if keywrapper is None: # Reading 100 config values can take ~1ms in testing. To avoid adding # this tax, stop reading keys as soon as we encounter a non-existent # entry (in lexicographic order). break elif (keywrapper.crypto_type == CryptoKeyType.CSEK and keywrapper.crypto_key_sha256 == key_sha256): return keywrapper.crypto_key def GetEncryptionKeyWrapper(boto_config): """Returns a CryptoKeyWrapper for the configured encryption key. Reads in the value of the "encryption_key" attribute in boto_config, and if present, verifies it is a valid base64-encoded string and returns a CryptoKeyWrapper for it. Args: boto_config: (boto.pyami.config.Config) The boto config in which to check for a matching encryption key. Returns: CryptoKeyWrapper for the specified encryption key, or None if no encryption key was specified in boto_config. """ encryption_key = boto_config.get('GSUtil', 'encryption_key', None) return CryptoKeyWrapper(encryption_key) if encryption_key else None def Base64Sha256FromBase64EncryptionKey(csek_encryption_key): if six.PY3: if not isinstance(csek_encryption_key, bytes): csek_encryption_key = csek_encryption_key.encode('ascii') decoded_bytes = base64.b64decode(csek_encryption_key) key_sha256 = _CalculateSha256FromString(decoded_bytes) sha256_bytes = binascii.unhexlify(key_sha256) sha256_base64 = base64.b64encode(sha256_bytes) return sha256_base64.replace(b'\n', b'') def ValidateCMEK(key): if not key: raise CommandException('KMS key is empty.') if key.startswith('/'): raise CommandException( 'KMS key should not start with leading slash (/): "%s"' % key) if not VALID_CMEK_RE().match(key): raise CommandException( 'Invalid KMS key name: "%s".\nKMS keys should follow the format ' '"projects/<project-id>/locations/<location>/keyRings/<keyring>/' 'cryptoKeys/<key-name>"' % key) def _CalculateSha256FromString(input_string): sha256_hash = sha256() sha256_hash.update(input_string) return sha256_hash.hexdigest() def _GetAndVerifyBase64EncryptionKey(boto_config): """Reads the encryption key from boto_config and ensures it is base64-encoded. Args: boto_config: (boto.pyami.config.Config) The boto config in which to check for a matching encryption key. Returns: (str) Base64-encoded encryption key string, or None if no encryption key exists in configuration. """ encryption_key = boto_config.get('GSUtil', 'encryption_key', None) if encryption_key: # Ensure the key has a valid encoding. try: base64.b64decode(encryption_key) except: raise CommandException( 'Configured encryption_key is not a valid base64 string. Please ' 'double-check your configuration and ensure the key is valid and in ' 'base64 format.') return encryption_key utils/__init__.py 0000644 00000000000 15025013616 0010002 0 ustar 00 utils/unit_util.py 0000644 00000017352 15025013616 0010301 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility methods that calculate, convert, and simplify units.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import math import re import six if six.PY3: long = int # Binary exponentiation strings. _EXP_STRINGS = [ (0, 'B', 'bit'), (10, 'KiB', 'Kibit', 'K'), (20, 'MiB', 'Mibit', 'M'), (30, 'GiB', 'Gibit', 'G'), (40, 'TiB', 'Tibit', 'T'), (50, 'PiB', 'Pibit', 'P'), (60, 'EiB', 'Eibit', 'E'), ] _EXP_TEN_STRING = [ (3, 'k'), (6, 'm'), (9, 'b'), (12, 't'), (15, 'q'), ] # Define this method before constants below, as some call it to init themselves. def _GenerateSuffixRegex(): """Creates a suffix regex for human-readable byte counts.""" human_bytes_re = r'(?P<num>\d*\.\d+|\d+)\s*(?P<suffix>%s)?' suffixes = [] suffix_to_si = {} for i, si in enumerate(_EXP_STRINGS): si_suffixes = [s.lower() for s in list(si)[1:]] for suffix in si_suffixes: suffix_to_si[suffix] = i suffixes.extend(si_suffixes) human_bytes_re %= '|'.join(suffixes) matcher = re.compile(human_bytes_re) return suffix_to_si, matcher # TODO: These should include the unit in the name, e.g. BYTES_PER_KIB, or # BYTES_IN_ONE_KIB. ONE_KIB = 1024 ONE_MIB = 1024 * ONE_KIB ONE_GIB = 1024 * ONE_MIB # TODO: Remove 2, 8, 10 MIB vars. TWO_MIB = 2 * ONE_MIB EIGHT_MIB = 8 * ONE_MIB TEN_MIB = 10 * ONE_MIB SECONDS_PER_DAY = long(60 * 60 * 24) SUFFIX_TO_SI, MATCH_HUMAN_BYTES = _GenerateSuffixRegex() def _RoundToNearestExponent(num): i = 0 while i + 1 < len(_EXP_STRINGS) and num >= (2**_EXP_STRINGS[i + 1][0]): i += 1 return i, round(float(num) / 2.0**_EXP_STRINGS[i][0], 2) def CalculateThroughput(total_bytes_transferred, total_elapsed_time): """Calculates throughput and checks for a small total_elapsed_time. Args: total_bytes_transferred: Total bytes transferred in a period of time. total_elapsed_time: The amount of time elapsed in seconds. Returns: The throughput as a float. """ if total_elapsed_time < 0.01: total_elapsed_time = 0.01 return float(total_bytes_transferred) / float(total_elapsed_time) def DecimalShort(num): """Creates a shorter string version for a given number of objects. Args: num: The number of objects to be shortened. Returns: shortened string version for this number. It takes the largest scale (thousand, million or billion) smaller than the number and divides it by that scale, indicated by a suffix with one decimal place. This will thus create a string of at most 6 characters, assuming num < 10^18. Example: 123456789 => 123.4m """ for divisor_exp, suffix in reversed(_EXP_TEN_STRING): if num >= 10**divisor_exp: quotient = '%.1lf' % (float(num) / 10**divisor_exp) return quotient + suffix return str(num) def DivideAndCeil(dividend, divisor): """Returns ceil(dividend / divisor). Takes care to avoid the pitfalls of floating point arithmetic that could otherwise yield the wrong result for large numbers. Args: dividend: Dividend for the operation. divisor: Divisor for the operation. Returns: Quotient. """ quotient = dividend // divisor if (dividend % divisor) != 0: quotient += 1 return quotient def HumanReadableToBytes(human_string): """Tries to convert a human-readable string to a number of bytes. Args: human_string: A string supplied by user, e.g. '1M', '3 GiB'. Returns: An integer containing the number of bytes. Raises: ValueError: on an invalid string. """ human_string = human_string.lower() m = MATCH_HUMAN_BYTES.match(human_string) if m: num = float(m.group('num')) if m.group('suffix'): power = _EXP_STRINGS[SUFFIX_TO_SI[m.group('suffix')]][0] num *= (2.0**power) num = int(round(num)) return num raise ValueError('Invalid byte string specified: %s' % human_string) def HumanReadableWithDecimalPlaces(number, decimal_places=1): """Creates a human readable format for bytes with fixed decimal places. Args: number: The number of bytes. decimal_places: The number of decimal places. Returns: String representing a readable format for number with decimal_places decimal places. """ number_format = MakeHumanReadable(number).split() num = str(int(round(10**decimal_places * float(number_format[0])))) if num == '0': number_format[0] = ('0' + (('.' + ('0' * decimal_places)) if decimal_places else '')) else: num_length = len(num) if decimal_places: num = (num[:num_length - decimal_places] + '.' + num[num_length - decimal_places:]) number_format[0] = num return ' '.join(number_format) def MakeBitsHumanReadable(num): """Generates human readable string for a number of bits. Args: num: The number, in bits. Returns: A string form of the number using bit size abbreviations (kbit, Mbit, etc.) """ i, rounded_val = _RoundToNearestExponent(num) return '%g %s' % (rounded_val, _EXP_STRINGS[i][2]) def MakeHumanReadable(num): """Generates human readable string for a number of bytes. Args: num: The number, in bytes. Returns: A string form of the number using size abbreviations (KiB, MiB, etc.). """ i, rounded_val = _RoundToNearestExponent(num) return '%g %s' % (rounded_val, _EXP_STRINGS[i][1]) def Percentile(values, percent, key=lambda x: x): """Find the percentile of a list of values. Taken from: http://code.activestate.com/recipes/511478/ Args: values: a list of numeric values. Note that the values MUST BE already sorted. percent: a float value from 0.0 to 1.0. key: optional key function to compute value from each element of the list of values. Returns: The percentile of the values. """ if not values: return None k = (len(values) - 1) * percent f = math.floor(k) c = math.ceil(k) if f == c: return key(values[int(k)]) d0 = key(values[int(f)]) * (c - k) d1 = key(values[int(c)]) * (k - f) return d0 + d1 def PrettyTime(remaining_time): """Creates a standard version for a given remaining time in seconds. Created over using strftime because strftime seems to be more suitable for a datetime object, rather than just a number of seconds remaining. Args: remaining_time: The number of seconds remaining as a float, or a string/None value indicating time was not correctly calculated. Returns: if remaining_time is a valid float, %H:%M:%D time remaining format with the nearest integer from remaining_time (%H might be higher than 23). Else, it returns the same message it received. """ remaining_time = int(round(remaining_time)) hours = remaining_time // 3600 if hours >= 100: # Too large to display with precision of minutes and seconds. # If over 1000, saying 999+ hours should be enough. return '%d+ hrs' % min(hours, 999) remaining_time -= (3600 * hours) minutes = remaining_time // 60 remaining_time -= (60 * minutes) seconds = remaining_time return (str('%02d' % hours) + ':' + str('%02d' % minutes) + ':' + str('%02d' % seconds)) utils/version_check.py 0000644 00000004324 15025013616 0011102 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility functions to ensure the correct version of Python is used.""" import sys from sys import version_info # Key: Int, supported Python major version # Value: Int, min supported Python minor version # We currently support Python >=3.5 MIN_SUPPORTED_PYTHON_VERSION = {3: 5} def check_python_version_support(): """Return an exception if running in an unsupported version of Python. This function compares the running version of cPython and against the list of supported python version. If the running version is less than any of the supported versions, return a Tuple of (False, Str(error message)) for the caller to handle. Minor versions of Python greater than those listed in the supported versions are allowed. Args: None Returns: Tuple(Boolean, String) A Tuple containing a Boolean and a String. The boolean represents if the version is supported, and the String will either be empty, or contain an error message. """ major = sys.version_info.major minor = sys.version_info.minor if major not in MIN_SUPPORTED_PYTHON_VERSION: return (False, 'Gsutil does not support running under Python{major}'.format( major=major)) if minor < MIN_SUPPORTED_PYTHON_VERSION[major]: lowest_minor = MIN_SUPPORTED_PYTHON_VERSION[major] return ( False, 'For Python{major}, gsutil requires Python{major}.{lowest_minor}+, but ' 'you are using Python{major}.{minor}'.format(major=major, minor=minor, lowest_minor=lowest_minor)) return (True, '') utils/ls_helper.py 0000644 00000043624 15025013616 0010243 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility functions and class for listing commands such as ls and du.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import fnmatch import sys import six from gslib.cloud_api import EncryptionException from gslib.exception import CommandException from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.storage_url import GenerationFromUrlAndString from gslib.utils.constants import S3_ACL_MARKER_GUID from gslib.utils.constants import S3_DELETE_MARKER_GUID from gslib.utils.constants import S3_MARKER_GUIDS from gslib.utils.constants import UTF8 from gslib.utils.system_util import IS_WINDOWS from gslib.utils.translation_helper import AclTranslation from gslib.utils import text_util from gslib.wildcard_iterator import StorageUrlFromString ENCRYPTED_FIELDS = [ 'md5Hash', 'crc32c', ] UNENCRYPTED_FULL_LISTING_FIELDS = [ 'acl', 'cacheControl', 'componentCount', 'contentDisposition', 'contentEncoding', 'contentLanguage', 'contentType', 'customTime', 'kmsKeyName', 'customerEncryption', 'etag', 'eventBasedHold', 'generation', 'metadata', 'metageneration', 'retentionExpirationTime', 'size', 'storageClass', 'temporaryHold', 'timeCreated', 'timeDeleted', 'timeStorageClassUpdated', 'updated', ] def MakeMetadataLine(label, value, indent=1): """Returns a string with a vertically aligned label and value. Labels of the same indentation level will start at the same column. Values will all start at the same column (unless the combined left-indent and label length is excessively long). If a value spans multiple lines, indentation will only be applied to the first line. Example output from several calls: Label1: Value (default indent of 1 was used) Sublabel1: Value (used indent of 2 here) Label2: Value Args: label: The label to print in the first column. value: The value to print in the second column. indent: (4 * indent) spaces will be placed before the label. Returns: A string with a vertically aligned label and value. """ return '{}{}'.format(((' ' * indent * 4) + label + ':').ljust(28), value) def PrintBucketHeader(bucket_listing_ref): # pylint: disable=unused-argument """Default function for printing headers for buckets. Header is printed prior to listing the contents of the bucket. Args: bucket_listing_ref: BucketListingRef of type BUCKET. """ pass def PrintDir(bucket_listing_ref): """Default function for printing buckets or prefixes. Args: bucket_listing_ref: BucketListingRef of type BUCKET or PREFIX. """ text_util.print_to_fd(bucket_listing_ref.url_string) # pylint: disable=unused-argument def PrintDirSummary(num_bytes, bucket_listing_ref): """Off-by-default function for printing buckets or prefix size summaries. Args: num_bytes: Number of bytes contained in the directory. bucket_listing_ref: BucketListingRef of type BUCKET or PREFIX. """ pass def PrintDirHeader(bucket_listing_ref): """Default function for printing headers for prefixes. Header is printed prior to listing the contents of the prefix. Args: bucket_listing_ref: BucketListingRef of type PREFIX. """ text_util.print_to_fd('{}:'.format(bucket_listing_ref.url_string)) def PrintNewLine(): """Default function for printing new lines between directories.""" text_util.print_to_fd() # pylint: disable=too-many-statements def PrintFullInfoAboutObject(bucket_listing_ref, incl_acl=True): """Print full info for given object (like what displays for gsutil ls -L). Args: bucket_listing_ref: BucketListingRef being listed. Must have ref_type OBJECT and a populated root_object with the desired fields. incl_acl: True if ACL info should be output. Returns: Tuple (number of objects, object_length) Raises: Exception: if calling bug encountered. """ url_str = bucket_listing_ref.url_string storage_url = StorageUrlFromString(url_str) obj = bucket_listing_ref.root_object if (obj.metadata and S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties): num_bytes = 0 num_objs = 0 url_str += '<DeleteMarker>' else: num_bytes = obj.size num_objs = 1 text_util.print_to_fd('{}:'.format(url_str)) if obj.timeCreated: text_util.print_to_fd( MakeMetadataLine('Creation time', obj.timeCreated.strftime('%a, %d %b %Y %H:%M:%S GMT'))) if obj.updated: text_util.print_to_fd( MakeMetadataLine('Update time', obj.updated.strftime('%a, %d %b %Y %H:%M:%S GMT'))) if (obj.timeStorageClassUpdated and obj.timeStorageClassUpdated != obj.timeCreated): text_util.print_to_fd( MakeMetadataLine( 'Storage class update time', obj.timeStorageClassUpdated.strftime('%a, %d %b %Y %H:%M:%S GMT'))) if obj.storageClass: text_util.print_to_fd(MakeMetadataLine('Storage class', obj.storageClass)) if obj.temporaryHold: text_util.print_to_fd(MakeMetadataLine('Temporary Hold', 'Enabled')) if obj.eventBasedHold: text_util.print_to_fd(MakeMetadataLine('Event-Based Hold', 'Enabled')) if obj.retentionExpirationTime: text_util.print_to_fd( MakeMetadataLine( 'Retention Expiration', obj.retentionExpirationTime.strftime('%a, %d %b %Y %H:%M:%S GMT'))) if obj.kmsKeyName: text_util.print_to_fd(MakeMetadataLine('KMS key', obj.kmsKeyName)) if obj.cacheControl: text_util.print_to_fd(MakeMetadataLine('Cache-Control', obj.cacheControl)) if obj.contentDisposition: text_util.print_to_fd( MakeMetadataLine('Content-Disposition', obj.contentDisposition)) if obj.contentEncoding: text_util.print_to_fd( MakeMetadataLine('Content-Encoding', obj.contentEncoding)) if obj.contentLanguage: text_util.print_to_fd( MakeMetadataLine('Content-Language', obj.contentLanguage)) text_util.print_to_fd(MakeMetadataLine('Content-Length', obj.size)) text_util.print_to_fd(MakeMetadataLine('Content-Type', obj.contentType)) if obj.componentCount: text_util.print_to_fd( MakeMetadataLine('Component-Count', obj.componentCount)) if obj.customTime: text_util.print_to_fd(MakeMetadataLine('Custom-Time', obj.customTime)) if obj.timeDeleted: text_util.print_to_fd( MakeMetadataLine('Noncurrent time', obj.timeDeleted.strftime('%a, %d %b %Y %H:%M:%S GMT'))) marker_props = {} if obj.metadata and obj.metadata.additionalProperties: non_marker_props = [] for add_prop in obj.metadata.additionalProperties: if add_prop.key not in S3_MARKER_GUIDS: non_marker_props.append(add_prop) else: marker_props[add_prop.key] = add_prop.value if non_marker_props: text_util.print_to_fd(MakeMetadataLine('Metadata', '')) for ap in non_marker_props: ap_key = '{}'.format(ap.key) ap_value = '{}'.format(ap.value) meta_data_line = MakeMetadataLine(ap_key, ap_value, indent=2) text_util.print_to_fd(meta_data_line) if obj.customerEncryption: if not obj.crc32c: text_util.print_to_fd(MakeMetadataLine('Hash (crc32c)', 'encrypted')) if not obj.md5Hash: text_util.print_to_fd(MakeMetadataLine('Hash (md5)', 'encrypted')) text_util.print_to_fd( MakeMetadataLine('Encryption algorithm', obj.customerEncryption.encryptionAlgorithm)) text_util.print_to_fd( MakeMetadataLine('Encryption key SHA256', obj.customerEncryption.keySha256)) if obj.crc32c: text_util.print_to_fd(MakeMetadataLine('Hash (crc32c)', obj.crc32c)) if obj.md5Hash: text_util.print_to_fd(MakeMetadataLine('Hash (md5)', obj.md5Hash)) text_util.print_to_fd(MakeMetadataLine('ETag', obj.etag.strip('"\''))) if obj.generation: generation_str = GenerationFromUrlAndString(storage_url, obj.generation) text_util.print_to_fd(MakeMetadataLine('Generation', generation_str)) if obj.metageneration: text_util.print_to_fd(MakeMetadataLine('Metageneration', obj.metageneration)) if incl_acl: # JSON API won't return acls as part of the response unless we have # full control scope if obj.acl: text_util.print_to_fd( MakeMetadataLine('ACL', AclTranslation.JsonFromMessage(obj.acl))) elif S3_ACL_MARKER_GUID in marker_props: text_util.print_to_fd( MakeMetadataLine('ACL', marker_props[S3_ACL_MARKER_GUID])) else: # Empty ACLs are possible with Bucket Policy Only and no longer imply # ACCESS DENIED anymore. text_util.print_to_fd(MakeMetadataLine('ACL', '[]')) return (num_objs, num_bytes) def PrintObject(bucket_listing_ref): """Default printing function for objects. Args: bucket_listing_ref: BucketListingRef of type OBJECT. Returns: (num_objects, num_bytes). """ try: text_util.print_to_fd(bucket_listing_ref.url_string) except IOError as e: # Windows throws an IOError 0 here for object names containing Unicode # chars. Ignore it. if not (IS_WINDOWS and e.errno == 0): raise return (1, 0) class LsHelper(object): """Helper class for ls and du.""" def __init__(self, iterator_func, logger, print_object_func=PrintObject, print_dir_func=PrintDir, print_dir_header_func=PrintDirHeader, print_bucket_header_func=PrintBucketHeader, print_dir_summary_func=PrintDirSummary, print_newline_func=PrintNewLine, all_versions=False, should_recurse=False, exclude_patterns=None, fields=('name',), list_subdir_contents=True): """Initializes the helper class to prepare for listing. Args: iterator_func: Function for instantiating iterator. Inputs- url_string- Url string to iterate on. May include wildcards. all_versions=False- If true, iterate over all object versions. logger: Logger for outputting warnings / errors. print_object_func: Function for printing objects. print_dir_func: Function for printing buckets/prefixes. print_dir_header_func: Function for printing header line for buckets or prefixes. print_bucket_header_func: Function for printing header line for buckets or prefixes. print_dir_summary_func: Function for printing size summaries about buckets/prefixes. print_newline_func: Function for printing new lines between dirs. all_versions: If true, list all object versions. should_recurse: If true, recursively listing buckets/prefixes. exclude_patterns: Patterns to exclude when listing. fields: Fields to request from bucket listings; this should include all fields that need to be populated in objects so they can be listed. Can be set to None to retrieve all object fields. Defaults to short listing fields. list_subdir_contents: If true, return the directory and any contents, otherwise return only the directory itself. """ self._iterator_func = iterator_func self.logger = logger self._print_object_func = print_object_func self._print_dir_func = print_dir_func self._print_dir_header_func = print_dir_header_func self._print_bucket_header_func = print_bucket_header_func self._print_dir_summary_func = print_dir_summary_func self._print_newline_func = print_newline_func self.all_versions = all_versions self.should_recurse = should_recurse self.exclude_patterns = exclude_patterns self.bucket_listing_fields = fields self.list_subdir_contents = list_subdir_contents def ExpandUrlAndPrint(self, url): """Iterates over the given URL and calls print functions. Args: url: StorageUrl to iterate over. Returns: (num_objects, num_bytes) total number of objects and bytes iterated. """ num_objects = 0 num_dirs = 0 num_bytes = 0 print_newline = False if url.IsBucket() or self.should_recurse: # IsBucket() implies a top-level listing. if url.IsBucket(): self._print_bucket_header_func(url) return self._RecurseExpandUrlAndPrint(url.url_string, print_initial_newline=False) else: # User provided a prefix or object URL, but can't tell which unless there # is a generation or we do a listing to see what matches. if url.HasGeneration(): iteration_url = url.url_string else: iteration_url = url.CreatePrefixUrl() top_level_iterator = PluralityCheckableIterator( self._iterator_func( iteration_url, all_versions=self.all_versions).IterAll( expand_top_level_buckets=True, bucket_listing_fields=self.bucket_listing_fields)) plurality = top_level_iterator.HasPlurality() try: top_level_iterator.PeekException() except EncryptionException: # Detailed listing on a single object can perform a GetObjectMetadata # call, which raises if a matching encryption key isn't found. # Re-iterate without requesting encrypted fields. top_level_iterator = PluralityCheckableIterator( self._iterator_func( url.CreatePrefixUrl(wildcard_suffix=None), all_versions=self.all_versions).IterAll( expand_top_level_buckets=True, bucket_listing_fields=UNENCRYPTED_FULL_LISTING_FIELDS)) plurality = top_level_iterator.HasPlurality() for blr in top_level_iterator: if self._MatchesExcludedPattern(blr): continue if blr.IsObject(): nd = 0 no, nb = self._print_object_func(blr) print_newline = True elif blr.IsPrefix(): if print_newline: self._print_newline_func() else: print_newline = True if plurality and self.list_subdir_contents: self._print_dir_header_func(blr) elif plurality and not self.list_subdir_contents: print_newline = False expansion_url_str = StorageUrlFromString( blr.url_string).CreatePrefixUrl( wildcard_suffix='*' if self.list_subdir_contents else None) nd, no, nb = self._RecurseExpandUrlAndPrint(expansion_url_str) self._print_dir_summary_func(nb, blr) else: # We handle all buckets at the top level, so this should never happen. raise CommandException( 'Sub-level iterator returned a CsBucketListingRef of type Bucket') num_objects += no num_dirs += nd num_bytes += nb return num_dirs, num_objects, num_bytes def _RecurseExpandUrlAndPrint(self, url_str, print_initial_newline=True): """Iterates over the given URL string and calls print functions. Args: url_str: String describing StorageUrl to iterate over. Must be of depth one or higher. print_initial_newline: If true, print a newline before recursively expanded prefixes. Returns: (num_objects, num_bytes) total number of objects and bytes iterated. """ num_objects = 0 num_dirs = 0 num_bytes = 0 for blr in self._iterator_func( '%s' % url_str, all_versions=self.all_versions).IterAll( expand_top_level_buckets=True, bucket_listing_fields=self.bucket_listing_fields): if self._MatchesExcludedPattern(blr): continue if blr.IsObject(): nd = 0 no, nb = self._print_object_func(blr) elif blr.IsPrefix(): if self.should_recurse: if print_initial_newline: self._print_newline_func() else: print_initial_newline = True self._print_dir_header_func(blr) expansion_url_str = StorageUrlFromString( blr.url_string).CreatePrefixUrl(wildcard_suffix='*') nd, no, nb = self._RecurseExpandUrlAndPrint(expansion_url_str) self._print_dir_summary_func(nb, blr) else: nd, no, nb = 1, 0, 0 self._print_dir_func(blr) else: # We handle all buckets at the top level, so this should never happen. raise CommandException( 'Sub-level iterator returned a bucketListingRef of type Bucket') num_dirs += nd num_objects += no num_bytes += nb return num_dirs, num_objects, num_bytes def _MatchesExcludedPattern(self, blr): """Checks bucket listing reference against patterns to exclude. Args: blr: BucketListingRef to check. Returns: True if reference matches a pattern and should be excluded. """ if self.exclude_patterns: tomatch = six.ensure_str(blr.url_string) for pattern in self.exclude_patterns: if fnmatch.fnmatch(tomatch, six.ensure_str(pattern)): return True return False discard_messages_queue.py 0000644 00000002453 15025013616 0011625 0 ustar 00 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Provides a message queue that discards all messages.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class DiscardMessagesQueue(object): """Emulates a Cloud API status queue but drops all messages. This is useful when you want to perform some operations but not have the UI thread display information about those ops (e.g. running a test or fetching the public gsutil tarball object's metadata to perform a version check). """ # pylint: disable=invalid-name, unused-argument def put(self, message=None, timeout=None): pass # pylint: enable=invalid-name, unused-argument vendored/boto/requirements-py33.txt 0000644 00000000026 15025013616 0013372 0 ustar 00 paramiko>=1.10.0,<2.4 vendored/boto/tests/unit/route53/test_zone.py 0000644 00000004764 15025013616 0015342 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.route53.zone import Zone from tests.compat import mock, unittest class TestZone(unittest.TestCase): def test_find_records(self): mock_connection = mock.Mock() zone = Zone(mock_connection, {}) zone.id = None rr_names = ['amazon.com', 'amazon.com', 'aws.amazon.com', 'aws.amazon.com'] mock_rrs = [] # Create some mock resource records. for rr_name in rr_names: mock_rr = mock.Mock() mock_rr.name = rr_name mock_rr.type = 'A' mock_rr.weight = None mock_rr.region = None mock_rrs.append(mock_rr) # Set the last resource record to ``None``. The ``find_records`` loop # should never hit this. mock_rrs[3] = None mock_connection.get_all_rrsets.return_value = mock_rrs mock_connection._make_qualified.return_value = 'amazon.com' # Ensure that the ``None`` type object was not iterated over. try: result_rrs = zone.find_records('amazon.com', 'A', all=True) except AttributeError as e: self.fail("find_records() iterated too far into resource" " record list.") # Determine that the resulting records are correct. self.assertEqual(result_rrs, [mock_rrs[0], mock_rrs[1]]) if __name__ == "__main__": unittest.main() vendored/boto/tests/unit/route53/test_connection.py 0000644 00000100041 15025013616 0016507 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.compat import mock import re import xml.dom.minidom from boto.exception import BotoServerError from boto.route53.connection import Route53Connection from boto.route53.exception import DNSServerError from boto.route53.healthcheck import HealthCheck from boto.route53.record import ResourceRecordSets, Record from boto.route53.zone import Zone from nose.plugins.attrib import attr from tests.unit import AWSMockServiceTestCase from boto.compat import six urllib = six.moves.urllib @attr(route53=True) class TestRoute53Connection(AWSMockServiceTestCase): connection_class = Route53Connection def setUp(self): super(TestRoute53Connection, self).setUp() self.calls = { 'count': 0, } def default_body(self): return b"""<Route53Result> <Message>It failed.</Message> </Route53Result> """ def test_typical_400(self): self.set_http_response(status_code=400, header=[ ['Code', 'AccessDenied'], ]) with self.assertRaises(DNSServerError) as err: self.service_connection.get_all_hosted_zones() self.assertTrue('It failed.' in str(err.exception)) def test_retryable_400_prior_request_not_complete(self): # Test ability to retry on ``PriorRequestNotComplete``. self.set_http_response(status_code=400, body="""<?xml version="1.0"?> <ErrorResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"><Error><Type>Sender</Type><Code>PriorRequestNotComplete</Code><Message>The request was rejected because Route 53 was still processing a prior request.</Message></Error><RequestId>12d222a0-f3d9-11e4-a611-c321a3a00f9c</RequestId></ErrorResponse> """) self.do_retry_handler() def test_retryable_400_throttling(self): # Test ability to rety on ``Throttling``. self.set_http_response(status_code=400, body="""<?xml version="1.0"?> <ErrorResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"><Error><Type>Sender</Type><Code>Throttling</Code><Message>Rate exceeded</Message></Error><RequestId>19d0a9a0-f3d9-11e4-a611-c321a3a00f9c</RequestId></ErrorResponse> """) self.do_retry_handler() @mock.patch('time.sleep') def do_retry_handler(self, sleep_mock): def incr_retry_handler(func): def _wrapper(*args, **kwargs): self.calls['count'] += 1 return func(*args, **kwargs) return _wrapper # Patch. orig_retry = self.service_connection._retry_handler self.service_connection._retry_handler = incr_retry_handler( orig_retry ) self.assertEqual(self.calls['count'], 0) # Retries get exhausted. with self.assertRaises(BotoServerError): self.service_connection.get_all_hosted_zones() self.assertEqual(self.calls['count'], 7) # Unpatch. self.service_connection._retry_handler = orig_retry def test_private_zone_invalid_vpc_400(self): self.set_http_response(status_code=400, header=[ ['Code', 'InvalidVPCId'], ]) with self.assertRaises(DNSServerError) as err: self.service_connection.create_hosted_zone("example.com.", private_zone=True) self.assertTrue('It failed.' in str(err.exception)) @attr(route53=True) class TestCreateZoneRoute53(AWSMockServiceTestCase): connection_class = Route53Connection def setUp(self): super(TestCreateZoneRoute53, self).setUp() def default_body(self): return b""" <CreateHostedZoneResponse xmlns="https://route53.amazonaws.com/doc/2012-02-29/"> <HostedZone> <Id>/hostedzone/Z11111</Id> <Name>example.com.</Name> <CallerReference>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</CallerReference> <Config> <Comment></Comment> <PrivateZone>false</PrivateZone> </Config> <ResourceRecordSetCount>2</ResourceRecordSetCount> </HostedZone> <ChangeInfo> <Id>/change/C1111111111111</Id> <Status>PENDING</Status> <SubmittedAt>2014-02-02T10:19:29.928Z</SubmittedAt> </ChangeInfo> <DelegationSet> <NameServers> <NameServer>ns-100.awsdns-01.com</NameServer> <NameServer>ns-1000.awsdns-01.co.uk</NameServer> <NameServer>ns-1000.awsdns-01.org</NameServer> <NameServer>ns-900.awsdns-01.net</NameServer> </NameServers> </DelegationSet> </CreateHostedZoneResponse> """ def test_create_zone(self): self.set_http_response(status_code=201) response = self.service_connection.create_zone("example.com.") self.assertTrue(isinstance(response, Zone)) self.assertEqual(response.id, "Z11111") self.assertEqual(response.name, "example.com.") def test_create_hosted_zone(self): self.set_http_response(status_code=201) response = self.service_connection.create_hosted_zone("example.com.", "my_ref", "a comment") self.assertEqual(response['CreateHostedZoneResponse'] ['DelegationSet']['NameServers'], ['ns-100.awsdns-01.com', 'ns-1000.awsdns-01.co.uk', 'ns-1000.awsdns-01.org', 'ns-900.awsdns-01.net']) self.assertEqual(response['CreateHostedZoneResponse'] ['HostedZone']['Config']['PrivateZone'], u'false') @attr(route53=True) class TestCreatePrivateZoneRoute53(AWSMockServiceTestCase): connection_class = Route53Connection def setUp(self): super(TestCreatePrivateZoneRoute53, self).setUp() def default_body(self): return b""" <CreateHostedZoneResponse xmlns="https://route53.amazonaws.com/doc/2012-02-29/"> <HostedZone> <Id>/hostedzone/Z11111</Id> <Name>example.com.</Name> <VPC> <VPCId>vpc-1a2b3c4d</VPCId> <VPCRegion>us-east-1</VPCRegion> </VPC> <CallerReference>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</CallerReference> <Config> <Comment></Comment> <PrivateZone>true</PrivateZone> </Config> <ResourceRecordSetCount>2</ResourceRecordSetCount> </HostedZone> <ChangeInfo> <Id>/change/C1111111111111</Id> <Status>PENDING</Status> <SubmittedAt>2014-02-02T10:19:29.928Z</SubmittedAt> </ChangeInfo> <DelegationSet> <NameServers> <NameServer>ns-100.awsdns-01.com</NameServer> <NameServer>ns-1000.awsdns-01.co.uk</NameServer> <NameServer>ns-1000.awsdns-01.org</NameServer> <NameServer>ns-900.awsdns-01.net</NameServer> </NameServers> </DelegationSet> </CreateHostedZoneResponse> """ def test_create_private_zone(self): self.set_http_response(status_code=201) r = self.service_connection.create_hosted_zone("example.com.", private_zone=True, vpc_id='vpc-1a2b3c4d', vpc_region='us-east-1' ) self.assertEqual(r['CreateHostedZoneResponse']['HostedZone'] ['Config']['PrivateZone'], u'true') self.assertEqual(r['CreateHostedZoneResponse']['HostedZone'] ['VPC']['VPCId'], u'vpc-1a2b3c4d') self.assertEqual(r['CreateHostedZoneResponse']['HostedZone'] ['VPC']['VPCRegion'], u'us-east-1') @attr(route53=True) class TestGetZoneRoute53(AWSMockServiceTestCase): connection_class = Route53Connection def setUp(self): super(TestGetZoneRoute53, self).setUp() def default_body(self): return b""" <ListHostedZonesResponse xmlns="https://route53.amazonaws.com/doc/2012-02-29/"> <HostedZones> <HostedZone> <Id>/hostedzone/Z1111</Id> <Name>example2.com.</Name> <CallerReference>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</CallerReference> <Config/> <ResourceRecordSetCount>3</ResourceRecordSetCount> </HostedZone> <HostedZone> <Id>/hostedzone/Z2222</Id> <Name>example1.com.</Name> <CallerReference>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeef</CallerReference> <Config/> <ResourceRecordSetCount>6</ResourceRecordSetCount> </HostedZone> <HostedZone> <Id>/hostedzone/Z3333</Id> <Name>example.com.</Name> <CallerReference>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeeg</CallerReference> <Config/> <ResourceRecordSetCount>6</ResourceRecordSetCount> </HostedZone> </HostedZones> <IsTruncated>false</IsTruncated> <MaxItems>100</MaxItems> </ListHostedZonesResponse> """ def test_list_zones(self): self.set_http_response(status_code=201) response = self.service_connection.get_all_hosted_zones() domains = ['example2.com.', 'example1.com.', 'example.com.'] print(response['ListHostedZonesResponse']['HostedZones'][0]) for d in response['ListHostedZonesResponse']['HostedZones']: print("Removing: %s" % d['Name']) domains.remove(d['Name']) self.assertEqual(domains, []) def test_get_zone(self): self.set_http_response(status_code=201) response = self.service_connection.get_zone('example.com.') self.assertTrue(isinstance(response, Zone)) self.assertEqual(response.name, "example.com.") @attr(route53=True) class TestGetHostedZoneRoute53(AWSMockServiceTestCase): connection_class = Route53Connection def setUp(self): super(TestGetHostedZoneRoute53, self).setUp() def default_body(self): return b""" <GetHostedZoneResponse xmlns="https://route53.amazonaws.com/doc/2012-02-29/"> <HostedZone> <Id>/hostedzone/Z1111</Id> <Name>example.com.</Name> <CallerReference>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</CallerReference> <Config/> <ResourceRecordSetCount>3</ResourceRecordSetCount> </HostedZone> <DelegationSet> <NameServers> <NameServer>ns-1000.awsdns-40.org</NameServer> <NameServer>ns-200.awsdns-30.com</NameServer> <NameServer>ns-900.awsdns-50.net</NameServer> <NameServer>ns-1000.awsdns-00.co.uk</NameServer> </NameServers> </DelegationSet> </GetHostedZoneResponse> """ def test_list_zones(self): self.set_http_response(status_code=201) response = self.service_connection.get_hosted_zone("Z1111") self.assertEqual(response['GetHostedZoneResponse'] ['HostedZone']['Id'], '/hostedzone/Z1111') self.assertEqual(response['GetHostedZoneResponse'] ['HostedZone']['Name'], 'example.com.') self.assertEqual(response['GetHostedZoneResponse'] ['DelegationSet']['NameServers'], ['ns-1000.awsdns-40.org', 'ns-200.awsdns-30.com', 'ns-900.awsdns-50.net', 'ns-1000.awsdns-00.co.uk']) @attr(route53=True) class TestGetAllRRSetsRoute53(AWSMockServiceTestCase): connection_class = Route53Connection def setUp(self): super(TestGetAllRRSetsRoute53, self).setUp() def default_body(self): return b""" <ListResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <ResourceRecordSets> <ResourceRecordSet> <Name>test.example.com.</Name> <Type>A</Type> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>10.0.0.1</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> <ResourceRecordSet> <Name>www.example.com.</Name> <Type>A</Type> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>10.0.0.2</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> <ResourceRecordSet> <Name>us-west-2-evaluate-health.example.com.</Name> <Type>A</Type> <SetIdentifier>latency-example-us-west-2-evaluate-health</SetIdentifier> <Region>us-west-2</Region> <AliasTarget> <HostedZoneId>ABCDEFG123456</HostedZoneId> <EvaluateTargetHealth>true</EvaluateTargetHealth> <DNSName>example-123456-evaluate-health.us-west-2.elb.amazonaws.com.</DNSName> </AliasTarget> <HealthCheckId>abcdefgh-abcd-abcd-abcd-abcdefghijkl</HealthCheckId> </ResourceRecordSet> <ResourceRecordSet> <Name>us-west-2-no-evaluate-health.example.com.</Name> <Type>A</Type> <SetIdentifier>latency-example-us-west-2-no-evaluate-health</SetIdentifier> <Region>us-west-2</Region> <AliasTarget> <HostedZoneId>ABCDEFG567890</HostedZoneId> <EvaluateTargetHealth>false</EvaluateTargetHealth> <DNSName>example-123456-no-evaluate-health.us-west-2.elb.amazonaws.com.</DNSName> </AliasTarget> <HealthCheckId>abcdefgh-abcd-abcd-abcd-abcdefghijkl</HealthCheckId> </ResourceRecordSet> <ResourceRecordSet> <Name>failover.example.com.</Name> <Type>A</Type> <SetIdentifier>failover-primary</SetIdentifier> <Failover>PRIMARY</Failover> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>10.0.0.4</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> <ResourceRecordSet> <Name>us-west-2-evaluate-health-healthcheck.example.com.</Name> <Type>A</Type> <SetIdentifier>latency-example-us-west-2-evaluate-health-healthcheck</SetIdentifier> <Region>us-west-2</Region> <AliasTarget> <HostedZoneId>ABCDEFG123456</HostedZoneId> <EvaluateTargetHealth>true</EvaluateTargetHealth> <DNSName>example-123456-evaluate-health-healthcheck.us-west-2.elb.amazonaws.com.</DNSName> </AliasTarget> <HealthCheckId>076a32f8-86f7-4c9e-9fa2-c163d5be67d9</HealthCheckId> </ResourceRecordSet> </ResourceRecordSets> <IsTruncated>false</IsTruncated> <MaxItems>100</MaxItems> </ListResourceRecordSetsResponse> """ def test_get_all_rr_sets(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_rrsets("Z1111", "A", "example.com.") self.assertIn(self.actual_request.path, ("/2013-04-01/hostedzone/Z1111/rrset?type=A&name=example.com.", "/2013-04-01/hostedzone/Z1111/rrset?name=example.com.&type=A")) self.assertTrue(isinstance(response, ResourceRecordSets)) self.assertEqual(response.hosted_zone_id, "Z1111") self.assertTrue(isinstance(response[0], Record)) self.assertTrue(response[0].name, "test.example.com.") self.assertTrue(response[0].ttl, "60") self.assertTrue(response[0].type, "A") evaluate_record = response[2] self.assertEqual(evaluate_record.name, 'us-west-2-evaluate-health.example.com.') self.assertEqual(evaluate_record.type, 'A') self.assertEqual(evaluate_record.identifier, 'latency-example-us-west-2-evaluate-health') self.assertEqual(evaluate_record.region, 'us-west-2') self.assertEqual(evaluate_record.alias_hosted_zone_id, 'ABCDEFG123456') self.assertTrue(evaluate_record.alias_evaluate_target_health) self.assertEqual(evaluate_record.alias_dns_name, 'example-123456-evaluate-health.us-west-2.elb.amazonaws.com.') evaluate_xml = evaluate_record.to_xml() self.assertTrue(evaluate_record.health_check, 'abcdefgh-abcd-abcd-abcd-abcdefghijkl') self.assertTrue('<EvaluateTargetHealth>true</EvaluateTargetHealth>' in evaluate_xml) no_evaluate_record = response[3] self.assertEqual(no_evaluate_record.name, 'us-west-2-no-evaluate-health.example.com.') self.assertEqual(no_evaluate_record.type, 'A') self.assertEqual(no_evaluate_record.identifier, 'latency-example-us-west-2-no-evaluate-health') self.assertEqual(no_evaluate_record.region, 'us-west-2') self.assertEqual(no_evaluate_record.alias_hosted_zone_id, 'ABCDEFG567890') self.assertFalse(no_evaluate_record.alias_evaluate_target_health) self.assertEqual(no_evaluate_record.alias_dns_name, 'example-123456-no-evaluate-health.us-west-2.elb.amazonaws.com.') no_evaluate_xml = no_evaluate_record.to_xml() self.assertTrue(no_evaluate_record.health_check, 'abcdefgh-abcd-abcd-abcd-abcdefghijkl') self.assertTrue('<EvaluateTargetHealth>false</EvaluateTargetHealth>' in no_evaluate_xml) failover_record = response[4] self.assertEqual(failover_record.name, 'failover.example.com.') self.assertEqual(failover_record.type, 'A') self.assertEqual(failover_record.identifier, 'failover-primary') self.assertEqual(failover_record.failover, 'PRIMARY') self.assertEqual(failover_record.ttl, '60') healthcheck_record = response[5] self.assertEqual(healthcheck_record.health_check, '076a32f8-86f7-4c9e-9fa2-c163d5be67d9') self.assertEqual(healthcheck_record.name, 'us-west-2-evaluate-health-healthcheck.example.com.') self.assertEqual(healthcheck_record.identifier, 'latency-example-us-west-2-evaluate-health-healthcheck') self.assertEqual(healthcheck_record.alias_dns_name, 'example-123456-evaluate-health-healthcheck.us-west-2.elb.amazonaws.com.') @attr(route53=True) class TestTruncatedGetAllRRSetsRoute53(AWSMockServiceTestCase): connection_class = Route53Connection def setUp(self): super(TestTruncatedGetAllRRSetsRoute53, self).setUp() def default_body(self): return b""" <ListResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <ResourceRecordSets> <ResourceRecordSet> <Name>example.com.</Name> <Type>NS</Type> <TTL>900</TTL> <ResourceRecords> <ResourceRecord> <Value>ns-91.awsdns-41.co.uk.</Value> </ResourceRecord> <ResourceRecord> <Value>ns-1929.awsdns-93.net.</Value> </ResourceRecord> <ResourceRecord> <Value>ns-12.awsdns-21.org.</Value> </ResourceRecord> <ResourceRecord> <Value>ns-102.awsdns-96.com.</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> <ResourceRecordSet> <Name>example.com.</Name> <Type>SOA</Type> <TTL>1800</TTL> <ResourceRecords> <ResourceRecord> <Value>ns-1929.awsdns-93.net. hostmaster.awsdns.net. 1 10800 3600 604800 1800</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> <ResourceRecordSet> <Name>wrr.example.com.</Name> <Type>A</Type> <SetIdentifier>primary</SetIdentifier> <Weight>100</Weight> <TTL>300</TTL> <ResourceRecords> <ResourceRecord><Value>127.0.0.1</Value></ResourceRecord> </ResourceRecords> </ResourceRecordSet> </ResourceRecordSets> <IsTruncated>true</IsTruncated> <NextRecordName>wrr.example.com.</NextRecordName> <NextRecordType>A</NextRecordType> <NextRecordIdentifier>secondary</NextRecordIdentifier> <MaxItems>3</MaxItems> </ListResourceRecordSetsResponse>""" def paged_body(self): return b""" <ListResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <ResourceRecordSets> <ResourceRecordSet> <Name>wrr.example.com.</Name> <Type>A</Type> <SetIdentifier>secondary</SetIdentifier> <Weight>50</Weight> <TTL>300</TTL> <ResourceRecords> <ResourceRecord><Value>127.0.0.2</Value></ResourceRecord> </ResourceRecords> </ResourceRecordSet> </ResourceRecordSets> <IsTruncated>false</IsTruncated> <MaxItems>3</MaxItems> </ListResourceRecordSetsResponse>""" def test_get_all_rr_sets(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_rrsets("Z1111", maxitems=3) # made first request self.assertEqual(self.actual_request.path, '/2013-04-01/hostedzone/Z1111/rrset?maxitems=3') # anticipate a second request when we page it self.set_http_response(status_code=200, body=self.paged_body()) # this should trigger another call to get_all_rrsets self.assertEqual(len(list(response)), 4) url_parts = urllib.parse.urlparse(self.actual_request.path) self.assertEqual(url_parts.path, '/2013-04-01/hostedzone/Z1111/rrset') self.assertEqual(urllib.parse.parse_qs(url_parts.query), dict(type=['A'], name=['wrr.example.com.'], identifier=['secondary'])) @attr(route53=True) class TestCreateHealthCheckRoute53IpAddress(AWSMockServiceTestCase): connection_class = Route53Connection def setUp(self): super(TestCreateHealthCheckRoute53IpAddress, self).setUp() def default_body(self): return b""" <CreateHealthCheckResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <HealthCheck> <Id>34778cf8-e31e-4974-bad0-b108bd1623d3</Id> <CallerReference>2fa48c8f-76ef-4253-9874-8bcb2b0d7694</CallerReference> <HealthCheckConfig> <IPAddress>74.125.228.81</IPAddress> <Port>443</Port> <Type>HTTPS_STR_MATCH</Type> <SearchString>OK</SearchString> <ResourcePath>/health_check</ResourcePath> <RequestInterval>30</RequestInterval> <FailureThreshold>3</FailureThreshold> </HealthCheckConfig> </HealthCheck> </CreateHealthCheckResponse> """ def test_create_health_check_ip_address(self): self.set_http_response(status_code=201) hc = HealthCheck(ip_addr='74.125.228.81', port=443, hc_type='HTTPS_STR_MATCH', resource_path='/health_check', string_match='OK') hc_xml = hc.to_xml() self.assertFalse('<FullyQualifiedDomainName>' in hc_xml) self.assertTrue('<IPAddress>' in hc_xml) response = self.service_connection.create_health_check(hc) hc_resp = response['CreateHealthCheckResponse']['HealthCheck']['HealthCheckConfig'] self.assertEqual(hc_resp['IPAddress'], '74.125.228.81') self.assertEqual(hc_resp['ResourcePath'], '/health_check') self.assertEqual(hc_resp['Type'], 'HTTPS_STR_MATCH') self.assertEqual(hc_resp['Port'], '443') self.assertEqual(hc_resp['ResourcePath'], '/health_check') self.assertEqual(hc_resp['SearchString'], 'OK') self.assertEqual(response['CreateHealthCheckResponse']['HealthCheck']['Id'], '34778cf8-e31e-4974-bad0-b108bd1623d3') @attr(route53=True) class TestGetCheckerIpRanges(AWSMockServiceTestCase): connection_class = Route53Connection def default_body(self): return b""" <GetCheckerIpRangesResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <CheckerIpRanges> <member>54.183.255.128/26</member> <member>54.228.16.0/26</member> <member>54.232.40.64/26</member> <member>177.71.207.128/26</member> <member>176.34.159.192/26</member> </CheckerIpRanges> </GetCheckerIpRangesResponse> """ def test_get_checker_ip_ranges(self): self.set_http_response(status_code=200) response = self.service_connection.get_checker_ip_ranges() ip_ranges = response['GetCheckerIpRangesResponse']['CheckerIpRanges'] self.assertEqual(len(ip_ranges), 5) self.assertIn('54.183.255.128/26', ip_ranges) self.assertIn('54.228.16.0/26', ip_ranges) self.assertIn('54.232.40.64/26', ip_ranges) self.assertIn('177.71.207.128/26', ip_ranges) self.assertIn('176.34.159.192/26', ip_ranges) @attr(route53=True) class TestCreateHealthCheckRoute53FQDN(AWSMockServiceTestCase): connection_class = Route53Connection def setUp(self): super(TestCreateHealthCheckRoute53FQDN, self).setUp() def default_body(self): return b""" <CreateHealthCheckResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <HealthCheck> <Id>f9abfe10-8d2a-4bbd-8f35-796f0f8572f2</Id> <CallerReference>3246ac17-b651-4295-a5c8-c132a59693d7</CallerReference> <HealthCheckConfig> <Port>443</Port> <Type>HTTPS</Type> <ResourcePath>/health_check</ResourcePath> <FullyQualifiedDomainName>example.com</FullyQualifiedDomainName> <RequestInterval>30</RequestInterval> <FailureThreshold>3</FailureThreshold> </HealthCheckConfig> </HealthCheck> </CreateHealthCheckResponse> """ def test_create_health_check_fqdn(self): self.set_http_response(status_code=201) hc = HealthCheck(ip_addr='', port=443, hc_type='HTTPS', resource_path='/health_check', fqdn='example.com') hc_xml = hc.to_xml() self.assertTrue('<FullyQualifiedDomainName>' in hc_xml) self.assertFalse('<IPAddress>' in hc_xml) response = self.service_connection.create_health_check(hc) hc_resp = response['CreateHealthCheckResponse']['HealthCheck']['HealthCheckConfig'] self.assertEqual(hc_resp['FullyQualifiedDomainName'], 'example.com') self.assertEqual(hc_resp['ResourcePath'], '/health_check') self.assertEqual(hc_resp['Type'], 'HTTPS') self.assertEqual(hc_resp['Port'], '443') self.assertEqual(hc_resp['ResourcePath'], '/health_check') self.assertEqual(response['CreateHealthCheckResponse']['HealthCheck']['Id'], 'f9abfe10-8d2a-4bbd-8f35-796f0f8572f2') @attr(route53=True) class TestChangeResourceRecordSetsRoute53(AWSMockServiceTestCase): connection_class = Route53Connection def setUp(self): super(TestChangeResourceRecordSetsRoute53, self).setUp() def default_body(self): return b""" <ChangeResourceRecordSetsResponse xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <ChangeInfo> <Id>/change/C1111111111111</Id> <Status>PENDING</Status> <SubmittedAt>2014-05-05T10:11:12.123Z</SubmittedAt> </ChangeInfo> </ChangeResourceRecordSetsResponse> """ def test_record_commit(self): rrsets = ResourceRecordSets(self.service_connection) rrsets.add_change_record('CREATE', Record('vanilla.example.com', 'A', 60, ['1.2.3.4'])) rrsets.add_change_record('CREATE', Record('alias.example.com', 'AAAA', alias_hosted_zone_id='Z123OTHER', alias_dns_name='target.other', alias_evaluate_target_health=True)) rrsets.add_change_record('CREATE', Record('wrr.example.com', 'CNAME', 60, ['cname.target'], weight=10, identifier='weight-1')) rrsets.add_change_record('CREATE', Record('lbr.example.com', 'TXT', 60, ['text record'], region='us-west-2', identifier='region-1')) rrsets.add_change_record('CREATE', Record('failover.example.com', 'A', 60, ['2.2.2.2'], health_check='hc-1234', failover='PRIMARY', identifier='primary')) changes_xml = rrsets.to_xml() # the whitespacing doesn't match exactly, so we'll pretty print and drop all new lines # not the best, but actual_xml = re.sub(r"\s*[\r\n]+", "\n", xml.dom.minidom.parseString(changes_xml).toprettyxml()) expected_xml = re.sub(r"\s*[\r\n]+", "\n", xml.dom.minidom.parseString(b""" <ChangeResourceRecordSetsRequest xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <ChangeBatch> <Comment>None</Comment> <Changes> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>vanilla.example.com</Name> <Type>A</Type> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>1.2.3.4</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>alias.example.com</Name> <Type>AAAA</Type> <AliasTarget> <HostedZoneId>Z123OTHER</HostedZoneId> <DNSName>target.other</DNSName> <EvaluateTargetHealth>true</EvaluateTargetHealth> </AliasTarget> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>wrr.example.com</Name> <Type>CNAME</Type> <SetIdentifier>weight-1</SetIdentifier> <Weight>10</Weight> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>cname.target</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>lbr.example.com</Name> <Type>TXT</Type> <SetIdentifier>region-1</SetIdentifier> <Region>us-west-2</Region> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>text record</Value> </ResourceRecord> </ResourceRecords> </ResourceRecordSet> </Change> <Change> <Action>CREATE</Action> <ResourceRecordSet> <Name>failover.example.com</Name> <Type>A</Type> <SetIdentifier>primary</SetIdentifier> <Failover>PRIMARY</Failover> <TTL>60</TTL> <ResourceRecords> <ResourceRecord> <Value>2.2.2.2</Value> </ResourceRecord> </ResourceRecords> <HealthCheckId>hc-1234</HealthCheckId> </ResourceRecordSet> </Change> </Changes> </ChangeBatch> </ChangeResourceRecordSetsRequest> """).toprettyxml()) # Note: the alias XML should not include the TTL, even if it's specified in the object model self.assertEqual(actual_xml, expected_xml) vendored/boto/tests/unit/route53/__init__.py 0000644 00000000000 15025013616 0015042 0 ustar 00 vendored/boto/tests/unit/kinesis/test_kinesis.py 0000644 00000005633 15025013616 0016167 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.compat import json from boto.kinesis.layer1 import KinesisConnection from tests.unit import AWSMockServiceTestCase class TestKinesis(AWSMockServiceTestCase): connection_class = KinesisConnection def default_body(self): return b'{}' def test_put_record_binary(self): self.set_http_response(status_code=200) self.service_connection.put_record('stream-name', b'\x00\x01\x02\x03\x04\x05', 'partition-key') body = json.loads(self.actual_request.body.decode('utf-8')) self.assertEqual(body['Data'], 'AAECAwQF') target = self.actual_request.headers['X-Amz-Target'] self.assertTrue('PutRecord' in target) def test_put_record_string(self): self.set_http_response(status_code=200) self.service_connection.put_record('stream-name', 'data', 'partition-key') body = json.loads(self.actual_request.body.decode('utf-8')) self.assertEqual(body['Data'], 'ZGF0YQ==') target = self.actual_request.headers['X-Amz-Target'] self.assertTrue('PutRecord' in target) def test_put_records(self): self.set_http_response(status_code=200) record_binary = { 'Data': b'\x00\x01\x02\x03\x04\x05', 'PartitionKey': 'partition-key' } record_str = { 'Data': 'data', 'PartitionKey': 'partition-key' } self.service_connection.put_records(stream_name='stream-name', records=[record_binary, record_str]) body = json.loads(self.actual_request.body.decode('utf-8')) self.assertEqual(body['Records'][0]['Data'], 'AAECAwQF') self.assertEqual(body['Records'][1]['Data'], 'ZGF0YQ==') target = self.actual_request.headers['X-Amz-Target'] self.assertTrue('PutRecord' in target) vendored/boto/tests/unit/kinesis/__init__.py 0000644 00000000000 15025013616 0015201 0 ustar 00 vendored/boto/tests/unit/dynamodb/test_layer2.py 0000644 00000011332 15025013616 0016041 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from mock import Mock from boto.dynamodb.layer2 import Layer2 from boto.dynamodb.table import Table, Schema DESCRIBE_TABLE = { "Table": { "CreationDateTime": 1.353526122785E9, "ItemCount": 1, "KeySchema": { "HashKeyElement": {"AttributeName": "foo", "AttributeType": "N"}}, "ProvisionedThroughput": { "NumberOfDecreasesToday": 0, "ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, "TableName": "footest", "TableSizeBytes": 21, "TableStatus": "ACTIVE"} } class TestTableConstruction(unittest.TestCase): def setUp(self): self.layer2 = Layer2('access_key', 'secret_key') self.api = Mock() self.layer2.layer1 = self.api def test_get_table(self): self.api.describe_table.return_value = DESCRIBE_TABLE table = self.layer2.get_table('footest') self.assertEqual(table.name, 'footest') self.assertEqual(table.create_time, 1353526122.785) self.assertEqual(table.status, 'ACTIVE') self.assertEqual(table.item_count, 1) self.assertEqual(table.size_bytes, 21) self.assertEqual(table.read_units, 5) self.assertEqual(table.write_units, 5) self.assertEqual(table.schema, Schema.create(hash_key=('foo', 'N'))) def test_create_table_without_api_call(self): table = self.layer2.table_from_schema( name='footest', schema=Schema.create(hash_key=('foo', 'N'))) self.assertEqual(table.name, 'footest') self.assertEqual(table.schema, Schema.create(hash_key=('foo', 'N'))) # describe_table is never called. self.assertEqual(self.api.describe_table.call_count, 0) def test_create_schema_with_hash_and_range(self): schema = self.layer2.create_schema('foo', int, 'bar', str) self.assertEqual(schema.hash_key_name, 'foo') self.assertEqual(schema.hash_key_type, 'N') self.assertEqual(schema.range_key_name, 'bar') self.assertEqual(schema.range_key_type, 'S') def test_create_schema_with_hash(self): schema = self.layer2.create_schema('foo', str) self.assertEqual(schema.hash_key_name, 'foo') self.assertEqual(schema.hash_key_type, 'S') self.assertIsNone(schema.range_key_name) self.assertIsNone(schema.range_key_type) class TestSchemaEquality(unittest.TestCase): def test_schema_equal(self): s1 = Schema.create(hash_key=('foo', 'N')) s2 = Schema.create(hash_key=('foo', 'N')) self.assertEqual(s1, s2) def test_schema_not_equal(self): s1 = Schema.create(hash_key=('foo', 'N')) s2 = Schema.create(hash_key=('bar', 'N')) s3 = Schema.create(hash_key=('foo', 'S')) self.assertNotEqual(s1, s2) self.assertNotEqual(s1, s3) def test_equal_with_hash_and_range(self): s1 = Schema.create(hash_key=('foo', 'N'), range_key=('bar', 'S')) s2 = Schema.create(hash_key=('foo', 'N'), range_key=('bar', 'S')) self.assertEqual(s1, s2) def test_schema_with_hash_and_range_not_equal(self): s1 = Schema.create(hash_key=('foo', 'N'), range_key=('bar', 'S')) s2 = Schema.create(hash_key=('foo', 'N'), range_key=('bar', 'N')) s3 = Schema.create(hash_key=('foo', 'S'), range_key=('baz', 'N')) s4 = Schema.create(hash_key=('bar', 'N'), range_key=('baz', 'N')) self.assertNotEqual(s1, s2) self.assertNotEqual(s1, s3) self.assertNotEqual(s1, s4) self.assertNotEqual(s2, s4) self.assertNotEqual(s3, s4) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/dynamodb/test_types.py 0000644 00000016147 15025013616 0016020 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from decimal import Decimal from tests.compat import unittest from boto.compat import six, json from boto.dynamodb import types from boto.dynamodb.exceptions import DynamoDBNumberError class TestDynamizer(unittest.TestCase): def setUp(self): pass def test_encoding_to_dynamodb(self): dynamizer = types.Dynamizer() self.assertEqual(dynamizer.encode('foo'), {'S': 'foo'}) self.assertEqual(dynamizer.encode(54), {'N': '54'}) self.assertEqual(dynamizer.encode(Decimal('1.1')), {'N': '1.1'}) self.assertEqual(dynamizer.encode(set([1, 2, 3])), {'NS': ['1', '2', '3']}) self.assertIn(dynamizer.encode(set(['foo', 'bar'])), ({'SS': ['foo', 'bar']}, {'SS': ['bar', 'foo']})) self.assertEqual(dynamizer.encode(types.Binary(b'\x01')), {'B': 'AQ=='}) self.assertEqual(dynamizer.encode(set([types.Binary(b'\x01')])), {'BS': ['AQ==']}) self.assertEqual(dynamizer.encode(['foo', 54, [1]]), {'L': [{'S': 'foo'}, {'N': '54'}, {'L': [{'N': '1'}]}]}) self.assertEqual(dynamizer.encode({'foo': 'bar', 'hoge': {'sub': 1}}), {'M': {'foo': {'S': 'bar'}, 'hoge': {'M': {'sub': {'N': '1'}}}}}) self.assertEqual(dynamizer.encode(None), {'NULL': True}) self.assertEqual(dynamizer.encode(False), {'BOOL': False}) def test_decoding_to_dynamodb(self): dynamizer = types.Dynamizer() self.assertEqual(dynamizer.decode({'S': 'foo'}), 'foo') self.assertEqual(dynamizer.decode({'N': '54'}), 54) self.assertEqual(dynamizer.decode({'N': '1.1'}), Decimal('1.1')) self.assertEqual(dynamizer.decode({'NS': ['1', '2', '3']}), set([1, 2, 3])) self.assertEqual(dynamizer.decode({'SS': ['foo', 'bar']}), set(['foo', 'bar'])) self.assertEqual(dynamizer.decode({'B': 'AQ=='}), types.Binary(b'\x01')) self.assertEqual(dynamizer.decode({'BS': ['AQ==']}), set([types.Binary(b'\x01')])) self.assertEqual(dynamizer.decode({'L': [{'S': 'foo'}, {'N': '54'}, {'L': [{'N': '1'}]}]}), ['foo', 54, [1]]) self.assertEqual(dynamizer.decode({'M': {'foo': {'S': 'bar'}, 'hoge': {'M': {'sub': {'N': '1'}}}}}), {'foo': 'bar', 'hoge': {'sub': 1}}) self.assertEqual(dynamizer.decode({'NULL': True}), None) self.assertEqual(dynamizer.decode({'BOOL': False}), False) def test_float_conversion_errors(self): dynamizer = types.Dynamizer() # When supporting decimals, certain floats will work: self.assertEqual(dynamizer.encode(1.25), {'N': '1.25'}) # And some will generate errors, which is why it's best # to just use Decimals directly: with self.assertRaises(DynamoDBNumberError): dynamizer.encode(1.1) def test_non_boolean_conversions(self): dynamizer = types.NonBooleanDynamizer() self.assertEqual(dynamizer.encode(True), {'N': '1'}) def test_lossy_float_conversions(self): dynamizer = types.LossyFloatDynamizer() # Just testing the differences here, specifically float conversions: self.assertEqual(dynamizer.encode(1.1), {'N': '1.1'}) self.assertEqual(dynamizer.decode({'N': '1.1'}), 1.1) self.assertEqual(dynamizer.encode(set([1.1])), {'NS': ['1.1']}) self.assertEqual(dynamizer.decode({'NS': ['1.1', '2.2', '3.3']}), set([1.1, 2.2, 3.3])) def test_decoding_full_doc(self): '''Simple List decoding that had caused some errors''' dynamizer = types.Dynamizer() doc = '{"__type__":{"S":"Story"},"company_tickers":{"SS":["NASDAQ-TSLA","NYSE-F","NYSE-GM"]},"modified_at":{"N":"1452525162"},"created_at":{"N":"1452525162"},"version":{"N":"1"},"categories":{"SS":["AUTOMTVE","LTRTR","MANUFCTU","PN","PRHYPE","TAXE","TJ","TL"]},"provider_categories":{"L":[{"S":"F"},{"S":"GM"},{"S":"TSLA"}]},"received_at":{"S":"2016-01-11T11:26:31Z"}}' output_doc = {'provider_categories': ['F', 'GM', 'TSLA'], '__type__': 'Story', 'company_tickers': set(['NASDAQ-TSLA', 'NYSE-GM', 'NYSE-F']), 'modified_at': Decimal('1452525162'), 'version': Decimal('1'), 'received_at': '2016-01-11T11:26:31Z', 'created_at': Decimal('1452525162'), 'categories': set(['LTRTR', 'TAXE', 'MANUFCTU', 'TL', 'TJ', 'AUTOMTVE', 'PRHYPE', 'PN'])} self.assertEqual(json.loads(doc, object_hook=dynamizer.decode), output_doc) class TestBinary(unittest.TestCase): def test_good_input(self): data = types.Binary(b'\x01') self.assertEqual(b'\x01', data) self.assertEqual(b'\x01', bytes(data)) def test_non_ascii_good_input(self): # Binary data that is out of ASCII range data = types.Binary(b'\x88') self.assertEqual(b'\x88', data) self.assertEqual(b'\x88', bytes(data)) @unittest.skipUnless(six.PY2, "Python 2 only") def test_bad_input(self): with self.assertRaises(TypeError): types.Binary(1) @unittest.skipUnless(six.PY3, "Python 3 only") def test_bytes_input(self): data = types.Binary(1) self.assertEqual(data, b'\x00') self.assertEqual(data.value, b'\x00') @unittest.skipUnless(six.PY2, "Python 2 only") def test_unicode_py2(self): # It's dirty. But remains for backward compatibility. data = types.Binary(u'\x01') self.assertEqual(data, b'\x01') self.assertEqual(bytes(data), b'\x01') # Delegate to built-in b'\x01' == u'\x01' # In Python 2.x these are considered equal self.assertEqual(data, u'\x01') # Check that the value field is of type bytes self.assertEqual(type(data.value), bytes) @unittest.skipUnless(six.PY3, "Python 3 only") def test_unicode_py3(self): with self.assertRaises(TypeError): types.Binary(u'\x01') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/dynamodb/__init__.py 0000644 00000000000 15025013616 0015331 0 ustar 00 vendored/boto/tests/unit/dynamodb/test_batch.py 0000644 00000007751 15025013616 0015736 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from boto.dynamodb.batch import Batch from boto.dynamodb.table import Table from boto.dynamodb.layer2 import Layer2 from boto.dynamodb.batch import BatchList DESCRIBE_TABLE_1 = { 'Table': { 'CreationDateTime': 1349910554.478, 'ItemCount': 1, 'KeySchema': {'HashKeyElement': {'AttributeName': u'foo', 'AttributeType': u'S'}}, 'ProvisionedThroughput': {'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10}, 'TableName': 'testtable', 'TableSizeBytes': 54, 'TableStatus': 'ACTIVE'} } DESCRIBE_TABLE_2 = { 'Table': { 'CreationDateTime': 1349910554.478, 'ItemCount': 1, 'KeySchema': {'HashKeyElement': {'AttributeName': u'baz', 'AttributeType': u'S'}, 'RangeKeyElement': {'AttributeName': 'myrange', 'AttributeType': 'N'}}, 'ProvisionedThroughput': {'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10}, 'TableName': 'testtable2', 'TableSizeBytes': 54, 'TableStatus': 'ACTIVE'} } class TestBatchObjects(unittest.TestCase): maxDiff = None def setUp(self): self.layer2 = Layer2('access_key', 'secret_key') self.table = Table(self.layer2, DESCRIBE_TABLE_1) self.table2 = Table(self.layer2, DESCRIBE_TABLE_2) def test_batch_to_dict(self): b = Batch(self.table, ['k1', 'k2'], attributes_to_get=['foo'], consistent_read=True) self.assertDictEqual( b.to_dict(), {'AttributesToGet': ['foo'], 'Keys': [{'HashKeyElement': {'S': 'k1'}}, {'HashKeyElement': {'S': 'k2'}}], 'ConsistentRead': True} ) def test_batch_consistent_read_defaults_to_false(self): b = Batch(self.table, ['k1']) self.assertDictEqual( b.to_dict(), {'Keys': [{'HashKeyElement': {'S': 'k1'}}], 'ConsistentRead': False} ) def test_batch_list_consistent_read(self): b = BatchList(self.layer2) b.add_batch(self.table, ['k1'], ['foo'], consistent_read=True) b.add_batch(self.table2, [('k2', 54)], ['bar'], consistent_read=False) self.assertDictEqual( b.to_dict(), {'testtable': {'AttributesToGet': ['foo'], 'Keys': [{'HashKeyElement': {'S': 'k1'}}], 'ConsistentRead': True}, 'testtable2': {'AttributesToGet': ['bar'], 'Keys': [{'HashKeyElement': {'S': 'k2'}, 'RangeKeyElement': {'N': '54'}}], 'ConsistentRead': False}}) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/sts/test_credentials.py 0000644 00000002071 15025013616 0016154 0 ustar 00 import unittest from boto.sts.credentials import Credentials class STSCredentialsTest(unittest.TestCase): sts = True def setUp(self): super(STSCredentialsTest, self).setUp() self.creds = Credentials() def test_to_dict(self): # This would fail miserably if ``Credentials.request_id`` hadn't been # explicitly set (no default). # Default. self.assertEqual(self.creds.to_dict(), { 'access_key': None, 'expiration': None, 'request_id': None, 'secret_key': None, 'session_token': None }) # Override. creds = Credentials() creds.access_key = 'something' creds.secret_key = 'crypto' creds.session_token = 'this' creds.expiration = 'way' creds.request_id = 'comes' self.assertEqual(creds.to_dict(), { 'access_key': 'something', 'expiration': 'way', 'request_id': 'comes', 'secret_key': 'crypto', 'session_token': 'this' }) vendored/boto/tests/unit/sts/test_connection.py 0000644 00000021401 15025013616 0016014 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from boto.sts.connection import STSConnection from tests.unit import AWSMockServiceTestCase class TestSecurityToken(AWSMockServiceTestCase): connection_class = STSConnection def create_service_connection(self, **kwargs): kwargs['security_token'] = 'token' return super(TestSecurityToken, self).create_service_connection(**kwargs) def test_security_token(self): self.assertEqual('token', self.service_connection.provider.security_token) class TestSTSConnection(AWSMockServiceTestCase): connection_class = STSConnection def setUp(self): super(TestSTSConnection, self).setUp() def default_body(self): return b""" <AssumeRoleResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/"> <AssumeRoleResult> <AssumedRoleUser> <Arn>arn:role</Arn> <AssumedRoleId>roleid:myrolesession</AssumedRoleId> </AssumedRoleUser> <Credentials> <SessionToken>session_token</SessionToken> <SecretAccessKey>secretkey</SecretAccessKey> <Expiration>2012-10-18T10:18:14.789Z</Expiration> <AccessKeyId>accesskey</AccessKeyId> </Credentials> </AssumeRoleResult> <ResponseMetadata> <RequestId>8b7418cb-18a8-11e2-a706-4bd22ca68ab7</RequestId> </ResponseMetadata> </AssumeRoleResponse> """ def test_assume_role(self): self.set_http_response(status_code=200) response = self.service_connection.assume_role('arn:role', 'mysession') self.assert_request_parameters( {'Action': 'AssumeRole', 'RoleArn': 'arn:role', 'RoleSessionName': 'mysession'}, ignore_params_values=['Version']) self.assertEqual(response.credentials.access_key, 'accesskey') self.assertEqual(response.credentials.secret_key, 'secretkey') self.assertEqual(response.credentials.session_token, 'session_token') self.assertEqual(response.user.arn, 'arn:role') self.assertEqual(response.user.assume_role_id, 'roleid:myrolesession') def test_assume_role_with_mfa(self): self.set_http_response(status_code=200) response = self.service_connection.assume_role( 'arn:role', 'mysession', mfa_serial_number='GAHT12345678', mfa_token='abc123' ) self.assert_request_parameters( {'Action': 'AssumeRole', 'RoleArn': 'arn:role', 'RoleSessionName': 'mysession', 'SerialNumber': 'GAHT12345678', 'TokenCode': 'abc123'}, ignore_params_values=['Version']) self.assertEqual(response.credentials.access_key, 'accesskey') self.assertEqual(response.credentials.secret_key, 'secretkey') self.assertEqual(response.credentials.session_token, 'session_token') self.assertEqual(response.user.arn, 'arn:role') self.assertEqual(response.user.assume_role_id, 'roleid:myrolesession') class TestSTSWebIdentityConnection(AWSMockServiceTestCase): connection_class = STSConnection def setUp(self): super(TestSTSWebIdentityConnection, self).setUp() def default_body(self): return b""" <AssumeRoleWithWebIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/"> <AssumeRoleWithWebIdentityResult> <SubjectFromWebIdentityToken> amzn1.account.AF6RHO7KZU5XRVQJGXK6HB56KR2A </SubjectFromWebIdentityToken> <AssumedRoleUser> <Arn> arn:aws:sts::000240903217:assumed-role/FederatedWebIdentityRole/app1 </Arn> <AssumedRoleId> AROACLKWSDQRAOFQC3IDI:app1 </AssumedRoleId> </AssumedRoleUser> <Credentials> <SessionToken> AQoDYXdzEE0a8ANXXXXXXXXNO1ewxE5TijQyp+IPfnyowF </SessionToken> <SecretAccessKey> secretkey </SecretAccessKey> <Expiration> 2013-05-14T23:00:23Z </Expiration> <AccessKeyId> accesskey </AccessKeyId> </Credentials> </AssumeRoleWithWebIdentityResult> <ResponseMetadata> <RequestId>ad4156e9-bce1-11e2-82e6-6b6ef249e618</RequestId> </ResponseMetadata> </AssumeRoleWithWebIdentityResponse> """ def test_assume_role_with_web_identity(self): arn = 'arn:aws:iam::000240903217:role/FederatedWebIdentityRole' wit = 'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' self.set_http_response(status_code=200) response = self.service_connection.assume_role_with_web_identity( role_arn=arn, role_session_name='guestuser', web_identity_token=wit, provider_id='www.amazon.com', ) self.assert_request_parameters({ 'RoleSessionName': 'guestuser', 'RoleArn': arn, 'WebIdentityToken': wit, 'ProviderId': 'www.amazon.com', 'Action': 'AssumeRoleWithWebIdentity' }, ignore_params_values=[ 'Version' ]) self.assertEqual( response.credentials.access_key.strip(), 'accesskey' ) self.assertEqual( response.credentials.secret_key.strip(), 'secretkey' ) self.assertEqual( response.credentials.session_token.strip(), 'AQoDYXdzEE0a8ANXXXXXXXXNO1ewxE5TijQyp+IPfnyowF' ) self.assertEqual( response.user.arn.strip(), 'arn:aws:sts::000240903217:assumed-role/FederatedWebIdentityRole/app1' ) self.assertEqual( response.user.assume_role_id.strip(), 'AROACLKWSDQRAOFQC3IDI:app1' ) class TestSTSSAMLConnection(AWSMockServiceTestCase): connection_class = STSConnection def setUp(self): super(TestSTSSAMLConnection, self).setUp() def default_body(self): return b""" <AssumeRoleWithSAMLResponse xmlns="https://sts.amazonaws.com/doc/ 2011-06-15/"> <AssumeRoleWithSAMLResult> <Credentials> <SessionToken>session_token</SessionToken> <SecretAccessKey>secretkey</SecretAccessKey> <Expiration>2011-07-15T23:28:33.359Z</Expiration> <AccessKeyId>accesskey</AccessKeyId> </Credentials> <AssumedRoleUser> <Arn>arn:role</Arn> <AssumedRoleId>roleid:myrolesession</AssumedRoleId> </AssumedRoleUser> <PackedPolicySize>6</PackedPolicySize> </AssumeRoleWithSAMLResult> <ResponseMetadata> <RequestId>c6104cbe-af31-11e0-8154-cbc7ccf896c7</RequestId> </ResponseMetadata> </AssumeRoleWithSAMLResponse> """ def test_assume_role_with_saml(self): arn = 'arn:aws:iam::000240903217:role/Test' principal = 'arn:aws:iam::000240903217:role/Principal' assertion = 'test' self.set_http_response(status_code=200) response = self.service_connection.assume_role_with_saml( role_arn=arn, principal_arn=principal, saml_assertion=assertion ) self.assert_request_parameters({ 'RoleArn': arn, 'PrincipalArn': principal, 'SAMLAssertion': assertion, 'Action': 'AssumeRoleWithSAML' }, ignore_params_values=[ 'Version' ]) self.assertEqual(response.credentials.access_key, 'accesskey') self.assertEqual(response.credentials.secret_key, 'secretkey') self.assertEqual(response.credentials.session_token, 'session_token') self.assertEqual(response.user.arn, 'arn:role') self.assertEqual(response.user.assume_role_id, 'roleid:myrolesession') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/sts/__init__.py 0000644 00000000000 15025013616 0014345 0 ustar 00 vendored/boto/tests/unit/test_exception.py 0000644 00000012150 15025013616 0015043 0 ustar 00 from tests.unit import unittest from boto.exception import BotoServerError, S3CreateError, JSONResponseError from httpretty import HTTPretty, httprettified class TestBotoServerError(unittest.TestCase): def test_botoservererror_basics(self): bse = BotoServerError('400', 'Bad Request') self.assertEqual(bse.status, '400') self.assertEqual(bse.reason, 'Bad Request') def test_message_elb_xml(self): # This test XML response comes from #509 xml = """ <ErrorResponse xmlns="http://elasticloadbalancing.amazonaws.com/doc/2011-11-15/"> <Error> <Type>Sender</Type> <Code>LoadBalancerNotFound</Code> <Message>Cannot find Load Balancer webapp-balancer2</Message> </Error> <RequestId>093f80d0-4473-11e1-9234-edce8ec08e2d</RequestId> </ErrorResponse>""" bse = BotoServerError('400', 'Bad Request', body=xml) self.assertEqual(bse.error_message, 'Cannot find Load Balancer webapp-balancer2') self.assertEqual(bse.error_message, bse.message) self.assertEqual(bse.request_id, '093f80d0-4473-11e1-9234-edce8ec08e2d') self.assertEqual(bse.error_code, 'LoadBalancerNotFound') self.assertEqual(bse.status, '400') self.assertEqual(bse.reason, 'Bad Request') def test_message_sd_xml(self): # Sample XML response from: https://forums.aws.amazon.com/thread.jspa?threadID=87393 xml = """ <Response> <Errors> <Error> <Code>AuthorizationFailure</Code> <Message>Session does not have permission to perform (sdb:CreateDomain) on resource (arn:aws:sdb:us-east-1:xxxxxxx:domain/test_domain). Contact account owner.</Message> <BoxUsage>0.0055590278</BoxUsage> </Error> </Errors> <RequestID>e73bb2bb-63e3-9cdc-f220-6332de66dbbe</RequestID> </Response>""" bse = BotoServerError('403', 'Forbidden', body=xml) self.assertEqual( bse.error_message, 'Session does not have permission to perform (sdb:CreateDomain) on ' 'resource (arn:aws:sdb:us-east-1:xxxxxxx:domain/test_domain). ' 'Contact account owner.') self.assertEqual(bse.error_message, bse.message) self.assertEqual(bse.box_usage, '0.0055590278') self.assertEqual(bse.error_code, 'AuthorizationFailure') self.assertEqual(bse.status, '403') self.assertEqual(bse.reason, 'Forbidden') @httprettified def test_xmlns_not_loaded(self): xml = '<ErrorResponse xmlns="http://elasticloadbalancing.amazonaws.com/doc/2011-11-15/">' bse = BotoServerError('403', 'Forbidden', body=xml) self.assertEqual([], HTTPretty.latest_requests) @httprettified def test_xml_entity_not_loaded(self): xml = '<!DOCTYPE Message [<!ENTITY xxe SYSTEM "http://aws.amazon.com/">]><Message>error:&xxe;</Message>' bse = BotoServerError('403', 'Forbidden', body=xml) self.assertEqual([], HTTPretty.latest_requests) def test_message_storage_create_error(self): # This test value comes from https://answers.launchpad.net/duplicity/+question/150801 xml = """<?xml version="1.0" encoding="UTF-8"?> <Error> <Code>BucketAlreadyOwnedByYou</Code> <Message>Your previous request to create the named bucket succeeded and you already own it.</Message> <BucketName>cmsbk</BucketName> <RequestId>FF8B86A32CC3FE4F</RequestId> <HostId>6ENGL3DT9f0n7Tkv4qdKIs/uBNCMMA6QUFapw265WmodFDluP57esOOkecp55qhh</HostId> </Error> """ s3ce = S3CreateError('409', 'Conflict', body=xml) self.assertEqual(s3ce.bucket, 'cmsbk') self.assertEqual(s3ce.error_code, 'BucketAlreadyOwnedByYou') self.assertEqual(s3ce.status, '409') self.assertEqual(s3ce.reason, 'Conflict') self.assertEqual( s3ce.error_message, 'Your previous request to create the named bucket succeeded ' 'and you already own it.') self.assertEqual(s3ce.error_message, s3ce.message) self.assertEqual(s3ce.request_id, 'FF8B86A32CC3FE4F') def test_message_json_response_error(self): # This test comes from https://forums.aws.amazon.com/thread.jspa?messageID=374936 body = { '__type': 'com.amazon.coral.validate#ValidationException', 'message': 'The attempted filter operation is not supported ' 'for the provided filter argument count'} jre = JSONResponseError('400', 'Bad Request', body=body) self.assertEqual(jre.status, '400') self.assertEqual(jre.reason, 'Bad Request') self.assertEqual(jre.error_message, body['message']) self.assertEqual(jre.error_message, jre.message) self.assertEqual(jre.code, 'ValidationException') self.assertEqual(jre.code, jre.error_code) def test_message_not_xml(self): body = 'This is not XML' bse = BotoServerError('400', 'Bad Request', body=body) self.assertEqual(bse.error_message, 'This is not XML') def test_getters(self): body = "This is the body" bse = BotoServerError('400', 'Bad Request', body=body) self.assertEqual(bse.code, bse.error_code) self.assertEqual(bse.message, bse.error_message) vendored/boto/tests/unit/test_connect_to_region.py 0000644 00000057155 15025013616 0016561 0 ustar 00 # Copyright (c) 2017 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import os from tests.unit import unittest class TestConnectAwslambda(unittest.TestCase): def test_connect_to_region(self): from boto.awslambda import connect_to_region from boto.awslambda.layer1 import AWSLambdaConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, AWSLambdaConnection) self.assertEqual(connection.host, 'lambda.us-east-1.amazonaws.com') class TestConnectBeanstalk(unittest.TestCase): def test_connect_to_region(self): from boto.beanstalk import connect_to_region from boto.beanstalk.layer1 import Layer1 connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, Layer1) self.assertEqual( connection.host, 'elasticbeanstalk.us-east-1.amazonaws.com') class TestConnectCloudformation(unittest.TestCase): def test_connect_to_region(self): from boto.cloudformation import connect_to_region from boto.cloudformation.connection import CloudFormationConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, CloudFormationConnection) self.assertEqual( connection.host, 'cloudformation.us-east-1.amazonaws.com') class TestConnectCloudHsm(unittest.TestCase): def test_connect_to_region(self): from boto.cloudhsm import connect_to_region from boto.cloudhsm.layer1 import CloudHSMConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, CloudHSMConnection) self.assertEqual(connection.host, 'cloudhsm.us-east-1.amazonaws.com') class TestCloudsearchConnection(unittest.TestCase): def test_connect_to_region(self): from boto.cloudsearch import connect_to_region from boto.cloudsearch.layer1 import Layer1 connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, Layer1) self.assertEqual( connection.host, 'cloudsearch.us-east-1.amazonaws.com') class TestCloudsearch2Connection(unittest.TestCase): def test_connect_to_region(self): from boto.cloudsearch2 import connect_to_region from boto.cloudsearch2.layer1 import CloudSearchConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, CloudSearchConnection) self.assertEqual( connection.host, 'cloudsearch.us-east-1.amazonaws.com') class TestCloudsearchDomainConnection(unittest.TestCase): def test_connect_to_region(self): from boto.cloudsearchdomain import connect_to_region from boto.cloudsearchdomain.layer1 import CloudSearchDomainConnection host = 'mycustomdomain.us-east-1.amazonaws.com' connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar', host=host) self.assertIsInstance(connection, CloudSearchDomainConnection) self.assertEqual(connection.host, host) class TestCloudTrailConnection(unittest.TestCase): def test_connect_to_region(self): from boto.cloudtrail import connect_to_region from boto.cloudtrail.layer1 import CloudTrailConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, CloudTrailConnection) self.assertEqual(connection.host, 'cloudtrail.us-east-1.amazonaws.com') class TestCodeDeployConnection(unittest.TestCase): def test_connect_to_region(self): from boto.codedeploy import connect_to_region from boto.codedeploy.layer1 import CodeDeployConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, CodeDeployConnection) self.assertEqual(connection.host, 'codedeploy.us-east-1.amazonaws.com') class TestCognitoIdentityConnection(unittest.TestCase): def test_connect_to_region(self): from boto.cognito.identity import connect_to_region from boto.cognito.identity.layer1 import CognitoIdentityConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, CognitoIdentityConnection) self.assertEqual( connection.host, 'cognito-identity.us-east-1.amazonaws.com') class TestCognitoSyncConnection(unittest.TestCase): def test_connect_to_region(self): from boto.cognito.sync import connect_to_region from boto.cognito.sync.layer1 import CognitoSyncConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, CognitoSyncConnection) self.assertEqual( connection.host, 'cognito-sync.us-east-1.amazonaws.com') class TestConfigserviceConnection(unittest.TestCase): def test_connect_to_region(self): from boto.configservice import connect_to_region from boto.configservice.layer1 import ConfigServiceConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, ConfigServiceConnection) self.assertEqual(connection.host, 'config.us-east-1.amazonaws.com') class TestDatapipelineConnection(unittest.TestCase): def test_connect_to_region(self): from boto.datapipeline import connect_to_region from boto.datapipeline.layer1 import DataPipelineConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, DataPipelineConnection) self.assertEqual( connection.host, 'datapipeline.us-east-1.amazonaws.com') class TestDirectconnectConnection(unittest.TestCase): def test_connect_to_region(self): from boto.directconnect import connect_to_region from boto.directconnect.layer1 import DirectConnectConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, DirectConnectConnection) self.assertEqual( connection.host, 'directconnect.us-east-1.amazonaws.com') class TestDynamodbConnection(unittest.TestCase): def test_connect_to_region(self): from boto.dynamodb import connect_to_region from boto.dynamodb.layer2 import Layer2 connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, Layer2) self.assertEqual( connection.layer1.host, 'dynamodb.us-east-1.amazonaws.com') class TestDynamodb2Connection(unittest.TestCase): def test_connect_to_region(self): from boto.dynamodb2 import connect_to_region from boto.dynamodb2.layer1 import DynamoDBConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, DynamoDBConnection) self.assertEqual(connection.host, 'dynamodb.us-east-1.amazonaws.com') def test_connect_to_unkown_region(self): from boto.dynamodb2 import connect_to_region from boto.dynamodb2.layer1 import DynamoDBConnection os.environ['BOTO_USE_ENDPOINT_HEURISTICS'] = 'True' connection = connect_to_region( 'us-east-45', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, DynamoDBConnection) self.assertEqual(connection.host, 'dynamodb.us-east-45.amazonaws.com') class TestEC2Connection(unittest.TestCase): def test_connect_to_region(self): from boto.ec2 import connect_to_region from boto.ec2.connection import EC2Connection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, EC2Connection) self.assertEqual(connection.host, 'ec2.us-east-1.amazonaws.com') class TestAutoscaleConnection(unittest.TestCase): def test_connect_to_region(self): from boto.ec2.autoscale import connect_to_region from boto.ec2.autoscale import AutoScaleConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, AutoScaleConnection) self.assertEqual( connection.host, 'autoscaling.us-east-1.amazonaws.com') class TestCloudwatchConnection(unittest.TestCase): def test_connect_to_region(self): from boto.ec2.cloudwatch import connect_to_region from boto.ec2.cloudwatch import CloudWatchConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, CloudWatchConnection) self.assertEqual(connection.host, 'monitoring.us-east-1.amazonaws.com') class TestElbConnection(unittest.TestCase): def test_connect_to_region(self): from boto.ec2.elb import connect_to_region from boto.ec2.elb import ELBConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, ELBConnection) self.assertEqual( connection.host, 'elasticloadbalancing.us-east-1.amazonaws.com') class TestEc2ContainerserviceConnection(unittest.TestCase): def test_connect_to_region(self): from boto.ec2containerservice import connect_to_region import boto.ec2containerservice.layer1 as layer1 connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance( connection, layer1.EC2ContainerServiceConnection) self.assertEqual(connection.host, 'ecs.us-east-1.amazonaws.com') class TestElasticacheConnection(unittest.TestCase): def test_connect_to_region(self): from boto.elasticache import connect_to_region from boto.elasticache.layer1 import ElastiCacheConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, ElastiCacheConnection) self.assertEqual( connection.host, 'elasticache.us-east-1.amazonaws.com') class TestElastictranscoderConnection(unittest.TestCase): def test_connect_to_region(self): from boto.elastictranscoder import connect_to_region from boto.elastictranscoder.layer1 import ElasticTranscoderConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, ElasticTranscoderConnection) self.assertEqual( connection.host, 'elastictranscoder.us-east-1.amazonaws.com') class TestEmrConnection(unittest.TestCase): def test_connect_to_region(self): from boto.emr import connect_to_region from boto.emr.connection import EmrConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, EmrConnection) self.assertEqual( connection.host, 'elasticmapreduce.us-east-1.amazonaws.com') class TestGlacierConnection(unittest.TestCase): def test_connect_to_region(self): from boto.glacier import connect_to_region from boto.glacier.layer2 import Layer2 connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, Layer2) self.assertEqual( connection.layer1.host, 'glacier.us-east-1.amazonaws.com') class TestIamConnection(unittest.TestCase): def assert_connection(self, region, host): from boto.iam import connect_to_region from boto.iam.connection import IAMConnection connection = connect_to_region(region, aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, IAMConnection) self.assertEqual(connection.host, host) def test_connect_to_region(self): self.assert_connection('us-east-1', 'iam.amazonaws.com') def test_connect_to_universal_region(self): self.assert_connection('universal', 'iam.amazonaws.com') class TestKinesisConnection(unittest.TestCase): def test_connect_to_region(self): from boto.kinesis import connect_to_region from boto.kinesis.layer1 import KinesisConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, KinesisConnection) self.assertEqual(connection.host, 'kinesis.us-east-1.amazonaws.com') class TestLogsConnection(unittest.TestCase): def test_connect_to_region(self): from boto.logs import connect_to_region from boto.logs.layer1 import CloudWatchLogsConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, CloudWatchLogsConnection) self.assertEqual( connection.host, 'logs.us-east-1.amazonaws.com') class TestMachinelearningConnection(unittest.TestCase): def test_connect_to_region(self): from boto.machinelearning import connect_to_region from boto.machinelearning.layer1 import MachineLearningConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, MachineLearningConnection) self.assertEqual( connection.host, 'machinelearning.us-east-1.amazonaws.com') class TestOpsworksConnection(unittest.TestCase): def test_connect_to_region(self): from boto.opsworks import connect_to_region from boto.opsworks.layer1 import OpsWorksConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, OpsWorksConnection) self.assertEqual( connection.host, 'opsworks.us-east-1.amazonaws.com') class TestRdsConnection(unittest.TestCase): def test_connect_to_region(self): from boto.rds import connect_to_region from boto.rds import RDSConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, RDSConnection) self.assertEqual( connection.host, 'rds.amazonaws.com') class TestRds2Connection(unittest.TestCase): def test_connect_to_region(self): from boto.rds2 import connect_to_region from boto.rds2.layer1 import RDSConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, RDSConnection) self.assertEqual( connection.host, 'rds.amazonaws.com') class TestRedshiftConnection(unittest.TestCase): def test_connect_to_region(self): from boto.redshift import connect_to_region from boto.redshift.layer1 import RedshiftConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, RedshiftConnection) self.assertEqual( connection.host, 'redshift.us-east-1.amazonaws.com') class TestRoute53Connection(unittest.TestCase): def assert_connection(self, region, host): from boto.route53 import connect_to_region from boto.route53.connection import Route53Connection connection = connect_to_region(region, aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, Route53Connection) self.assertEqual(connection.host, host) def test_connect_to_region(self): self.assert_connection('us-east-1', 'route53.amazonaws.com') def test_connect_to_universal_region(self): self.assert_connection('universal', 'route53.amazonaws.com') class TestRoute53DomainsConnection(unittest.TestCase): def test_connect_to_region(self): from boto.route53.domains import connect_to_region from boto.route53.domains.layer1 import Route53DomainsConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, Route53DomainsConnection) self.assertEqual( connection.host, 'route53domains.us-east-1.amazonaws.com') class TestS3Connection(unittest.TestCase): def test_connect_to_region(self): from boto.s3 import connect_to_region from boto.s3.connection import S3Connection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, S3Connection) self.assertEqual(connection.host, 's3.amazonaws.com') def test_connect_to_custom_host(self): from boto.s3 import connect_to_region from boto.s3.connection import S3Connection host = 'mycustomhost.example.com' connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar', host=host) self.assertIsInstance(connection, S3Connection) self.assertEqual(connection.host, host) class TestSdbConnection(unittest.TestCase): def test_connect_to_region(self): from boto.sdb import connect_to_region from boto.sdb.connection import SDBConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, SDBConnection) self.assertEqual(connection.host, 'sdb.amazonaws.com') class TestSesConnection(unittest.TestCase): def test_connect_to_region(self): from boto.ses import connect_to_region from boto.ses.connection import SESConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, SESConnection) self.assertEqual(connection.host, 'email.us-east-1.amazonaws.com') class TestSnsConnection(unittest.TestCase): def test_connect_to_region(self): from boto.sns import connect_to_region from boto.sns.connection import SNSConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, SNSConnection) self.assertEqual(connection.host, 'sns.us-east-1.amazonaws.com') class TestSqsConnection(unittest.TestCase): def test_connect_to_region(self): from boto.sqs import connect_to_region from boto.sqs.connection import SQSConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, SQSConnection) self.assertEqual(connection.host, 'queue.amazonaws.com') class TestStsConnection(unittest.TestCase): def test_connect_to_region(self): from boto.sts import connect_to_region from boto.sts.connection import STSConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, STSConnection) self.assertEqual(connection.host, 'sts.amazonaws.com') class TestSupportConnection(unittest.TestCase): def test_connect_to_region(self): from boto.support import connect_to_region from boto.support.layer1 import SupportConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, SupportConnection) self.assertEqual(connection.host, 'support.us-east-1.amazonaws.com') class TestSwfConnection(unittest.TestCase): def test_connect_to_region(self): from boto.swf import connect_to_region from boto.swf.layer1 import Layer1 connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, Layer1) self.assertEqual(connection.host, 'swf.us-east-1.amazonaws.com') class TestVpcConnection(unittest.TestCase): def test_connect_to_region(self): from boto.vpc import connect_to_region from boto.vpc import VPCConnection connection = connect_to_region('us-east-1', aws_access_key_id='foo', aws_secret_access_key='bar') self.assertIsInstance(connection, VPCConnection) self.assertEqual(connection.host, 'ec2.us-east-1.amazonaws.com') vendored/boto/tests/unit/cloudfront/test_invalidation.py 0000644 00000001521 15025013616 0017705 0 ustar 00 #!/usr/bin/env python from tests.compat import unittest import boto.cloudfront as cf class CFInvalidationTest(unittest.TestCase): cloudfront = True def test_wildcard_escape(self): """ Test that wildcards are retained as literals See: http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#invalidation-specifying-objects-paths """ batch = cf.invalidation.InvalidationBatch() self.assertEqual(batch.escape("/*"), "/*") self.assertEqual(batch.escape("/foo*"), "/foo*") self.assertEqual(batch.escape("/foo/bar/*"), "/foo/bar/*") self.assertEqual(batch.escape("/nowildcard"), "/nowildcard") self.assertEqual(batch.escape("/other special characters"), "/other%20special%20characters") if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/cloudfront/test_signed_urls.py 0000644 00000040646 15025013616 0017555 0 ustar 00 import tempfile import unittest from boto.compat import StringIO, six, json from textwrap import dedent from boto.cloudfront.distribution import Distribution class CloudfrontSignedUrlsTest(unittest.TestCase): cloudfront = True notdefault = True def setUp(self): self.pk_str = dedent(""" -----BEGIN RSA PRIVATE KEY----- MIICXQIBAAKBgQDA7ki9gI/lRygIoOjV1yymgx6FYFlzJ+z1ATMaLo57nL57AavW hb68HYY8EA0GJU9xQdMVaHBogF3eiCWYXSUZCWM/+M5+ZcdQraRRScucmn6g4EvY 2K4W2pxbqH8vmUikPxir41EeBPLjMOzKvbzzQy9e/zzIQVREKSp/7y1mywIDAQAB AoGABc7mp7XYHynuPZxChjWNJZIq+A73gm0ASDv6At7F8Vi9r0xUlQe/v0AQS3yc N8QlyR4XMbzMLYk3yjxFDXo4ZKQtOGzLGteCU2srANiLv26/imXA8FVidZftTAtL viWQZBVPTeYIA69ATUYPEq0a5u5wjGyUOij9OWyuy01mbPkCQQDluYoNpPOekQ0Z WrPgJ5rxc8f6zG37ZVoDBiexqtVShIF5W3xYuWhW5kYb0hliYfkq15cS7t9m95h3 1QJf/xI/AkEA1v9l/WN1a1N3rOK4VGoCokx7kR2SyTMSbZgF9IWJNOugR/WZw7HT njipO3c9dy1Ms9pUKwUF46d7049ck8HwdQJARgrSKuLWXMyBH+/l1Dx/I4tXuAJI rlPyo+VmiOc7b5NzHptkSHEPfR9s1OK0VqjknclqCJ3Ig86OMEtEFBzjZQJBAKYz 470hcPkaGk7tKYAgP48FvxRsnzeooptURW5E+M+PQ2W9iDPPOX9739+Xi02hGEWF B0IGbQoTRFdE4VVcPK0CQQCeS84lODlC0Y2BZv2JxW3Osv/WkUQ4dslfAQl1T303 7uwwr7XTroMv8dIFQIPreoPhRKmd/SbJzbiKfS/4QDhU -----END RSA PRIVATE KEY----- """) self.pk_id = "PK123456789754" self.dist = Distribution() self.canned_policy = ( '{"Statement":[{"Resource":' '"http://d604721fxaaqy9.cloudfront.net/horizon.jpg' '?large=yes&license=yes",' '"Condition":{"DateLessThan":{"AWS:EpochTime":1258237200}}}]}') self.custom_policy_1 = ( '{ \n' ' "Statement": [{ \n' ' "Resource":"http://d604721fxaaqy9.cloudfront.net/training/*", \n' ' "Condition":{ \n' ' "IpAddress":{"AWS:SourceIp":"145.168.143.0/24"}, \n' ' "DateLessThan":{"AWS:EpochTime":1258237200} \n' ' } \n' ' }] \n' '}\n') self.custom_policy_2 = ( '{ \n' ' "Statement": [{ \n' ' "Resource":"http://*", \n' ' "Condition":{ \n' ' "IpAddress":{"AWS:SourceIp":"216.98.35.1/32"},\n' ' "DateGreaterThan":{"AWS:EpochTime":1241073790},\n' ' "DateLessThan":{"AWS:EpochTime":1255674716}\n' ' } \n' ' }] \n' '}\n') def test_encode_custom_policy_1(self): """ Test base64 encoding custom policy 1 from Amazon's documentation. """ expected = ("eyAKICAgIlN0YXRlbWVudCI6IFt7IAogICAgICAiUmVzb3VyY2Ui" "OiJodHRwOi8vZDYwNDcyMWZ4YWFxeTkuY2xvdWRmcm9udC5uZXQv" "dHJhaW5pbmcvKiIsIAogICAgICAiQ29uZGl0aW9uIjp7IAogICAg" "ICAgICAiSXBBZGRyZXNzIjp7IkFXUzpTb3VyY2VJcCI6IjE0NS4x" "NjguMTQzLjAvMjQifSwgCiAgICAgICAgICJEYXRlTGVzc1RoYW4i" "OnsiQVdTOkVwb2NoVGltZSI6MTI1ODIzNzIwMH0gICAgICAKICAg" "ICAgfSAKICAgfV0gCn0K") encoded = self.dist._url_base64_encode(self.custom_policy_1) self.assertEqual(expected, encoded) def test_encode_custom_policy_2(self): """ Test base64 encoding custom policy 2 from Amazon's documentation. """ expected = ("eyAKICAgIlN0YXRlbWVudCI6IFt7IAogICAgICAiUmVzb3VyY2Ui" "OiJodHRwOi8vKiIsIAogICAgICAiQ29uZGl0aW9uIjp7IAogICAg" "ICAgICAiSXBBZGRyZXNzIjp7IkFXUzpTb3VyY2VJcCI6IjIxNi45" "OC4zNS4xLzMyIn0sCiAgICAgICAgICJEYXRlR3JlYXRlclRoYW4i" "OnsiQVdTOkVwb2NoVGltZSI6MTI0MTA3Mzc5MH0sCiAgICAgICAg" "ICJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTI1NTY3" "NDcxNn0KICAgICAgfSAKICAgfV0gCn0K") encoded = self.dist._url_base64_encode(self.custom_policy_2) self.assertEqual(expected, encoded) def test_sign_canned_policy(self): """ Test signing the canned policy from amazon's cloudfront documentation. """ expected = ("Nql641NHEUkUaXQHZINK1FZ~SYeUSoBJMxjdgqrzIdzV2gyEXPDN" "v0pYdWJkflDKJ3xIu7lbwRpSkG98NBlgPi4ZJpRRnVX4kXAJK6td" "Nx6FucDB7OVqzcxkxHsGFd8VCG1BkC-Afh9~lOCMIYHIaiOB6~5j" "t9w2EOwi6sIIqrg_") sig = self.dist._sign_string(self.canned_policy, private_key_string=self.pk_str) encoded_sig = self.dist._url_base64_encode(sig) self.assertEqual(expected, encoded_sig) def test_sign_canned_policy_pk_file(self): """ Test signing the canned policy from amazon's cloudfront documentation with a file object. """ expected = ("Nql641NHEUkUaXQHZINK1FZ~SYeUSoBJMxjdgqrzIdzV2gyEXPDN" "v0pYdWJkflDKJ3xIu7lbwRpSkG98NBlgPi4ZJpRRnVX4kXAJK6td" "Nx6FucDB7OVqzcxkxHsGFd8VCG1BkC-Afh9~lOCMIYHIaiOB6~5j" "t9w2EOwi6sIIqrg_") pk_file = tempfile.TemporaryFile() pk_file.write(self.pk_str) pk_file.seek(0) sig = self.dist._sign_string(self.canned_policy, private_key_file=pk_file) encoded_sig = self.dist._url_base64_encode(sig) self.assertEqual(expected, encoded_sig) def test_sign_canned_policy_pk_file_name(self): """ Test signing the canned policy from amazon's cloudfront documentation with a file name. """ expected = ("Nql641NHEUkUaXQHZINK1FZ~SYeUSoBJMxjdgqrzIdzV2gyEXPDN" "v0pYdWJkflDKJ3xIu7lbwRpSkG98NBlgPi4ZJpRRnVX4kXAJK6td" "Nx6FucDB7OVqzcxkxHsGFd8VCG1BkC-Afh9~lOCMIYHIaiOB6~5j" "t9w2EOwi6sIIqrg_") pk_file = tempfile.NamedTemporaryFile() pk_file.write(self.pk_str) pk_file.flush() sig = self.dist._sign_string(self.canned_policy, private_key_file=pk_file.name) encoded_sig = self.dist._url_base64_encode(sig) self.assertEqual(expected, encoded_sig) def test_sign_canned_policy_pk_file_like(self): """ Test signing the canned policy from amazon's cloudfront documentation with a file-like object (not a subclass of 'file' type) """ expected = ("Nql641NHEUkUaXQHZINK1FZ~SYeUSoBJMxjdgqrzIdzV2gyEXPDN" "v0pYdWJkflDKJ3xIu7lbwRpSkG98NBlgPi4ZJpRRnVX4kXAJK6td" "Nx6FucDB7OVqzcxkxHsGFd8VCG1BkC-Afh9~lOCMIYHIaiOB6~5j" "t9w2EOwi6sIIqrg_") pk_file = StringIO() pk_file.write(self.pk_str) pk_file.seek(0) sig = self.dist._sign_string(self.canned_policy, private_key_file=pk_file) encoded_sig = self.dist._url_base64_encode(sig) self.assertEqual(expected, encoded_sig) def test_sign_canned_policy_unicode(self): """ Test signing the canned policy from amazon's cloudfront documentation. """ expected = ("Nql641NHEUkUaXQHZINK1FZ~SYeUSoBJMxjdgqrzIdzV2gyEXPDN" "v0pYdWJkflDKJ3xIu7lbwRpSkG98NBlgPi4ZJpRRnVX4kXAJK6td" "Nx6FucDB7OVqzcxkxHsGFd8VCG1BkC-Afh9~lOCMIYHIaiOB6~5j" "t9w2EOwi6sIIqrg_") unicode_policy = six.text_type(self.canned_policy) sig = self.dist._sign_string(unicode_policy, private_key_string=self.pk_str) encoded_sig = self.dist._url_base64_encode(sig) self.assertEqual(expected, encoded_sig) def test_sign_custom_policy_1(self): """ Test signing custom policy 1 from amazon's cloudfront documentation. """ expected = ("cPFtRKvUfYNYmxek6ZNs6vgKEZP6G3Cb4cyVt~FjqbHOnMdxdT7e" "T6pYmhHYzuDsFH4Jpsctke2Ux6PCXcKxUcTIm8SO4b29~1QvhMl-" "CIojki3Hd3~Unxjw7Cpo1qRjtvrimW0DPZBZYHFZtiZXsaPt87yB" "P9GWnTQoaVysMxQ_") sig = self.dist._sign_string(self.custom_policy_1, private_key_string=self.pk_str) encoded_sig = self.dist._url_base64_encode(sig) self.assertEqual(expected, encoded_sig) def test_sign_custom_policy_2(self): """ Test signing custom policy 2 from amazon's cloudfront documentation. """ expected = ("rc~5Qbbm8EJXjUTQ6Cn0LAxR72g1DOPrTmdtfbWVVgQNw0q~KHUA" "mBa2Zv1Wjj8dDET4XSL~Myh44CLQdu4dOH~N9huH7QfPSR~O4tIO" "S1WWcP~2JmtVPoQyLlEc8YHRCuN3nVNZJ0m4EZcXXNAS-0x6Zco2" "SYx~hywTRxWR~5Q_") sig = self.dist._sign_string(self.custom_policy_2, private_key_string=self.pk_str) encoded_sig = self.dist._url_base64_encode(sig) self.assertEqual(expected, encoded_sig) def test_create_canned_policy(self): """ Test that a canned policy is generated correctly. """ url = "http://1234567.cloudfront.com/test_resource.mp3?dog=true" expires = 999999 policy = self.dist._canned_policy(url, expires) policy = json.loads(policy) self.assertEqual(1, len(policy.keys())) statements = policy["Statement"] self.assertEqual(1, len(statements)) statement = statements[0] resource = statement["Resource"] self.assertEqual(url, resource) condition = statement["Condition"] self.assertEqual(1, len(condition.keys())) date_less_than = condition["DateLessThan"] self.assertEqual(1, len(date_less_than.keys())) aws_epoch_time = date_less_than["AWS:EpochTime"] self.assertEqual(expires, aws_epoch_time) def test_custom_policy_expires_and_policy_url(self): """ Test that a custom policy can be created with an expire time and an arbitrary URL. """ url = "http://1234567.cloudfront.com/*" expires = 999999 policy = self.dist._custom_policy(url, expires=expires) policy = json.loads(policy) self.assertEqual(1, len(policy.keys())) statements = policy["Statement"] self.assertEqual(1, len(statements)) statement = statements[0] resource = statement["Resource"] self.assertEqual(url, resource) condition = statement["Condition"] self.assertEqual(1, len(condition.keys())) date_less_than = condition["DateLessThan"] self.assertEqual(1, len(date_less_than.keys())) aws_epoch_time = date_less_than["AWS:EpochTime"] self.assertEqual(expires, aws_epoch_time) def test_custom_policy_valid_after(self): """ Test that a custom policy can be created with a valid-after time and an arbitrary URL. """ url = "http://1234567.cloudfront.com/*" valid_after = 999999 policy = self.dist._custom_policy(url, valid_after=valid_after) policy = json.loads(policy) self.assertEqual(1, len(policy.keys())) statements = policy["Statement"] self.assertEqual(1, len(statements)) statement = statements[0] resource = statement["Resource"] self.assertEqual(url, resource) condition = statement["Condition"] self.assertEqual(2, len(condition.keys())) date_less_than = condition["DateLessThan"] date_greater_than = condition["DateGreaterThan"] self.assertEqual(1, len(date_greater_than.keys())) aws_epoch_time = date_greater_than["AWS:EpochTime"] self.assertEqual(valid_after, aws_epoch_time) def test_custom_policy_ip_address(self): """ Test that a custom policy can be created with an IP address and an arbitrary URL. """ url = "http://1234567.cloudfront.com/*" ip_range = "192.168.0.1" policy = self.dist._custom_policy(url, ip_address=ip_range) policy = json.loads(policy) self.assertEqual(1, len(policy.keys())) statements = policy["Statement"] self.assertEqual(1, len(statements)) statement = statements[0] resource = statement["Resource"] self.assertEqual(url, resource) condition = statement["Condition"] self.assertEqual(2, len(condition.keys())) ip_address = condition["IpAddress"] self.assertTrue("DateLessThan" in condition) self.assertEqual(1, len(ip_address.keys())) source_ip = ip_address["AWS:SourceIp"] self.assertEqual("%s/32" % ip_range, source_ip) def test_custom_policy_ip_range(self): """ Test that a custom policy can be created with an IP address and an arbitrary URL. """ url = "http://1234567.cloudfront.com/*" ip_range = "192.168.0.0/24" policy = self.dist._custom_policy(url, ip_address=ip_range) policy = json.loads(policy) self.assertEqual(1, len(policy.keys())) statements = policy["Statement"] self.assertEqual(1, len(statements)) statement = statements[0] resource = statement["Resource"] self.assertEqual(url, resource) condition = statement["Condition"] self.assertEqual(2, len(condition.keys())) self.assertTrue("DateLessThan" in condition) ip_address = condition["IpAddress"] self.assertEqual(1, len(ip_address.keys())) source_ip = ip_address["AWS:SourceIp"] self.assertEqual(ip_range, source_ip) def test_custom_policy_all(self): """ Test that a custom policy can be created with an IP address and an arbitrary URL. """ url = "http://1234567.cloudfront.com/test.txt" expires = 999999 valid_after = 111111 ip_range = "192.168.0.0/24" policy = self.dist._custom_policy(url, expires=expires, valid_after=valid_after, ip_address=ip_range) policy = json.loads(policy) self.assertEqual(1, len(policy.keys())) statements = policy["Statement"] self.assertEqual(1, len(statements)) statement = statements[0] resource = statement["Resource"] self.assertEqual(url, resource) condition = statement["Condition"] self.assertEqual(3, len(condition.keys())) #check expires condition date_less_than = condition["DateLessThan"] self.assertEqual(1, len(date_less_than.keys())) aws_epoch_time = date_less_than["AWS:EpochTime"] self.assertEqual(expires, aws_epoch_time) #check valid_after condition date_greater_than = condition["DateGreaterThan"] self.assertEqual(1, len(date_greater_than.keys())) aws_epoch_time = date_greater_than["AWS:EpochTime"] self.assertEqual(valid_after, aws_epoch_time) #check source ip address condition ip_address = condition["IpAddress"] self.assertEqual(1, len(ip_address.keys())) source_ip = ip_address["AWS:SourceIp"] self.assertEqual(ip_range, source_ip) def test_params_canned_policy(self): """ Test the correct params are generated for a canned policy. """ url = "http://d604721fxaaqy9.cloudfront.net/horizon.jpg?large=yes&license=yes" expire_time = 1258237200 expected_sig = ("Nql641NHEUkUaXQHZINK1FZ~SYeUSoBJMxjdgqrzIdzV2gyE" "XPDNv0pYdWJkflDKJ3xIu7lbwRpSkG98NBlgPi4ZJpRRnVX4" "kXAJK6tdNx6FucDB7OVqzcxkxHsGFd8VCG1BkC-Afh9~lOCM" "IYHIaiOB6~5jt9w2EOwi6sIIqrg_") signed_url_params = self.dist._create_signing_params(url, self.pk_id, expire_time, private_key_string=self.pk_str) self.assertEqual(3, len(signed_url_params)) self.assertEqual(signed_url_params["Expires"], "1258237200") self.assertEqual(signed_url_params["Signature"], expected_sig) self.assertEqual(signed_url_params["Key-Pair-Id"], "PK123456789754") def test_canned_policy(self): """ Generate signed url from the Example Canned Policy in Amazon's documentation. """ url = "http://d604721fxaaqy9.cloudfront.net/horizon.jpg?large=yes&license=yes" expire_time = 1258237200 expected_url = "http://d604721fxaaqy9.cloudfront.net/horizon.jpg?large=yes&license=yes&Expires=1258237200&Signature=Nql641NHEUkUaXQHZINK1FZ~SYeUSoBJMxjdgqrzIdzV2gyEXPDNv0pYdWJkflDKJ3xIu7lbwRpSkG98NBlgPi4ZJpRRnVX4kXAJK6tdNx6FucDB7OVqzcxkxHsGFd8VCG1BkC-Afh9~lOCMIYHIaiOB6~5jt9w2EOwi6sIIqrg_&Key-Pair-Id=PK123456789754" signed_url = self.dist.create_signed_url( url, self.pk_id, expire_time, private_key_string=self.pk_str) self.assertEqual(expected_url, signed_url) vendored/boto/tests/unit/cloudfront/test_connection.py 0000644 00000023010 15025013616 0017360 0 ustar 00 from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.cloudfront import CloudFrontConnection from boto.cloudfront.distribution import Distribution, DistributionConfig, DistributionSummary from boto.cloudfront.origin import CustomOrigin class TestCloudFrontConnection(AWSMockServiceTestCase): connection_class = CloudFrontConnection def setUp(self): super(TestCloudFrontConnection, self).setUp() def test_get_all_distributions(self): body = b""" <DistributionList xmlns="http://cloudfront.amazonaws.com/doc/2010-11-01/"> <Marker></Marker> <MaxItems>100</MaxItems> <IsTruncated>false</IsTruncated> <DistributionSummary> <Id>EEEEEEEEEEEEE</Id> <Status>InProgress</Status> <LastModifiedTime>2014-02-03T11:03:41.087Z</LastModifiedTime> <DomainName>abcdef12345678.cloudfront.net</DomainName> <CustomOrigin> <DNSName>example.com</DNSName> <HTTPPort>80</HTTPPort> <HTTPSPort>443</HTTPSPort> <OriginProtocolPolicy>http-only</OriginProtocolPolicy> </CustomOrigin> <CNAME>static.example.com</CNAME> <Enabled>true</Enabled> </DistributionSummary> </DistributionList> """ self.set_http_response(status_code=200, body=body) response = self.service_connection.get_all_distributions() self.assertTrue(isinstance(response, list)) self.assertEqual(len(response), 1) self.assertTrue(isinstance(response[0], DistributionSummary)) self.assertEqual(response[0].id, "EEEEEEEEEEEEE") self.assertEqual(response[0].domain_name, "abcdef12345678.cloudfront.net") self.assertEqual(response[0].status, "InProgress") self.assertEqual(response[0].cnames, ["static.example.com"]) self.assertEqual(response[0].enabled, True) self.assertTrue(isinstance(response[0].origin, CustomOrigin)) self.assertEqual(response[0].origin.dns_name, "example.com") self.assertEqual(response[0].origin.http_port, 80) self.assertEqual(response[0].origin.https_port, 443) self.assertEqual(response[0].origin.origin_protocol_policy, 'http-only') def test_get_distribution_config(self): body = b""" <DistributionConfig xmlns="http://cloudfront.amazonaws.com/doc/2010-11-01/"> <CustomOrigin> <DNSName>example.com</DNSName> <HTTPPort>80</HTTPPort> <HTTPSPort>443</HTTPSPort> <OriginProtocolPolicy>http-only</OriginProtocolPolicy> </CustomOrigin> <CallerReference>1234567890123</CallerReference> <CNAME>static.example.com</CNAME> <Enabled>true</Enabled> </DistributionConfig> """ self.set_http_response(status_code=200, body=body, header={"Etag": "AABBCC"}) response = self.service_connection.get_distribution_config('EEEEEEEEEEEEE') self.assertTrue(isinstance(response, DistributionConfig)) self.assertTrue(isinstance(response.origin, CustomOrigin)) self.assertEqual(response.origin.dns_name, "example.com") self.assertEqual(response.origin.http_port, 80) self.assertEqual(response.origin.https_port, 443) self.assertEqual(response.origin.origin_protocol_policy, "http-only") self.assertEqual(response.cnames, ["static.example.com"]) self.assertTrue(response.enabled) self.assertEqual(response.etag, "AABBCC") def test_set_distribution_config(self): get_body = b""" <DistributionConfig xmlns="http://cloudfront.amazonaws.com/doc/2010-11-01/"> <CustomOrigin> <DNSName>example.com</DNSName> <HTTPPort>80</HTTPPort> <HTTPSPort>443</HTTPSPort> <OriginProtocolPolicy>http-only</OriginProtocolPolicy> </CustomOrigin> <CallerReference>1234567890123</CallerReference> <CNAME>static.example.com</CNAME> <Enabled>true</Enabled> </DistributionConfig> """ put_body = b""" <Distribution xmlns="http://cloudfront.amazonaws.com/doc/2010-11-01/"> <Id>EEEEEE</Id> <Status>InProgress</Status> <LastModifiedTime>2014-02-04T10:47:53.493Z</LastModifiedTime> <InProgressInvalidationBatches>0</InProgressInvalidationBatches> <DomainName>d2000000000000.cloudfront.net</DomainName> <DistributionConfig> <CustomOrigin> <DNSName>example.com</DNSName> <HTTPPort>80</HTTPPort> <HTTPSPort>443</HTTPSPort> <OriginProtocolPolicy>match-viewer</OriginProtocolPolicy> </CustomOrigin> <CallerReference>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</CallerReference> <Comment>this is a comment</Comment> <Enabled>false</Enabled> </DistributionConfig> </Distribution> """ self.set_http_response(status_code=200, body=get_body, header={"Etag": "AA"}) conf = self.service_connection.get_distribution_config('EEEEEEE') self.set_http_response(status_code=200, body=put_body, header={"Etag": "AABBCCD"}) conf.comment = 'this is a comment' response = self.service_connection.set_distribution_config('EEEEEEE', conf.etag, conf) self.assertEqual(response, "AABBCCD") def test_get_distribution_info(self): body = b""" <Distribution xmlns="http://cloudfront.amazonaws.com/doc/2010-11-01/"> <Id>EEEEEEEEEEEEE</Id> <Status>InProgress</Status> <LastModifiedTime>2014-02-03T11:03:41.087Z</LastModifiedTime> <InProgressInvalidationBatches>0</InProgressInvalidationBatches> <DomainName>abcdef12345678.cloudfront.net</DomainName> <DistributionConfig> <CustomOrigin> <DNSName>example.com</DNSName> <HTTPPort>80</HTTPPort> <HTTPSPort>443</HTTPSPort> <OriginProtocolPolicy>http-only</OriginProtocolPolicy> </CustomOrigin> <CallerReference>1111111111111</CallerReference> <CNAME>static.example.com</CNAME> <Enabled>true</Enabled> </DistributionConfig> </Distribution> """ self.set_http_response(status_code=200, body=body) response = self.service_connection.get_distribution_info('EEEEEEEEEEEEE') self.assertTrue(isinstance(response, Distribution)) self.assertTrue(isinstance(response.config, DistributionConfig)) self.assertTrue(isinstance(response.config.origin, CustomOrigin)) self.assertEqual(response.config.origin.dns_name, "example.com") self.assertEqual(response.config.origin.http_port, 80) self.assertEqual(response.config.origin.https_port, 443) self.assertEqual(response.config.origin.origin_protocol_policy, "http-only") self.assertEqual(response.config.cnames, ["static.example.com"]) self.assertTrue(response.config.enabled) self.assertEqual(response.id, "EEEEEEEEEEEEE") self.assertEqual(response.status, "InProgress") self.assertEqual(response.domain_name, "abcdef12345678.cloudfront.net") self.assertEqual(response.in_progress_invalidation_batches, 0) def test_create_distribution(self): body = b""" <Distribution xmlns="http://cloudfront.amazonaws.com/doc/2010-11-01/"> <Id>EEEEEEEEEEEEEE</Id> <Status>InProgress</Status> <LastModifiedTime>2014-02-04T10:34:07.873Z</LastModifiedTime> <InProgressInvalidationBatches>0</InProgressInvalidationBatches> <DomainName>d2000000000000.cloudfront.net</DomainName> <DistributionConfig> <CustomOrigin> <DNSName>example.com</DNSName> <HTTPPort>80</HTTPPort> <HTTPSPort>443</HTTPSPort> <OriginProtocolPolicy>match-viewer</OriginProtocolPolicy> </CustomOrigin> <CallerReference>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</CallerReference> <Comment>example.com distribution</Comment> <Enabled>false</Enabled> </DistributionConfig> </Distribution> """ self.set_http_response(status_code=201, body=body) origin = CustomOrigin("example.com", origin_protocol_policy="match_viewer") response = self.service_connection.create_distribution(origin, enabled=False, comment="example.com distribution") self.assertTrue(isinstance(response, Distribution)) self.assertTrue(isinstance(response.config, DistributionConfig)) self.assertTrue(isinstance(response.config.origin, CustomOrigin)) self.assertEqual(response.config.origin.dns_name, "example.com") self.assertEqual(response.config.origin.http_port, 80) self.assertEqual(response.config.origin.https_port, 443) self.assertEqual(response.config.origin.origin_protocol_policy, "match-viewer") self.assertEqual(response.config.cnames, []) self.assertTrue(not response.config.enabled) self.assertEqual(response.id, "EEEEEEEEEEEEEE") self.assertEqual(response.status, "InProgress") self.assertEqual(response.domain_name, "d2000000000000.cloudfront.net") self.assertEqual(response.in_progress_invalidation_batches, 0) vendored/boto/tests/unit/cloudfront/test_invalidation_list.py 0000644 00000007723 15025013616 0020752 0 ustar 00 #!/usr/bin/env python import random import string from tests.compat import unittest, mock import boto RESPONSE_TEMPLATE = r""" <InvalidationList> <Marker/> <NextMarker>%(next_marker)s</NextMarker> <MaxItems>%(max_items)s</MaxItems> <IsTruncated>%(is_truncated)s</IsTruncated> %(inval_summaries)s </InvalidationList> """ INVAL_SUMMARY_TEMPLATE = r""" <InvalidationSummary> <Id>%(cfid)s</Id> <Status>%(status)s</Status> </InvalidationSummary> """ class CFInvalidationListTest(unittest.TestCase): cloudfront = True def setUp(self): self.cf = boto.connect_cloudfront('aws.aws_access_key_id', 'aws.aws_secret_access_key') def _get_random_id(self, length=14): return ''.join([random.choice(string.ascii_letters) for i in range(length)]) def _group_iter(self, iterator, n): accumulator = [] for item in iterator: accumulator.append(item) if len(accumulator) == n: yield accumulator accumulator = [] if len(accumulator) != 0: yield accumulator def _get_mock_responses(self, num, max_items): max_items = min(max_items, 100) cfid_groups = list(self._group_iter([self._get_random_id() for i in range(num)], max_items)) cfg = dict(status='Completed', max_items=max_items, next_marker='') responses = [] is_truncated = 'true' for i, group in enumerate(cfid_groups): next_marker = group[-1] if (i + 1) == len(cfid_groups): is_truncated = 'false' next_marker = '' invals = '' cfg.update(dict(next_marker=next_marker, is_truncated=is_truncated)) for cfid in group: cfg.update(dict(cfid=cfid)) invals += INVAL_SUMMARY_TEMPLATE % cfg cfg.update(dict(inval_summaries=invals)) mock_response = mock.Mock() mock_response.read.return_value = (RESPONSE_TEMPLATE % cfg).encode('utf-8') mock_response.status = 200 responses.append(mock_response) return responses def test_manual_pagination(self, num_invals=30, max_items=4): """ Test that paginating manually works properly """ self.assertGreater(num_invals, max_items) responses = self._get_mock_responses(num=num_invals, max_items=max_items) self.cf.make_request = mock.Mock(side_effect=responses) ir = self.cf.get_invalidation_requests('dist-id-here', max_items=max_items) all_invals = list(ir) self.assertEqual(len(all_invals), max_items) while ir.is_truncated: ir = self.cf.get_invalidation_requests('dist-id-here', marker=ir.next_marker, max_items=max_items) invals = list(ir) self.assertLessEqual(len(invals), max_items) all_invals.extend(invals) remainder = num_invals % max_items if remainder != 0: self.assertEqual(len(invals), remainder) self.assertEqual(len(all_invals), num_invals) def test_auto_pagination(self, num_invals=1024): """ Test that auto-pagination works properly """ max_items = 100 self.assertGreaterEqual(num_invals, max_items) responses = self._get_mock_responses(num=num_invals, max_items=max_items) self.cf.make_request = mock.Mock(side_effect=responses) ir = self.cf.get_invalidation_requests('dist-id-here') self.assertEqual(len(ir._inval_cache), max_items) self.assertEqual(len(list(ir)), num_invals) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/cloudfront/__init__.py 0000644 00000000000 15025013616 0015713 0 ustar 00 vendored/boto/tests/unit/cloudfront/test_distribution.py 0000644 00000001143 15025013616 0017743 0 ustar 00 import unittest from boto.cloudfront.distribution import DistributionConfig from boto.cloudfront.logging import LoggingInfo class CloudfrontDistributionTest(unittest.TestCase): cloudfront = True def setUp(self): self.dist = DistributionConfig() def test_logging(self): # Default. self.assertEqual(self.dist.logging, None) # Override. lo = LoggingInfo(bucket='whatever', prefix='override_') dist = DistributionConfig(logging=lo) self.assertEqual(dist.logging.bucket, 'whatever') self.assertEqual(dist.logging.prefix, 'override_') vendored/boto/tests/unit/rds2/test_connection.py 0000644 00000021146 15025013616 0016063 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.ec2.securitygroup import SecurityGroup from boto.rds2.layer1 import RDSConnection class TestRDS2Connection(AWSMockServiceTestCase): connection_class = RDSConnection def setUp(self): super(TestRDS2Connection, self).setUp() def default_body(self): return """{ "DescribeDBInstancesResponse": { "DescribeDBInstancesResult": { "DBInstances": [{ "DBInstance": { "Iops": 2000, "BackupRetentionPeriod": 1, "MultiAZ": false, "DBInstanceStatus": "backing-up", "DBInstanceIdentifier": "mydbinstance2", "PreferredBackupWindow": "10:30-11:00", "PreferredMaintenanceWindow": "wed:06:30-wed:07:00", "OptionGroupMembership": { "OptionGroupName": "default:mysql-5-5", "Status": "in-sync" }, "AvailabilityZone": "us-west-2b", "ReadReplicaDBInstanceIdentifiers": null, "Engine": "mysql", "PendingModifiedValues": null, "LicenseModel": "general-public-license", "DBParameterGroups": [{ "DBParameterGroup": { "ParameterApplyStatus": "in-sync", "DBParameterGroupName": "default.mysql5.5" } }], "Endpoint": { "Port": 3306, "Address": "mydbinstance2.c0hjqouvn9mf.us-west-2.rds.amazonaws.com" }, "EngineVersion": "5.5.27", "DBSecurityGroups": [{ "DBSecurityGroup": { "Status": "active", "DBSecurityGroupName": "default" } }], "VpcSecurityGroups": [{ "VpcSecurityGroupMembership": { "VpcSecurityGroupId": "sg-1", "Status": "active" } }], "DBName": "mydb2", "AutoMinorVersionUpgrade": true, "InstanceCreateTime": "2012-10-03T22:01:51.047Z", "AllocatedStorage": 200, "DBInstanceClass": "db.m1.large", "MasterUsername": "awsuser", "StatusInfos": [{ "DBInstanceStatusInfo": { "Message": null, "Normal": true, "Status": "replicating", "StatusType": "read replication" } }], "DBSubnetGroup": { "VpcId": "990524496922", "SubnetGroupStatus": "Complete", "DBSubnetGroupDescription": "My modified DBSubnetGroup", "DBSubnetGroupName": "mydbsubnetgroup", "Subnets": [{ "Subnet": { "SubnetStatus": "Active", "SubnetIdentifier": "subnet-7c5b4115", "SubnetAvailabilityZone": { "Name": "us-east-1c" } }, "Subnet": { "SubnetStatus": "Active", "SubnetIdentifier": "subnet-7b5b4112", "SubnetAvailabilityZone": { "Name": "us-east-1b" } }, "Subnet": { "SubnetStatus": "Active", "SubnetIdentifier": "subnet-3ea6bd57", "SubnetAvailabilityZone": { "Name": "us-east-1d" } } }] } } }] } } }""" def test_describe_db_instances(self): self.set_http_response(status_code=200) response = self.service_connection.describe_db_instances('instance_id') self.assertEqual(len(response), 1) self.assert_request_parameters({ 'Action': 'DescribeDBInstances', 'ContentType': 'JSON', 'DBInstanceIdentifier': 'instance_id', }, ignore_params_values=['Version']) db = response['DescribeDBInstancesResponse']\ ['DescribeDBInstancesResult']['DBInstances'][0]\ ['DBInstance'] self.assertEqual(db['DBInstanceIdentifier'], 'mydbinstance2') self.assertEqual(db['InstanceCreateTime'], '2012-10-03T22:01:51.047Z') self.assertEqual(db['Engine'], 'mysql') self.assertEqual(db['DBInstanceStatus'], 'backing-up') self.assertEqual(db['AllocatedStorage'], 200) self.assertEqual(db['Endpoint']['Port'], 3306) self.assertEqual(db['DBInstanceClass'], 'db.m1.large') self.assertEqual(db['MasterUsername'], 'awsuser') self.assertEqual(db['AvailabilityZone'], 'us-west-2b') self.assertEqual(db['BackupRetentionPeriod'], 1) self.assertEqual(db['PreferredBackupWindow'], '10:30-11:00') self.assertEqual(db['PreferredMaintenanceWindow'], 'wed:06:30-wed:07:00') self.assertEqual(db['MultiAZ'], False) self.assertEqual(db['Iops'], 2000) self.assertEqual(db['PendingModifiedValues'], None) self.assertEqual( db['DBParameterGroups'][0]['DBParameterGroup']\ ['DBParameterGroupName'], 'default.mysql5.5' ) self.assertEqual( db['DBSecurityGroups'][0]['DBSecurityGroup']['DBSecurityGroupName'], 'default' ) self.assertEqual( db['DBSecurityGroups'][0]['DBSecurityGroup']['Status'], 'active' ) self.assertEqual(len(db['StatusInfos']), 1) self.assertEqual( db['StatusInfos'][0]['DBInstanceStatusInfo']['Message'], None ) self.assertEqual( db['StatusInfos'][0]['DBInstanceStatusInfo']['Normal'], True ) self.assertEqual( db['StatusInfos'][0]['DBInstanceStatusInfo']['Status'], 'replicating' ) self.assertEqual( db['StatusInfos'][0]['DBInstanceStatusInfo']['StatusType'], 'read replication' ) self.assertEqual( db['VpcSecurityGroups'][0]['VpcSecurityGroupMembership']['Status'], 'active' ) self.assertEqual( db['VpcSecurityGroups'][0]['VpcSecurityGroupMembership']\ ['VpcSecurityGroupId'], 'sg-1' ) self.assertEqual(db['LicenseModel'], 'general-public-license') self.assertEqual(db['EngineVersion'], '5.5.27') self.assertEqual(db['AutoMinorVersionUpgrade'], True) self.assertEqual( db['DBSubnetGroup']['DBSubnetGroupName'], 'mydbsubnetgroup' ) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/rds2/__init__.py 0000644 00000000000 15025013616 0014406 0 ustar 00 vendored/boto/tests/unit/s3/test_key.py 0000644 00000027032 15025013616 0014167 0 ustar 00 #!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import io from tests.compat import mock, unittest from tests.unit import AWSMockServiceTestCase from boto.compat import StringIO from boto.exception import BotoServerError from boto.exception import ResumableDownloadException from boto.exception import ResumableTransferDisposition from boto.s3.connection import S3Connection from boto.s3.bucket import Bucket from boto.s3.key import Key class TestS3Key(AWSMockServiceTestCase): connection_class = S3Connection def setUp(self): super(TestS3Key, self).setUp() def default_body(self): return "default body" def test_unicode_name(self): k = Key() k.name = u'Österreich' print(repr(k)) def test_when_no_restore_header_present(self): self.set_http_response(status_code=200) b = Bucket(self.service_connection, 'mybucket') k = b.get_key('myglacierkey') self.assertIsNone(k.ongoing_restore) self.assertIsNone(k.expiry_date) def test_restore_header_with_ongoing_restore(self): self.set_http_response( status_code=200, header=[('x-amz-restore', 'ongoing-request="true"')]) b = Bucket(self.service_connection, 'mybucket') k = b.get_key('myglacierkey') self.assertTrue(k.ongoing_restore) self.assertIsNone(k.expiry_date) def test_restore_completed(self): self.set_http_response( status_code=200, header=[('x-amz-restore', 'ongoing-request="false", ' 'expiry-date="Fri, 21 Dec 2012 00:00:00 GMT"')]) b = Bucket(self.service_connection, 'mybucket') k = b.get_key('myglacierkey') self.assertFalse(k.ongoing_restore) self.assertEqual(k.expiry_date, 'Fri, 21 Dec 2012 00:00:00 GMT') def test_delete_key_return_key(self): self.set_http_response(status_code=204, body='') b = Bucket(self.service_connection, 'mybucket') key = b.delete_key('fookey') self.assertIsNotNone(key) def test_storage_class(self): self.set_http_response(status_code=200) b = Bucket(self.service_connection, 'mybucket') k = b.get_key('fookey') # Mock out the bucket object - we really only care about calls # to list. k.bucket = mock.MagicMock() # Default behavior doesn't call list k.set_contents_from_string('test') k.bucket.list.assert_not_called() # Direct access calls list to get the real value if unset, # and still defaults to STANDARD if unavailable. sc_value = k.storage_class self.assertEqual(sc_value, 'STANDARD') k.bucket.list.assert_called_with(k.name.encode('utf-8')) k.bucket.list.reset_mock() # Setting manually doesn't call list k.storage_class = 'GLACIER' k.set_contents_from_string('test') k.bucket.list.assert_not_called() def test_change_storage_class(self): self.set_http_response(status_code=200) b = Bucket(self.service_connection, 'mybucket') k = b.get_key('fookey') # Mock out Key.copy so we can record calls to it k.copy = mock.MagicMock() # Mock out the bucket so we don't actually need to have fake responses k.bucket = mock.MagicMock() k.bucket.name = 'mybucket' self.assertEqual(k.storage_class, 'STANDARD') # The default change_storage_class call should result in a copy to our # bucket k.change_storage_class('REDUCED_REDUNDANCY') k.copy.assert_called_with( 'mybucket', 'fookey', reduced_redundancy=True, preserve_acl=True, validate_dst_bucket=True, ) def test_change_storage_class_new_bucket(self): self.set_http_response(status_code=200) b = Bucket(self.service_connection, 'mybucket') k = b.get_key('fookey') # Mock out Key.copy so we can record calls to it k.copy = mock.MagicMock() # Mock out the bucket so we don't actually need to have fake responses k.bucket = mock.MagicMock() k.bucket.name = 'mybucket' self.assertEqual(k.storage_class, 'STANDARD') # Specifying a different dst_bucket should result in a copy to the new # bucket k.copy.reset_mock() k.change_storage_class('REDUCED_REDUNDANCY', dst_bucket='yourbucket') k.copy.assert_called_with( 'yourbucket', 'fookey', reduced_redundancy=True, preserve_acl=True, validate_dst_bucket=True, ) def test_download_succeeds(self): test_case_headers = [ [('Content-Length', '5')], [('Content-Range', 'bytes 15-19/100')], ] for headers in test_case_headers: with self.subTest(headers=headers): head_object_response = self.create_response( status_code=200, header=headers) media_response = self.create_response( status_code=200, header=headers) media_response.read.side_effect = [ b'12345', b'', b'', # Key.close calls read an additional time. ] self.https_connection.getresponse.side_effect = [ head_object_response, media_response ] bucket = Bucket(self.service_connection, 'bucket') key = bucket.get_key('object') output_stream = io.BytesIO() key.get_file(output_stream) output_stream.seek(0) self.assertEqual(output_stream.read(), b'12345') def test_download_raises_retriable_error_with_truncated_stream(self): test_case_headers = [ [('Content-Length', '5')], [('Content-Range', 'bytes 15-19/100')], ] for headers in test_case_headers: with self.subTest(headers=headers): head_object_response = self.create_response( status_code=200, header=headers) media_response = self.create_response( status_code=200, header=headers) media_response.read.side_effect = [ # Stream is truncated and returns < 5 bytes. b'1234', b'', ] self.https_connection.getresponse.side_effect = [ head_object_response, media_response ] bucket = Bucket(self.service_connection, 'bucket') key = bucket.get_key('object') with self.assertRaisesRegex( ResumableDownloadException, 'Download stream truncated. Received 4 of 5 bytes.' ) as context: key.get_file(io.BytesIO()) self.assertEqual( context.exception.disposition, ResumableTransferDisposition.WAIT_BEFORE_RETRY) def counter(fn): def _wrapper(*args, **kwargs): _wrapper.count += 1 return fn(*args, **kwargs) _wrapper.count = 0 return _wrapper class TestS3KeyRetries(AWSMockServiceTestCase): connection_class = S3Connection @mock.patch('time.sleep') def test_500_retry(self, sleep_mock): self.set_http_response(status_code=500) b = Bucket(self.service_connection, 'mybucket') k = b.new_key('test_failure') fail_file = StringIO('This will attempt to retry.') with self.assertRaises(BotoServerError): k.send_file(fail_file) @mock.patch('time.sleep') def test_400_timeout(self, sleep_mock): weird_timeout_body = "<Error><Code>RequestTimeout</Code></Error>" self.set_http_response(status_code=400, body=weird_timeout_body) b = Bucket(self.service_connection, 'mybucket') k = b.new_key('test_failure') fail_file = StringIO('This will pretend to be chunk-able.') k.should_retry = counter(k.should_retry) self.assertEqual(k.should_retry.count, 0) with self.assertRaises(BotoServerError): k.send_file(fail_file) self.assertTrue(k.should_retry.count, 1) @mock.patch('time.sleep') def test_502_bad_gateway(self, sleep_mock): weird_timeout_body = "<Error><Code>BadGateway</Code></Error>" self.set_http_response(status_code=502, body=weird_timeout_body) b = Bucket(self.service_connection, 'mybucket') k = b.new_key('test_failure') fail_file = StringIO('This will pretend to be chunk-able.') k.should_retry = counter(k.should_retry) self.assertEqual(k.should_retry.count, 0) with self.assertRaises(BotoServerError): k.send_file(fail_file) self.assertTrue(k.should_retry.count, 1) @mock.patch('time.sleep') def test_504_gateway_timeout(self, sleep_mock): weird_timeout_body = "<Error><Code>GatewayTimeout</Code></Error>" self.set_http_response(status_code=504, body=weird_timeout_body) b = Bucket(self.service_connection, 'mybucket') k = b.new_key('test_failure') fail_file = StringIO('This will pretend to be chunk-able.') k.should_retry = counter(k.should_retry) self.assertEqual(k.should_retry.count, 0) with self.assertRaises(BotoServerError): k.send_file(fail_file) self.assertTrue(k.should_retry.count, 1) def test_should_not_raise_kms_related_integrity_errors(self): self.set_http_response(status_code=200, header=[ ('x-amz-server-side-encryption-aws-kms-key-id', 'key'), ('etag', 'not equal to key.md5')]) bucket = Bucket(self.service_connection, 'mybucket') key = bucket.new_key('test_kms') file_content = StringIO('Some content to upload.') # Should not raise errors related to integrity checks: key.send_file(file_content) class TestFileError(unittest.TestCase): def test_file_error(self): key = Key() class CustomException(Exception): pass key.get_contents_to_file = mock.Mock( side_effect=CustomException('File blew up!')) # Ensure our exception gets raised instead of a file or IO error with self.assertRaises(CustomException): key.get_contents_to_filename('foo.txt') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/s3/test_keyfile.py 0000644 00000010755 15025013616 0015033 0 ustar 00 # Copyright 2013 Google Inc. # Copyright 2011, Nexenta Systems Inc. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import os import unittest from boto.s3.keyfile import KeyFile from tests.integration.s3.mock_storage_service import MockConnection from tests.integration.s3.mock_storage_service import MockBucket class KeyfileTest(unittest.TestCase): def setUp(self): service_connection = MockConnection() self.contents = '0123456789' bucket = MockBucket(service_connection, 'mybucket') key = bucket.new_key('mykey') key.set_contents_from_string(self.contents) self.keyfile = KeyFile(key) def tearDown(self): self.keyfile.close() def testReadFull(self): self.assertEqual(self.keyfile.read(len(self.contents)), self.contents) def testReadPartial(self): self.assertEqual(self.keyfile.read(5), self.contents[:5]) self.assertEqual(self.keyfile.read(5), self.contents[5:]) def testTell(self): self.assertEqual(self.keyfile.tell(), 0) self.keyfile.read(4) self.assertEqual(self.keyfile.tell(), 4) self.keyfile.read(6) self.assertEqual(self.keyfile.tell(), 10) self.keyfile.close() try: self.keyfile.tell() except ValueError as e: self.assertEqual(str(e), 'I/O operation on closed file') def testSeek(self): self.assertEqual(self.keyfile.read(4), self.contents[:4]) self.keyfile.seek(0) self.assertEqual(self.keyfile.read(4), self.contents[:4]) self.keyfile.seek(5) self.assertEqual(self.keyfile.read(5), self.contents[5:]) # Seeking negative should raise. try: self.keyfile.seek(-5) except IOError as e: self.assertEqual(str(e), 'Invalid argument') # Reading past end of file is supposed to return empty string. self.keyfile.read(10) self.assertEqual(self.keyfile.read(20), '') # Seeking past end of file is supposed to silently work. self.keyfile.seek(50) self.assertEqual(self.keyfile.tell(), 50) self.assertEqual(self.keyfile.read(1), '') def testSeekEnd(self): self.assertEqual(self.keyfile.read(4), self.contents[:4]) self.keyfile.seek(0, os.SEEK_END) self.assertEqual(self.keyfile.read(1), '') self.keyfile.seek(-1, os.SEEK_END) self.assertEqual(self.keyfile.tell(), 9) self.assertEqual(self.keyfile.read(1), '9') # Test attempt to seek backwards past the start from the end. try: self.keyfile.seek(-100, os.SEEK_END) except IOError as e: self.assertEqual(str(e), 'Invalid argument') def testSeekCur(self): self.assertEqual(self.keyfile.read(1), self.contents[0]) self.keyfile.seek(1, os.SEEK_CUR) self.assertEqual(self.keyfile.tell(), 2) self.assertEqual(self.keyfile.read(4), self.contents[2:6]) def testSetEtag(self): # Make sure both bytes and strings work as contents. This is one of the # very few places Boto uses the mock key object. # https://github.com/GoogleCloudPlatform/gsutil/issues/214#issuecomment-49906044 self.keyfile.key.data = b'test' self.keyfile.key.set_etag() self.assertEqual(self.keyfile.key.etag, '098f6bcd4621d373cade4e832627b4f6') self.keyfile.key.etag = None self.keyfile.key.data = 'test' self.keyfile.key.set_etag() self.assertEqual(self.keyfile.key.etag, '098f6bcd4621d373cade4e832627b4f6') vendored/boto/tests/unit/s3/test_lifecycle.py 0000644 00000015672 15025013616 0015345 0 ustar 00 # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import AWSMockServiceTestCase from boto.s3.connection import S3Connection from boto.s3.bucket import Bucket from boto.s3.lifecycle import Rule, Lifecycle, Transition class TestS3LifeCycle(AWSMockServiceTestCase): connection_class = S3Connection def default_body(self): return """ <LifecycleConfiguration> <Rule> <ID>rule-1</ID> <Prefix>prefix/foo</Prefix> <Status>Enabled</Status> <Transition> <Days>30</Days> <StorageClass>GLACIER</StorageClass> </Transition> <Expiration> <Days>365</Days> </Expiration> </Rule> <Rule> <ID>rule-2</ID> <Prefix>prefix/bar</Prefix> <Status>Disabled</Status> <Transition> <Date>2012-12-31T00:00:000Z</Date> <StorageClass>STANDARD_IA</StorageClass> </Transition> <Expiration> <Date>2012-12-31T00:00:000Z</Date> </Expiration> </Rule> <Rule> <ID>multiple-transitions</ID> <Prefix></Prefix> <Status>Enabled</Status> <Transition> <Days>30</Days> <StorageClass>STANDARD_IA</StorageClass> </Transition> <Transition> <Days>90</Days> <StorageClass>GLACIER</StorageClass> </Transition> </Rule> </LifecycleConfiguration> """ def _get_bucket_lifecycle_config(self): self.set_http_response(status_code=200) bucket = Bucket(self.service_connection, 'mybucket') return bucket.get_lifecycle_config() def test_lifecycle_response_contains_all_rules(self): self.assertEqual(len(self._get_bucket_lifecycle_config()), 3) def test_parse_lifecycle_id(self): rule = self._get_bucket_lifecycle_config()[0] self.assertEqual(rule.id, 'rule-1') def test_parse_lifecycle_prefix(self): rule = self._get_bucket_lifecycle_config()[0] self.assertEqual(rule.prefix, 'prefix/foo') def test_parse_lifecycle_no_prefix(self): rule = self._get_bucket_lifecycle_config()[2] self.assertEqual(rule.prefix, '') def test_parse_lifecycle_enabled(self): rule = self._get_bucket_lifecycle_config()[0] self.assertEqual(rule.status, 'Enabled') def test_parse_lifecycle_disabled(self): rule = self._get_bucket_lifecycle_config()[1] self.assertEqual(rule.status, 'Disabled') def test_parse_expiration_days(self): rule = self._get_bucket_lifecycle_config()[0] self.assertEqual(rule.expiration.days, 365) def test_parse_expiration_date(self): rule = self._get_bucket_lifecycle_config()[1] self.assertEqual(rule.expiration.date, '2012-12-31T00:00:000Z') def test_parse_expiration_not_required(self): rule = self._get_bucket_lifecycle_config()[2] self.assertIsNone(rule.expiration) def test_parse_transition_days(self): transition = self._get_bucket_lifecycle_config()[0].transition[0] self.assertEqual(transition.days, 30) self.assertIsNone(transition.date) def test_parse_transition_days_deprecated(self): transition = self._get_bucket_lifecycle_config()[0].transition self.assertEqual(transition.days, 30) self.assertIsNone(transition.date) def test_parse_transition_date(self): transition = self._get_bucket_lifecycle_config()[1].transition[0] self.assertEqual(transition.date, '2012-12-31T00:00:000Z') self.assertIsNone(transition.days) def test_parse_transition_date_deprecated(self): transition = self._get_bucket_lifecycle_config()[1].transition self.assertEqual(transition.date, '2012-12-31T00:00:000Z') self.assertIsNone(transition.days) def test_parse_storage_class_standard_ia(self): transition = self._get_bucket_lifecycle_config()[1].transition[0] self.assertEqual(transition.storage_class, 'STANDARD_IA') def test_parse_storage_class_glacier(self): transition = self._get_bucket_lifecycle_config()[0].transition[0] self.assertEqual(transition.storage_class, 'GLACIER') def test_parse_storage_class_deprecated(self): transition = self._get_bucket_lifecycle_config()[1].transition self.assertEqual(transition.storage_class, 'STANDARD_IA') def test_parse_multiple_lifecycle_rules(self): transition = self._get_bucket_lifecycle_config()[2].transition self.assertEqual(len(transition), 2) def test_expiration_with_no_transition(self): lifecycle = Lifecycle() lifecycle.add_rule('myid', 'prefix', 'Enabled', 30) xml = lifecycle.to_xml() self.assertIn('<Expiration><Days>30</Days></Expiration>', xml) def test_expiration_is_optional(self): t = Transition(days=30, storage_class='GLACIER') r = Rule('myid', 'prefix', 'Enabled', expiration=None, transition=t) xml = r.to_xml() self.assertIn( '<Transition><StorageClass>GLACIER</StorageClass><Days>30</Days>', xml) def test_transition_is_optional(self): r = Rule('myid', 'prefix', 'Enabled') xml = r.to_xml() self.assertEqual( '<Rule><ID>myid</ID><Prefix>prefix</Prefix><Status>Enabled</Status></Rule>', xml) def test_expiration_and_transition(self): t = Transition(date='2012-11-30T00:00:000Z', storage_class='GLACIER') r = Rule('myid', 'prefix', 'Enabled', expiration=30, transition=t) xml = r.to_xml() self.assertIn( '<Transition><StorageClass>GLACIER</StorageClass>' '<Date>2012-11-30T00:00:000Z</Date>', xml) self.assertIn('<Expiration><Days>30</Days></Expiration>', xml) vendored/boto/tests/unit/s3/test_bucketlistresultset.py 0000644 00000004626 15025013616 0017527 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2016 Mitch Garnaat http://garnaat.org/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. from mock import patch, Mock import unittest from boto.s3.bucket import ResultSet from boto.s3.bucketlistresultset import multipart_upload_lister from boto.s3.bucketlistresultset import versioned_bucket_lister class S3BucketListResultSetTest (unittest.TestCase): def _test_patched_lister_encoding(self, inner_method, outer_method): bucket = Mock() call_args = [] first = ResultSet() first.append('foo') first.next_key_marker = 'a+b' first.is_truncated = True second = ResultSet() second.append('bar') second.is_truncated = False pages = [first, second] def return_pages(**kwargs): call_args.append(kwargs) return pages.pop(0) setattr(bucket, inner_method, return_pages) results = list(outer_method(bucket, encoding_type='url')) self.assertEqual(['foo', 'bar'], results) self.assertEqual('a b', call_args[1]['key_marker']) def test_list_object_versions_with_url_encoding(self): self._test_patched_lister_encoding( 'get_all_versions', versioned_bucket_lister) def test_list_multipart_upload_with_url_encoding(self): self._test_patched_lister_encoding( 'get_all_multipart_uploads', multipart_upload_lister) vendored/boto/tests/unit/s3/test_connection.py 0000644 00000065657 15025013616 0015555 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.compat import mock from tests.compat import unittest from tests.unit import AWSMockServiceTestCase from tests.unit import MockServiceWithConfigTestCase from boto.connection import AWSAuthConnection from boto.s3.connection import S3Connection, HostRequiredError from boto.s3.connection import S3ResponseError, Bucket class TestSignatureAlteration(AWSMockServiceTestCase): connection_class = S3Connection def test_unchanged(self): self.assertEqual( self.service_connection._required_auth_capability(), ['hmac-v4-s3'] ) def test_switched(self): conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host='s3.cn-north-1.amazonaws.com.cn' ) self.assertEqual( conn._required_auth_capability(), ['hmac-v4-s3'] ) class TestAnon(MockServiceWithConfigTestCase): connection_class = S3Connection def test_generate_url(self): conn = self.connection_class( anon=True, host='s3.amazonaws.com' ) url = conn.generate_url(0, 'GET', bucket='examplebucket', key='test.txt') self.assertNotIn('Signature=', url) def test_anon_default_taken_from_config_opt(self): self.config = { 's3': { # Value must be a string for `config.getbool` to not crash. 'no_sign_request': 'True', } } conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host='s3.amazonaws.com', ) url = conn.generate_url( 0, 'GET', bucket='examplebucket', key='test.txt') self.assertNotIn('Signature=', url) def test_explicit_anon_arg_overrides_config_value(self): self.config = { 's3': { # Value must be a string for `config.getbool` to not crash. 'no_sign_request': 'True', } } conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host='s3.amazonaws.com', anon=False ) url = conn.generate_url( 0, 'GET', bucket='examplebucket', key='test.txt') self.assertIn('Signature=', url) class TestPresigned(MockServiceWithConfigTestCase): connection_class = S3Connection def test_presign_respect_query_auth(self): self.config = { 's3': { 'use-sigv4': False, } } conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host='s3.amazonaws.com' ) url_enabled = conn.generate_url(86400, 'GET', bucket='examplebucket', key='test.txt', query_auth=True) url_disabled = conn.generate_url(86400, 'GET', bucket='examplebucket', key='test.txt', query_auth=False) self.assertIn('Signature=', url_enabled) self.assertNotIn('Signature=', url_disabled) class TestSigV4HostError(MockServiceWithConfigTestCase): connection_class = S3Connection def test_historical_behavior(self): self.assertEqual( self.service_connection._required_auth_capability(), ['hmac-v4-s3'] ) self.assertEqual(self.service_connection.host, 's3.amazonaws.com') def test_sigv4_opt_in(self): host_value = 's3.cn-north-1.amazonaws.com.cn' # Switch it at the config, so we can check to see how the host is # handled. self.config = { 's3': { 'use-sigv4': True, } } # Ensure passing a ``host`` in the connection args still works. conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host=host_value ) self.assertEqual( conn._required_auth_capability(), ['hmac-v4-s3'] ) self.assertEqual( conn.host, host_value ) # Ensure that the host is populated from our config if one is not # provided when creating a connection. self.config = { 's3': { 'host': host_value, 'use-sigv4': True, } } conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more' ) self.assertEqual( conn._required_auth_capability(), ['hmac-v4-s3'] ) self.assertEqual( conn.host, host_value ) class TestSigV4Presigned(MockServiceWithConfigTestCase): connection_class = S3Connection def test_sigv4_presign(self): self.config = { 's3': { 'use-sigv4': True, } } conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host='s3.amazonaws.com' ) # Here we force an input iso_date to ensure we always get the # same signature. url = conn.generate_url_sigv4(86400, 'GET', bucket='examplebucket', key='test.txt', iso_date='20140625T000000Z') self.assertIn( 'a937f5fbc125d98ac8f04c49e0204ea1526a7b8ca058000a54c192457be05b7d', url) def test_sigv4_presign_respects_is_secure(self): self.config = { 's3': { 'use-sigv4': True, } } conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host='s3.amazonaws.com', is_secure=True, ) url = conn.generate_url_sigv4(86400, 'GET', bucket='examplebucket', key='test.txt') self.assertTrue(url.startswith( 'https://examplebucket.s3.amazonaws.com/test.txt?')) conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host='s3.amazonaws.com', is_secure=False, ) url = conn.generate_url_sigv4(86400, 'GET', bucket='examplebucket', key='test.txt') self.assertTrue(url.startswith( 'http://examplebucket.s3.amazonaws.com/test.txt?')) def test_sigv4_presign_optional_params(self): self.config = { 's3': { 'use-sigv4': True, } } conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', security_token='token', host='s3.amazonaws.com' ) url = conn.generate_url_sigv4(86400, 'GET', bucket='examplebucket', key='test.txt', version_id=2) self.assertIn('VersionId=2', url) self.assertIn('X-Amz-Security-Token=token', url) def test_sigv4_presign_respect_query_auth(self): self.config = { 's3': { 'use-sigv4': True, } } conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host='s3.amazonaws.com' ) url_enabled = conn.generate_url(86400, 'GET', bucket='examplebucket', key='test.txt', query_auth=True) url_disabled = conn.generate_url(86400, 'GET', bucket='examplebucket', key='test.txt', query_auth=False) self.assertIn('Signature=', url_enabled) self.assertNotIn('Signature=', url_disabled) def test_sigv4_presign_headers(self): self.config = { 's3': { 'use-sigv4': True, } } conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host='s3.amazonaws.com' ) headers = {'x-amz-meta-key': 'val'} url = conn.generate_url_sigv4(86400, 'GET', bucket='examplebucket', key='test.txt', headers=headers) self.assertIn('host', url) self.assertIn('x-amz-meta-key', url) def test_sigv4_presign_response_headers(self): self.config = { 's3': { 'use-sigv4': True, } } conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', host='s3.amazonaws.com' ) response_headers = {'response-content-disposition': 'attachment; filename="file.ext"'} url = conn.generate_url_sigv4(86400, 'GET', bucket='examplebucket', key='test.txt', response_headers=response_headers) self.assertIn('host', url) self.assertIn('response-content-disposition', url) class TestUnicodeCallingFormat(AWSMockServiceTestCase): connection_class = S3Connection def default_body(self): return """<?xml version="1.0" encoding="UTF-8"?> <ListAllMyBucketsResult xmlns="http://doc.s3.amazonaws.com/2006-03-01"> <Owner> <ID>bcaf1ffd86f461ca5fb16fd081034f</ID> <DisplayName>webfile</DisplayName> </Owner> <Buckets> <Bucket> <Name>quotes</Name> <CreationDate>2006-02-03T16:45:09.000Z</CreationDate> </Bucket> <Bucket> <Name>samples</Name> <CreationDate>2006-02-03T16:41:58.000Z</CreationDate> </Bucket> </Buckets> </ListAllMyBucketsResult>""" def create_service_connection(self, **kwargs): kwargs['calling_format'] = u'boto.s3.connection.OrdinaryCallingFormat' return super(TestUnicodeCallingFormat, self).create_service_connection(**kwargs) def test_unicode_calling_format(self): self.set_http_response(status_code=200) self.service_connection.get_all_buckets() class TestHeadBucket(AWSMockServiceTestCase): connection_class = S3Connection def default_body(self): # HEAD requests always have an empty body. return "" def test_head_bucket_success(self): self.set_http_response(status_code=200) buck = self.service_connection.head_bucket('my-test-bucket') self.assertTrue(isinstance(buck, Bucket)) self.assertEqual(buck.name, 'my-test-bucket') def test_head_bucket_forbidden(self): self.set_http_response(status_code=403) with self.assertRaises(S3ResponseError) as cm: self.service_connection.head_bucket('cant-touch-this') err = cm.exception self.assertEqual(err.status, 403) self.assertEqual(err.error_code, 'AccessDenied') self.assertEqual(err.message, 'Access Denied') def test_head_bucket_notfound(self): self.set_http_response(status_code=404) with self.assertRaises(S3ResponseError) as cm: self.service_connection.head_bucket('totally-doesnt-exist') err = cm.exception self.assertEqual(err.status, 404) self.assertEqual(err.error_code, 'NoSuchBucket') self.assertEqual(err.message, 'The specified bucket does not exist') def test_head_bucket_other(self): self.set_http_response(status_code=405) with self.assertRaises(S3ResponseError) as cm: self.service_connection.head_bucket('you-broke-it') err = cm.exception self.assertEqual(err.status, 405) # We don't have special-cases for this error status. self.assertEqual(err.error_code, None) self.assertEqual(err.message, '') RETRY_REGION_BYTES = b'us-east-2' AUTHORIZATION_HEADER_MALFORMED=( b'<Error><Code>AuthorizationHeaderMalformed</Code><Message>The ' b'authorization header is malformed; the region \'us-east-1\' ' b'is wrong; expecting \'%s\'</Message>' b'<Region>us-east-2</Region><RequestId>asdf</RequestId>' b'<HostId>asdf</HostId></Error>' ) % RETRY_REGION_BYTES ILLEGAL_LOCATION_CONSTRAINT_SPECIFIED_REGION = ( b'<Error><Code>IllegalLocationConstraintException</Code>' b'<Message>The %s location constraint is incompatible ' b'for the region specific endpoint this request was sent to.' b'</Message><RequestId>asdf</RequestId>' b'<HostId>asdf</HostId></Error>' ) % RETRY_REGION_BYTES PERMANENT_REDIRECT = ( b'<Error><Code>PermanentRedirect</Code><Message>The bucket you ' b'are attempting to access must be addressed using the ' b'specified endpoint. Please send all future requests to this ' b'endpoint.</Message><Endpoint>' b'bucket.s3.%s.amazonaws.com</Endpoint>' b'<Bucket>bucket</Bucket><RequestId>asdf</RequestId>' b'<HostId>asdf</HostId></Error>' ) % RETRY_REGION_BYTES ILLEGAL_LOCATION_CONSTRAINT_UNSPECIFIED_REGION = ( b'<Error><Code>IllegalLocationConstraintException</Code>' b'<Message>The unspecified location constraint is incompatible ' b'for the region specific endpoint this request was sent to.' b'</Message><RequestId>asdf</RequestId>' b'<HostId>asdf</HostId></Error>' ) # This output cannot come from the API, but it's here to test # that strings not matching the regex handling this error are # handled gracefully. ILLEGAL_LOCATION_CONSTRAINT_MALFORMED = ( b'<Error><Code>IllegalLocationConstraintException</Code>' b'<Message>Some random string.</Message><RequestId>asdf</RequestId>' b'<HostId>asdf</HostId></Error>' ) ERRORS_WITH_REGION_IN_BODY = ( (400, AUTHORIZATION_HEADER_MALFORMED), (400, ILLEGAL_LOCATION_CONSTRAINT_SPECIFIED_REGION), (301, PERMANENT_REDIRECT) ) ERRORS_WITHOUT_REGION_IN_BODY = ( (400, ILLEGAL_LOCATION_CONSTRAINT_UNSPECIFIED_REGION), (400, ILLEGAL_LOCATION_CONSTRAINT_MALFORMED) ) class TestMakeRequestRetriesWithCorrectHost(AWSMockServiceTestCase): def setUp(self): self.connection = AWSAuthConnection('s3.amazonaws.com') self.non_retriable_code = 404 self.retry_status_codes = [301, 400] self.success_response = self.create_response(200) self.default_host = 'bucket.s3.amazonaws.com' self.retry_region = RETRY_REGION_BYTES.decode('utf-8') self.default_retried_host = ( 'bucket.s3.%s.amazonaws.com' % self.retry_region ) self.test_headers = [('x-amz-bucket-region', self.retry_region)] def test_non_retriable_status_returns_original_response(self): with mock.patch.object(self.connection, '_mexe') as mocked_mexe: error_response = self.create_response(self.non_retriable_code) mocked_mexe.side_effect = [error_response] response = self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(response, error_response) # called_once_with does not compare equality correctly with # HTTPResponse objects. self.assertEqual(mocked_mexe.call_count, 1) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_host ) def test_non_retriable_host_returns_original_response(self): for code in self.retry_status_codes: with mock.patch.object(self.connection, '_mexe') as mocked_mexe: error_response = self.create_response(code) mocked_mexe.side_effect = [error_response] other_host = 'bucket.some-other-provider.com' response = self.connection.make_request( 'HEAD', '/', host=other_host) self.assertEqual(response, error_response) self.assertEqual(mocked_mexe.call_count, 1) self.assertEqual( mocked_mexe.call_args[0][0].host, other_host ) def test_non_retriable_status_raises_original_exception(self): with mock.patch.object(self.connection, '_mexe') as mocked_mexe: error_response = S3ResponseError(self.non_retriable_code, 'reason') mocked_mexe.side_effect = [error_response] with self.assertRaises(S3ResponseError) as cm: self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(cm.exception, error_response) self.assertEqual(mocked_mexe.call_count, 1) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_host ) def test_non_retriable_host_raises_original_exception(self): with mock.patch.object(self.connection, '_mexe') as mocked_mexe: error_response = S3ResponseError(self.non_retriable_code, 'reason') mocked_mexe.side_effect = [error_response] other_host = 'bucket.some-other-provider.com' with self.assertRaises(S3ResponseError) as cm: self.connection.make_request( 'HEAD', '/', host=other_host) self.assertEqual(cm.exception, error_response) self.assertEqual(mocked_mexe.call_count, 1) self.assertEqual( mocked_mexe.call_args[0][0].host, other_host ) def test_response_retries_from_callable_headers(self): for code in self.retry_status_codes: with mock.patch.object(self.connection, '_mexe') as mocked_mexe: mocked_mexe.side_effect = [ self.create_response(code, header=self.test_headers), self.success_response ] response = self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(response, self.success_response) self.assertEqual(mocked_mexe.call_count, 2) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_retried_host ) def test_retry_changes_host_with_region(self): with mock.patch.object(self.connection, '_mexe') as mocked_mexe: # Assume 400 with callable headers results uses the same url # manipulation as all of the other successful cases. mocked_mexe.side_effect = [ self.create_response(400, header=self.test_headers), self.success_response ] response = self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(response, self.success_response) self.assertEqual(mocked_mexe.call_count, 2) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_retried_host ) def test_retry_changes_host_with_multiple_s3_occurrences(self): with mock.patch.object(self.connection, '_mexe') as mocked_mexe: # Assume 400 with callable headers results uses the same url # manipulation as all of the other successful cases. mocked_mexe.side_effect = [ self.create_response(400, header=self.test_headers), self.success_response ] response = self.connection.make_request( 'HEAD', '/', host='a.s3.a.s3.amazonaws.com') self.assertEqual(response, self.success_response) self.assertEqual(mocked_mexe.call_count, 2) self.assertEqual( mocked_mexe.call_args[0][0].host, 'a.s3.a.s3.us-east-2.amazonaws.com' ) def test_retry_changes_host_with_s3_in_region(self): with mock.patch.object(self.connection, '_mexe') as mocked_mexe: # Assume 400 with callable headers results uses the same url # manipulation as all of the other successful cases. mocked_mexe.side_effect = [ self.create_response(400, header=self.test_headers), self.success_response ] response = self.connection.make_request( 'HEAD', '/', host='bucket.s3.asdf-s3.amazonaws.com') self.assertEqual(response, self.success_response) self.assertEqual(mocked_mexe.call_count, 2) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_retried_host ) def test_response_body_parsed_for_region(self): for code, body in ERRORS_WITH_REGION_IN_BODY: with mock.patch.object(self.connection, '_mexe') as mocked_mexe: mocked_mexe.side_effect = [ self.create_response(code, body=body), self.success_response ] response = self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(response, self.success_response) self.assertEqual(mocked_mexe.call_count, 2) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_retried_host ) def test_error_body_parsed_for_region(self): for code, body in ERRORS_WITH_REGION_IN_BODY: with mock.patch.object(self.connection, '_mexe') as mocked_mexe: mocked_mexe.side_effect = [ S3ResponseError(code, 'reason', body=body), self.success_response ] response = self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(response, self.success_response) self.assertEqual(mocked_mexe.call_count, 2) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_retried_host ) def test_response_without_region_header_retries_from_bucket_head(self): for code in self.retry_status_codes: with mock.patch.object(self.connection, '_mexe') as mocked_mexe: mocked_mexe.side_effect = [ self.create_response(code), self.create_response(200, header=self.test_headers), self.success_response ] response = self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(response, self.success_response) self.assertEqual(mocked_mexe.call_count, 3) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_retried_host ) def test_response_body_without_region_sends_bucket_head(self): for code, body in ERRORS_WITHOUT_REGION_IN_BODY: with mock.patch.object(self.connection, '_mexe') as mocked_mexe: mocked_mexe.side_effect = [ self.create_response(code, body=body), self.create_response(200, header=self.test_headers), self.success_response ] response = self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(response, self.success_response) self.assertEqual(mocked_mexe.call_count, 3) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_retried_host ) def test_error_body_without_region_retries_from_bucket_head_request(self): for code, body in ERRORS_WITHOUT_REGION_IN_BODY: with mock.patch.object(self.connection, '_mexe') as mocked_mexe: mocked_mexe.side_effect = [ S3ResponseError(code, 'reason', body=body), self.create_response(200, header=self.test_headers), self.success_response ] response = self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(response, self.success_response) self.assertEqual(mocked_mexe.call_count, 3) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_retried_host ) def test_retry_head_request_lacks_region_returns_original_response(self): for code in self.retry_status_codes: with mock.patch.object(self.connection, '_mexe') as mocked_mexe: error_response = self.create_response(code) mocked_mexe.side_effect = [ error_response, self.create_response(200, header=[]) # no region in header. ] response = self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(response, error_response) self.assertEqual(mocked_mexe.call_count, 2) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_host ) def test_retry_head_request_lacks_region_raises_original_exception(self): for code in self.retry_status_codes: with mock.patch.object(self.connection, '_mexe') as mocked_mexe: error_response = S3ResponseError(code, 'reason') mocked_mexe.side_effect = [ error_response, self.create_response(200, header=[]) ] with self.assertRaises(S3ResponseError) as cm: response = self.connection.make_request( 'HEAD', '/', host=self.default_host) self.assertEqual(cm.exception, error_response) self.assertEqual(mocked_mexe.call_count, 2) self.assertEqual( mocked_mexe.call_args[0][0].host, self.default_host ) if __name__ == "__main__": unittest.main() vendored/boto/tests/unit/s3/test_website.py 0000644 00000022003 15025013616 0015032 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest import xml.dom.minidom import xml.sax from boto.s3.website import WebsiteConfiguration from boto.s3.website import RedirectLocation from boto.s3.website import RoutingRules from boto.s3.website import Condition from boto.s3.website import RoutingRules from boto.s3.website import RoutingRule from boto.s3.website import Redirect from boto import handler def pretty_print_xml(text): text = ''.join(t.strip() for t in text.splitlines()) x = xml.dom.minidom.parseString(text) return x.toprettyxml() class TestS3WebsiteConfiguration(unittest.TestCase): maxDiff = None def setUp(self): pass def tearDown(self): pass def test_suffix_only(self): config = WebsiteConfiguration(suffix='index.html') xml = config.to_xml() self.assertIn( '<IndexDocument><Suffix>index.html</Suffix></IndexDocument>', xml) def test_suffix_and_error(self): config = WebsiteConfiguration(suffix='index.html', error_key='error.html') xml = config.to_xml() self.assertIn( '<ErrorDocument><Key>error.html</Key></ErrorDocument>', xml) def test_redirect_all_request_to_with_just_host(self): location = RedirectLocation(hostname='example.com') config = WebsiteConfiguration(redirect_all_requests_to=location) xml = config.to_xml() self.assertIn( ('<RedirectAllRequestsTo><HostName>' 'example.com</HostName></RedirectAllRequestsTo>'), xml) def test_redirect_all_requests_with_protocol(self): location = RedirectLocation(hostname='example.com', protocol='https') config = WebsiteConfiguration(redirect_all_requests_to=location) xml = config.to_xml() self.assertIn( ('<RedirectAllRequestsTo><HostName>' 'example.com</HostName><Protocol>https</Protocol>' '</RedirectAllRequestsTo>'), xml) def test_routing_rules_key_prefix(self): x = pretty_print_xml # This rule redirects requests for docs/* to documentation/* rules = RoutingRules() condition = Condition(key_prefix='docs/') redirect = Redirect(replace_key_prefix='documents/') rules.add_rule(RoutingRule(condition, redirect)) config = WebsiteConfiguration(suffix='index.html', routing_rules=rules) xml = config.to_xml() expected_xml = """<?xml version="1.0" encoding="UTF-8"?> <WebsiteConfiguration xmlns='http://s3.amazonaws.com/doc/2006-03-01/'> <IndexDocument> <Suffix>index.html</Suffix> </IndexDocument> <RoutingRules> <RoutingRule> <Condition> <KeyPrefixEquals>docs/</KeyPrefixEquals> </Condition> <Redirect> <ReplaceKeyPrefixWith>documents/</ReplaceKeyPrefixWith> </Redirect> </RoutingRule> </RoutingRules> </WebsiteConfiguration> """ self.assertEqual(x(expected_xml), x(xml)) def test_routing_rules_to_host_on_404(self): x = pretty_print_xml # Another example from the docs: # Redirect requests to a specific host in the event of a 404. # Also, the redirect inserts a report-404/. For example, # if you request a page ExamplePage.html and it results # in a 404, the request is routed to a page report-404/ExamplePage.html rules = RoutingRules() condition = Condition(http_error_code=404) redirect = Redirect(hostname='example.com', replace_key_prefix='report-404/') rules.add_rule(RoutingRule(condition, redirect)) config = WebsiteConfiguration(suffix='index.html', routing_rules=rules) xml = config.to_xml() expected_xml = """<?xml version="1.0" encoding="UTF-8"?> <WebsiteConfiguration xmlns='http://s3.amazonaws.com/doc/2006-03-01/'> <IndexDocument> <Suffix>index.html</Suffix> </IndexDocument> <RoutingRules> <RoutingRule> <Condition> <HttpErrorCodeReturnedEquals>404</HttpErrorCodeReturnedEquals> </Condition> <Redirect> <HostName>example.com</HostName> <ReplaceKeyPrefixWith>report-404/</ReplaceKeyPrefixWith> </Redirect> </RoutingRule> </RoutingRules> </WebsiteConfiguration> """ self.assertEqual(x(expected_xml), x(xml)) def test_key_prefix(self): x = pretty_print_xml rules = RoutingRules() condition = Condition(key_prefix="images/") redirect = Redirect(replace_key='folderdeleted.html') rules.add_rule(RoutingRule(condition, redirect)) config = WebsiteConfiguration(suffix='index.html', routing_rules=rules) xml = config.to_xml() expected_xml = """<?xml version="1.0" encoding="UTF-8"?> <WebsiteConfiguration xmlns='http://s3.amazonaws.com/doc/2006-03-01/'> <IndexDocument> <Suffix>index.html</Suffix> </IndexDocument> <RoutingRules> <RoutingRule> <Condition> <KeyPrefixEquals>images/</KeyPrefixEquals> </Condition> <Redirect> <ReplaceKeyWith>folderdeleted.html</ReplaceKeyWith> </Redirect> </RoutingRule> </RoutingRules> </WebsiteConfiguration> """ self.assertEqual(x(expected_xml), x(xml)) def test_builders(self): x = pretty_print_xml # This is a more declarative way to create rules. # First the long way. rules = RoutingRules() condition = Condition(http_error_code=404) redirect = Redirect(hostname='example.com', replace_key_prefix='report-404/') rules.add_rule(RoutingRule(condition, redirect)) xml = rules.to_xml() # Then the more concise way. rules2 = RoutingRules().add_rule( RoutingRule.when(http_error_code=404).then_redirect( hostname='example.com', replace_key_prefix='report-404/')) xml2 = rules2.to_xml() self.assertEqual(x(xml), x(xml2)) def test_parse_xml(self): x = pretty_print_xml xml_in = """<?xml version="1.0" encoding="UTF-8"?> <WebsiteConfiguration xmlns='http://s3.amazonaws.com/doc/2006-03-01/'> <IndexDocument> <Suffix>index.html</Suffix> </IndexDocument> <ErrorDocument> <Key>error.html</Key> </ErrorDocument> <RoutingRules> <RoutingRule> <Condition> <KeyPrefixEquals>docs/</KeyPrefixEquals> </Condition> <Redirect> <Protocol>https</Protocol> <HostName>www.example.com</HostName> <ReplaceKeyWith>documents/</ReplaceKeyWith> <HttpRedirectCode>302</HttpRedirectCode> </Redirect> </RoutingRule> <RoutingRule> <Condition> <HttpErrorCodeReturnedEquals>404</HttpErrorCodeReturnedEquals> </Condition> <Redirect> <HostName>example.com</HostName> <ReplaceKeyPrefixWith>report-404/</ReplaceKeyPrefixWith> </Redirect> </RoutingRule> </RoutingRules> </WebsiteConfiguration> """ webconfig = WebsiteConfiguration() h = handler.XmlHandler(webconfig, None) xml.sax.parseString(xml_in.encode('utf-8'), h) xml_out = webconfig.to_xml() self.assertEqual(x(xml_in), x(xml_out)) vendored/boto/tests/unit/s3/test_tagging.py 0000644 00000002766 15025013616 0015026 0 ustar 00 from tests.unit import AWSMockServiceTestCase from boto.s3.connection import S3Connection from boto.s3.bucket import Bucket from boto.s3.tagging import Tag class TestS3Tagging(AWSMockServiceTestCase): connection_class = S3Connection def default_body(self): return """ <Tagging> <TagSet> <Tag> <Key>Project</Key> <Value>Project One</Value> </Tag> <Tag> <Key>User</Key> <Value>jsmith</Value> </Tag> </TagSet> </Tagging> """ def test_parse_tagging_response(self): self.set_http_response(status_code=200) b = Bucket(self.service_connection, 'mybucket') api_response = b.get_tags() # The outer list is a list of tag sets. self.assertEqual(len(api_response), 1) # The inner list is a list of tags. self.assertEqual(len(api_response[0]), 2) self.assertEqual(api_response[0][0].key, 'Project') self.assertEqual(api_response[0][0].value, 'Project One') self.assertEqual(api_response[0][1].key, 'User') self.assertEqual(api_response[0][1].value, 'jsmith') def test_tag_equality(self): t1 = Tag('foo', 'bar') t2 = Tag('foo', 'bar') t3 = Tag('foo', 'baz') t4 = Tag('baz', 'bar') self.assertEqual(t1, t2) self.assertNotEqual(t1, t3) self.assertNotEqual(t1, t4) vendored/boto/tests/unit/s3/test_uri.py 0000644 00000030330 15025013616 0014171 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2013 Google, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import boto import tempfile from boto.exception import InvalidUriError from boto import storage_uri from boto.compat import urllib from boto.s3.keyfile import KeyFile from tests.integration.s3.mock_storage_service import MockBucket from tests.integration.s3.mock_storage_service import MockBucketStorageUri from tests.integration.s3.mock_storage_service import MockConnection from tests.unit import unittest """Unit tests for StorageUri interface.""" class UriTest(unittest.TestCase): def test_provider_uri(self): for prov in ('gs', 's3'): uri_str = '%s://' % prov uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual(prov, uri.scheme) self.assertEqual(uri_str, uri.uri) self.assertFalse(hasattr(uri, 'versionless_uri')) self.assertEqual('', uri.bucket_name) self.assertEqual('', uri.object_name) self.assertEqual(None, uri.version_id) self.assertEqual(None, uri.generation) self.assertEqual(uri.names_provider(), True) self.assertEqual(uri.names_container(), True) self.assertEqual(uri.names_bucket(), False) self.assertEqual(uri.names_object(), False) self.assertEqual(uri.names_directory(), False) self.assertEqual(uri.names_file(), False) self.assertEqual(uri.is_stream(), False) self.assertEqual(uri.is_version_specific, False) def test_bucket_uri_no_trailing_slash(self): for prov in ('gs', 's3'): uri_str = '%s://bucket' % prov uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual(prov, uri.scheme) self.assertEqual('%s/' % uri_str, uri.uri) self.assertFalse(hasattr(uri, 'versionless_uri')) self.assertEqual('bucket', uri.bucket_name) self.assertEqual('', uri.object_name) self.assertEqual(None, uri.version_id) self.assertEqual(None, uri.generation) self.assertEqual(uri.names_provider(), False) self.assertEqual(uri.names_container(), True) self.assertEqual(uri.names_bucket(), True) self.assertEqual(uri.names_object(), False) self.assertEqual(uri.names_directory(), False) self.assertEqual(uri.names_file(), False) self.assertEqual(uri.is_stream(), False) self.assertEqual(uri.is_version_specific, False) def test_bucket_uri_with_trailing_slash(self): for prov in ('gs', 's3'): uri_str = '%s://bucket/' % prov uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual(prov, uri.scheme) self.assertEqual(uri_str, uri.uri) self.assertFalse(hasattr(uri, 'versionless_uri')) self.assertEqual('bucket', uri.bucket_name) self.assertEqual('', uri.object_name) self.assertEqual(None, uri.version_id) self.assertEqual(None, uri.generation) self.assertEqual(uri.names_provider(), False) self.assertEqual(uri.names_container(), True) self.assertEqual(uri.names_bucket(), True) self.assertEqual(uri.names_object(), False) self.assertEqual(uri.names_directory(), False) self.assertEqual(uri.names_file(), False) self.assertEqual(uri.is_stream(), False) self.assertEqual(uri.is_version_specific, False) def test_non_versioned_object_uri(self): for prov in ('gs', 's3'): uri_str = '%s://bucket/obj/a/b' % prov uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual(prov, uri.scheme) self.assertEqual(uri_str, uri.uri) self.assertEqual(uri_str, uri.versionless_uri) self.assertEqual('bucket', uri.bucket_name) self.assertEqual('obj/a/b', uri.object_name) self.assertEqual(None, uri.version_id) self.assertEqual(None, uri.generation) self.assertEqual(uri.names_provider(), False) self.assertEqual(uri.names_container(), False) self.assertEqual(uri.names_bucket(), False) self.assertEqual(uri.names_object(), True) self.assertEqual(uri.names_directory(), False) self.assertEqual(uri.names_file(), False) self.assertEqual(uri.is_stream(), False) self.assertEqual(uri.is_version_specific, False) def test_versioned_gs_object_uri(self): uri_str = 'gs://bucket/obj/a/b#1359908801674000' uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual('gs', uri.scheme) self.assertEqual(uri_str, uri.uri) self.assertEqual('gs://bucket/obj/a/b', uri.versionless_uri) self.assertEqual('bucket', uri.bucket_name) self.assertEqual('obj/a/b', uri.object_name) self.assertEqual(None, uri.version_id) self.assertEqual(1359908801674000, uri.generation) self.assertEqual(uri.names_provider(), False) self.assertEqual(uri.names_container(), False) self.assertEqual(uri.names_bucket(), False) self.assertEqual(uri.names_object(), True) self.assertEqual(uri.names_directory(), False) self.assertEqual(uri.names_file(), False) self.assertEqual(uri.is_stream(), False) self.assertEqual(uri.is_version_specific, True) def test_versioned_gs_object_uri_with_legacy_generation_value(self): uri_str = 'gs://bucket/obj/a/b#1' uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual('gs', uri.scheme) self.assertEqual(uri_str, uri.uri) self.assertEqual('gs://bucket/obj/a/b', uri.versionless_uri) self.assertEqual('bucket', uri.bucket_name) self.assertEqual('obj/a/b', uri.object_name) self.assertEqual(None, uri.version_id) self.assertEqual(1, uri.generation) self.assertEqual(uri.names_provider(), False) self.assertEqual(uri.names_container(), False) self.assertEqual(uri.names_bucket(), False) self.assertEqual(uri.names_object(), True) self.assertEqual(uri.names_directory(), False) self.assertEqual(uri.names_file(), False) self.assertEqual(uri.is_stream(), False) self.assertEqual(uri.is_version_specific, True) def test_roundtrip_versioned_gs_object_uri_parsed(self): uri_str = 'gs://bucket/obj#1359908801674000' uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) roundtrip_uri = boto.storage_uri(uri.uri, validate=False, suppress_consec_slashes=False) self.assertEqual(uri.uri, roundtrip_uri.uri) self.assertEqual(uri.is_version_specific, True) def test_versioned_s3_object_uri(self): uri_str = 's3://bucket/obj/a/b#eMuM0J15HkJ9QHlktfNP5MfA.oYR2q6S' uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual('s3', uri.scheme) self.assertEqual(uri_str, uri.uri) self.assertEqual('s3://bucket/obj/a/b', uri.versionless_uri) self.assertEqual('bucket', uri.bucket_name) self.assertEqual('obj/a/b', uri.object_name) self.assertEqual('eMuM0J15HkJ9QHlktfNP5MfA.oYR2q6S', uri.version_id) self.assertEqual(None, uri.generation) self.assertEqual(uri.names_provider(), False) self.assertEqual(uri.names_container(), False) self.assertEqual(uri.names_bucket(), False) self.assertEqual(uri.names_object(), True) self.assertEqual(uri.names_directory(), False) self.assertEqual(uri.names_file(), False) self.assertEqual(uri.is_stream(), False) self.assertEqual(uri.is_version_specific, True) def test_explicit_file_uri(self): tmp_dir = tempfile.tempdir or '' uri_str = 'file://%s' % urllib.request.pathname2url(tmp_dir) uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual('file', uri.scheme) self.assertEqual(uri_str, uri.uri) self.assertFalse(hasattr(uri, 'versionless_uri')) self.assertEqual('', uri.bucket_name) self.assertEqual(tmp_dir, uri.object_name) self.assertFalse(hasattr(uri, 'version_id')) self.assertFalse(hasattr(uri, 'generation')) self.assertFalse(hasattr(uri, 'is_version_specific')) self.assertEqual(uri.names_provider(), False) self.assertEqual(uri.names_bucket(), False) # Don't check uri.names_container(), uri.names_directory(), # uri.names_file(), or uri.names_object(), because for file URIs these # functions look at the file system and apparently unit tests run # chroot'd. self.assertEqual(uri.is_stream(), False) def test_implicit_file_uri(self): tmp_dir = tempfile.tempdir or '' uri_str = '%s' % urllib.request.pathname2url(tmp_dir) uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual('file', uri.scheme) self.assertEqual('file://%s' % tmp_dir, uri.uri) self.assertFalse(hasattr(uri, 'versionless_uri')) self.assertEqual('', uri.bucket_name) self.assertEqual(tmp_dir, uri.object_name) self.assertFalse(hasattr(uri, 'version_id')) self.assertFalse(hasattr(uri, 'generation')) self.assertFalse(hasattr(uri, 'is_version_specific')) self.assertEqual(uri.names_provider(), False) self.assertEqual(uri.names_bucket(), False) # Don't check uri.names_container(), uri.names_directory(), # uri.names_file(), or uri.names_object(), because for file URIs these # functions look at the file system and apparently unit tests run # chroot'd. self.assertEqual(uri.is_stream(), False) def test_gs_object_uri_contains_sharp_not_matching_version_syntax(self): uri_str = 'gs://bucket/obj#13a990880167400' uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual('gs', uri.scheme) self.assertEqual(uri_str, uri.uri) self.assertEqual('gs://bucket/obj#13a990880167400', uri.versionless_uri) self.assertEqual('bucket', uri.bucket_name) self.assertEqual('obj#13a990880167400', uri.object_name) self.assertEqual(None, uri.version_id) self.assertEqual(None, uri.generation) self.assertEqual(uri.names_provider(), False) self.assertEqual(uri.names_container(), False) self.assertEqual(uri.names_bucket(), False) self.assertEqual(uri.names_object(), True) self.assertEqual(uri.names_directory(), False) self.assertEqual(uri.names_file(), False) self.assertEqual(uri.is_stream(), False) self.assertEqual(uri.is_version_specific, False) def test_file_containing_colon(self): uri_str = 'abc:def' uri = boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) self.assertEqual('file', uri.scheme) self.assertEqual('file://%s' % uri_str, uri.uri) def test_invalid_scheme(self): uri_str = 'mars://bucket/object' try: boto.storage_uri(uri_str, validate=False, suppress_consec_slashes=False) except InvalidUriError as e: self.assertIn('Unrecognized scheme', e.message) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/s3/test_cors_configuration.py 0000644 00000004635 15025013616 0017300 0 ustar 00 #!/usr/bin/env python import unittest from boto.s3.cors import CORSConfiguration CORS_BODY_1 = ( '<CORSConfiguration>' '<CORSRule>' '<AllowedMethod>PUT</AllowedMethod>' '<AllowedMethod>POST</AllowedMethod>' '<AllowedMethod>DELETE</AllowedMethod>' '<AllowedOrigin>http://www.example.com</AllowedOrigin>' '<AllowedHeader>*</AllowedHeader>' '<ExposeHeader>x-amz-server-side-encryption</ExposeHeader>' '<MaxAgeSeconds>3000</MaxAgeSeconds>' '<ID>foobar_rule</ID>' '</CORSRule>' '</CORSConfiguration>') CORS_BODY_2 = ( '<CORSConfiguration>' '<CORSRule>' '<AllowedMethod>PUT</AllowedMethod>' '<AllowedMethod>POST</AllowedMethod>' '<AllowedMethod>DELETE</AllowedMethod>' '<AllowedOrigin>http://www.example.com</AllowedOrigin>' '<AllowedHeader>*</AllowedHeader>' '<ExposeHeader>x-amz-server-side-encryption</ExposeHeader>' '<MaxAgeSeconds>3000</MaxAgeSeconds>' '</CORSRule>' '<CORSRule>' '<AllowedMethod>GET</AllowedMethod>' '<AllowedOrigin>*</AllowedOrigin>' '<AllowedHeader>*</AllowedHeader>' '<MaxAgeSeconds>3000</MaxAgeSeconds>' '</CORSRule>' '</CORSConfiguration>') CORS_BODY_3 = ( '<CORSConfiguration>' '<CORSRule>' '<AllowedMethod>GET</AllowedMethod>' '<AllowedOrigin>*</AllowedOrigin>' '</CORSRule>' '</CORSConfiguration>') class TestCORSConfiguration(unittest.TestCase): def test_one_rule_with_id(self): cfg = CORSConfiguration() cfg.add_rule(['PUT', 'POST', 'DELETE'], 'http://www.example.com', allowed_header='*', max_age_seconds=3000, expose_header='x-amz-server-side-encryption', id='foobar_rule') self.assertEqual(cfg.to_xml(), CORS_BODY_1) def test_two_rules(self): cfg = CORSConfiguration() cfg.add_rule(['PUT', 'POST', 'DELETE'], 'http://www.example.com', allowed_header='*', max_age_seconds=3000, expose_header='x-amz-server-side-encryption') cfg.add_rule('GET', '*', allowed_header='*', max_age_seconds=3000) self.assertEqual(cfg.to_xml(), CORS_BODY_2) def test_minimal(self): cfg = CORSConfiguration() cfg.add_rule('GET', '*') self.assertEqual(cfg.to_xml(), CORS_BODY_3) if __name__ == "__main__": unittest.main() vendored/boto/tests/unit/s3/test_bucket.py 0000644 00000020152 15025013616 0014650 0 ustar 00 # -*- coding: utf-8 -*- from mock import patch import xml.dom.minidom from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.exception import BotoClientError from boto.s3.connection import Location, S3Connection from boto.s3.bucket import Bucket from boto.s3.deletemarker import DeleteMarker from boto.s3.key import Key from boto.s3.multipart import MultiPartUpload from boto.s3.prefix import Prefix class TestS3Bucket(AWSMockServiceTestCase): connection_class = S3Connection def setUp(self): super(TestS3Bucket, self).setUp() def test_bucket_create_bucket(self): self.set_http_response(status_code=200) bucket = self.service_connection.create_bucket('mybucket_create') self.assertEqual(bucket.name, 'mybucket_create') def test_bucket_create_eu_central_1_location(self): self.set_http_response(status_code=200) bucket = self.service_connection.create_bucket( 'eu_central_1_bucket', location=Location.EUCentral1 ) self.assertEqual(bucket.name, 'eu_central_1_bucket') def test_bucket_constructor(self): self.set_http_response(status_code=200) bucket = Bucket(self.service_connection, 'mybucket_constructor') self.assertEqual(bucket.name, 'mybucket_constructor') def test_bucket_basics(self): self.set_http_response(status_code=200) bucket = self.service_connection.create_bucket('mybucket') self.assertEqual(bucket.__repr__(), '<Bucket: mybucket>') def test_bucket_new_key(self): self.set_http_response(status_code=200) bucket = self.service_connection.create_bucket('mybucket') key = bucket.new_key('mykey') self.assertEqual(key.bucket, bucket) self.assertEqual(key.key, 'mykey') def test_bucket_new_key_missing_name(self): self.set_http_response(status_code=200) bucket = self.service_connection.create_bucket('mybucket') with self.assertRaises(ValueError): key = bucket.new_key('') def test_bucket_delete_key_missing_name(self): self.set_http_response(status_code=200) bucket = self.service_connection.create_bucket('mybucket') with self.assertRaises(ValueError): key = bucket.delete_key('') def test_bucket_kwargs_misspelling(self): self.set_http_response(status_code=200) bucket = self.service_connection.create_bucket('mybucket') with self.assertRaises(TypeError): bucket.get_all_keys(delimeter='foo') def test__get_all_query_args(self): bukket = Bucket() # Default. qa = bukket._get_all_query_args({}) self.assertEqual(qa, '') # Default with initial. qa = bukket._get_all_query_args({}, 'initial=1') self.assertEqual(qa, 'initial=1') # Single param. qa = bukket._get_all_query_args({ 'foo': 'true' }) self.assertEqual(qa, 'foo=true') # Single param with initial. qa = bukket._get_all_query_args({ 'foo': 'true' }, 'initial=1') self.assertEqual(qa, 'initial=1&foo=true') # Multiple params with all the weird cases. multiple_params = { 'foo': 'true', # Ensure Unicode chars get encoded. 'bar': '☃', # Ensure unicode strings with non-ascii characters get encoded 'baz': u'χ', # Underscores are bad, m'kay? 'some_other': 'thing', # Change the variant of ``max-keys``. 'maxkeys': 0, # ``None`` values get excluded. 'notthere': None, # Empty values also get excluded. 'notpresenteither': '', } qa = bukket._get_all_query_args(multiple_params) self.assertEqual( qa, 'bar=%E2%98%83&baz=%CF%87&foo=true&max-keys=0&some-other=thing' ) # Multiple params with initial. qa = bukket._get_all_query_args(multiple_params, 'initial=1') self.assertEqual( qa, 'initial=1&bar=%E2%98%83&baz=%CF%87&foo=true&max-keys=0&some-other=thing' ) @patch.object(S3Connection, 'head_bucket') def test_bucket_copy_key_no_validate(self, mock_head_bucket): self.set_http_response(status_code=200) bucket = self.service_connection.create_bucket('mybucket') self.assertFalse(mock_head_bucket.called) self.service_connection.get_bucket('mybucket', validate=True) self.assertTrue(mock_head_bucket.called) mock_head_bucket.reset_mock() self.assertFalse(mock_head_bucket.called) try: bucket.copy_key('newkey', 'srcbucket', 'srckey', preserve_acl=True) except: # Will throw because of empty response. pass self.assertFalse(mock_head_bucket.called) @patch.object(Bucket, '_get_all') def test_bucket_encoding(self, mock_get_all): self.set_http_response(status_code=200) bucket = self.service_connection.get_bucket('mybucket') # First, without the encoding. mock_get_all.reset_mock() bucket.get_all_keys() mock_get_all.assert_called_with( [ ('Contents', Key), ('CommonPrefixes', Prefix) ], '', None ) # Now the variants with the encoding. mock_get_all.reset_mock() bucket.get_all_keys(encoding_type='url') mock_get_all.assert_called_with( [ ('Contents', Key), ('CommonPrefixes', Prefix) ], '', None, encoding_type='url' ) mock_get_all.reset_mock() bucket.get_all_versions(encoding_type='url') mock_get_all.assert_called_with( [ ('Version', Key), ('CommonPrefixes', Prefix), ('DeleteMarker', DeleteMarker), ], 'versions', None, encoding_type='url' ) mock_get_all.reset_mock() bucket.get_all_multipart_uploads(encoding_type='url') mock_get_all.assert_called_with( [ ('Upload', MultiPartUpload), ('CommonPrefixes', Prefix) ], 'uploads', None, encoding_type='url' ) @patch.object(Bucket, 'get_all_keys') @patch.object(Bucket, '_get_key_internal') def test_bucket_get_key_no_validate(self, mock_gki, mock_gak): self.set_http_response(status_code=200) bucket = self.service_connection.get_bucket('mybucket') key = bucket.get_key('mykey', validate=False) self.assertEqual(len(mock_gki.mock_calls), 0) self.assertTrue(isinstance(key, Key)) self.assertEqual(key.name, 'mykey') with self.assertRaises(BotoClientError): bucket.get_key( 'mykey', version_id='something', validate=False ) def acl_policy(self): return """<?xml version="1.0" encoding="UTF-8"?> <AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> <Owner> <ID>owner_id</ID> <DisplayName>owner_display_name</DisplayName> </Owner> <AccessControlList> <Grant> <Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser"> <ID>grantee_id</ID> <DisplayName>grantee_display_name</DisplayName> </Grantee> <Permission>FULL_CONTROL</Permission> </Grant> </AccessControlList> </AccessControlPolicy>""" def test_bucket_acl_policy_namespace(self): self.set_http_response(status_code=200) bucket = self.service_connection.get_bucket('mybucket') self.set_http_response(status_code=200, body=self.acl_policy()) policy = bucket.get_acl() xml_policy = policy.to_xml() document = xml.dom.minidom.parseString(xml_policy) namespace = document.documentElement.namespaceURI self.assertEqual(namespace, 'http://s3.amazonaws.com/doc/2006-03-01/') vendored/boto/tests/unit/s3/__init__.py 0000644 00000000000 15025013616 0014061 0 ustar 00 vendored/boto/tests/unit/ec2containerservice/test_connection.py 0000644 00000003575 15025013616 0021154 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. # All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from mock import Mock from tests.unit import unittest import boto.ec2containerservice from boto.ec2containerservice.layer1 import EC2ContainerServiceConnection from boto.compat import http_client class TestConnectToRegion(unittest.TestCase): def setUp(self): self.https_connection = Mock(spec=http_client.HTTPSConnection) self.https_connection_factory = ( Mock(return_value=self.https_connection), ()) def test_aws_region(self): ecs = boto.ec2containerservice.connect_to_region('us-east-1', https_connection_factory=self.https_connection_factory, aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key' ) self.assertIsInstance(ecs, EC2ContainerServiceConnection) vendored/boto/tests/unit/ec2containerservice/__init__.py 0000644 00000000000 15025013616 0017471 0 ustar 00 vendored/boto/tests/unit/beanstalk/test_exception.py 0000644 00000004045 15025013616 0017013 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. # All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. from boto.beanstalk.exception import simple from tests.compat import unittest class FakeError(object): def __init__(self, code, status, reason, body): self.code = code self.status = status self.reason = reason self.body = body class TestExceptions(unittest.TestCase): def test_exception_class_names(self): # Create exception from class name error = FakeError('TooManyApplications', 400, 'foo', 'bar') exception = simple(error) self.assertEqual(exception.__class__.__name__, 'TooManyApplications') # Create exception from class name + 'Exception' as seen from the # live service today error = FakeError('TooManyApplicationsException', 400, 'foo', 'bar') exception = simple(error) self.assertEqual(exception.__class__.__name__, 'TooManyApplications') # Make sure message body is present self.assertEqual(exception.message, 'bar') vendored/boto/tests/unit/beanstalk/test_layer1.py 0000644 00000015473 15025013616 0016221 0 ustar 00 #!/usr/bin/env python import json from tests.unit import AWSMockServiceTestCase from boto.beanstalk.layer1 import Layer1 # These tests are just checking the basic structure of # the Elastic Beanstalk code, by picking a few calls # and verifying we get the expected results with mocked # responses. The integration tests actually verify the # API calls interact with the service correctly. class TestListAvailableSolutionStacks(AWSMockServiceTestCase): connection_class = Layer1 def default_body(self): return json.dumps( {u'ListAvailableSolutionStacksResponse': {u'ListAvailableSolutionStacksResult': {u'SolutionStackDetails': [ {u'PermittedFileTypes': [u'war', u'zip'], u'SolutionStackName': u'32bit Amazon Linux running Tomcat 7'}, {u'PermittedFileTypes': [u'zip'], u'SolutionStackName': u'32bit Amazon Linux running PHP 5.3'}], u'SolutionStacks': [u'32bit Amazon Linux running Tomcat 7', u'32bit Amazon Linux running PHP 5.3']}, u'ResponseMetadata': {u'RequestId': u'request_id'}}}).encode('utf-8') def test_list_available_solution_stacks(self): self.set_http_response(status_code=200) api_response = self.service_connection.list_available_solution_stacks() stack_details = api_response['ListAvailableSolutionStacksResponse']\ ['ListAvailableSolutionStacksResult']\ ['SolutionStackDetails'] solution_stacks = api_response['ListAvailableSolutionStacksResponse']\ ['ListAvailableSolutionStacksResult']\ ['SolutionStacks'] self.assertEqual(solution_stacks, [u'32bit Amazon Linux running Tomcat 7', u'32bit Amazon Linux running PHP 5.3']) # These are the parameters that are actually sent to the CloudFormation # service. self.assert_request_parameters({ 'Action': 'ListAvailableSolutionStacks', 'ContentType': 'JSON', 'Version': '2010-12-01', }) class TestCreateApplicationVersion(AWSMockServiceTestCase): connection_class = Layer1 def default_body(self): return json.dumps({ 'CreateApplicationVersionResponse': {u'CreateApplicationVersionResult': {u'ApplicationVersion': {u'ApplicationName': u'application1', u'DateCreated': 1343067094.342, u'DateUpdated': 1343067094.342, u'Description': None, u'SourceBundle': {u'S3Bucket': u'elasticbeanstalk-us-east-1', u'S3Key': u'resources/elasticbeanstalk-sampleapp.war'}, u'VersionLabel': u'version1'}}}}).encode('utf-8') def test_create_application_version(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_application_version( 'application1', 'version1', s3_bucket='mybucket', s3_key='mykey', auto_create_application=True) app_version = api_response['CreateApplicationVersionResponse']\ ['CreateApplicationVersionResult']\ ['ApplicationVersion'] self.assert_request_parameters({ 'Action': 'CreateApplicationVersion', 'ContentType': 'JSON', 'Version': '2010-12-01', 'ApplicationName': 'application1', 'AutoCreateApplication': 'true', 'SourceBundle.S3Bucket': 'mybucket', 'SourceBundle.S3Key': 'mykey', 'VersionLabel': 'version1', }) self.assertEqual(app_version['ApplicationName'], 'application1') self.assertEqual(app_version['VersionLabel'], 'version1') class TestCreateEnvironment(AWSMockServiceTestCase): connection_class = Layer1 def default_body(self): return json.dumps({}).encode('utf-8') def test_create_environment(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_environment( 'application1', 'environment1', 'version1', '32bit Amazon Linux running Tomcat 7', option_settings=[ ('aws:autoscaling:launchconfiguration', 'Ec2KeyName', 'mykeypair'), ('aws:elasticbeanstalk:application:environment', 'ENVVAR', 'VALUE1')]) self.assert_request_parameters({ 'Action': 'CreateEnvironment', 'ApplicationName': 'application1', 'EnvironmentName': 'environment1', 'TemplateName': '32bit Amazon Linux running Tomcat 7', 'ContentType': 'JSON', 'Version': '2010-12-01', 'VersionLabel': 'version1', 'OptionSettings.member.1.Namespace': 'aws:autoscaling:launchconfiguration', 'OptionSettings.member.1.OptionName': 'Ec2KeyName', 'OptionSettings.member.1.Value': 'mykeypair', 'OptionSettings.member.2.Namespace': 'aws:elasticbeanstalk:application:environment', 'OptionSettings.member.2.OptionName': 'ENVVAR', 'OptionSettings.member.2.Value': 'VALUE1', }) def test_create_environment_with_tier(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_environment( 'application1', 'environment1', 'version1', '32bit Amazon Linux running Tomcat 7', option_settings=[ ('aws:autoscaling:launchconfiguration', 'Ec2KeyName', 'mykeypair'), ('aws:elasticbeanstalk:application:environment', 'ENVVAR', 'VALUE1')], tier_name='Worker', tier_type='SQS/HTTP', tier_version='1.0') self.assert_request_parameters({ 'Action': 'CreateEnvironment', 'ApplicationName': 'application1', 'EnvironmentName': 'environment1', 'TemplateName': '32bit Amazon Linux running Tomcat 7', 'ContentType': 'JSON', 'Version': '2010-12-01', 'VersionLabel': 'version1', 'OptionSettings.member.1.Namespace': 'aws:autoscaling:launchconfiguration', 'OptionSettings.member.1.OptionName': 'Ec2KeyName', 'OptionSettings.member.1.Value': 'mykeypair', 'OptionSettings.member.2.Namespace': 'aws:elasticbeanstalk:application:environment', 'OptionSettings.member.2.OptionName': 'ENVVAR', 'OptionSettings.member.2.Value': 'VALUE1', 'Tier.Name': 'Worker', 'Tier.Type': 'SQS/HTTP', 'Tier.Version': '1.0', }) vendored/boto/tests/unit/beanstalk/__init__.py 0000644 00000000000 15025013616 0015500 0 ustar 00 vendored/boto/tests/unit/iam/test_policy.py 0000644 00000060543 15025013616 0015123 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2015 Shaun Brady. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.compat import json from boto.iam.connection import IAMConnection from tests.unit import AWSMockServiceTestCase class TestCreatePolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <CreatePolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <CreatePolicyResult> <Policy> <PolicyName>S3-read-only-example-bucket</PolicyName> <DefaultVersionId>v1</DefaultVersionId> <PolicyId>AGPACKCEVSQ6C2EXAMPLE</PolicyId> <Path>/</Path> <Arn>arn:aws:iam::123456789012:policy/S3-read-only-example-bucket</Arn> <AttachmentCount>0</AttachmentCount> <CreateDate>2014-09-15T17:36:14.673Z</CreateDate> <UpdateDate>2014-09-15T17:36:14.673Z</UpdateDate> </Policy> </CreatePolicyResult> <ResponseMetadata> <RequestId>ca64c9e1-3cfe-11e4-bfad-8d1c6EXAMPLE</RequestId> </ResponseMetadata> </CreatePolicyResponse> """ def test_create_policy(self): self.set_http_response(status_code=200) policy_doc = """ { "Version": "2012-10-17", "Statement": [ { "Sid": "Stmt1430948004000", "Effect": "Deny", "Action": [ "s3:*" ], "Resource": [ "*" ] } ] } """ response = self.service_connection.create_policy( 'S3-read-only-example-bucket', policy_doc) self.assert_request_parameters( {'Action': 'CreatePolicy', 'PolicyDocument': policy_doc, 'Path': '/', 'PolicyName': 'S3-read-only-example-bucket'}, ignore_params_values=['Version']) self.assertEqual(response['create_policy_response'] ['create_policy_result'] ['policy'] ['policy_name'], 'S3-read-only-example-bucket') class TestCreatePolicyVersion(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <CreatePolicyVersionResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <CreatePolicyVersionResult> <PolicyVersion> <IsDefaultVersion>true</IsDefaultVersion> <VersionId>v2</VersionId> <CreateDate>2014-09-15T19:58:59.430Z</CreateDate> </PolicyVersion> </CreatePolicyVersionResult> <ResponseMetadata> <RequestId>bb551b92-3d12-11e4-bfad-8d1c6EXAMPLE</RequestId> </ResponseMetadata> </CreatePolicyVersionResponse> """ def test_create_policy_version(self): self.set_http_response(status_code=200) policy_doc = """ { "Version": "2012-10-17", "Statement": [ { "Sid": "Stmt1430948004000", "Effect": "Deny", "Action": [ "s3:*" ], "Resource": [ "*" ] } ] } """ response = self.service_connection.create_policy_version( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', policy_doc, set_as_default=True) self.assert_request_parameters( {'Action': 'CreatePolicyVersion', 'PolicyDocument': policy_doc, 'SetAsDefault': 'true', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket'}, ignore_params_values=['Version']) self.assertEqual(response['create_policy_version_response'] ['create_policy_version_result'] ['policy_version'] ['is_default_version'], 'true') class TestDeletePolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <DeletePolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ResponseMetadata> <RequestId>4706281b-3d19-11e4-a4a0-cffb9EXAMPLE</RequestId> </ResponseMetadata> </DeletePolicyResponse> """ def test_delete_policy(self): self.set_http_response(status_code=200) response = self.service_connection.delete_policy( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket') self.assert_request_parameters( {'Action': 'DeletePolicy', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket'}, ignore_params_values=['Version']) self.assertEqual('request_id' in response['delete_policy_response'] ['response_metadata'], True) class TestDeletePolicyVersion(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <DeletePolicyVersionResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ResponseMetadata> <RequestId>268e1556-3d19-11e4-a4a0-cffb9EXAMPLE</RequestId> </ResponseMetadata> </DeletePolicyVersionResponse> """ def test_delete_policy_version(self): self.set_http_response(status_code=200) response = self.service_connection.delete_policy_version( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'v1') self.assert_request_parameters( {'Action': 'DeletePolicyVersion', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'VersionId': 'v1'}, ignore_params_values=['Version']) self.assertEqual('request_id' in response['delete_policy_version_response'] ['response_metadata'], True) class TestGetPolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <GetPolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <GetPolicyResult> <Policy> <PolicyName>S3-read-only-example-bucket</PolicyName> <DefaultVersionId>v1</DefaultVersionId> <PolicyId>AGPACKCEVSQ6C2EXAMPLE</PolicyId> <Path>/</Path> <Arn>arn:aws:iam::123456789012:policy/S3-read-only-example-bucket</Arn> <AttachmentCount>9</AttachmentCount> <CreateDate>2014-09-15T17:36:14Z</CreateDate> <UpdateDate>2014-09-15T20:31:47Z</UpdateDate> <Description>My Awesome Policy</Description> </Policy> </GetPolicyResult> <ResponseMetadata> <RequestId>684f0917-3d22-11e4-a4a0-cffb9EXAMPLE</RequestId> </ResponseMetadata> </GetPolicyResponse> """ def test_get_policy(self): self.set_http_response(status_code=200) response = self.service_connection.get_policy( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket') self.assert_request_parameters( {'Action': 'GetPolicy', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket'}, ignore_params_values=['Version']) self.assertEqual(response['get_policy_response'] ['get_policy_result'] ['policy'] ['arn'], 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket') self.assertEqual(response['get_policy_response'] ['get_policy_result'] ['policy'] ['description'], 'My Awesome Policy') class TestGetPolicyVersion(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <GetPolicyVersionResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <GetPolicyVersionResult> <PolicyVersion> <Document> {"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:Get*","s3:List*"], "Resource":["arn:aws:s3:::EXAMPLE-BUCKET","arn:aws:s3:::EXAMPLE-BUCKET/*"]}]} </Document> <IsDefaultVersion>true</IsDefaultVersion> <VersionId>v1</VersionId> <CreateDate>2014-09-15T20:31:47Z</CreateDate> </PolicyVersion> </GetPolicyVersionResult> <ResponseMetadata> <RequestId>d472f28e-3d23-11e4-a4a0-cffb9EXAMPLE</RequestId> </ResponseMetadata> </GetPolicyVersionResponse> """ def test_get_policy_version(self): self.set_http_response(status_code=200) response = self.service_connection.get_policy_version( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'v1') self.assert_request_parameters( {'Action': 'GetPolicyVersion', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'VersionId': 'v1'}, ignore_params_values=['Version']) self.assertEqual(response['get_policy_version_response'] ['get_policy_version_result'] ['policy_version'] ['version_id'], 'v1') class TestListPolicies(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <ListPoliciesResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ListPoliciesResult> <IsTruncated>true</IsTruncated> <Marker>EXAMPLEkakv9BCuUNFDtxWSyfzetYwEx2ADc8dnzfvERF5S6YMvXKx41t6gCl/eeaCX3Jo94/bKqezEAg8TEVS99EKFLxm3jtbpl25FDWEXAMPLE </Marker> <Policies> <member> <PolicyName>ExamplePolicy</PolicyName> <DefaultVersionId>v1</DefaultVersionId> <PolicyId>AGPACKCEVSQ6C2EXAMPLE</PolicyId> <Path>/</Path> <Arn>arn:aws:iam::123456789012:policy/ExamplePolicy</Arn> <AttachmentCount>2</AttachmentCount> <CreateDate>2014-09-15T17:36:14Z</CreateDate> <UpdateDate>2014-09-15T20:31:47Z</UpdateDate> </member> <member> <PolicyName>PowerUserAccess</PolicyName> <DefaultVersionId>v1</DefaultVersionId> <PolicyId>AGPACKCEVSQ6C2EXAMPLE</PolicyId> <Path>/</Path> <Arn>arn:aws:iam::aws:policy/PowerUserAccess</Arn> <AttachmentCount>0</AttachmentCount> <CreateDate>2014-08-21T20:25:01Z</CreateDate> <UpdateDate>2014-08-21T20:25:01Z</UpdateDate> </member> <member> <PolicyName>AdministratorAccess</PolicyName> <DefaultVersionId>v1</DefaultVersionId> <PolicyId>AGPACKCEVSQ6C2EXAMPLE</PolicyId> <Path>/</Path> <Arn>arn:aws:iam::aws:policy/AdministratorAccess</Arn> <AttachmentCount>1</AttachmentCount> <CreateDate>2014-08-21T20:11:25Z</CreateDate> <UpdateDate>2014-08-21T20:11:25Z</UpdateDate> </member> <member> <PolicyName>ReadOnlyAccess</PolicyName> <DefaultVersionId>v1</DefaultVersionId> <PolicyId>AGPACKCEVSQ6C2EXAMPLE</PolicyId> <Path>/</Path> <Arn>arn:aws:iam::aws:policy/ReadOnlyAccess</Arn> <AttachmentCount>6</AttachmentCount> <CreateDate>2014-08-21T20:31:44Z</CreateDate> <UpdateDate>2014-08-21T20:31:44Z</UpdateDate> </member> </Policies> </ListPoliciesResult> <ResponseMetadata> <RequestId>6207e832-3eb7-11e4-9d0d-6f969EXAMPLE</RequestId> </ResponseMetadata> </ListPoliciesResponse> """ def test_list_policies(self): self.set_http_response(status_code=200) response = self.service_connection.list_policies( max_items=4) self.assert_request_parameters( {'Action': 'ListPolicies', 'MaxItems': 4}, ignore_params_values=['Version']) self.assertEqual(len(response['list_policies_response'] ['list_policies_result'] ['policies']), 4) self.assertEqual(response['list_policies_response'] ['list_policies_result'] ['is_truncated'], 'true') class TestListPolicyVersions(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <ListPolicyVersionsResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ListPolicyVersionsResult> <Versions> <member> <IsDefaultVersion>false</IsDefaultVersion> <VersionId>v3</VersionId> <CreateDate>2014-09-17T22:32:43Z</CreateDate> </member> <member> <IsDefaultVersion>true</IsDefaultVersion> <VersionId>v2</VersionId> <CreateDate>2014-09-15T20:31:47Z</CreateDate> </member> <member> <IsDefaultVersion>false</IsDefaultVersion> <VersionId>v1</VersionId> <CreateDate>2014-09-15T17:36:14Z</CreateDate> </member> </Versions> <IsTruncated>false</IsTruncated> </ListPolicyVersionsResult> <ResponseMetadata> <RequestId>a31d1a86-3eba-11e4-9d0d-6f969EXAMPLE</RequestId> </ResponseMetadata> </ListPolicyVersionsResponse> """ def test_list_policy_versions(self): self.set_http_response(status_code=200) response = self.service_connection.list_policy_versions( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', max_items=3) self.assert_request_parameters( {'Action': 'ListPolicyVersions', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'MaxItems': 3}, ignore_params_values=['Version']) self.assertEqual(len(response['list_policy_versions_response'] ['list_policy_versions_result'] ['versions']), 3) class TestSetDefaultPolicyVersion(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <SetDefaultPolicyVersionResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ResponseMetadata> <RequestId>35f241af-3ebc-11e4-9d0d-6f969EXAMPLE</RequestId> </ResponseMetadata> </SetDefaultPolicyVersionResponse> """ def test_set_default_policy_version(self): self.set_http_response(status_code=200) response = self.service_connection.set_default_policy_version( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'v1') self.assert_request_parameters( {'Action': 'SetDefaultPolicyVersion', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'VersionId': 'v1'}, ignore_params_values=['Version']) self.assertEqual('request_id' in response['set_default_policy_version_response'] ['response_metadata'], True) class TestListEntitiesForPolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <ListEntitiesForPolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ListEntitiesForPolicyResult> <PolicyRoles> <member> <RoleName>DevRole</RoleName> </member> </PolicyRoles> <PolicyGroups> <member> <GroupName>Dev</GroupName> </member> </PolicyGroups> <IsTruncated>false</IsTruncated> <PolicyUsers> <member> <UserName>Alice</UserName> </member> <member> <UserName>Bob</UserName> </member> </PolicyUsers> </ListEntitiesForPolicyResult> <ResponseMetadata> <RequestId>eb358e22-9d1f-11e4-93eb-190ecEXAMPLE</RequestId> </ResponseMetadata> </ListEntitiesForPolicyResponse> """ def test_list_entities_for_policy(self): self.set_http_response(status_code=200) response = self.service_connection.list_entities_for_policy( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket') self.assert_request_parameters( {'Action': 'ListEntitiesForPolicy', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket'}, ignore_params_values=['Version']) self.assertEqual(len(response['list_entities_for_policy_response'] ['list_entities_for_policy_result'] ['policy_roles']), 1) self.assertEqual(len(response['list_entities_for_policy_response'] ['list_entities_for_policy_result'] ['policy_groups']), 1) self.assertEqual(len(response['list_entities_for_policy_response'] ['list_entities_for_policy_result'] ['policy_users']), 2) self.assertEqual({'user_name': 'Alice'} in response['list_entities_for_policy_response'] ['list_entities_for_policy_result'] ['policy_users'], True) class TestAttachGroupPolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <AttachGroupPolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ResponseMetadata> <RequestId>f8a7b7b9-3d01-11e4-bfad-8d1c6EXAMPLE</RequestId> </ResponseMetadata> </AttachGroupPolicyResponse> """ def test_attach_group_policy(self): self.set_http_response(status_code=200) response = self.service_connection.attach_group_policy( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'Dev') self.assert_request_parameters( {'Action': 'AttachGroupPolicy', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'GroupName': 'Dev'}, ignore_params_values=['Version']) self.assertEqual('request_id' in response['attach_group_policy_response'] ['response_metadata'], True) class TestAttachRolePolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <AttachRolePolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ResponseMetadata> <RequestId>37a87673-3d07-11e4-bfad-8d1c6EXAMPLE</RequestId> </ResponseMetadata> </AttachRolePolicyResponse> """ def test_attach_role_policy(self): self.set_http_response(status_code=200) response = self.service_connection.attach_role_policy( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'DevRole') self.assert_request_parameters( {'Action': 'AttachRolePolicy', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'RoleName': 'DevRole'}, ignore_params_values=['Version']) self.assertEqual('request_id' in response['attach_role_policy_response'] ['response_metadata'], True) class TestAttachUserPolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <AttachUserPolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ResponseMetadata> <RequestId>ed7e72d3-3d07-11e4-bfad-8d1c6EXAMPLE</RequestId> </ResponseMetadata> </AttachUserPolicyResponse> """ def test_attach_user_policy(self): self.set_http_response(status_code=200) response = self.service_connection.attach_user_policy( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'Alice') self.assert_request_parameters( {'Action': 'AttachUserPolicy', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'UserName': 'Alice'}, ignore_params_values=['Version']) self.assertEqual('request_id' in response['attach_user_policy_response'] ['response_metadata'], True) class TestDetachGroupPolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <DetachGroupPolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ResponseMetadata> <RequestId>d4faa7aa-3d1d-11e4-a4a0-cffb9EXAMPLE</RequestId> </ResponseMetadata> </DetachGroupPolicyResponse> """ def test_detach_group_policy(self): self.set_http_response(status_code=200) response = self.service_connection.detach_group_policy( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'Dev') self.assert_request_parameters( {'Action': 'DetachGroupPolicy', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'GroupName': 'Dev'}, ignore_params_values=['Version']) self.assertEqual('request_id' in response['detach_group_policy_response'] ['response_metadata'], True) class TestDetachRolePolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <DetachRolePolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ResponseMetadata> <RequestId>4c80ccf4-3d1e-11e4-a4a0-cffb9EXAMPLE</RequestId> </ResponseMetadata> </DetachRolePolicyResponse> """ def test_detach_role_policy(self): self.set_http_response(status_code=200) response = self.service_connection.detach_role_policy( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'DevRole') self.assert_request_parameters( {'Action': 'DetachRolePolicy', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'RoleName': 'DevRole'}, ignore_params_values=['Version']) self.assertEqual('request_id' in response['detach_role_policy_response'] ['response_metadata'], True) class TestDetachUserPolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <DetachUserPolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ResponseMetadata> <RequestId>85ba31fa-3d1f-11e4-a4a0-cffb9EXAMPLE</RequestId> </ResponseMetadata> </DetachUserPolicyResponse> """ def test_detach_user_policy(self): self.set_http_response(status_code=200) response = self.service_connection.detach_user_policy( 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'Alice') self.assert_request_parameters( {'Action': 'DetachUserPolicy', 'PolicyArn': 'arn:aws:iam::123456789012:policy/S3-read-only-example-bucket', 'UserName': 'Alice'}, ignore_params_values=['Version']) self.assertEqual('request_id' in response['detach_user_policy_response'] ['response_metadata'], True) vendored/boto/tests/unit/iam/test_connection.py 0000644 00000045531 15025013616 0015763 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from base64 import b64decode from boto.compat import json from boto.iam.connection import IAMConnection from tests.unit import AWSMockServiceTestCase class TestCreateSamlProvider(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <CreateSAMLProviderResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <CreateSAMLProviderResult> <SAMLProviderArn>arn</SAMLProviderArn> </CreateSAMLProviderResult> <ResponseMetadata> <RequestId>29f47818-99f5-11e1-a4c3-27EXAMPLE804</RequestId> </ResponseMetadata> </CreateSAMLProviderResponse> """ def test_create_saml_provider(self): self.set_http_response(status_code=200) response = self.service_connection.create_saml_provider('document', 'name') self.assert_request_parameters( {'Action': 'CreateSAMLProvider', 'SAMLMetadataDocument': 'document', 'Name': 'name'}, ignore_params_values=['Version']) self.assertEqual(response['create_saml_provider_response'] ['create_saml_provider_result'] ['saml_provider_arn'], 'arn') class TestListSamlProviders(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <ListSAMLProvidersResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ListSAMLProvidersResult> <SAMLProviderList> <member> <Arn>arn:aws:iam::123456789012:instance-profile/application_abc/component_xyz/Database</Arn> <ValidUntil>2032-05-09T16:27:11Z</ValidUntil> <CreateDate>2012-05-09T16:27:03Z</CreateDate> </member> <member> <Arn>arn:aws:iam::123456789012:instance-profile/application_abc/component_xyz/Webserver</Arn> <ValidUntil>2015-03-11T13:11:02Z</ValidUntil> <CreateDate>2012-05-09T16:27:11Z</CreateDate> </member> </SAMLProviderList> </ListSAMLProvidersResult> <ResponseMetadata> <RequestId>fd74fa8d-99f3-11e1-a4c3-27EXAMPLE804</RequestId> </ResponseMetadata> </ListSAMLProvidersResponse> """ def test_list_saml_providers(self): self.set_http_response(status_code=200) response = self.service_connection.list_saml_providers() self.assert_request_parameters( {'Action': 'ListSAMLProviders'}, ignore_params_values=['Version']) self.assertEqual(response.saml_provider_list, [ {'arn': 'arn:aws:iam::123456789012:instance-profile/application_abc/component_xyz/Database', 'valid_until': '2032-05-09T16:27:11Z', 'create_date': '2012-05-09T16:27:03Z'}, {'arn': 'arn:aws:iam::123456789012:instance-profile/application_abc/component_xyz/Webserver', 'valid_until': '2015-03-11T13:11:02Z', 'create_date': '2012-05-09T16:27:11Z'}]) class TestGetSamlProvider(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <GetSAMLProviderResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <GetSAMLProviderResult> <CreateDate>2012-05-09T16:27:11Z</CreateDate> <ValidUntil>2015-12-31T211:59:59Z</ValidUntil> <SAMLMetadataDocument>Pd9fexDssTkRgGNqs...DxptfEs==</SAMLMetadataDocument> </GetSAMLProviderResult> <ResponseMetadata> <RequestId>29f47818-99f5-11e1-a4c3-27EXAMPLE804</RequestId> </ResponseMetadata> </GetSAMLProviderResponse> """ def test_get_saml_provider(self): self.set_http_response(status_code=200) self.service_connection.get_saml_provider('arn') self.assert_request_parameters( { 'Action': 'GetSAMLProvider', 'SAMLProviderArn': 'arn' }, ignore_params_values=['Version']) class TestUpdateSamlProvider(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <UpdateSAMLProviderResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <UpdateSAMLProviderResult> <SAMLProviderArn>arn:aws:iam::123456789012:saml-metadata/MyUniversity</SAMLProviderArn> </UpdateSAMLProviderResult> <ResponseMetadata> <RequestId>29f47818-99f5-11e1-a4c3-27EXAMPLE804</RequestId> </ResponseMetadata> </UpdateSAMLProviderResponse> """ def test_update_saml_provider(self): self.set_http_response(status_code=200) self.service_connection.update_saml_provider('arn', 'doc') self.assert_request_parameters( { 'Action': 'UpdateSAMLProvider', 'SAMLMetadataDocument': 'doc', 'SAMLProviderArn': 'arn' }, ignore_params_values=['Version']) class TestDeleteSamlProvider(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return "" def test_delete_saml_provider(self): self.set_http_response(status_code=200) self.service_connection.delete_saml_provider('arn') self.assert_request_parameters( { 'Action': 'DeleteSAMLProvider', 'SAMLProviderArn': 'arn' }, ignore_params_values=['Version']) class TestCreateRole(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <CreateRoleResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <CreateRoleResult> <Role> <Path>/application_abc/component_xyz/</Path> <Arn>arn:aws:iam::123456789012:role/application_abc/component_xyz/S3Access</Arn> <RoleName>S3Access</RoleName> <AssumeRolePolicyDocument>{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":["ec2.amazonaws.com"]},"Action":["sts:AssumeRole"]}]}</AssumeRolePolicyDocument> <CreateDate>2012-05-08T23:34:01.495Z</CreateDate> <RoleId>AROADBQP57FF2AEXAMPLE</RoleId> </Role> </CreateRoleResult> <ResponseMetadata> <RequestId>4a93ceee-9966-11e1-b624-b1aEXAMPLE7c</RequestId> </ResponseMetadata> </CreateRoleResponse> """ def test_create_role_default(self): self.set_http_response(status_code=200) self.service_connection.create_role('a_name') self.assert_request_parameters( {'Action': 'CreateRole', 'RoleName': 'a_name'}, ignore_params_values=['Version', 'AssumeRolePolicyDocument']) self.assertDictEqual(json.loads(self.actual_request.params["AssumeRolePolicyDocument"]), {"Statement": [{"Action": ["sts:AssumeRole"], "Effect": "Allow", "Principal": {"Service": ["ec2.amazonaws.com"]}}]}) def test_create_role_default_cn_north(self): self.set_http_response(status_code=200) self.service_connection.host = 'iam.cn-north-1.amazonaws.com.cn' self.service_connection.create_role('a_name') self.assert_request_parameters( {'Action': 'CreateRole', 'RoleName': 'a_name'}, ignore_params_values=['Version', 'AssumeRolePolicyDocument']) self.assertDictEqual(json.loads(self.actual_request.params["AssumeRolePolicyDocument"]), {"Statement": [{"Action": ["sts:AssumeRole"], "Effect": "Allow", "Principal": {"Service": ["ec2.amazonaws.com.cn"]}}]}) def test_create_role_string_policy(self): self.set_http_response(status_code=200) self.service_connection.create_role( 'a_name', # Historical usage. assume_role_policy_document='{"hello": "policy"}' ) self.assert_request_parameters( {'Action': 'CreateRole', 'AssumeRolePolicyDocument': '{"hello": "policy"}', 'RoleName': 'a_name'}, ignore_params_values=['Version']) def test_create_role_data_policy(self): self.set_http_response(status_code=200) self.service_connection.create_role( 'a_name', # With plain data, we should dump it for them. assume_role_policy_document={"hello": "policy"} ) self.assert_request_parameters( {'Action': 'CreateRole', 'AssumeRolePolicyDocument': '{"hello": "policy"}', 'RoleName': 'a_name'}, ignore_params_values=['Version']) class TestGetSigninURL(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <ListAccountAliasesResponse> <ListAccountAliasesResult> <IsTruncated>false</IsTruncated> <AccountAliases> <member>foocorporation</member> <member>anotherunused</member> </AccountAliases> </ListAccountAliasesResult> <ResponseMetadata> <RequestId>c5a076e9-f1b0-11df-8fbe-45274EXAMPLE</RequestId> </ResponseMetadata> </ListAccountAliasesResponse> """ def test_get_signin_url_default(self): self.set_http_response(status_code=200) url = self.service_connection.get_signin_url() self.assertEqual( url, 'https://foocorporation.signin.aws.amazon.com/console/ec2' ) def test_get_signin_url_s3(self): self.set_http_response(status_code=200) url = self.service_connection.get_signin_url(service='s3') self.assertEqual( url, 'https://foocorporation.signin.aws.amazon.com/console/s3' ) def test_get_signin_url_cn_north(self): self.set_http_response(status_code=200) self.service_connection.host = 'iam.cn-north-1.amazonaws.com.cn' url = self.service_connection.get_signin_url() self.assertEqual( url, 'https://foocorporation.signin.amazonaws.cn/console/ec2' ) class TestGetSigninURLNoAliases(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <ListAccountAliasesResponse> <ListAccountAliasesResult> <IsTruncated>false</IsTruncated> <AccountAliases></AccountAliases> </ListAccountAliasesResult> <ResponseMetadata> <RequestId>c5a076e9-f1b0-11df-8fbe-45274EXAMPLE</RequestId> </ResponseMetadata> </ListAccountAliasesResponse> """ def test_get_signin_url_no_aliases(self): self.set_http_response(status_code=200) with self.assertRaises(Exception): self.service_connection.get_signin_url() class TestGenerateCredentialReport(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <GenerateCredentialReportResponse> <GenerateCredentialReportResult> <State>COMPLETE</State> </GenerateCredentialReportResult> <ResponseMetadata> <RequestId>b62e22a3-0da1-11e4-ba55-0990EXAMPLE</RequestId> </ResponseMetadata> </GenerateCredentialReportResponse> """ def test_generate_credential_report(self): self.set_http_response(status_code=200) response = self.service_connection.generate_credential_report() self.assertEqual(response['generate_credential_report_response'] ['generate_credential_report_result'] ['state'], 'COMPLETE') class TestGetCredentialReport(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <GetCredentialReportResponse> <ResponseMetadata> <RequestId>99e60e9a-0db5-11e4-94d4-b764EXAMPLE</RequestId> </ResponseMetadata> <GetCredentialReportResult> <Content>RXhhbXBsZQ==</Content> <ReportFormat>text/csv</ReportFormat> <GeneratedTime>2014-07-17T11:09:11Z</GeneratedTime> </GetCredentialReportResult> </GetCredentialReportResponse> """ def test_get_credential_report(self): self.set_http_response(status_code=200) response = self.service_connection.get_credential_report() b64decode(response['get_credential_report_response'] ['get_credential_report_result'] ['content']) class TestCreateVirtualMFADevice(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <CreateVirtualMFADeviceResponse> <CreateVirtualMFADeviceResult> <VirtualMFADevice> <SerialNumber>arn:aws:iam::123456789012:mfa/ExampleName</SerialNumber> <Base32StringSeed>2K5K5XTLA7GGE75TQLYEXAMPLEEXAMPLEEXAMPLECHDFW4KJYZ6 UFQ75LL7COCYKM</Base32StringSeed> <QRCodePNG>89504E470D0A1A0AASDFAHSDFKJKLJFKALSDFJASDF</QRCodePNG> </VirtualMFADevice> </CreateVirtualMFADeviceResult> <ResponseMetadata> <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId> </ResponseMetadata> </CreateVirtualMFADeviceResponse> """ def test_create_virtual_mfa_device(self): self.set_http_response(status_code=200) response = self.service_connection.create_virtual_mfa_device('/', 'ExampleName') self.assert_request_parameters( {'Path': '/', 'VirtualMFADeviceName': 'ExampleName', 'Action': 'CreateVirtualMFADevice'}, ignore_params_values=['Version']) self.assertEqual(response['create_virtual_mfa_device_response'] ['create_virtual_mfa_device_result'] ['virtual_mfa_device'] ['serial_number'], 'arn:aws:iam::123456789012:mfa/ExampleName') class TestGetAccountPasswordPolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <GetAccountPasswordPolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <GetAccountPasswordPolicyResult> <PasswordPolicy> <AllowUsersToChangePassword>true</AllowUsersToChangePassword> <RequireUppercaseCharacters>true</RequireUppercaseCharacters> <RequireSymbols>true</RequireSymbols> <ExpirePasswords>false</ExpirePasswords> <PasswordReusePrevention>12</PasswordReusePrevention> <RequireLowercaseCharacters>true</RequireLowercaseCharacters> <MaxPasswordAge>90</MaxPasswordAge> <HardExpiry>false</HardExpiry> <RequireNumbers>true</RequireNumbers> <MinimumPasswordLength>12</MinimumPasswordLength> </PasswordPolicy> </GetAccountPasswordPolicyResult> <ResponseMetadata> <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId> </ResponseMetadata> </GetAccountPasswordPolicyResponse> """ def test_get_account_password_policy(self): self.set_http_response(status_code=200) response = self.service_connection.get_account_password_policy() self.assert_request_parameters( { 'Action': 'GetAccountPasswordPolicy', }, ignore_params_values=['Version']) self.assertEqual(response['get_account_password_policy_response'] ['get_account_password_policy_result']['password_policy'] ['minimum_password_length'], '12') class TestUpdateAccountPasswordPolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <UpdateAccountPasswordPolicyResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/"> <ResponseMetadata> <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId> </ResponseMetadata> </UpdateAccountPasswordPolicyResponse> """ def test_update_account_password_policy(self): self.set_http_response(status_code=200) response = self.service_connection.update_account_password_policy(minimum_password_length=88) self.assert_request_parameters( { 'Action': 'UpdateAccountPasswordPolicy', 'MinimumPasswordLength': 88 }, ignore_params_values=['Version']) class TestDeleteAccountPasswordPolicy(AWSMockServiceTestCase): connection_class = IAMConnection def default_body(self): return b""" <DeleteAccountPasswordPolicyResponse> <ResponseMetadata> <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId> </ResponseMetadata> </DeleteAccountPasswordPolicyResponse> """ def test_delete_account_password_policy(self): self.set_http_response(status_code=200) response = self.service_connection.delete_account_password_policy() self.assert_request_parameters( { 'Action': 'DeleteAccountPasswordPolicy' }, ignore_params_values=['Version']) vendored/boto/tests/unit/iam/__init__.py 0000644 00000000000 15025013616 0014302 0 ustar 00 vendored/boto/tests/unit/test_endpoints.py 0000644 00000032667 15025013616 0015067 0 ustar 00 # Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at # # http://aws.amazon.com/apache2.0/ # # or in the "license" file accompanying this file. This file is # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import mock import os import json from nose.tools import assert_equal from tests.unit import unittest import boto from boto.endpoints import BotoEndpointResolver from boto.endpoints import StaticEndpointBuilder class BaseEndpointResolverTest(unittest.TestCase): def _endpoint_data(self): return { 'partitions': [ { 'partition': 'aws', 'dnsSuffix': 'amazonaws.com', 'regionRegex': r"^(us|eu)-\w+$", 'defaults': { 'hostname': '{service}.{region}.{dnsSuffix}' }, 'regions': { 'us-foo': {'regionName': 'a'}, 'us-bar': {'regionName': 'b'}, 'eu-baz': {'regionName': 'd'} }, 'services': { 'ec2': { 'endpoints': { 'us-foo': {}, 'us-bar': {}, 'eu-baz': {}, 'd': {} } }, 's3': { 'defaults': { 'sslCommonName': \ '{service}.{region}.{dnsSuffix}' }, 'endpoints': { 'us-foo': { 'sslCommonName': \ '{region}.{service}.{dnsSuffix}' }, 'us-bar': {}, 'eu-baz': {'hostname': 'foo'} } }, 'not-regionalized': { 'isRegionalized': False, 'partitionEndpoint': 'aws', 'endpoints': { 'aws': {'hostname': 'not-regionalized'}, 'us-foo': {}, 'eu-baz': {} } }, 'non-partition': { 'partitionEndpoint': 'aws', 'endpoints': { 'aws': {'hostname': 'host'}, 'us-foo': {} } }, 'merge': { 'defaults': { 'signatureVersions': ['v2'], 'protocols': ['http'] }, 'endpoints': { 'us-foo': {'signatureVersions': ['v4']}, 'us-bar': {'protocols': ['https']} } } } }, { 'partition': 'foo', 'dnsSuffix': 'foo.com', 'regionRegex': r"^(foo)-\w+$", 'defaults': { 'hostname': '{service}.{region}.{dnsSuffix}', 'protocols': ['http'], 'foo': 'bar' }, 'regions': { 'foo-1': {'regionName': '1'}, 'foo-2': {'regionName': '2'}, 'foo-3': {'regionName': '3'} }, 'services': { 'ec2': { 'endpoints': { 'foo-1': { 'foo': 'baz' }, 'foo-2': {}, 'foo-3': {} } } } } ] } class TestBotoEndpointResolver(BaseEndpointResolverTest): def test_get_all_available_regions(self): resolver = BotoEndpointResolver(self._endpoint_data()) regions = sorted(resolver.get_all_available_regions('ec2')) expected_regions = sorted([ 'us-bar', 'eu-baz', 'us-foo', 'foo-1', 'foo-2', 'foo-3']) self.assertEqual(regions, expected_regions) def test_get_all_regions_on_non_regional_service(self): resolver = BotoEndpointResolver(self._endpoint_data()) regions = sorted( resolver.get_all_available_regions('not-regionalized')) expected_regions = sorted(['us-foo', 'us-bar', 'eu-baz']) self.assertEqual(regions, expected_regions) def test_get_all_regions_with_renames(self): rename_map = {'ec3': 'ec2'} resolver = BotoEndpointResolver( endpoint_data=self._endpoint_data(), service_rename_map=rename_map ) regions = sorted(resolver.get_all_available_regions('ec2')) expected_regions = sorted([ 'us-bar', 'eu-baz', 'us-foo', 'foo-1', 'foo-2', 'foo-3']) self.assertEqual(regions, expected_regions) def test_resolve_hostname(self): resolver = BotoEndpointResolver(self._endpoint_data()) hostname = resolver.resolve_hostname('ec2', 'us-foo') expected_hostname = 'ec2.us-foo.amazonaws.com' self.assertEqual(hostname, expected_hostname) def test_resolve_hostname_with_rename(self): rename_map = {'ec3': 'ec2'} resolver = BotoEndpointResolver( endpoint_data=self._endpoint_data(), service_rename_map=rename_map ) hostname = resolver.resolve_hostname('ec3', 'us-foo') expected_hostname = 'ec2.us-foo.amazonaws.com' self.assertEqual(hostname, expected_hostname) def test_resolve_hostname_with_ssl_common_name(self): resolver = BotoEndpointResolver(self._endpoint_data()) hostname = resolver.resolve_hostname('s3', 'us-foo') expected_hostname = 'us-foo.s3.amazonaws.com' self.assertEqual(hostname, expected_hostname) def test_resolve_hostname_on_invalid_region_prefix(self): resolver = BotoEndpointResolver(self._endpoint_data()) hostname = resolver.resolve_hostname('s3', 'fake-west-1') self.assertIsNone(hostname) def test_get_available_services(self): resolver = BotoEndpointResolver(self._endpoint_data()) services = sorted(resolver.get_available_services()) expected_services = sorted([ 'ec2', 's3', 'not-regionalized', 'non-partition', 'merge']) self.assertEqual(services, expected_services) def test_get_available_services_with_renames(self): rename_map = {'ec3': 'ec2'} resolver = BotoEndpointResolver( endpoint_data=self._endpoint_data(), service_rename_map=rename_map ) services = sorted(resolver.get_available_services()) expected_services = sorted([ 'ec3', 's3', 'not-regionalized', 'non-partition', 'merge']) self.assertEqual(services, expected_services) class TestStaticEndpointBuilder(unittest.TestCase): def setUp(self): self.resolver = mock.Mock(spec=BotoEndpointResolver) self.builder = StaticEndpointBuilder(self.resolver) def test_build_single_service(self): regions = ['mars-west-1', 'moon-darkside-1'] self.resolver.get_all_available_regions.return_value = regions self.resolver.resolve_hostname.side_effect = [ 'fake-service.mars-west-1.amazonaws.com', 'fake-service.moon-darkside-1.amazonaws.com' ] endpoints = self.builder.build_static_endpoints(['fake-service']) expected_endpoints = {'fake-service': { 'mars-west-1': 'fake-service.mars-west-1.amazonaws.com', 'moon-darkside-1': 'fake-service.moon-darkside-1.amazonaws.com' }} self.assertEqual(endpoints, expected_endpoints) def test_build_multiple_services(self): regions = [['mars-west-1', 'moon-darkside-1'], ['mars-west-1']] self.resolver.get_all_available_regions.side_effect = regions self.resolver.resolve_hostname.side_effect = [ 'fake-service.mars-west-1.amazonaws.com', 'fake-service.moon-darkside-1.amazonaws.com', 'sample-service.mars-west-1.amazonaws.com' ] services = ['fake-service', 'sample-service'] endpoints = self.builder.build_static_endpoints(services) expected_endpoints = { 'fake-service': { 'mars-west-1': 'fake-service.mars-west-1.amazonaws.com', 'moon-darkside-1': 'fake-service.moon-darkside-1.amazonaws.com' }, 'sample-service': { 'mars-west-1': 'sample-service.mars-west-1.amazonaws.com' } } self.assertEqual(endpoints, expected_endpoints) def test_build_all_services(self): regions = [['mars-west-1', 'moon-darkside-1'], ['mars-west-1']] self.resolver.get_all_available_regions.side_effect = regions self.resolver.resolve_hostname.side_effect = [ 'fake-service.mars-west-1.amazonaws.com', 'fake-service.moon-darkside-1.amazonaws.com', 'sample-service.mars-west-1.amazonaws.com' ] # Set the list of available services on the resolver so it doesn't # have to be specified on the call to build_static_endpoints services = ['fake-service', 'sample-service'] self.resolver.get_available_services.return_value = services endpoints = self.builder.build_static_endpoints() expected_endpoints = { 'fake-service': { 'mars-west-1': 'fake-service.mars-west-1.amazonaws.com', 'moon-darkside-1': 'fake-service.moon-darkside-1.amazonaws.com' }, 'sample-service': { 'mars-west-1': 'sample-service.mars-west-1.amazonaws.com' } } self.assertEqual(endpoints, expected_endpoints) def test_backwards_compatibility(): # Tests backwards compatibility with the old endpoint generation tooling. # There are snapshots of the two endpoint formats at a single point in # time. Given the snapshot of the new endpoint file, this asserts that # we can generate the exact data in the snapshot of the old endpoints file. # A single 'test' is generated for each service. data_dir = os.path.join(os.path.dirname(__file__), 'data') old_endpoints_file = os.path.join(data_dir, 'old_endpoints.json') new_endpoints_file = os.path.join(data_dir, 'new_endpoints.json') with open(old_endpoints_file) as f: old_endpoints = json.load(f) with open(new_endpoints_file) as f: new_endpoints = json.load(f) resolver = BotoEndpointResolver(new_endpoints) builder = StaticEndpointBuilder(resolver) services = resolver.get_available_services() built = builder.build_static_endpoints(services) for service in services: old = old_endpoints[service] new = built[service] case = EndpointTestCase(service, old, new) yield case.run class EndpointTestCase(object): def __init__(self, service, old_endpoints, new_endpoints): self.service = service self.old_endpoints = old_endpoints self.new_endpoints = new_endpoints def run(self): assert_equal(self.old_endpoints, self.new_endpoints) def test_no_lost_endpoints(): # This makes sure that a bad sync doesn't cause us to loose any services data_dir = os.path.join(os.path.dirname(__file__), 'data') old_endpoints_file = os.path.join(data_dir, 'old_endpoints.json') with open(old_endpoints_file) as f: old_endpoints = json.load(f) with open(boto.ENDPOINTS_PATH) as f: new_endpoints = json.load(f) builder = StaticEndpointBuilder(BotoEndpointResolver(new_endpoints)) built = builder.build_static_endpoints() # Assert no services are lost for service, service_endpoints in old_endpoints.items(): new_service_endpoints = built.get(service, {}) for region, regional_endpoint in service_endpoints.items(): new_regional_endpoint = new_service_endpoints.get(region) case = EndpointPreservedTestCase( service, region, regional_endpoint, new_regional_endpoint) yield case.run class EndpointPreservedTestCase(object): def __init__(self, service_name, region_name, old_endpoint, new_endpoint): self.service_name = service_name self.region_name = region_name self.old_endpoint = old_endpoint self.new_endpoint = new_endpoint def run(self): message = "Endpoint for %s in %s does not match snapshot: %s != %s" % ( self.service_name, self.region_name, self.new_endpoint, self.old_endpoint ) assert_equal(self.old_endpoint, self.new_endpoint, message) vendored/boto/tests/unit/test_connection.py 0000644 00000056655 15025013616 0015226 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import os import socket from tests.compat import mock, unittest from httpretty import HTTPretty from boto import UserAgent from boto.compat import json, parse_qs from boto.connection import AWSQueryConnection, AWSAuthConnection, HTTPRequest from boto.exception import BotoServerError from boto.regioninfo import RegionInfo class TestListParamsSerialization(unittest.TestCase): maxDiff = None def setUp(self): self.connection = AWSQueryConnection('access_key', 'secret_key') def test_complex_list_serialization(self): # This example is taken from the doc string of # build_complex_list_params. params = {} self.connection.build_complex_list_params( params, [('foo', 'bar', 'baz'), ('foo2', 'bar2', 'baz2')], 'ParamName.member', ('One', 'Two', 'Three')) self.assertDictEqual({ 'ParamName.member.1.One': 'foo', 'ParamName.member.1.Two': 'bar', 'ParamName.member.1.Three': 'baz', 'ParamName.member.2.One': 'foo2', 'ParamName.member.2.Two': 'bar2', 'ParamName.member.2.Three': 'baz2', }, params) def test_simple_list_serialization(self): params = {} self.connection.build_list_params( params, ['foo', 'bar', 'baz'], 'ParamName.member') self.assertDictEqual({ 'ParamName.member.1': 'foo', 'ParamName.member.2': 'bar', 'ParamName.member.3': 'baz', }, params) class MockAWSService(AWSQueryConnection): """ Fake AWS Service This is used to test the AWSQueryConnection object is behaving properly. """ APIVersion = '2012-01-01' def _required_auth_capability(self): return ['sign-v2'] def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, host=None, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, region=None, path='/', api_version=None, security_token=None, validate_certs=True, profile_name=None): self.region = region if host is None: host = self.region.endpoint AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, host, debug, https_connection_factory, path, security_token, validate_certs=validate_certs, profile_name=profile_name) class TestAWSAuthConnection(unittest.TestCase): def test_get_path(self): conn = AWSAuthConnection( 'mockservice.cc-zone-1.amazonaws.com', aws_access_key_id='access_key', aws_secret_access_key='secret', suppress_consec_slashes=False ) # Test some sample paths for mangling. self.assertEqual(conn.get_path('/'), '/') self.assertEqual(conn.get_path('image.jpg'), '/image.jpg') self.assertEqual(conn.get_path('folder/image.jpg'), '/folder/image.jpg') self.assertEqual(conn.get_path('folder//image.jpg'), '/folder//image.jpg') # Ensure leading slashes aren't removed. # See https://github.com/boto/boto/issues/1387 self.assertEqual(conn.get_path('/folder//image.jpg'), '/folder//image.jpg') self.assertEqual(conn.get_path('/folder////image.jpg'), '/folder////image.jpg') self.assertEqual(conn.get_path('///folder////image.jpg'), '///folder////image.jpg') def test_connection_behind_proxy(self): os.environ['http_proxy'] = "http://john.doe:p4ssw0rd@127.0.0.1:8180" conn = AWSAuthConnection( 'mockservice.cc-zone-1.amazonaws.com', aws_access_key_id='access_key', aws_secret_access_key='secret', suppress_consec_slashes=False ) self.assertEqual(conn.proxy, '127.0.0.1') self.assertEqual(conn.proxy_user, 'john.doe') self.assertEqual(conn.proxy_pass, 'p4ssw0rd') self.assertEqual(conn.proxy_port, '8180') del os.environ['http_proxy'] def test_get_proxy_url_with_auth(self): conn = AWSAuthConnection( 'mockservice.cc-zone-1.amazonaws.com', aws_access_key_id='access_key', aws_secret_access_key='secret', suppress_consec_slashes=False, proxy="127.0.0.1", proxy_user="john.doe", proxy_pass="p4ssw0rd", proxy_port="8180" ) self.assertEqual(conn.get_proxy_url_with_auth(), 'http://john.doe:p4ssw0rd@127.0.0.1:8180') def test_build_base_http_request_noproxy(self): os.environ['no_proxy'] = 'mockservice.cc-zone-1.amazonaws.com' conn = AWSAuthConnection( 'mockservice.cc-zone-1.amazonaws.com', aws_access_key_id='access_key', aws_secret_access_key='secret', suppress_consec_slashes=False, proxy="127.0.0.1", proxy_user="john.doe", proxy_pass="p4ssw0rd", proxy_port="8180" ) request = conn.build_base_http_request('GET', '/', None) del os.environ['no_proxy'] self.assertEqual(request.path, '/') def test_connection_behind_proxy_without_explicit_port(self): os.environ['http_proxy'] = "http://127.0.0.1" conn = AWSAuthConnection( 'mockservice.cc-zone-1.amazonaws.com', aws_access_key_id='access_key', aws_secret_access_key='secret', suppress_consec_slashes=False, port=8180 ) self.assertEqual(conn.proxy, '127.0.0.1') self.assertEqual(conn.proxy_port, 8180) del os.environ['http_proxy'] @mock.patch.object(socket, 'create_connection') @mock.patch('boto.compat.http_client.HTTPResponse') @mock.patch('boto.connection.ssl', autospec=True) def test_proxy_ssl_with_verification(self, ssl_mock, http_response_mock, create_connection_mock): type(http_response_mock.return_value).status = mock.PropertyMock( return_value=200) conn = AWSAuthConnection( 'mockservice.s3.amazonaws.com', aws_access_key_id='access_key', aws_secret_access_key='secret', suppress_consec_slashes=False, proxy_port=80 ) conn.https_validate_certificates = True dummy_cert = { 'subjectAltName': (('DNS', 's3.amazonaws.com'), ('DNS', '*.s3.amazonaws.com')), } mock_sock = mock.Mock() create_connection_mock.return_value = mock_sock mock_sslSock = mock.Mock() mock_sslSock.getpeercert.return_value = dummy_cert mock_context = mock.Mock() mock_context.wrap_socket.return_value = mock_sslSock ssl_mock.create_default_context.return_value = mock_context # Attempt to call proxy_ssl and make sure it works conn.proxy_ssl('mockservice.s3.amazonaws.com', 80) mock_sslSock.getpeercert.assert_called_once_with() mock_context.wrap_socket.assert_called_once_with( mock_sock, server_hostname='mockservice.s3.amazonaws.com') # this tests the proper setting of the host_header in v4 signing def test_host_header_with_nonstandard_port(self): # test standard port first conn = V4AuthConnection( 'testhost', aws_access_key_id='access_key', aws_secret_access_key='secret') request = conn.build_base_http_request( method='POST', path='/', auth_path=None, params=None, headers=None, data='', host=None) conn.set_host_header(request) self.assertEqual(request.headers['Host'], 'testhost') # next, test non-standard port conn = V4AuthConnection( 'testhost', aws_access_key_id='access_key', aws_secret_access_key='secret', port=8773) request = conn.build_base_http_request( method='POST', path='/', auth_path=None, params=None, headers=None, data='', host=None) conn.set_host_header(request) self.assertEqual(request.headers['Host'], 'testhost:8773') class V4AuthConnection(AWSAuthConnection): def __init__(self, host, aws_access_key_id, aws_secret_access_key, port=443): AWSAuthConnection.__init__( self, host, aws_access_key_id, aws_secret_access_key, port=port) def _required_auth_capability(self): return ['hmac-v4'] class TestAWSQueryConnection(unittest.TestCase): def setUp(self): self.region = RegionInfo( name='cc-zone-1', endpoint='mockservice.cc-zone-1.amazonaws.com', connection_cls=MockAWSService) HTTPretty.enable() def tearDown(self): HTTPretty.disable() class TestAWSQueryConnectionSimple(TestAWSQueryConnection): def test_query_connection_basis(self): HTTPretty.register_uri(HTTPretty.POST, 'https://%s/' % self.region.endpoint, json.dumps({'test': 'secure'}), content_type='application/json') conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret') self.assertEqual(conn.host, 'mockservice.cc-zone-1.amazonaws.com') def test_query_connection_noproxy(self): HTTPretty.register_uri(HTTPretty.POST, 'https://%s/' % self.region.endpoint, json.dumps({'test': 'secure'}), content_type='application/json') os.environ['no_proxy'] = self.region.endpoint conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret', proxy="NON_EXISTENT_HOSTNAME", proxy_port="3128") resp = conn.make_request('myCmd', {'par1': 'foo', 'par2': 'baz'}, "/", "POST") del os.environ['no_proxy'] args = parse_qs(HTTPretty.last_request.body) self.assertEqual(args[b'AWSAccessKeyId'], [b'access_key']) def test_query_connection_noproxy_nosecure(self): HTTPretty.register_uri(HTTPretty.POST, 'https://%s/' % self.region.endpoint, json.dumps({'test': 'insecure'}), content_type='application/json') os.environ['no_proxy'] = self.region.endpoint conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret', proxy="NON_EXISTENT_HOSTNAME", proxy_port="3128", is_secure=False) resp = conn.make_request('myCmd', {'par1': 'foo', 'par2': 'baz'}, "/", "POST") del os.environ['no_proxy'] args = parse_qs(HTTPretty.last_request.body) self.assertEqual(args[b'AWSAccessKeyId'], [b'access_key']) def test_single_command(self): HTTPretty.register_uri(HTTPretty.POST, 'https://%s/' % self.region.endpoint, json.dumps({'test': 'secure'}), content_type='application/json') conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret') resp = conn.make_request('myCmd', {'par1': 'foo', 'par2': 'baz'}, "/", "POST") args = parse_qs(HTTPretty.last_request.body) self.assertEqual(args[b'AWSAccessKeyId'], [b'access_key']) self.assertEqual(args[b'SignatureMethod'], [b'HmacSHA256']) self.assertEqual(args[b'Version'], [conn.APIVersion.encode('utf-8')]) self.assertEqual(args[b'par1'], [b'foo']) self.assertEqual(args[b'par2'], [b'baz']) self.assertEqual(resp.read(), b'{"test": "secure"}') def test_multi_commands(self): """Check connection re-use""" HTTPretty.register_uri(HTTPretty.POST, 'https://%s/' % self.region.endpoint, json.dumps({'test': 'secure'}), content_type='application/json') conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret') resp1 = conn.make_request('myCmd1', {'par1': 'foo', 'par2': 'baz'}, "/", "POST") body1 = parse_qs(HTTPretty.last_request.body) resp2 = conn.make_request('myCmd2', {'par3': 'bar', 'par4': 'narf'}, "/", "POST") body2 = parse_qs(HTTPretty.last_request.body) self.assertEqual(body1[b'par1'], [b'foo']) self.assertEqual(body1[b'par2'], [b'baz']) with self.assertRaises(KeyError): body1[b'par3'] self.assertEqual(body2[b'par3'], [b'bar']) self.assertEqual(body2[b'par4'], [b'narf']) with self.assertRaises(KeyError): body2['par1'] self.assertEqual(resp1.read(), b'{"test": "secure"}') self.assertEqual(resp2.read(), b'{"test": "secure"}') def test_non_secure(self): HTTPretty.register_uri(HTTPretty.POST, 'http://%s/' % self.region.endpoint, json.dumps({'test': 'normal'}), content_type='application/json') conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret', is_secure=False) resp = conn.make_request('myCmd1', {'par1': 'foo', 'par2': 'baz'}, "/", "POST") self.assertEqual(resp.read(), b'{"test": "normal"}') def test_alternate_port(self): HTTPretty.register_uri(HTTPretty.POST, 'http://%s:8080/' % self.region.endpoint, json.dumps({'test': 'alternate'}), content_type='application/json') conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret', port=8080, is_secure=False) resp = conn.make_request('myCmd1', {'par1': 'foo', 'par2': 'baz'}, "/", "POST") self.assertEqual(resp.read(), b'{"test": "alternate"}') def test_temp_failure(self): responses = [HTTPretty.Response(body="{'test': 'fail'}", status=500), HTTPretty.Response(body="{'test': 'success'}", status=200)] HTTPretty.register_uri(HTTPretty.POST, 'https://%s/temp_fail/' % self.region.endpoint, responses=responses) conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret') resp = conn.make_request('myCmd1', {'par1': 'foo', 'par2': 'baz'}, '/temp_fail/', 'POST') self.assertEqual(resp.read(), b"{'test': 'success'}") def test_unhandled_exception(self): HTTPretty.register_uri(HTTPretty.POST, 'https://%s/temp_exception/' % self.region.endpoint, responses=[]) def fake_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None): raise socket.timeout('fake error') socket.create_connection = fake_connection conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret') conn.num_retries = 0 with self.assertRaises(socket.error): resp = conn.make_request('myCmd1', {'par1': 'foo', 'par2': 'baz'}, '/temp_exception/', 'POST') def test_connection_close(self): """Check connection re-use after close header is received""" HTTPretty.register_uri(HTTPretty.POST, 'https://%s/' % self.region.endpoint, json.dumps({'test': 'secure'}), content_type='application/json', connection='close') conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret') def mock_put_conn(*args, **kwargs): raise Exception('put_http_connection should not be called!') conn.put_http_connection = mock_put_conn resp1 = conn.make_request('myCmd1', {'par1': 'foo', 'par2': 'baz'}, "/", "POST") # If we've gotten this far then no exception was raised # by attempting to put the connection back into the pool # Now let's just confirm the close header was actually # set or we have another problem. self.assertEqual(resp1.getheader('connection'), 'close') def test_port_pooling(self): conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret', port=8080) # Pick a connection, then put it back con1 = conn.get_http_connection(conn.host, conn.port, conn.is_secure) conn.put_http_connection(conn.host, conn.port, conn.is_secure, con1) # Pick another connection, which hopefully is the same yet again con2 = conn.get_http_connection(conn.host, conn.port, conn.is_secure) conn.put_http_connection(conn.host, conn.port, conn.is_secure, con2) self.assertEqual(con1, con2) # Change the port and make sure a new connection is made conn.port = 8081 con3 = conn.get_http_connection(conn.host, conn.port, conn.is_secure) conn.put_http_connection(conn.host, conn.port, conn.is_secure, con3) self.assertNotEqual(con1, con3) class TestAWSQueryStatus(TestAWSQueryConnection): def test_get_status(self): HTTPretty.register_uri(HTTPretty.GET, 'https://%s/status' % self.region.endpoint, '<status>ok</status>', content_type='text/xml') conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret') resp = conn.get_status('getStatus', {'par1': 'foo', 'par2': 'baz'}, 'status') self.assertEqual(resp, "ok") def test_get_status_blank_error(self): HTTPretty.register_uri(HTTPretty.GET, 'https://%s/status' % self.region.endpoint, '', content_type='text/xml') conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret') with self.assertRaises(BotoServerError): resp = conn.get_status('getStatus', {'par1': 'foo', 'par2': 'baz'}, 'status') def test_get_status_error(self): HTTPretty.register_uri(HTTPretty.GET, 'https://%s/status' % self.region.endpoint, '<status>error</status>', content_type='text/xml', status=400) conn = self.region.connect(aws_access_key_id='access_key', aws_secret_access_key='secret') with self.assertRaises(BotoServerError): resp = conn.get_status('getStatus', {'par1': 'foo', 'par2': 'baz'}, 'status') class TestHTTPRequest(unittest.TestCase): def test_user_agent_not_url_encoded(self): headers = {'Some-Header': u'should be encoded \u2713', 'User-Agent': UserAgent} request = HTTPRequest('PUT', 'https', 'amazon.com', 443, None, None, {}, headers, 'Body') mock_connection = mock.Mock() # Create a method that preserves the headers at the time of # authorization. def mock_add_auth(req, **kwargs): mock_connection.headers_at_auth = req.headers.copy() mock_connection._auth_handler.add_auth = mock_add_auth request.authorize(mock_connection) # Ensure the headers at authorization are as expected i.e. # the user agent header was not url encoded but the other header was. self.assertEqual(mock_connection.headers_at_auth, {'Some-Header': 'should be encoded %E2%9C%93', 'User-Agent': UserAgent}) def test_content_length_str(self): request = HTTPRequest('PUT', 'https', 'amazon.com', 443, None, None, {}, {}, 'Body') mock_connection = mock.Mock() request.authorize(mock_connection) # Ensure Content-Length header is a str. This is more explicit than # relying on other code cast the value later. (Python 2.7.0, for # example, assumes headers are of type str.) self.assertIsInstance(request.headers['Content-Length'], str) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/sqs/test_connection.py 0000644 00000030761 15025013616 0016022 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import AWSMockServiceTestCase, MockServiceWithConfigTestCase from tests.compat import mock from boto.sqs.connection import SQSConnection from boto.sqs.regioninfo import SQSRegionInfo from boto.sqs.message import RawMessage from boto.sqs.queue import Queue from boto.connection import AWSQueryConnection from nose.plugins.attrib import attr class SQSAuthParams(AWSMockServiceTestCase): connection_class = SQSConnection def setUp(self): super(SQSAuthParams, self).setUp() def default_body(self): return """<?xml version="1.0"?> <CreateQueueResponse> <CreateQueueResult> <QueueUrl> https://queue.amazonaws.com/599169622985/myqueue1 </QueueUrl> </CreateQueueResult> <ResponseMetadata> <RequestId>54d4c94d-2307-54a8-bb27-806a682a5abd</RequestId> </ResponseMetadata> </CreateQueueResponse>""" @attr(sqs=True) def test_auth_service_name_override(self): self.set_http_response(status_code=200) # We can use the auth_service_name to change what service # name to use for the credential scope for sigv4. self.service_connection.auth_service_name = 'service_override' self.service_connection.create_queue('my_queue') # Note the service_override value instead. self.assertIn('us-east-1/service_override/aws4_request', self.actual_request.headers['Authorization']) @attr(sqs=True) def test_class_attribute_can_set_service_name(self): self.set_http_response(status_code=200) # The SQS class has an 'AuthServiceName' param of 'sqs': self.assertEqual(self.service_connection.AuthServiceName, 'sqs') self.service_connection.create_queue('my_queue') # And because of this, the value of 'sqs' will be used instead of # 'queue' for the credential scope: self.assertIn('us-east-1/sqs/aws4_request', self.actual_request.headers['Authorization']) @attr(sqs=True) def test_auth_region_name_is_automatically_updated(self): region = SQSRegionInfo(name='us-west-2', endpoint='us-west-2.queue.amazonaws.com') self.service_connection = SQSConnection( https_connection_factory=self.https_connection_factory, aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key', region=region) self.initialize_service_connection() self.set_http_response(status_code=200) self.service_connection.create_queue('my_queue') # Note the region name below is 'us-west-2'. self.assertIn('us-west-2/sqs/aws4_request', self.actual_request.headers['Authorization']) @attr(sqs=True) def test_set_get_auth_service_and_region_names(self): self.service_connection.auth_service_name = 'service_name' self.service_connection.auth_region_name = 'region_name' self.assertEqual(self.service_connection.auth_service_name, 'service_name') self.assertEqual(self.service_connection.auth_region_name, 'region_name') @attr(sqs=True) def test_get_queue_with_owner_account_id_returns_queue(self): self.set_http_response(status_code=200) self.service_connection.create_queue('my_queue') self.service_connection.get_queue('my_queue', '599169622985') assert 'QueueOwnerAWSAccountId' in self.actual_request.params.keys() self.assertEqual(self.actual_request.params['QueueOwnerAWSAccountId'], '599169622985') class SQSProfileName(MockServiceWithConfigTestCase): connection_class = SQSConnection profile_name = 'prod' def setUp(self): super(SQSProfileName, self).setUp() self.config = { "profile prod": { 'aws_access_key_id': 'access_key', 'aws_secret_access_key': 'secret_access', } } @attr(sqs=True) def test_profile_name_gets_passed(self): region = SQSRegionInfo(name='us-west-2', endpoint='us-west-2.queue.amazonaws.com') self.service_connection = SQSConnection( https_connection_factory=self.https_connection_factory, region=region, profile_name=self.profile_name) self.initialize_service_connection() self.set_http_response(status_code=200) self.assertEqual(self.service_connection.profile_name, self.profile_name) class SQSMessageAttributesParsing(AWSMockServiceTestCase): connection_class = SQSConnection def default_body(self): return """<?xml version="1.0"?> <ReceiveMessageResponse xmlns="http://queue.amazonaws.com/doc/2012-11-05/"> <ReceiveMessageResult> <Message> <Body>This is a test</Body> <ReceiptHandle>+eXJYhj5rDql5hp2VwGkXvQVsefdjAlsQe5EGS57gyORPB48KwP1d/3Rfy4DrQXt+MgfRPHUCUH36xL9+Ol/UWD/ylKrrWhiXSY0Ip4EsI8jJNTo/aneEjKE/iZnz/nL8MFP5FmMj8PbDAy5dgvAqsdvX1rm8Ynn0bGnQLJGfH93cLXT65p6Z/FDyjeBN0M+9SWtTcuxOIcMdU8NsoFIwm/6mLWgWAV46OhlYujzvyopCvVwsj+Y8jLEpdSSvTQHNlQEaaY/V511DqAvUwru2p0ZbW7ZzcbhUTn6hHkUROo=</ReceiptHandle> <MD5OfBody>ce114e4501d2f4e2dcea3e17b546f339</MD5OfBody> <MessageAttribute> <Name>Count</Name> <Value> <DataType>Number</DataType> <StringValue>1</StringValue> </Value> </MessageAttribute> <MessageAttribute> <Name>Foo</Name> <Value> <DataType>String</DataType> <StringValue>Bar</StringValue> </Value> </MessageAttribute> <MessageId>7049431b-e5f6-430b-93c4-ded53864d02b</MessageId> <MD5OfMessageAttributes>324758f82d026ac6ec5b31a3b192d1e3</MD5OfMessageAttributes> </Message> </ReceiveMessageResult> <ResponseMetadata> <RequestId>73f978f2-400b-5460-8d38-3316e39e79c6</RequestId> </ResponseMetadata> </ReceiveMessageResponse>""" @attr(sqs=True) def test_message_attribute_response(self): self.set_http_response(status_code=200) queue = Queue( url='http://sqs.us-east-1.amazonaws.com/123456789012/testQueue/', message_class=RawMessage) message = self.service_connection.receive_message(queue)[0] self.assertEqual(message.get_body(), 'This is a test') self.assertEqual(message.id, '7049431b-e5f6-430b-93c4-ded53864d02b') self.assertEqual(message.md5, 'ce114e4501d2f4e2dcea3e17b546f339') self.assertEqual(message.md5_message_attributes, '324758f82d026ac6ec5b31a3b192d1e3') mattributes = message.message_attributes self.assertEqual(len(mattributes.keys()), 2) self.assertEqual(mattributes['Count']['data_type'], 'Number') self.assertEqual(mattributes['Foo']['string_value'], 'Bar') class SQSSendMessageAttributes(AWSMockServiceTestCase): connection_class = SQSConnection def default_body(self): return """<SendMessageResponse> <SendMessageResult> <MD5OfMessageBody> fafb00f5732ab283681e124bf8747ed1 </MD5OfMessageBody> <MD5OfMessageAttributes> 3ae8f24a165a8cedc005670c81a27295 </MD5OfMessageAttributes> <MessageId> 5fea7756-0ea4-451a-a703-a558b933e274 </MessageId> </SendMessageResult> <ResponseMetadata> <RequestId> 27daac76-34dd-47df-bd01-1f6e873584a0 </RequestId> </ResponseMetadata> </SendMessageResponse> """ @attr(sqs=True) def test_send_message_attributes(self): self.set_http_response(status_code=200) queue = Queue( url='http://sqs.us-east-1.amazonaws.com/123456789012/testQueue/', message_class=RawMessage) self.service_connection.send_message(queue, 'Test message', message_attributes={ 'name1': { 'data_type': 'String', 'string_value': 'Bob' }, 'name2': { 'data_type': 'Number', 'string_value': '1' } }) self.assert_request_parameters({ 'Action': 'SendMessage', 'MessageAttribute.1.Name': 'name1', 'MessageAttribute.1.Value.DataType': 'String', 'MessageAttribute.1.Value.StringValue': 'Bob', 'MessageAttribute.2.Name': 'name2', 'MessageAttribute.2.Value.DataType': 'Number', 'MessageAttribute.2.Value.StringValue': '1', 'MessageBody': 'Test message', 'Version': '2012-11-05' }) class SQSSendBatchMessageAttributes(AWSMockServiceTestCase): connection_class = SQSConnection def default_body(self): return """<SendMessageBatchResponse> <SendMessageBatchResult> <SendMessageBatchResultEntry> <Id>test_msg_001</Id> <MessageId>0a5231c7-8bff-4955-be2e-8dc7c50a25fa</MessageId> <MD5OfMessageBody>0e024d309850c78cba5eabbeff7cae71</MD5OfMessageBody> </SendMessageBatchResultEntry> <SendMessageBatchResultEntry> <Id>test_msg_002</Id> <MessageId>15ee1ed3-87e7-40c1-bdaa-2e49968ea7e9</MessageId> <MD5OfMessageBody>7fb8146a82f95e0af155278f406862c2</MD5OfMessageBody> <MD5OfMessageAttributes>295c5fa15a51aae6884d1d7c1d99ca50</MD5OfMessageAttributes> </SendMessageBatchResultEntry> </SendMessageBatchResult> <ResponseMetadata> <RequestId>ca1ad5d0-8271-408b-8d0f-1351bf547e74</RequestId> </ResponseMetadata> </SendMessageBatchResponse> """ @attr(sqs=True) def test_send_message_attributes(self): self.set_http_response(status_code=200) queue = Queue( url='http://sqs.us-east-1.amazonaws.com/123456789012/testQueue/', message_class=RawMessage) message1 = (1, 'Message 1', 0, {'name1': {'data_type': 'String', 'string_value': 'foo'}}) message2 = (2, 'Message 2', 0, {'name2': {'data_type': 'Number', 'string_value': '1'}}) self.service_connection.send_message_batch(queue, (message1, message2)) self.assert_request_parameters({ 'Action': 'SendMessageBatch', 'SendMessageBatchRequestEntry.1.DelaySeconds': 0, 'SendMessageBatchRequestEntry.1.Id': 1, 'SendMessageBatchRequestEntry.1.MessageAttribute.1.Name': 'name1', 'SendMessageBatchRequestEntry.1.MessageAttribute.1.Value.DataType': 'String', 'SendMessageBatchRequestEntry.1.MessageAttribute.1.Value.StringValue': 'foo', 'SendMessageBatchRequestEntry.1.MessageBody': 'Message 1', 'SendMessageBatchRequestEntry.2.DelaySeconds': 0, 'SendMessageBatchRequestEntry.2.Id': 2, 'SendMessageBatchRequestEntry.2.MessageAttribute.1.Name': 'name2', 'SendMessageBatchRequestEntry.2.MessageAttribute.1.Value.DataType': 'Number', 'SendMessageBatchRequestEntry.2.MessageAttribute.1.Value.StringValue': '1', 'SendMessageBatchRequestEntry.2.MessageBody': 'Message 2', 'Version': '2012-11-05' }) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/sqs/test_message.py 0000644 00000010257 15025013616 0015305 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from boto.sqs.message import MHMessage from boto.sqs.message import RawMessage from boto.sqs.message import Message from boto.sqs.bigmessage import BigMessage from boto.exception import SQSDecodeError from nose.plugins.attrib import attr class TestMHMessage(unittest.TestCase): @attr(sqs=True) def test_contains(self): msg = MHMessage() msg.update({'hello': 'world'}) self.assertTrue('hello' in msg) class DecodeExceptionRaisingMessage(RawMessage): @attr(sqs=True) def decode(self, message): raise SQSDecodeError('Sample decode error', self) class TestEncodeMessage(unittest.TestCase): @attr(sqs=True) def test_message_id_available(self): import xml.sax from boto.resultset import ResultSet from boto.handler import XmlHandler sample_value = 'abcdef' body = """<?xml version="1.0"?> <ReceiveMessageResponse> <ReceiveMessageResult> <Message> <Body>%s</Body> <ReceiptHandle>%s</ReceiptHandle> <MessageId>%s</MessageId> </Message> </ReceiveMessageResult> </ReceiveMessageResponse>""" % tuple([sample_value] * 3) rs = ResultSet([('Message', DecodeExceptionRaisingMessage)]) h = XmlHandler(rs, None) with self.assertRaises(SQSDecodeError) as context: xml.sax.parseString(body.encode('utf-8'), h) message = context.exception.message self.assertEqual(message.id, sample_value) self.assertEqual(message.receipt_handle, sample_value) @attr(sqs=True) def test_encode_bytes_message(self): message = Message() body = b'\x00\x01\x02\x03\x04\x05' message.set_body(body) self.assertEqual(message.get_body_encoded(), 'AAECAwQF') @attr(sqs=True) def test_encode_string_message(self): message = Message() body = 'hello world' message.set_body(body) self.assertEqual(message.get_body_encoded(), 'aGVsbG8gd29ybGQ=') class TestBigMessage(unittest.TestCase): @attr(sqs=True) def test_s3url_parsing(self): msg = BigMessage() # Try just a bucket name bucket, key = msg._get_bucket_key('s3://foo') self.assertEqual(bucket, 'foo') self.assertEqual(key, None) # Try just a bucket name with trailing "/" bucket, key = msg._get_bucket_key('s3://foo/') self.assertEqual(bucket, 'foo') self.assertEqual(key, None) # Try a bucket and a key bucket, key = msg._get_bucket_key('s3://foo/bar') self.assertEqual(bucket, 'foo') self.assertEqual(key, 'bar') # Try a bucket and a key with "/" bucket, key = msg._get_bucket_key('s3://foo/bar/fie/baz') self.assertEqual(bucket, 'foo') self.assertEqual(key, 'bar/fie/baz') # Try it with no s3:// prefix with self.assertRaises(SQSDecodeError) as context: bucket, key = msg._get_bucket_key('foo/bar') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/sqs/__init__.py 0000644 00000000000 15025013616 0014342 0 ustar 00 vendored/boto/tests/unit/sqs/test_queue.py 0000644 00000003630 15025013616 0015002 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from mock import Mock from boto.sqs.queue import Queue from nose.plugins.attrib import attr class TestQueue(unittest.TestCase): @attr(sqs=True) def test_queue_arn(self): connection = Mock() connection.region.name = 'us-east-1' q = Queue( connection=connection, url='https://sqs.us-east-1.amazonaws.com/id/queuename') self.assertEqual(q.arn, 'arn:aws:sqs:us-east-1:id:queuename') @attr(sqs=True) def test_queue_name(self): connection = Mock() connection.region.name = 'us-east-1' q = Queue( connection=connection, url='https://sqs.us-east-1.amazonaws.com/id/queuename') self.assertEqual(q.name, 'queuename') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/manage/test_ssh.py 0000644 00000003724 15025013616 0015101 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # try: import paramiko from boto.manage.cmdshell import SSHClient except ImportError: paramiko = None SSHClient = None from tests.compat import mock, unittest class TestSSHTimeout(unittest.TestCase): @unittest.skipIf(not paramiko, 'Paramiko missing') def test_timeout(self): client_tmp = paramiko.SSHClient def client_mock(): client = client_tmp() client.connect = mock.Mock(name='connect') return client paramiko.SSHClient = client_mock paramiko.RSAKey.from_private_key_file = mock.Mock() server = mock.Mock() test = SSHClient(server) self.assertEqual(test._ssh_client.connect.call_args[1]['timeout'], None) test2 = SSHClient(server, timeout=30) self.assertEqual(test2._ssh_client.connect.call_args[1]['timeout'], 30) vendored/boto/tests/unit/manage/__init__.py 0000644 00000000000 15025013616 0014764 0 ustar 00 vendored/boto/tests/unit/provider/__init__.py 0000644 00000000000 15025013616 0015366 0 ustar 00 vendored/boto/tests/unit/provider/test_provider.py 0000644 00000043620 15025013616 0016537 0 ustar 00 #!/usr/bin/env python from datetime import datetime, timedelta, timezone from tests.compat import mock, unittest import os from boto import provider from boto.compat import expanduser from boto.exception import InvalidInstanceMetadataError INSTANCE_CONFIG = { 'allowall': { u'AccessKeyId': u'iam_access_key', u'Code': u'Success', u'Expiration': u'2012-09-01T03:57:34Z', u'LastUpdated': u'2012-08-31T21:43:40Z', u'SecretAccessKey': u'iam_secret_key', u'Token': u'iam_token', u'Type': u'AWS-HMAC' } } class TestProvider(unittest.TestCase): def setUp(self): self.environ = {} self.config = {} self.shared_config = {} self.metadata_patch = mock.patch('boto.utils.get_instance_metadata') self.config_patch = mock.patch('boto.provider.config.get', self.get_config) self.has_config_patch = mock.patch('boto.provider.config.has_option', self.has_config) self.config_object_patch = mock.patch.object( provider.Config, 'get', self.get_shared_config) self.has_config_object_patch = mock.patch.object( provider.Config, 'has_option', self.has_shared_config) self.environ_patch = mock.patch('os.environ', self.environ) self.get_instance_metadata = self.metadata_patch.start() self.get_instance_metadata.return_value = None self.config_patch.start() self.has_config_patch.start() self.config_object_patch.start() self.has_config_object_patch.start() self.environ_patch.start() def tearDown(self): self.metadata_patch.stop() self.config_patch.stop() self.has_config_patch.stop() self.config_object_patch.stop() self.has_config_object_patch.stop() self.environ_patch.stop() def has_config(self, section_name, key): try: self.config[section_name][key] return True except KeyError: return False def get_config(self, section_name, key): try: return self.config[section_name][key] except KeyError: return None def has_shared_config(self, section_name, key): try: self.shared_config[section_name][key] return True except KeyError: return False def get_shared_config(self, section_name, key): try: return self.shared_config[section_name][key] except KeyError: return None def test_passed_in_values_are_used(self): p = provider.Provider('aws', 'access_key', 'secret_key', 'security_token') self.assertEqual(p.access_key, 'access_key') self.assertEqual(p.secret_key, 'secret_key') self.assertEqual(p.security_token, 'security_token') def test_environment_variables_are_used(self): self.environ['AWS_ACCESS_KEY_ID'] = 'env_access_key' self.environ['AWS_SECRET_ACCESS_KEY'] = 'env_secret_key' p = provider.Provider('aws') self.assertEqual(p.access_key, 'env_access_key') self.assertEqual(p.secret_key, 'env_secret_key') self.assertIsNone(p.security_token) def test_environment_variable_aws_security_token(self): self.environ['AWS_ACCESS_KEY_ID'] = 'env_access_key' self.environ['AWS_SECRET_ACCESS_KEY'] = 'env_secret_key' self.environ['AWS_SECURITY_TOKEN'] = 'env_security_token' p = provider.Provider('aws') self.assertEqual(p.access_key, 'env_access_key') self.assertEqual(p.secret_key, 'env_secret_key') self.assertEqual(p.security_token, 'env_security_token') def test_no_credentials_provided(self): p = provider.Provider( 'aws', provider.NO_CREDENTIALS_PROVIDED, provider.NO_CREDENTIALS_PROVIDED, provider.NO_CREDENTIALS_PROVIDED ) self.assertEqual(p.access_key, provider.NO_CREDENTIALS_PROVIDED) self.assertEqual(p.secret_key, provider.NO_CREDENTIALS_PROVIDED) self.assertEqual(p.security_token, provider.NO_CREDENTIALS_PROVIDED) def test_config_profile_values_are_used(self): self.config = { 'profile dev': { 'aws_access_key_id': 'dev_access_key', 'aws_secret_access_key': 'dev_secret_key', }, 'profile prod': { 'aws_access_key_id': 'prod_access_key', 'aws_secret_access_key': 'prod_secret_key', }, 'profile prod_withtoken': { 'aws_access_key_id': 'prod_access_key', 'aws_secret_access_key': 'prod_secret_key', 'aws_security_token': 'prod_token', }, 'Credentials': { 'aws_access_key_id': 'default_access_key', 'aws_secret_access_key': 'default_secret_key' } } p = provider.Provider('aws', profile_name='prod') self.assertEqual(p.access_key, 'prod_access_key') self.assertEqual(p.secret_key, 'prod_secret_key') p = provider.Provider('aws', profile_name='prod_withtoken') self.assertEqual(p.access_key, 'prod_access_key') self.assertEqual(p.secret_key, 'prod_secret_key') self.assertEqual(p.security_token, 'prod_token') q = provider.Provider('aws', profile_name='dev') self.assertEqual(q.access_key, 'dev_access_key') self.assertEqual(q.secret_key, 'dev_secret_key') def test_config_missing_profile(self): # None of these default profiles should be loaded! self.shared_config = { 'default': { 'aws_access_key_id': 'shared_access_key', 'aws_secret_access_key': 'shared_secret_key', } } self.config = { 'Credentials': { 'aws_access_key_id': 'default_access_key', 'aws_secret_access_key': 'default_secret_key' } } with self.assertRaises(provider.ProfileNotFoundError): provider.Provider('aws', profile_name='doesntexist') def test_config_values_are_used(self): self.config = { 'Credentials': { 'aws_access_key_id': 'cfg_access_key', 'aws_secret_access_key': 'cfg_secret_key', } } p = provider.Provider('aws') self.assertEqual(p.access_key, 'cfg_access_key') self.assertEqual(p.secret_key, 'cfg_secret_key') self.assertIsNone(p.security_token) def test_config_value_security_token_is_used(self): self.config = { 'Credentials': { 'aws_access_key_id': 'cfg_access_key', 'aws_secret_access_key': 'cfg_secret_key', 'aws_security_token': 'cfg_security_token', } } p = provider.Provider('aws') self.assertEqual(p.access_key, 'cfg_access_key') self.assertEqual(p.secret_key, 'cfg_secret_key') self.assertEqual(p.security_token, 'cfg_security_token') def test_keyring_is_used(self): self.config = { 'Credentials': { 'aws_access_key_id': 'cfg_access_key', 'keyring': 'test', } } import sys try: import keyring imported = True except ImportError: sys.modules['keyring'] = keyring = type(mock)('keyring', '') imported = False try: with mock.patch('keyring.get_password', create=True): keyring.get_password.side_effect = ( lambda kr, login: kr+login+'pw') p = provider.Provider('aws') self.assertEqual(p.access_key, 'cfg_access_key') self.assertEqual(p.secret_key, 'testcfg_access_keypw') self.assertIsNone(p.security_token) finally: if not imported: del sys.modules['keyring'] def test_passed_in_values_beat_env_vars(self): self.environ['AWS_ACCESS_KEY_ID'] = 'env_access_key' self.environ['AWS_SECRET_ACCESS_KEY'] = 'env_secret_key' self.environ['AWS_SECURITY_TOKEN'] = 'env_security_token' p = provider.Provider('aws', 'access_key', 'secret_key') self.assertEqual(p.access_key, 'access_key') self.assertEqual(p.secret_key, 'secret_key') self.assertEqual(p.security_token, None) def test_env_vars_beat_shared_creds_values(self): self.environ['AWS_ACCESS_KEY_ID'] = 'env_access_key' self.environ['AWS_SECRET_ACCESS_KEY'] = 'env_secret_key' self.shared_config = { 'default': { 'aws_access_key_id': 'shared_access_key', 'aws_secret_access_key': 'shared_secret_key', } } p = provider.Provider('aws') self.assertEqual(p.access_key, 'env_access_key') self.assertEqual(p.secret_key, 'env_secret_key') self.assertIsNone(p.security_token) def test_shared_creds_beat_config_values(self): self.shared_config = { 'default': { 'aws_access_key_id': 'shared_access_key', 'aws_secret_access_key': 'shared_secret_key', } } self.config = { 'Credentials': { 'aws_access_key_id': 'cfg_access_key', 'aws_secret_access_key': 'cfg_secret_key', } } p = provider.Provider('aws') self.assertEqual(p.access_key, 'shared_access_key') self.assertEqual(p.secret_key, 'shared_secret_key') self.assertIsNone(p.security_token) def test_shared_creds_profile_beats_defaults(self): self.shared_config = { 'default': { 'aws_access_key_id': 'shared_access_key', 'aws_secret_access_key': 'shared_secret_key', }, 'foo': { 'aws_access_key_id': 'foo_access_key', 'aws_secret_access_key': 'foo_secret_key', } } p = provider.Provider('aws', profile_name='foo') self.assertEqual(p.access_key, 'foo_access_key') self.assertEqual(p.secret_key, 'foo_secret_key') self.assertIsNone(p.security_token) def test_env_profile_loads_profile(self): self.environ['AWS_PROFILE'] = 'foo' self.shared_config = { 'default': { 'aws_access_key_id': 'shared_access_key', 'aws_secret_access_key': 'shared_secret_key', }, 'foo': { 'aws_access_key_id': 'shared_access_key_foo', 'aws_secret_access_key': 'shared_secret_key_foo', } } self.config = { 'profile foo': { 'aws_access_key_id': 'cfg_access_key_foo', 'aws_secret_access_key': 'cfg_secret_key_foo', }, 'Credentials': { 'aws_access_key_id': 'cfg_access_key', 'aws_secret_access_key': 'cfg_secret_key', } } p = provider.Provider('aws') self.assertEqual(p.access_key, 'shared_access_key_foo') self.assertEqual(p.secret_key, 'shared_secret_key_foo') self.assertIsNone(p.security_token) self.shared_config = {} p = provider.Provider('aws') self.assertEqual(p.access_key, 'cfg_access_key_foo') self.assertEqual(p.secret_key, 'cfg_secret_key_foo') self.assertIsNone(p.security_token) def test_env_vars_security_token_beats_config_values(self): self.environ['AWS_ACCESS_KEY_ID'] = 'env_access_key' self.environ['AWS_SECRET_ACCESS_KEY'] = 'env_secret_key' self.environ['AWS_SECURITY_TOKEN'] = 'env_security_token' self.shared_config = { 'default': { 'aws_access_key_id': 'shared_access_key', 'aws_secret_access_key': 'shared_secret_key', 'aws_security_token': 'shared_security_token', } } self.config = { 'Credentials': { 'aws_access_key_id': 'cfg_access_key', 'aws_secret_access_key': 'cfg_secret_key', 'aws_security_token': 'cfg_security_token', } } p = provider.Provider('aws') self.assertEqual(p.access_key, 'env_access_key') self.assertEqual(p.secret_key, 'env_secret_key') self.assertEqual(p.security_token, 'env_security_token') self.environ.clear() p = provider.Provider('aws') self.assertEqual(p.security_token, 'shared_security_token') self.shared_config.clear() p = provider.Provider('aws') self.assertEqual(p.security_token, 'cfg_security_token') def test_metadata_server_credentials(self): self.get_instance_metadata.return_value = INSTANCE_CONFIG p = provider.Provider('aws') self.assertEqual(p.access_key, 'iam_access_key') self.assertEqual(p.secret_key, 'iam_secret_key') self.assertEqual(p.security_token, 'iam_token') self.assertEqual( self.get_instance_metadata.call_args[1]['data'], 'meta-data/iam/security-credentials/') def test_metadata_server_returns_bad_type(self): self.get_instance_metadata.return_value = { 'rolename': [], } with self.assertRaises(InvalidInstanceMetadataError): p = provider.Provider('aws') def test_metadata_server_returns_empty_string(self): self.get_instance_metadata.return_value = { 'rolename': '' } with self.assertRaises(InvalidInstanceMetadataError): p = provider.Provider('aws') def test_metadata_server_returns_missing_keys(self): self.get_instance_metadata.return_value = { 'allowall': { u'AccessKeyId': u'iam_access_key', # Missing SecretAccessKey. u'Token': u'iam_token', u'Expiration': u'2012-09-01T03:57:34Z', } } with self.assertRaises(InvalidInstanceMetadataError): p = provider.Provider('aws') def test_refresh_credentials(self): now = datetime.now(tz=timezone.utc).replace(tzinfo=None) first_expiration = (now + timedelta(seconds=10)).strftime( "%Y-%m-%dT%H:%M:%SZ") credentials = { u'AccessKeyId': u'first_access_key', u'Code': u'Success', u'Expiration': first_expiration, u'LastUpdated': u'2012-08-31T21:43:40Z', u'SecretAccessKey': u'first_secret_key', u'Token': u'first_token', u'Type': u'AWS-HMAC' } instance_config = {'allowall': credentials} self.get_instance_metadata.return_value = instance_config p = provider.Provider('aws') self.assertEqual(p.access_key, 'first_access_key') self.assertEqual(p.secret_key, 'first_secret_key') self.assertEqual(p.security_token, 'first_token') self.assertIsNotNone(p._credential_expiry_time) # Now set the expiration to something in the past. expired = now - timedelta(seconds=20) p._credential_expiry_time = expired credentials['AccessKeyId'] = 'second_access_key' credentials['SecretAccessKey'] = 'second_secret_key' credentials['Token'] = 'second_token' self.get_instance_metadata.return_value = instance_config # Now upon attribute access, the credentials should be updated. self.assertEqual(p.access_key, 'second_access_key') self.assertEqual(p.secret_key, 'second_secret_key') self.assertEqual(p.security_token, 'second_token') @mock.patch('boto.provider.config.getint') @mock.patch('boto.provider.config.getfloat') def test_metadata_config_params(self, config_float, config_int): config_int.return_value = 10 config_float.return_value = 4.0 self.get_instance_metadata.return_value = INSTANCE_CONFIG p = provider.Provider('aws') self.assertEqual(p.access_key, 'iam_access_key') self.assertEqual(p.secret_key, 'iam_secret_key') self.assertEqual(p.security_token, 'iam_token') self.get_instance_metadata.assert_called_with( timeout=4.0, num_retries=10, data='meta-data/iam/security-credentials/') def test_provider_google(self): self.environ['GS_ACCESS_KEY_ID'] = 'env_access_key' self.environ['GS_SECRET_ACCESS_KEY'] = 'env_secret_key' self.shared_config = { 'default': { 'gs_access_key_id': 'shared_access_key', 'gs_secret_access_key': 'shared_secret_key', } } self.config = { 'Credentials': { 'gs_access_key_id': 'cfg_access_key', 'gs_secret_access_key': 'cfg_secret_key', } } p = provider.Provider('google') self.assertEqual(p.access_key, 'env_access_key') self.assertEqual(p.secret_key, 'env_secret_key') self.environ.clear() p = provider.Provider('google') self.assertEqual(p.access_key, 'shared_access_key') self.assertEqual(p.secret_key, 'shared_secret_key') self.shared_config.clear() p = provider.Provider('google') self.assertEqual(p.access_key, 'cfg_access_key') self.assertEqual(p.secret_key, 'cfg_secret_key') @mock.patch('os.path.isfile', return_value=True) @mock.patch.object(provider.Config, 'load_from_path') def test_shared_config_loading(self, load_from_path, exists): provider.Provider('aws') path = os.path.join(expanduser('~'), '.aws', 'credentials') exists.assert_called_once_with(path) load_from_path.assert_called_once_with(path) exists.reset_mock() load_from_path.reset_mock() provider.Provider('google') path = os.path.join(expanduser('~'), '.google', 'credentials') exists.assert_called_once_with(path) load_from_path.assert_called_once_with(path) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/cloudsearchdomain/test_cloudsearchdomain.py 0000644 00000011405 15025013616 0022217 0 ustar 00 #!/usr/bin env python import json import mock from tests.unit import AWSMockServiceTestCase from boto.cloudsearch2.domain import Domain from boto.cloudsearch2.layer1 import CloudSearchConnection from boto.cloudsearchdomain.layer1 import CloudSearchDomainConnection class CloudSearchDomainConnectionTest(AWSMockServiceTestCase): connection_class = CloudSearchDomainConnection domain_status = """{ "SearchInstanceType": null, "DomainId": "1234567890/demo", "DomainName": "demo", "Deleted": false, "SearchInstanceCount": 0, "Created": true, "SearchService": { "Endpoint": "search-demo.us-east-1.cloudsearch.amazonaws.com" }, "RequiresIndexDocuments": false, "Processing": false, "DocService": { "Endpoint": "doc-demo.us-east-1.cloudsearch.amazonaws.com" }, "ARN": "arn:aws:cs:us-east-1:1234567890:domain/demo", "SearchPartitionCount": 0 }""" def create_service_connection(self, **kwargs): if kwargs.get('host', None) is None: kwargs['host'] = 'search-demo.us-east-1.cloudsearch.amazonaws.com' return super(CloudSearchDomainConnectionTest, self).\ create_service_connection(**kwargs) def test_get_search_service(self): layer1 = CloudSearchConnection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key', sign_request=True) domain = Domain(layer1=layer1, data=json.loads(self.domain_status)) search_service = domain.get_search_service() self.assertEqual(search_service.sign_request, True) def test_get_document_service(self): layer1 = CloudSearchConnection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key', sign_request=True) domain = Domain(layer1=layer1, data=json.loads(self.domain_status)) document_service = domain.get_document_service() self.assertEqual(document_service.sign_request, True) def test_search_with_auth(self): layer1 = CloudSearchConnection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key', sign_request=True) domain = Domain(layer1=layer1, data=json.loads(self.domain_status)) search_service = domain.get_search_service() response = { 'rank': '-text_relevance', 'match-expr': "Test", 'hits': { 'found': 30, 'start': 0, 'hit': { 'id': '12341', 'fields': { 'title': 'Document 1', 'rank': 1 } } }, 'status': { 'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08', 'time-ms': 2, 'cpu-time-ms': 0 } } self.set_http_response(status_code=200, body=json.dumps(response).encode('utf-8')) search_service.domain_connection = self.service_connection resp = search_service.search() headers = self.actual_request.headers self.assertIsNotNone(headers.get('Authorization')) def test_upload_documents_with_auth(self): layer1 = CloudSearchConnection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key', sign_request=True) domain = Domain(layer1=layer1, data=json.loads(self.domain_status)) document_service = domain.get_document_service() response = { 'status': 'success', 'adds': 1, 'deletes': 0, } document = { "id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"] } self.set_http_response(status_code=200, body=json.dumps(response).encode('utf-8')) document_service.domain_connection = self.service_connection document_service.add("1234", document) resp = document_service.commit() headers = self.actual_request.headers self.assertIsNotNone(headers.get('Authorization')) def test_no_host_provided(self): # A host must be provided or a error is thrown. with self.assertRaises(ValueError): CloudSearchDomainConnection( aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key' ) vendored/boto/tests/unit/cloudsearchdomain/__init__.py 0000644 00000000000 15025013616 0017220 0 ustar 00 vendored/boto/tests/unit/swf/test_layer2_base.py 0000644 00000001772 15025013616 0016044 0 ustar 00 import boto.swf.layer2 from boto.swf.layer2 import SWFBase from tests.unit import unittest from mock import Mock MOCK_DOMAIN = 'Mock' MOCK_ACCESS_KEY = 'inheritable access key' MOCK_SECRET_KEY = 'inheritable secret key' MOCK_REGION = 'Mock Region' class TestBase(unittest.TestCase): """ Test for SWFBase. """ def setUp(self): boto.swf.layer2.Layer1 = Mock() self.swf_base = SWFBase( domain=MOCK_DOMAIN, aws_access_key_id=MOCK_ACCESS_KEY, aws_secret_access_key=MOCK_SECRET_KEY, region=MOCK_REGION ) def test_instantiation(self): self.assertEqual(MOCK_DOMAIN, self.swf_base.domain) self.assertEqual(MOCK_ACCESS_KEY, self.swf_base.aws_access_key_id) self.assertEqual(MOCK_SECRET_KEY, self.swf_base.aws_secret_access_key) self.assertEqual(MOCK_REGION, self.swf_base.region) boto.swf.layer2.Layer1.assert_called_with( MOCK_ACCESS_KEY, MOCK_SECRET_KEY, region=MOCK_REGION) vendored/boto/tests/unit/swf/test_layer2_types.py 0000644 00000003310 15025013616 0016264 0 ustar 00 import boto.swf.layer2 from boto.swf.layer2 import ActivityType, WorkflowType, WorkflowExecution from tests.unit import unittest from mock import Mock, ANY class TestTypes(unittest.TestCase): def setUp(self): boto.swf.layer2.Layer1 = Mock() def test_workflow_type_register_defaults(self): wf_type = WorkflowType(name='name', domain='test', version='1') wf_type.register() wf_type._swf.register_workflow_type.assert_called_with('test', 'name', '1', default_execution_start_to_close_timeout=ANY, default_task_start_to_close_timeout=ANY, default_child_policy=ANY ) def test_activity_type_register_defaults(self): act_type = ActivityType(name='name', domain='test', version='1') act_type.register() act_type._swf.register_activity_type.assert_called_with('test', 'name', '1', default_task_heartbeat_timeout=ANY, default_task_schedule_to_close_timeout=ANY, default_task_schedule_to_start_timeout=ANY, default_task_start_to_close_timeout=ANY ) def test_workflow_type_start_execution(self): wf_type = WorkflowType(name='name', domain='test', version='1') run_id = '122aJcg6ic7MRAkjDRzLBsqU/R49qt5D0LPHycT/6ArN4=' wf_type._swf.start_workflow_execution.return_value = {'runId': run_id} execution = wf_type.start(task_list='hello_world') self.assertIsInstance(execution, WorkflowExecution) self.assertEqual(wf_type.name, execution.name) self.assertEqual(wf_type.version, execution.version) self.assertEqual(run_id, execution.runId) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/swf/test_layer2_domain.py 0000644 00000013552 15025013616 0016400 0 ustar 00 import boto.swf.layer2 from boto.swf.layer2 import Domain, ActivityType, WorkflowType, WorkflowExecution from tests.unit import unittest from mock import Mock class TestDomain(unittest.TestCase): def setUp(self): boto.swf.layer2.Layer1 = Mock() self.domain = Domain(name='test-domain', description='My test domain') self.domain.aws_access_key_id = 'inheritable access key' self.domain.aws_secret_access_key = 'inheritable secret key' self.domain.region = 'test-region' def test_domain_instantiation(self): self.assertEqual('test-domain', self.domain.name) self.assertEqual('My test domain', self.domain.description) def test_domain_list_activities(self): self.domain._swf.list_activity_types.return_value = { 'typeInfos': [{'activityType': {'name': 'DeleteLocalFile', 'version': '1.0'}, 'creationDate': 1332853651.235, 'status': 'REGISTERED'}, {'activityType': {'name': 'DoUpdate', 'version': 'test'}, 'creationDate': 1333463734.528, 'status': 'REGISTERED'}, {'activityType': {'name': 'GrayscaleTransform', 'version': '1.0'}, 'creationDate': 1332853651.18, 'status': 'REGISTERED'}, {'activityType': {'name': 'S3Download', 'version': '1.0'}, 'creationDate': 1332853651.264, 'status': 'REGISTERED'}, {'activityType': {'name': 'S3Upload', 'version': '1.0'}, 'creationDate': 1332853651.314, 'status': 'REGISTERED'}, {'activityType': {'name': 'SepiaTransform', 'version': '1.1'}, 'creationDate': 1333373797.734, 'status': 'REGISTERED'}]} expected_names = ('DeleteLocalFile', 'GrayscaleTransform', 'S3Download', 'S3Upload', 'SepiaTransform', 'DoUpdate') activity_types = self.domain.activities() self.assertEqual(6, len(activity_types)) for activity_type in activity_types: self.assertIsInstance(activity_type, ActivityType) self.assertTrue(activity_type.name in expected_names) self.assertEqual(self.domain.region, activity_type.region) def test_domain_list_workflows(self): self.domain._swf.list_workflow_types.return_value = { 'typeInfos': [{'creationDate': 1332853651.136, 'description': 'Image processing sample workflow type', 'status': 'REGISTERED', 'workflowType': {'name': 'ProcessFile', 'version': '1.0'}}, {'creationDate': 1333551719.89, 'status': 'REGISTERED', 'workflowType': {'name': 'test_workflow_name', 'version': 'v1'}}]} expected_names = ('ProcessFile', 'test_workflow_name') workflow_types = self.domain.workflows() self.assertEqual(2, len(workflow_types)) for workflow_type in workflow_types: self.assertIsInstance(workflow_type, WorkflowType) self.assertTrue(workflow_type.name in expected_names) self.assertEqual(self.domain.aws_access_key_id, workflow_type.aws_access_key_id) self.assertEqual(self.domain.aws_secret_access_key, workflow_type.aws_secret_access_key) self.assertEqual(self.domain.name, workflow_type.domain) self.assertEqual(self.domain.region, workflow_type.region) def test_domain_list_executions(self): self.domain._swf.list_open_workflow_executions.return_value = { 'executionInfos': [{'cancelRequested': False, 'execution': {'runId': '12OeDTyoD27TDaafViz/QIlCHrYzspZmDgj0coIfjm868=', 'workflowId': 'ProcessFile-1.0-1378933928'}, 'executionStatus': 'OPEN', 'startTimestamp': 1378933928.676, 'workflowType': {'name': 'ProcessFile', 'version': '1.0'}}, {'cancelRequested': False, 'execution': {'runId': '12GwBkx4hH6t2yaIh8LYxy5HyCM6HcyhDKePJCg0/ciJk=', 'workflowId': 'ProcessFile-1.0-1378933927'}, 'executionStatus': 'OPEN', 'startTimestamp': 1378933927.919, 'workflowType': {'name': 'ProcessFile', 'version': '1.0'}}, {'cancelRequested': False, 'execution': {'runId': '12oRG3vEWrQ7oYBV+Bqi33Fht+ZRCYTt+tOdn5kLVcwKI=', 'workflowId': 'ProcessFile-1.0-1378933926'}, 'executionStatus': 'OPEN', 'startTimestamp': 1378933927.04, 'workflowType': {'name': 'ProcessFile', 'version': '1.0'}}, {'cancelRequested': False, 'execution': {'runId': '12qrdcpYmad2cjnqJcM4Njm3qrCGvmRFR1wwQEt+a2ako=', 'workflowId': 'ProcessFile-1.0-1378933874'}, 'executionStatus': 'OPEN', 'startTimestamp': 1378933874.956, 'workflowType': {'name': 'ProcessFile', 'version': '1.0'}}]} executions = self.domain.executions() self.assertEqual(4, len(executions)) for wf_execution in executions: self.assertIsInstance(wf_execution, WorkflowExecution) self.assertEqual(self.domain.aws_access_key_id, wf_execution.aws_access_key_id) self.assertEqual(self.domain.aws_secret_access_key, wf_execution.aws_secret_access_key) self.assertEqual(self.domain.name, wf_execution.domain) self.assertEqual(self.domain.region, wf_execution.region) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/swf/test_layer1_decisions.py 0000644 00000002234 15025013616 0017103 0 ustar 00 from tests.unit import unittest import boto.swf.layer1_decisions class TestDecisions(unittest.TestCase): def setUp(self): self.decisions = boto.swf.layer1_decisions.Layer1Decisions() def assert_data(self, *data): self.assertEqual(self.decisions._data, list(data)) def test_continue_as_new_workflow_execution(self): self.decisions.continue_as_new_workflow_execution( child_policy='TERMINATE', execution_start_to_close_timeout='10', input='input', tag_list=['t1', 't2'], task_list='tasklist', start_to_close_timeout='20', workflow_type_version='v2' ) self.assert_data({ 'decisionType': 'ContinueAsNewWorkflowExecution', 'continueAsNewWorkflowExecutionDecisionAttributes': { 'childPolicy': 'TERMINATE', 'executionStartToCloseTimeout': '10', 'input': 'input', 'tagList': ['t1', 't2'], 'taskList': {'name': 'tasklist'}, 'taskStartToCloseTimeout': '20', 'workflowTypeVersion': 'v2', } }) vendored/boto/tests/unit/swf/test_layer2_actors.py 0000644 00000007711 15025013616 0016424 0 ustar 00 import boto.swf.layer2 from boto.swf.layer2 import Decider, ActivityWorker from tests.unit import unittest from mock import Mock class TestActors(unittest.TestCase): def setUp(self): boto.swf.layer2.Layer1 = Mock() self.worker = ActivityWorker(name='test-worker', domain='test', task_list='test_list') self.decider = Decider(name='test-worker', domain='test', task_list='test_list') self.worker._swf = Mock() self.decider._swf = Mock() def test_decider_pass_tasktoken(self): self.decider._swf.poll_for_decision_task.return_value = { 'events': [{'eventId': 1, 'eventTimestamp': 1379019427.953, 'eventType': 'WorkflowExecutionStarted', 'workflowExecutionStartedEventAttributes': { 'childPolicy': 'TERMINATE', 'executionStartToCloseTimeout': '3600', 'parentInitiatedEventId': 0, 'taskList': {'name': 'test_list'}, 'taskStartToCloseTimeout': '123', 'workflowType': {'name': 'test_workflow_name', 'version': 'v1'}}}, {'decisionTaskScheduledEventAttributes': {'startToCloseTimeout': '123', 'taskList': {'name': 'test_list'}}, 'eventId': 2, 'eventTimestamp': 1379019427.953, 'eventType': 'DecisionTaskScheduled'}, {'decisionTaskStartedEventAttributes': {'scheduledEventId': 2}, 'eventId': 3, 'eventTimestamp': 1379019495.585, 'eventType': 'DecisionTaskStarted'}], 'previousStartedEventId': 0, 'startedEventId': 3, 'taskToken': 'my_specific_task_token', 'workflowExecution': {'runId': 'fwr243dsa324132jmflkfu0943tr09=', 'workflowId': 'test_workflow_name-v1-1379019427'}, 'workflowType': {'name': 'test_workflow_name', 'version': 'v1'}} self.decider.poll() self.decider.complete() self.decider._swf.respond_decision_task_completed.assert_called_with('my_specific_task_token', None) self.assertEqual('my_specific_task_token', self.decider.last_tasktoken) def test_worker_pass_tasktoken(self): task_token = 'worker_task_token' self.worker._swf.poll_for_activity_task.return_value = { 'activityId': 'SomeActivity-1379020713', 'activityType': {'name': 'SomeActivity', 'version': '1.0'}, 'startedEventId': 6, 'taskToken': task_token, 'workflowExecution': {'runId': '12T026NzGK5c4eMti06N9O3GHFuTDaNyA+8LFtoDkAwfE=', 'workflowId': 'MyWorkflow-1.0-1379020705'}} self.worker.poll() self.worker.cancel(details='Cancelling!') self.worker.complete(result='Done!') self.worker.fail(reason='Failure!') self.worker.heartbeat() self.worker._swf.respond_activity_task_canceled.assert_called_with(task_token, 'Cancelling!') self.worker._swf.respond_activity_task_completed.assert_called_with(task_token, 'Done!') self.worker._swf.respond_activity_task_failed.assert_called_with(task_token, None, 'Failure!') self.worker._swf.record_activity_task_heartbeat.assert_called_with(task_token, None) def test_actor_poll_without_tasklist_override(self): self.worker.poll() self.decider.poll() self.worker._swf.poll_for_activity_task.assert_called_with('test', 'test_list') self.decider._swf.poll_for_decision_task.assert_called_with('test', 'test_list') def test_worker_override_tasklist(self): self.worker.poll(task_list='some_other_tasklist') self.worker._swf.poll_for_activity_task.assert_called_with('test', 'some_other_tasklist') def test_decider_override_tasklist(self): self.decider.poll(task_list='some_other_tasklist') self.decider._swf.poll_for_decision_task.assert_called_with('test', 'some_other_tasklist') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/swf/__init__.py 0000644 00000000000 15025013616 0014333 0 ustar 00 vendored/boto/tests/unit/test_endpoints.json 0000644 00000000104 15025013616 0015365 0 ustar 00 { "ec2": { "test-1": "ec2.test-1.amazonaws.com" } } vendored/boto/tests/unit/cloudsearch/test_document.py 0000644 00000027375 15025013616 0017176 0 ustar 00 #!/usr/bin env python from tests.unit import unittest from httpretty import HTTPretty from mock import MagicMock import json from boto.cloudsearch.document import DocumentServiceConnection from boto.cloudsearch.document import CommitMismatchError, EncodingError, \ ContentTooLongError, DocumentServiceConnection, SearchServiceException import boto class CloudSearchDocumentTest(unittest.TestCase): def setUp(self): HTTPretty.enable() HTTPretty.register_uri( HTTPretty.POST, ("http://doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com/" "2011-02-01/documents/batch"), body=json.dumps(self.response).encode('utf-8'), content_type="application/json") def tearDown(self): HTTPretty.disable() class CloudSearchDocumentSingleTest(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 1, 'deletes': 0, } def test_cloudsearch_add_basics(self): """ Check that a simple add document actually sends an add document request to AWS. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", 10, {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8'))[0] self.assertEqual(args['lang'], 'en') self.assertEqual(args['type'], 'add') def test_cloudsearch_add_single_basic(self): """ Check that a simple add document sends correct document metadata to AWS. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", 10, {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8'))[0] self.assertEqual(args['id'], '1234') self.assertEqual(args['version'], 10) self.assertEqual(args['type'], 'add') def test_cloudsearch_add_single_fields(self): """ Check that a simple add document sends the actual document to AWS. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", 10, {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8'))[0] self.assertEqual(args['fields']['category'], ['cat_a', 'cat_b', 'cat_c']) self.assertEqual(args['fields']['id'], '1234') self.assertEqual(args['fields']['title'], 'Title 1') def test_cloudsearch_add_single_result(self): """ Check that the reply from adding a single document is correctly parsed. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", 10, {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) doc = document.commit() self.assertEqual(doc.status, 'success') self.assertEqual(doc.adds, 1) self.assertEqual(doc.deletes, 0) self.assertEqual(doc.doc_service, document) class CloudSearchDocumentMultipleAddTest(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 3, 'deletes': 0, } objs = { '1234': { 'version': 10, 'fields': {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}}, '1235': { 'version': 11, 'fields': {"id": "1235", "title": "Title 2", "category": ["cat_b", "cat_c", "cat_d"]}}, '1236': { 'version': 12, 'fields': {"id": "1236", "title": "Title 3", "category": ["cat_e", "cat_f", "cat_g"]}}, } def test_cloudsearch_add_basics(self): """Check that multiple documents are added correctly to AWS""" document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") for (key, obj) in self.objs.items(): document.add(key, obj['version'], obj['fields']) document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8')) for arg in args: self.assertTrue(arg['id'] in self.objs) self.assertEqual(arg['version'], self.objs[arg['id']]['version']) self.assertEqual(arg['fields']['id'], self.objs[arg['id']]['fields']['id']) self.assertEqual(arg['fields']['title'], self.objs[arg['id']]['fields']['title']) self.assertEqual(arg['fields']['category'], self.objs[arg['id']]['fields']['category']) def test_cloudsearch_add_results(self): """ Check that the result from adding multiple documents is parsed correctly. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") for (key, obj) in self.objs.items(): document.add(key, obj['version'], obj['fields']) doc = document.commit() self.assertEqual(doc.status, 'success') self.assertEqual(doc.adds, len(self.objs)) self.assertEqual(doc.deletes, 0) class CloudSearchDocumentDelete(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 0, 'deletes': 1, } def test_cloudsearch_delete(self): """ Test that the request for a single document deletion is done properly. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.delete("5", "10") document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8'))[0] self.assertEqual(args['version'], '10') self.assertEqual(args['type'], 'delete') self.assertEqual(args['id'], '5') def test_cloudsearch_delete_results(self): """ Check that the result of a single document deletion is parsed properly. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.delete("5", "10") doc = document.commit() self.assertEqual(doc.status, 'success') self.assertEqual(doc.adds, 0) self.assertEqual(doc.deletes, 1) class CloudSearchDocumentDeleteMultiple(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 0, 'deletes': 2, } def test_cloudsearch_delete_multiples(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.delete("5", "10") document.delete("6", "11") document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8')) self.assertEqual(len(args), 2) for arg in args: self.assertEqual(arg['type'], 'delete') if arg['id'] == '5': self.assertEqual(arg['version'], '10') elif arg['id'] == '6': self.assertEqual(arg['version'], '11') else: # Unknown result out of AWS that shouldn't be there self.assertTrue(False) class CloudSearchSDFManipulation(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 1, 'deletes': 0, } def test_cloudsearch_initial_sdf_is_blank(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") self.assertEqual(document.get_sdf(), '[]') def test_cloudsearch_single_document_sdf(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", 10, {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) self.assertNotEqual(document.get_sdf(), '[]') document.clear_sdf() self.assertEqual(document.get_sdf(), '[]') class CloudSearchBadSDFTesting(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 1, 'deletes': 0, } def test_cloudsearch_erroneous_sdf(self): original = boto.log.error boto.log.error = MagicMock() document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", 10, {"id": "1234", "title": None, "category": ["cat_a", "cat_b", "cat_c"]}) document.commit() self.assertNotEqual(len(boto.log.error.call_args_list), 1) boto.log.error = original class CloudSearchDocumentErrorBadUnicode(CloudSearchDocumentTest): response = { 'status': 'error', 'adds': 0, 'deletes': 0, 'errors': [{'message': 'Illegal Unicode character in document'}] } def test_fake_bad_unicode(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", 10, {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) self.assertRaises(EncodingError, document.commit) class CloudSearchDocumentErrorDocsTooBig(CloudSearchDocumentTest): response = { 'status': 'error', 'adds': 0, 'deletes': 0, 'errors': [{'message': 'The Content-Length is too long'}] } def test_fake_docs_too_big(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", 10, {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) self.assertRaises(ContentTooLongError, document.commit) class CloudSearchDocumentErrorMismatch(CloudSearchDocumentTest): response = { 'status': 'error', 'adds': 0, 'deletes': 0, 'errors': [{'message': 'Something went wrong'}] } def test_fake_failure(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", 10, {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) self.assertRaises(CommitMismatchError, document.commit) class CloudSearchDocumentsErrorMissingAdds(CloudSearchDocumentTest): response = { 'status': 'error', 'deletes': 0, 'errors': [{'message': 'Unknown error message'}] } def test_fake_failure(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", 10, {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) self.assertRaises(SearchServiceException, document.commit) vendored/boto/tests/unit/cloudsearch/test_exceptions.py 0000644 00000002670 15025013616 0017530 0 ustar 00 from boto.compat import json from tests.compat import mock, unittest from tests.unit.cloudsearch.test_search import HOSTNAME, \ CloudSearchSearchBaseTest from boto.cloudsearch.search import SearchConnection, SearchServiceException def fake_loads_value_error(content, *args, **kwargs): """Callable to generate a fake ValueError""" raise ValueError("HAHAHA! Totally not simplejson & you gave me bad JSON.") def fake_loads_json_error(content, *args, **kwargs): """Callable to generate a fake JSONDecodeError""" raise json.JSONDecodeError('Using simplejson & you gave me bad JSON.', '', 0) class CloudSearchJSONExceptionTest(CloudSearchSearchBaseTest): response = b'{}' def test_no_simplejson_value_error(self): with mock.patch.object(json, 'loads', fake_loads_value_error): search = SearchConnection(endpoint=HOSTNAME) with self.assertRaisesRegex(SearchServiceException, 'non-json'): search.search(q='test') @unittest.skipUnless(hasattr(json, 'JSONDecodeError'), 'requires simplejson') def test_simplejson_jsondecodeerror(self): with mock.patch.object(json, 'loads', fake_loads_json_error): search = SearchConnection(endpoint=HOSTNAME) with self.assertRaisesRegex(SearchServiceException, 'non-json'): search.search(q='test') vendored/boto/tests/unit/cloudsearch/test_connection.py 0000644 00000020501 15025013616 0017477 0 ustar 00 #!/usr/bin env python from tests.unit import AWSMockServiceTestCase from boto.cloudsearch.domain import Domain from boto.cloudsearch.layer1 import Layer1 class TestCloudSearchCreateDomain(AWSMockServiceTestCase): connection_class = Layer1 def default_body(self): return b""" <CreateDomainResponse xmlns="http://cloudsearch.amazonaws.com/doc/2011-02-01"> <CreateDomainResult> <DomainStatus> <SearchPartitionCount>0</SearchPartitionCount> <SearchService> <Arn>arn:aws:cs:us-east-1:1234567890:search/demo</Arn> <Endpoint>search-demo-userdomain.us-east-1.cloudsearch.amazonaws.com</Endpoint> </SearchService> <NumSearchableDocs>0</NumSearchableDocs> <Created>true</Created> <DomainId>1234567890/demo</DomainId> <Processing>false</Processing> <SearchInstanceCount>0</SearchInstanceCount> <DomainName>demo</DomainName> <RequiresIndexDocuments>false</RequiresIndexDocuments> <Deleted>false</Deleted> <DocService> <Arn>arn:aws:cs:us-east-1:1234567890:doc/demo</Arn> <Endpoint>doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com</Endpoint> </DocService> </DomainStatus> </CreateDomainResult> <ResponseMetadata> <RequestId>00000000-0000-0000-0000-000000000000</RequestId> </ResponseMetadata> </CreateDomainResponse> """ def test_create_domain(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') self.assert_request_parameters({ 'Action': 'CreateDomain', 'DomainName': 'demo', 'Version': '2011-02-01', }) def test_cloudsearch_connect_result_endpoints(self): """Check that endpoints & ARNs are correctly returned from AWS""" self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') domain = Domain(self, api_response) self.assertEqual(domain.doc_service_arn, "arn:aws:cs:us-east-1:1234567890:doc/demo") self.assertEqual( domain.doc_service_endpoint, "doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") self.assertEqual(domain.search_service_arn, "arn:aws:cs:us-east-1:1234567890:search/demo") self.assertEqual( domain.search_service_endpoint, "search-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") def test_cloudsearch_connect_result_statuses(self): """Check that domain statuses are correctly returned from AWS""" self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') domain = Domain(self, api_response) self.assertEqual(domain.created, True) self.assertEqual(domain.processing, False) self.assertEqual(domain.requires_index_documents, False) self.assertEqual(domain.deleted, False) def test_cloudsearch_connect_result_details(self): """Check that the domain information is correctly returned from AWS""" self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') domain = Domain(self, api_response) self.assertEqual(domain.id, "1234567890/demo") self.assertEqual(domain.name, "demo") def test_cloudsearch_documentservice_creation(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') domain = Domain(self, api_response) document = domain.get_document_service() self.assertEqual( document.endpoint, "doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") def test_cloudsearch_searchservice_creation(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') domain = Domain(self, api_response) search = domain.get_search_service() self.assertEqual( search.endpoint, "search-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") class CloudSearchConnectionDeletionTest(AWSMockServiceTestCase): connection_class = Layer1 def default_body(self): return b""" <DeleteDomainResponse xmlns="http://cloudsearch.amazonaws.com/doc/2011-02-01"> <DeleteDomainResult> <DomainStatus> <SearchPartitionCount>0</SearchPartitionCount> <SearchService> <Arn>arn:aws:cs:us-east-1:1234567890:search/demo</Arn> <Endpoint>search-demo-userdomain.us-east-1.cloudsearch.amazonaws.com</Endpoint> </SearchService> <NumSearchableDocs>0</NumSearchableDocs> <Created>true</Created> <DomainId>1234567890/demo</DomainId> <Processing>false</Processing> <SearchInstanceCount>0</SearchInstanceCount> <DomainName>demo</DomainName> <RequiresIndexDocuments>false</RequiresIndexDocuments> <Deleted>false</Deleted> <DocService> <Arn>arn:aws:cs:us-east-1:1234567890:doc/demo</Arn> <Endpoint>doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com</Endpoint> </DocService> </DomainStatus> </DeleteDomainResult> <ResponseMetadata> <RequestId>00000000-0000-0000-0000-000000000000</RequestId> </ResponseMetadata> </DeleteDomainResponse> """ def test_cloudsearch_deletion(self): """ Check that the correct arguments are sent to AWS when creating a cloudsearch connection. """ self.set_http_response(status_code=200) api_response = self.service_connection.delete_domain('demo') self.assert_request_parameters({ 'Action': 'DeleteDomain', 'DomainName': 'demo', 'Version': '2011-02-01', }) class CloudSearchConnectionIndexDocumentTest(AWSMockServiceTestCase): connection_class = Layer1 def default_body(self): return b""" <IndexDocumentsResponse xmlns="http://cloudsearch.amazonaws.com/doc/2011-02-01"> <IndexDocumentsResult> <FieldNames> <member>average_score</member> <member>brand_id</member> <member>colors</member> <member>context</member> <member>context_owner</member> <member>created_at</member> <member>creator_id</member> <member>description</member> <member>file_size</member> <member>format</member> <member>has_logo</member> <member>has_messaging</member> <member>height</member> <member>image_id</member> <member>ingested_from</member> <member>is_advertising</member> <member>is_photo</member> <member>is_reviewed</member> <member>modified_at</member> <member>subject_date</member> <member>tags</member> <member>title</member> <member>width</member> </FieldNames> </IndexDocumentsResult> <ResponseMetadata> <RequestId>eb2b2390-6bbd-11e2-ab66-93f3a90dcf2a</RequestId> </ResponseMetadata> </IndexDocumentsResponse> """ def test_cloudsearch_index_documents(self): """ Check that the correct arguments are sent to AWS when indexing a domain. """ self.set_http_response(status_code=200) api_response = self.service_connection.index_documents('demo') self.assert_request_parameters({ 'Action': 'IndexDocuments', 'DomainName': 'demo', 'Version': '2011-02-01', }) def test_cloudsearch_index_documents_resp(self): """ Check that the AWS response is being parsed correctly when indexing a domain. """ self.set_http_response(status_code=200) api_response = self.service_connection.index_documents('demo') self.assertEqual(api_response, ['average_score', 'brand_id', 'colors', 'context', 'context_owner', 'created_at', 'creator_id', 'description', 'file_size', 'format', 'has_logo', 'has_messaging', 'height', 'image_id', 'ingested_from', 'is_advertising', 'is_photo', 'is_reviewed', 'modified_at', 'subject_date', 'tags', 'title', 'width']) vendored/boto/tests/unit/cloudsearch/test_search.py 0000644 00000032635 15025013616 0016620 0 ustar 00 #!/usr/bin env python from tests.compat import mock, unittest from httpretty import HTTPretty import json import requests from boto.cloudsearch.search import SearchConnection, SearchServiceException from boto.compat import six, map HOSTNAME = "search-demo-userdomain.us-east-1.cloudsearch.amazonaws.com" FULL_URL = 'http://%s/2011-02-01/search' % HOSTNAME class CloudSearchSearchBaseTest(unittest.TestCase): hits = [ { 'id': '12341', 'title': 'Document 1', }, { 'id': '12342', 'title': 'Document 2', }, { 'id': '12343', 'title': 'Document 3', }, { 'id': '12344', 'title': 'Document 4', }, { 'id': '12345', 'title': 'Document 5', }, { 'id': '12346', 'title': 'Document 6', }, { 'id': '12347', 'title': 'Document 7', }, ] content_type = "text/xml" response_status = 200 def get_args(self, requestline): (_, request, _) = requestline.split(b" ") (_, request) = request.split(b"?", 1) args = six.moves.urllib.parse.parse_qs(request) return args def setUp(self): HTTPretty.enable() body = self.response if not isinstance(body, bytes): body = json.dumps(body).encode('utf-8') HTTPretty.register_uri(HTTPretty.GET, FULL_URL, body=body, content_type=self.content_type, status=self.response_status) def tearDown(self): HTTPretty.disable() class CloudSearchSearchTest(CloudSearchSearchBaseTest): response = { 'rank': '-text_relevance', 'match-expr': "Test", 'hits': { 'found': 30, 'start': 0, 'hit': CloudSearchSearchBaseTest.hits }, 'info': { 'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08', 'time-ms': 2, 'cpu-time-ms': 0 } } def test_cloudsearch_qsearch(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test') args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'q'], [b"Test"]) self.assertEqual(args[b'start'], [b"0"]) self.assertEqual(args[b'size'], [b"10"]) def test_cloudsearch_bqsearch(self): search = SearchConnection(endpoint=HOSTNAME) search.search(bq="'Test'") args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'bq'], [b"'Test'"]) def test_cloudsearch_search_details(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', size=50, start=20) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'q'], [b"Test"]) self.assertEqual(args[b'size'], [b"50"]) self.assertEqual(args[b'start'], [b"20"]) def test_cloudsearch_facet_single(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', facet=["Author"]) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet'], [b"Author"]) def test_cloudsearch_facet_multiple(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', facet=["author", "cat"]) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet'], [b"author,cat"]) def test_cloudsearch_facet_constraint_single(self): search = SearchConnection(endpoint=HOSTNAME) search.search( q='Test', facet_constraints={'author': "'John Smith','Mark Smith'"}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet-author-constraints'], [b"'John Smith','Mark Smith'"]) def test_cloudsearch_facet_constraint_multiple(self): search = SearchConnection(endpoint=HOSTNAME) search.search( q='Test', facet_constraints={'author': "'John Smith','Mark Smith'", 'category': "'News','Reviews'"}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet-author-constraints'], [b"'John Smith','Mark Smith'"]) self.assertEqual(args[b'facet-category-constraints'], [b"'News','Reviews'"]) def test_cloudsearch_facet_sort_single(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', facet_sort={'author': 'alpha'}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet-author-sort'], [b'alpha']) def test_cloudsearch_facet_sort_multiple(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', facet_sort={'author': 'alpha', 'cat': 'count'}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet-author-sort'], [b'alpha']) self.assertEqual(args[b'facet-cat-sort'], [b'count']) def test_cloudsearch_top_n_single(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', facet_top_n={'author': 5}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet-author-top-n'], [b'5']) def test_cloudsearch_top_n_multiple(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', facet_top_n={'author': 5, 'cat': 10}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet-author-top-n'], [b'5']) self.assertEqual(args[b'facet-cat-top-n'], [b'10']) def test_cloudsearch_rank_single(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', rank=["date"]) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'rank'], [b'date']) def test_cloudsearch_rank_multiple(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', rank=["date", "score"]) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'rank'], [b'date,score']) def test_cloudsearch_result_fields_single(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', return_fields=['author']) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'return-fields'], [b'author']) def test_cloudsearch_result_fields_multiple(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', return_fields=['author', 'title']) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'return-fields'], [b'author,title']) def test_cloudsearch_t_field_single(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', t={'year': '2001..2007'}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b't-year'], [b'2001..2007']) def test_cloudsearch_t_field_multiple(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', t={'year': '2001..2007', 'score': '10..50'}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b't-year'], [b'2001..2007']) self.assertEqual(args[b't-score'], [b'10..50']) def test_cloudsearch_results_meta(self): """Check returned metadata is parsed correctly""" search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test') # These rely on the default response which is fed into HTTPretty self.assertEqual(results.rank, "-text_relevance") self.assertEqual(results.match_expression, "Test") def test_cloudsearch_results_info(self): """Check num_pages_needed is calculated correctly""" search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test') # This relies on the default response which is fed into HTTPretty self.assertEqual(results.num_pages_needed, 3.0) def test_cloudsearch_results_matched(self): """ Check that information objects are passed back through the API correctly. """ search = SearchConnection(endpoint=HOSTNAME) query = search.build_query(q='Test') results = search(query) self.assertEqual(results.search_service, search) self.assertEqual(results.query, query) def test_cloudsearch_results_hits(self): """Check that documents are parsed properly from AWS""" search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test') hits = list(map(lambda x: x['id'], results.docs)) # This relies on the default response which is fed into HTTPretty self.assertEqual( hits, ["12341", "12342", "12343", "12344", "12345", "12346", "12347"]) def test_cloudsearch_results_iterator(self): """Check the results iterator""" search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test') results_correct = iter(["12341", "12342", "12343", "12344", "12345", "12346", "12347"]) for x in results: self.assertEqual(x['id'], next(results_correct)) def test_cloudsearch_results_internal_consistancy(self): """Check the documents length matches the iterator details""" search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test') self.assertEqual(len(results), len(results.docs)) def test_cloudsearch_search_nextpage(self): """Check next page query is correct""" search = SearchConnection(endpoint=HOSTNAME) query1 = search.build_query(q='Test') query2 = search.build_query(q='Test') results = search(query2) self.assertEqual(results.next_page().query.start, query1.start + query1.size) self.assertEqual(query1.q, query2.q) class CloudSearchSearchFacetTest(CloudSearchSearchBaseTest): response = { 'rank': '-text_relevance', 'match-expr': "Test", 'hits': { 'found': 30, 'start': 0, 'hit': CloudSearchSearchBaseTest.hits }, 'info': { 'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08', 'time-ms': 2, 'cpu-time-ms': 0 }, 'facets': { 'tags': {}, 'animals': {'constraints': [{'count': '2', 'value': 'fish'}, {'count': '1', 'value': 'lions'}]}, } } def test_cloudsearch_search_facets(self): #self.response['facets'] = {'tags': {}} search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test', facet=['tags']) self.assertTrue('tags' not in results.facets) self.assertEqual(results.facets['animals'], {u'lions': u'1', u'fish': u'2'}) class CloudSearchNonJsonTest(CloudSearchSearchBaseTest): response = b'<html><body><h1>500 Internal Server Error</h1></body></html>' response_status = 500 content_type = 'text/xml' def test_response(self): search = SearchConnection(endpoint=HOSTNAME) with self.assertRaises(SearchServiceException): search.search(q='Test') class CloudSearchUnauthorizedTest(CloudSearchSearchBaseTest): response = b'<html><body><h1>403 Forbidden</h1>foo bar baz</body></html>' response_status = 403 content_type = 'text/html' def test_response(self): search = SearchConnection(endpoint=HOSTNAME) with self.assertRaisesRegex(SearchServiceException, 'foo bar baz'): search.search(q='Test') class FakeResponse(object): status_code = 405 content = b'' class CloudSearchConnectionTest(unittest.TestCase): cloudsearch = True def setUp(self): super(CloudSearchConnectionTest, self).setUp() self.conn = SearchConnection( endpoint='test-domain.cloudsearch.amazonaws.com' ) def test_expose_additional_error_info(self): mpo = mock.patch.object fake = FakeResponse() fake.content = b'Nopenopenope' # First, in the case of a non-JSON, non-403 error. with mpo(requests, 'get', return_value=fake) as mock_request: with self.assertRaises(SearchServiceException) as cm: self.conn.search(q='not_gonna_happen') self.assertTrue('non-json response' in str(cm.exception)) self.assertTrue('Nopenopenope' in str(cm.exception)) # Then with JSON & an 'error' key within. fake.content = json.dumps({ 'error': "Something went wrong. Oops." }).encode('utf-8') with mpo(requests, 'get', return_value=fake) as mock_request: with self.assertRaises(SearchServiceException) as cm: self.conn.search(q='no_luck_here') self.assertTrue('Unknown error' in str(cm.exception)) self.assertTrue('went wrong. Oops' in str(cm.exception)) vendored/boto/tests/unit/cloudsearch/__init__.py 0000644 00000000001 15025013616 0016031 0 ustar 00 vendored/boto/tests/unit/mws/test_response.py 0000644 00000021166 15025013616 0015520 0 ustar 00 #!/usr/bin/env python from boto.mws.connection import MWSConnection from boto.mws.response import (ResponseFactory, ResponseElement, Element, MemberList, ElementList, SimpleList) from tests.unit import AWSMockServiceTestCase from boto.compat import filter, map from tests.compat import unittest class TestMWSResponse(AWSMockServiceTestCase): connection_class = MWSConnection mws = True def test_parsing_nested_elements(self): class Test9one(ResponseElement): Nest = Element() Zoom = Element() class Test9Result(ResponseElement): Item = Element(Test9one) text = b"""<Test9Response><Test9Result> <Item> <Foo>Bar</Foo> <Nest> <Zip>Zap</Zip> <Zam>Zoo</Zam> </Nest> <Bif>Bam</Bif> </Item> </Test9Result></Test9Response>""" obj = self.check_issue(Test9Result, text) Item = obj._result.Item useful = lambda x: not x[0].startswith('_') nest = dict(filter(useful, Item.Nest.__dict__.items())) self.assertEqual(nest, dict(Zip='Zap', Zam='Zoo')) useful = lambda x: not x[0].startswith('_') and not x[0] == 'Nest' item = dict(filter(useful, Item.__dict__.items())) self.assertEqual(item, dict(Foo='Bar', Bif='Bam', Zoom=None)) def test_parsing_member_list_specification(self): class Test8extra(ResponseElement): Foo = SimpleList() class Test8Result(ResponseElement): Item = MemberList(SimpleList) Extra = MemberList(Test8extra) text = b"""<Test8Response><Test8Result> <Item> <member>0</member> <member>1</member> <member>2</member> <member>3</member> </Item> <Extra> <member><Foo>4</Foo><Foo>5</Foo></member> <member></member> <member><Foo>6</Foo><Foo>7</Foo></member> </Extra> </Test8Result></Test8Response>""" obj = self.check_issue(Test8Result, text) self.assertSequenceEqual( list(map(int, obj._result.Item)), list(range(4)), ) self.assertSequenceEqual( list(map(lambda x: list(map(int, x.Foo)), obj._result.Extra)), [[4, 5], [], [6, 7]], ) def test_parsing_nested_lists(self): class Test7Result(ResponseElement): Item = MemberList(Nest=MemberList(), List=ElementList(Simple=SimpleList())) text = b"""<Test7Response><Test7Result> <Item> <member> <Value>One</Value> <Nest> <member><Data>2</Data></member> <member><Data>4</Data></member> <member><Data>6</Data></member> </Nest> </member> <member> <Value>Two</Value> <Nest> <member><Data>1</Data></member> <member><Data>3</Data></member> <member><Data>5</Data></member> </Nest> <List> <Simple>4</Simple> <Simple>5</Simple> <Simple>6</Simple> </List> <List> <Simple>7</Simple> <Simple>8</Simple> <Simple>9</Simple> </List> </member> <member> <Value>Six</Value> <List> <Complex>Foo</Complex> <Simple>1</Simple> <Simple>2</Simple> <Simple>3</Simple> </List> <List> <Complex>Bar</Complex> </List> </member> </Item> </Test7Result></Test7Response>""" obj = self.check_issue(Test7Result, text) item = obj._result.Item self.assertEqual(len(item), 3) nests = [z.Nest for z in filter(lambda x: x.Nest, item)] self.assertSequenceEqual( [[y.Data for y in nest] for nest in nests], [[u'2', u'4', u'6'], [u'1', u'3', u'5']], ) self.assertSequenceEqual( [element.Simple for element in item[1].List], [[u'4', u'5', u'6'], [u'7', u'8', u'9']], ) self.assertSequenceEqual( item[-1].List[0].Simple, ['1', '2', '3'], ) self.assertEqual(item[-1].List[1].Simple, []) self.assertSequenceEqual( [e.Value for e in obj._result.Item], ['One', 'Two', 'Six'], ) def test_parsing_member_list(self): class Test6Result(ResponseElement): Item = MemberList() text = b"""<Test6Response><Test6Result> <Item> <member><Value>One</Value></member> <member><Value>Two</Value> <Error>Four</Error> </member> <member><Value>Six</Value></member> </Item> </Test6Result></Test6Response>""" obj = self.check_issue(Test6Result, text) self.assertSequenceEqual( [e.Value for e in obj._result.Item], ['One', 'Two', 'Six'], ) self.assertTrue(obj._result.Item[1].Error == 'Four') with self.assertRaises(AttributeError) as e: obj._result.Item[2].Error def test_parsing_empty_member_list(self): class Test5Result(ResponseElement): Item = MemberList(Nest=MemberList()) text = b"""<Test5Response><Test5Result> <Item/> </Test5Result></Test5Response>""" obj = self.check_issue(Test5Result, text) self.assertSequenceEqual(obj._result.Item, []) def test_parsing_missing_member_list(self): class Test4Result(ResponseElement): Item = MemberList(NestedItem=MemberList()) text = b"""<Test4Response><Test4Result> </Test4Result></Test4Response>""" obj = self.check_issue(Test4Result, text) self.assertSequenceEqual(obj._result.Item, []) def test_parsing_element_lists(self): class Test1Result(ResponseElement): Item = ElementList() text = b"""<Test1Response><Test1Result> <Item><Foo>Bar</Foo></Item> <Item><Zip>Bif</Zip></Item> <Item><Foo>Baz</Foo> <Zam>Zoo</Zam></Item> </Test1Result></Test1Response>""" obj = self.check_issue(Test1Result, text) self.assertTrue(len(obj._result.Item) == 3) elements = lambda x: getattr(x, 'Foo', getattr(x, 'Zip', '?')) elements = list(map(elements, obj._result.Item)) self.assertSequenceEqual(elements, ['Bar', 'Bif', 'Baz']) def test_parsing_missing_lists(self): class Test2Result(ResponseElement): Item = ElementList() text = b"""<Test2Response><Test2Result> </Test2Result></Test2Response>""" obj = self.check_issue(Test2Result, text) self.assertEqual(obj._result.Item, []) def test_parsing_simple_lists(self): class Test3Result(ResponseElement): Item = SimpleList() text = b"""<Test3Response><Test3Result> <Item>Bar</Item> <Item>Bif</Item> <Item>Baz</Item> </Test3Result></Test3Response>""" obj = self.check_issue(Test3Result, text) self.assertSequenceEqual(obj._result.Item, ['Bar', 'Bif', 'Baz']) def check_issue(self, klass, text): action = klass.__name__[:-len('Result')] factory = ResponseFactory(scopes=[{klass.__name__: klass}]) parser = factory(action, connection=self.service_connection) return self.service_connection._parse_response(parser, 'text/xml', text) if __name__ == "__main__": unittest.main() vendored/boto/tests/unit/mws/test_connection.py 0000644 00000020411 15025013616 0016011 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.mws.connection import MWSConnection, api_call_map, destructure_object from boto.mws.response import (ResponseElement, GetFeedSubmissionListResult, ResponseFactory) from boto.exception import BotoServerError from tests.compat import unittest from tests.unit import AWSMockServiceTestCase from mock import MagicMock class TestMWSConnection(AWSMockServiceTestCase): connection_class = MWSConnection mws = True def default_body(self): return b"""<?xml version="1.0"?> <GetFeedSubmissionListResponse xmlns="http://mws.amazonservices.com/ doc/2009-01-01/"> <GetFeedSubmissionListResult> <NextToken>2YgYW55IGNhcm5hbCBwbGVhc3VyZS4=</NextToken> <HasNext>true</HasNext> <FeedSubmissionInfo> <FeedSubmissionId>2291326430</FeedSubmissionId> <FeedType>_POST_PRODUCT_DATA_</FeedType> <SubmittedDate>2009-02-20T02:10:35+00:00</SubmittedDate> <FeedProcessingStatus>_SUBMITTED_</FeedProcessingStatus> </FeedSubmissionInfo> </GetFeedSubmissionListResult> <ResponseMetadata> <RequestId>1105b931-6f1c-4480-8e97-f3b467840a9e</RequestId> </ResponseMetadata> </GetFeedSubmissionListResponse>""" def default_body_error(self): return b"""<?xml version="1.0" encoding="UTF-8"?> <ErrorResponse xmlns="http://mws.amazonaws.com/doc/2009-01-01/"> <!--1 or more repetitions:--> <Error> <Type>Sender</Type> <Code>string</Code> <Message>string</Message> <Detail> <!--You may enter ANY elements at this point--> <AnyElement xmlns=""/> </Detail> </Error> <RequestId>string</RequestId> </ErrorResponse>""" def test_destructure_object(self): # Test that parsing of user input to Amazon input works. response = ResponseElement() response.C = 'four' response.D = 'five' inputs = [ ('A', 'B'), ['B', 'A'], set(['C']), False, 'String', {'A': 'one', 'B': 'two'}, response, {'A': 'one', 'B': 'two', 'C': [{'D': 'four', 'E': 'five'}, {'F': 'six', 'G': 'seven'}]}, ] outputs = [ {'Prefix.1': 'A', 'Prefix.2': 'B'}, {'Prefix.1': 'B', 'Prefix.2': 'A'}, {'Prefix.1': 'C'}, {'Prefix': 'false'}, {'Prefix': 'String'}, {'Prefix.A': 'one', 'Prefix.B': 'two'}, {'Prefix.C': 'four', 'Prefix.D': 'five'}, {'Prefix.A': 'one', 'Prefix.B': 'two', 'Prefix.C.member.1.D': 'four', 'Prefix.C.member.1.E': 'five', 'Prefix.C.member.2.F': 'six', 'Prefix.C.member.2.G': 'seven'} ] for user, amazon in zip(inputs, outputs): result = {} members = user is inputs[-1] destructure_object(user, result, prefix='Prefix', members=members) self.assertEqual(result, amazon) def test_decorator_order(self): for action, func in api_call_map.items(): func = getattr(self.service_connection, func) decs = [func.__name__] while func: i = 0 if not hasattr(func, '__closure__'): func = getattr(func, '__wrapped__', None) continue while i < len(func.__closure__): value = func.__closure__[i].cell_contents if hasattr(value, '__call__'): if 'requires' == value.__name__: self.assertTrue(not decs or decs[-1] == 'requires') decs.append(value.__name__) i += 1 func = getattr(func, '__wrapped__', None) def test_built_api_call_map(self): # Ensure that the map is populated. # It starts empty, but the decorators should add to it as they're # applied. As of 2013/10/21, there were 52 calls (with more likely # to be added), so let's simply ensure there are enough there. self.assertTrue(len(api_call_map.keys()) > 50) def test_method_for(self): # First, ensure that the map is in "right enough" state. self.assertTrue('GetFeedSubmissionList' in api_call_map) # Make sure we can find the correct method. func = self.service_connection.method_for('GetFeedSubmissionList') # Ensure the right name was found. self.assertTrue(callable(func)) ideal = self.service_connection.get_feed_submission_list self.assertEqual(func, ideal) # Check a non-existent action. func = self.service_connection.method_for('NotHereNorThere') self.assertEqual(func, None) def test_response_factory(self): connection = self.service_connection body = self.default_body() action = 'GetFeedSubmissionList' parser = connection._response_factory(action, connection=connection) response = connection._parse_response(parser, 'text/xml', body) self.assertEqual(response._action, action) self.assertEqual(response.__class__.__name__, action + 'Response') self.assertEqual(response._result.__class__, GetFeedSubmissionListResult) class MyResult(GetFeedSubmissionListResult): _hello = '_world' scope = {'GetFeedSubmissionListResult': MyResult} connection._setup_factories([scope]) parser = connection._response_factory(action, connection=connection) response = connection._parse_response(parser, 'text/xml', body) self.assertEqual(response._action, action) self.assertEqual(response.__class__.__name__, action + 'Response') self.assertEqual(response._result.__class__, MyResult) self.assertEqual(response._result._hello, '_world') self.assertEqual(response._result.HasNext, 'true') def test_get_service_status(self): with self.assertRaises(AttributeError) as err: self.service_connection.get_service_status() self.assertTrue('products' in str(err.exception)) self.assertTrue('inventory' in str(err.exception)) self.assertTrue('feeds' in str(err.exception)) def test_post_request(self): self.service_connection._mexe = MagicMock( side_effect= BotoServerError(500, 'You request has bee throttled', body=self.default_body_error())) with self.assertRaises(BotoServerError) as err: self.service_connection.get_lowest_offer_listings_for_asin( MarketplaceId='12345', ASINList='ASIN12345', condition='Any', SellerId='1234', excludeme='True') self.assertTrue('throttled' in str(err.reason)) self.assertEqual(int(err.status), 200) def test_sandboxify(self): # Create one-off connection class that has self._sandboxed = True conn = MWSConnection(https_connection_factory=self.https_connection_factory, aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key', sandbox=True) self.assertEqual(conn._sandboxify('a/bogus/path'), 'a/bogus_Sandbox/path') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/mws/__init__.py 0000644 00000000000 15025013616 0014342 0 ustar 00 vendored/boto/tests/unit/auth/test_stsanon.py 0000644 00000006374 15025013616 0015506 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import copy from mock import Mock from tests.unit import unittest from boto.auth import STSAnonHandler from boto.connection import HTTPRequest class TestSTSAnonHandler(unittest.TestCase): def setUp(self): self.provider = Mock() self.provider.access_key = 'access_key' self.provider.secret_key = 'secret_key' self.request = HTTPRequest( method='GET', protocol='https', host='sts.amazonaws.com', port=443, path='/', auth_path=None, params={ 'Action': 'AssumeRoleWithWebIdentity', 'Version': '2011-06-15', 'RoleSessionName': 'web-identity-federation', 'ProviderId': '2012-06-01', 'WebIdentityToken': 'Atza|IQEBLjAsAhRkcxQ', }, headers={}, body='' ) def test_escape_value(self): auth = STSAnonHandler('sts.amazonaws.com', Mock(), self.provider) # This is changed from a previous version because this string is # being passed to the query string and query strings must # be url encoded. value = auth._escape_value('Atza|IQEBLjAsAhRkcxQ') self.assertEqual(value, 'Atza%7CIQEBLjAsAhRkcxQ') def test_build_query_string(self): auth = STSAnonHandler('sts.amazonaws.com', Mock(), self.provider) query_string = auth._build_query_string(self.request.params) self.assertEqual(query_string, 'Action=AssumeRoleWithWebIdentity' + \ '&ProviderId=2012-06-01&RoleSessionName=web-identity-federation' + \ '&Version=2011-06-15&WebIdentityToken=Atza%7CIQEBLjAsAhRkcxQ') def test_add_auth(self): auth = STSAnonHandler('sts.amazonaws.com', Mock(), self.provider) req = copy.copy(self.request) auth.add_auth(req) self.assertEqual(req.body, 'Action=AssumeRoleWithWebIdentity' + \ '&ProviderId=2012-06-01&RoleSessionName=web-identity-federation' + \ '&Version=2011-06-15&WebIdentityToken=Atza%7CIQEBLjAsAhRkcxQ') vendored/boto/tests/unit/auth/test_sigv4.py 0000644 00000067030 15025013616 0015051 0 ustar 00 # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import copy import pickle import os from tests.compat import unittest, mock from tests.unit import MockServiceWithConfigTestCase from nose.tools import assert_equal from boto.auth import HmacAuthV4Handler from boto.auth import S3HmacAuthV4Handler from boto.auth import detect_potential_s3sigv4 from boto.auth import detect_potential_sigv4 from boto.connection import HTTPRequest from boto.provider import Provider from boto.regioninfo import RegionInfo class TestSigV4Handler(unittest.TestCase): def setUp(self): self.provider = mock.Mock() self.provider.access_key = 'access_key' self.provider.secret_key = 'secret_key' self.request = HTTPRequest( 'POST', 'https', 'glacier.us-east-1.amazonaws.com', 443, '/-/vaults/foo/archives', None, {}, {'x-amz-glacier-version': '2012-06-01'}, '') def test_not_adding_empty_qs(self): self.provider.security_token = None auth = HmacAuthV4Handler('glacier.us-east-1.amazonaws.com', mock.Mock(), self.provider) req = copy.copy(self.request) auth.add_auth(req) self.assertEqual(req.path, '/-/vaults/foo/archives') def test_inner_whitespace_is_collapsed(self): auth = HmacAuthV4Handler('glacier.us-east-1.amazonaws.com', mock.Mock(), self.provider) self.request.headers['x-amz-archive-description'] = 'two spaces' self.request.headers['x-amz-quoted-string'] = ' "a b c" ' headers = auth.headers_to_sign(self.request) self.assertEqual(headers, {'Host': 'glacier.us-east-1.amazonaws.com', 'x-amz-archive-description': 'two spaces', 'x-amz-glacier-version': '2012-06-01', 'x-amz-quoted-string': ' "a b c" '}) # Note the single space between the "two spaces". self.assertEqual(auth.canonical_headers(headers), 'host:glacier.us-east-1.amazonaws.com\n' 'x-amz-archive-description:two spaces\n' 'x-amz-glacier-version:2012-06-01\n' 'x-amz-quoted-string:"a b c"') def test_canonical_query_string(self): auth = HmacAuthV4Handler('glacier.us-east-1.amazonaws.com', mock.Mock(), self.provider) request = HTTPRequest( 'GET', 'https', 'glacier.us-east-1.amazonaws.com', 443, '/-/vaults/foo/archives', None, {}, {'x-amz-glacier-version': '2012-06-01'}, '') request.params['Foo.1'] = 'aaa' request.params['Foo.10'] = 'zzz' query_string = auth.canonical_query_string(request) self.assertEqual(query_string, 'Foo.1=aaa&Foo.10=zzz') def test_query_string(self): auth = HmacAuthV4Handler('sns.us-east-1.amazonaws.com', mock.Mock(), self.provider) params = { 'Message': u'We \u2665 utf-8'.encode('utf-8'), } request = HTTPRequest( 'POST', 'https', 'sns.us-east-1.amazonaws.com', 443, '/', None, params, {}, '') query_string = auth.query_string(request) self.assertEqual(query_string, 'Message=We%20%E2%99%A5%20utf-8') def test_canonical_uri(self): auth = HmacAuthV4Handler('glacier.us-east-1.amazonaws.com', mock.Mock(), self.provider) request = HTTPRequest( 'GET', 'https', 'glacier.us-east-1.amazonaws.com', 443, 'x/./././x .html', None, {}, {'x-amz-glacier-version': '2012-06-01'}, '') canonical_uri = auth.canonical_uri(request) # This should be both normalized & urlencoded. self.assertEqual(canonical_uri, 'x/x%20.html') auth = HmacAuthV4Handler('glacier.us-east-1.amazonaws.com', mock.Mock(), self.provider) request = HTTPRequest( 'GET', 'https', 'glacier.us-east-1.amazonaws.com', 443, 'x/./././x/html/', None, {}, {'x-amz-glacier-version': '2012-06-01'}, '') canonical_uri = auth.canonical_uri(request) # Trailing slashes should be preserved. self.assertEqual(canonical_uri, 'x/x/html/') request = HTTPRequest( 'GET', 'https', 'glacier.us-east-1.amazonaws.com', 443, '/', None, {}, {'x-amz-glacier-version': '2012-06-01'}, '') canonical_uri = auth.canonical_uri(request) # There should not be two-slashes. self.assertEqual(canonical_uri, '/') # Make sure Windows-style slashes are converted properly request = HTTPRequest( 'GET', 'https', 'glacier.us-east-1.amazonaws.com', 443, '\\x\\x.html', None, {}, {'x-amz-glacier-version': '2012-06-01'}, '') canonical_uri = auth.canonical_uri(request) self.assertEqual(canonical_uri, '/x/x.html') def test_credential_scope(self): # test the AWS standard regions IAM endpoint auth = HmacAuthV4Handler('iam.amazonaws.com', mock.Mock(), self.provider) request = HTTPRequest( 'POST', 'https', 'iam.amazonaws.com', 443, '/', '/', {'Action': 'ListAccountAliases', 'Version': '2010-05-08'}, { 'Content-Length': '44', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'X-Amz-Date': '20130808T013210Z' }, 'Action=ListAccountAliases&Version=2010-05-08') credential_scope = auth.credential_scope(request) region_name = credential_scope.split('/')[1] self.assertEqual(region_name, 'us-east-1') # test the AWS GovCloud region IAM endpoint auth = HmacAuthV4Handler('iam.us-gov.amazonaws.com', mock.Mock(), self.provider) request = HTTPRequest( 'POST', 'https', 'iam.us-gov.amazonaws.com', 443, '/', '/', {'Action': 'ListAccountAliases', 'Version': '2010-05-08'}, { 'Content-Length': '44', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'X-Amz-Date': '20130808T013210Z' }, 'Action=ListAccountAliases&Version=2010-05-08') credential_scope = auth.credential_scope(request) region_name = credential_scope.split('/')[1] self.assertEqual(region_name, 'us-gov-west-1') # iam.us-west-1.amazonaws.com does not exist however this # covers the remaining region_name control structure for a # different region name auth = HmacAuthV4Handler('iam.us-west-1.amazonaws.com', mock.Mock(), self.provider) request = HTTPRequest( 'POST', 'https', 'iam.us-west-1.amazonaws.com', 443, '/', '/', {'Action': 'ListAccountAliases', 'Version': '2010-05-08'}, { 'Content-Length': '44', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'X-Amz-Date': '20130808T013210Z' }, 'Action=ListAccountAliases&Version=2010-05-08') credential_scope = auth.credential_scope(request) region_name = credential_scope.split('/')[1] self.assertEqual(region_name, 'us-west-1') # Test connections to custom locations, e.g. localhost:8080 auth = HmacAuthV4Handler('localhost', mock.Mock(), self.provider, service_name='iam') request = HTTPRequest( 'POST', 'http', 'localhost', 8080, '/', '/', {'Action': 'ListAccountAliases', 'Version': '2010-05-08'}, { 'Content-Length': '44', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'X-Amz-Date': '20130808T013210Z' }, 'Action=ListAccountAliases&Version=2010-05-08') credential_scope = auth.credential_scope(request) timestamp, region, service, v = credential_scope.split('/') self.assertEqual(region, 'localhost') self.assertEqual(service, 'iam') def test_headers_to_sign(self): auth = HmacAuthV4Handler('glacier.us-east-1.amazonaws.com', mock.Mock(), self.provider) request = HTTPRequest( 'GET', 'http', 'glacier.us-east-1.amazonaws.com', 80, 'x/./././x .html', None, {}, {'x-amz-glacier-version': '2012-06-01'}, '') headers = auth.headers_to_sign(request) # Port 80 & not secure excludes the port. self.assertEqual(headers['Host'], 'glacier.us-east-1.amazonaws.com') request = HTTPRequest( 'GET', 'https', 'glacier.us-east-1.amazonaws.com', 443, 'x/./././x .html', None, {}, {'x-amz-glacier-version': '2012-06-01'}, '') headers = auth.headers_to_sign(request) # SSL port excludes the port. self.assertEqual(headers['Host'], 'glacier.us-east-1.amazonaws.com') request = HTTPRequest( 'GET', 'https', 'glacier.us-east-1.amazonaws.com', 8080, 'x/./././x .html', None, {}, {'x-amz-glacier-version': '2012-06-01'}, '') headers = auth.headers_to_sign(request) # URL should include port. self.assertEqual(headers['Host'], 'glacier.us-east-1.amazonaws.com:8080') def test_region_and_service_can_be_overriden(self): auth = HmacAuthV4Handler('queue.amazonaws.com', mock.Mock(), self.provider) self.request.headers['X-Amz-Date'] = '20121121000000' auth.region_name = 'us-west-2' auth.service_name = 'sqs' scope = auth.credential_scope(self.request) self.assertEqual(scope, '20121121/us-west-2/sqs/aws4_request') def test_pickle_works(self): provider = Provider('aws', access_key='access_key', secret_key='secret_key') auth = HmacAuthV4Handler('queue.amazonaws.com', None, provider) # Pickle it! pickled = pickle.dumps(auth) # Now restore it auth2 = pickle.loads(pickled) self.assertEqual(auth.host, auth2.host) def test_bytes_header(self): auth = HmacAuthV4Handler('glacier.us-east-1.amazonaws.com', mock.Mock(), self.provider) request = HTTPRequest( 'GET', 'http', 'glacier.us-east-1.amazonaws.com', 80, 'x/./././x .html', None, {}, {'x-amz-glacier-version': '2012-06-01', 'x-amz-hash': b'f00'}, '') canonical = auth.canonical_request(request) self.assertIn('f00', canonical) class TestS3HmacAuthV4Handler(unittest.TestCase): def setUp(self): self.provider = mock.Mock() self.provider.access_key = 'access_key' self.provider.secret_key = 'secret_key' self.provider.security_token = 'sekret_tokens' self.request = HTTPRequest( 'GET', 'https', 's3-us-west-2.amazonaws.com', 443, '/awesome-bucket/?max-keys=0', None, {}, {}, '' ) self.awesome_bucket_request = HTTPRequest( method='GET', protocol='https', host='awesome-bucket.s3-us-west-2.amazonaws.com', port=443, path='/', auth_path=None, params={ 'max-keys': 0, }, headers={ 'User-Agent': 'Boto', 'X-AMZ-Content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'X-AMZ-Date': '20130605T193245Z', }, body='' ) self.auth = S3HmacAuthV4Handler( host='awesome-bucket.s3-us-west-2.amazonaws.com', config=mock.Mock(), provider=self.provider, region_name='s3-us-west-2' ) def test_clean_region_name(self): # Untouched. cleaned = self.auth.clean_region_name('us-west-2') self.assertEqual(cleaned, 'us-west-2') # Stripped of the ``s3-`` prefix. cleaned = self.auth.clean_region_name('s3-us-west-2') self.assertEqual(cleaned, 'us-west-2') # Untouched (classic). cleaned = self.auth.clean_region_name('s3.amazonaws.com') self.assertEqual(cleaned, 's3.amazonaws.com') # Untouched. cleaned = self.auth.clean_region_name('something-s3-us-west-2') self.assertEqual(cleaned, 'something-s3-us-west-2') def test_region_stripping(self): auth = S3HmacAuthV4Handler( host='s3-us-west-2.amazonaws.com', config=mock.Mock(), provider=self.provider ) self.assertEqual(auth.region_name, None) # What we wish we got. auth = S3HmacAuthV4Handler( host='s3-us-west-2.amazonaws.com', config=mock.Mock(), provider=self.provider, region_name='us-west-2' ) self.assertEqual(auth.region_name, 'us-west-2') # What we actually get (i.e. ``s3-us-west-2``). self.assertEqual(self.auth.region_name, 'us-west-2') def test_determine_region_name(self): name = self.auth.determine_region_name('s3-us-west-2.amazonaws.com') self.assertEqual(name, 'us-west-2') def test_canonical_uri(self): request = HTTPRequest( 'GET', 'https', 's3-us-west-2.amazonaws.com', 443, 'x/./././~x .html', None, {}, {}, '' ) canonical_uri = self.auth.canonical_uri(request) # S3 doesn't canonicalize the way other SigV4 services do. # This just urlencoded, no normalization of the path. self.assertEqual(canonical_uri, 'x/./././~x%20.html') def test_determine_service_name(self): # What we wish we got. name = self.auth.determine_service_name( 's3.us-west-2.amazonaws.com' ) self.assertEqual(name, 's3') # What we actually get. name = self.auth.determine_service_name( 's3-us-west-2.amazonaws.com' ) self.assertEqual(name, 's3') # What we wish we got with virtual hosting. name = self.auth.determine_service_name( 'bucket.s3.us-west-2.amazonaws.com' ) self.assertEqual(name, 's3') # What we actually get with virtual hosting. name = self.auth.determine_service_name( 'bucket.s3-us-west-2.amazonaws.com' ) self.assertEqual(name, 's3') def test_add_auth(self): # The side-effects sideshow. self.assertFalse('x-amz-content-sha256' in self.request.headers) self.auth.add_auth(self.request) self.assertTrue('x-amz-content-sha256' in self.request.headers) the_sha = self.request.headers['x-amz-content-sha256'] self.assertEqual( the_sha, 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' ) def test_host_header(self): host = self.auth.host_header( self.awesome_bucket_request.host, self.awesome_bucket_request ) self.assertEqual(host, 'awesome-bucket.s3-us-west-2.amazonaws.com') def test_canonical_query_string(self): qs = self.auth.canonical_query_string(self.awesome_bucket_request) self.assertEqual(qs, 'max-keys=0') def test_correct_handling_of_plus_sign(self): request = HTTPRequest( 'GET', 'https', 's3-us-west-2.amazonaws.com', 443, 'hello+world.txt', None, {}, {}, '' ) canonical_uri = self.auth.canonical_uri(request) # Ensure that things are properly quoted. self.assertEqual(canonical_uri, 'hello%2Bworld.txt') request = HTTPRequest( 'GET', 'https', 's3-us-west-2.amazonaws.com', 443, 'hello%2Bworld.txt', None, {}, {}, '' ) canonical_uri = self.auth.canonical_uri(request) # Verify double escaping hasn't occurred. self.assertEqual(canonical_uri, 'hello%2Bworld.txt') def test_mangle_path_and_params(self): request = HTTPRequest( method='GET', protocol='https', host='awesome-bucket.s3-us-west-2.amazonaws.com', port=443, # LOOK AT THIS PATH. JUST LOOK AT IT. path='/?delete&max-keys=0', auth_path=None, params={ 'key': 'why hello there', # This gets overwritten, to make sure back-compat is maintained. 'max-keys': 1, }, headers={ 'User-Agent': 'Boto', 'X-AMZ-Content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'X-AMZ-Date': '20130605T193245Z', }, body='' ) mod_req = self.auth.mangle_path_and_params(request) self.assertEqual(mod_req.path, '/?delete&max-keys=0') self.assertEqual(mod_req.auth_path, '/') self.assertEqual(mod_req.params, { 'max-keys': '0', 'key': 'why hello there', 'delete': '' }) def test_unicode_query_string(self): request = HTTPRequest( method='HEAD', protocol='https', host='awesome-bucket.s3-us-west-2.amazonaws.com', port=443, path=u'/?max-keys=1&prefix=El%20Ni%C3%B1o', auth_path=u'/awesome-bucket/?max-keys=1&prefix=El%20Ni%C3%B1o', params={}, headers={}, body='' ) mod_req = self.auth.mangle_path_and_params(request) self.assertEqual(mod_req.path, u'/?max-keys=1&prefix=El%20Ni%C3%B1o') self.assertEqual(mod_req.auth_path, u'/awesome-bucket/') self.assertEqual(mod_req.params, { u'max-keys': u'1', u'prefix': u'El Ni\xf1o', }) def test_canonical_request(self): expected = """GET / max-keys=0 host:awesome-bucket.s3-us-west-2.amazonaws.com user-agent:Boto x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 x-amz-date:20130605T193245Z host;user-agent;x-amz-content-sha256;x-amz-date e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855""" authed_req = self.auth.canonical_request(self.awesome_bucket_request) self.assertEqual(authed_req, expected) # Now the way ``boto.s3`` actually sends data. request = copy.copy(self.awesome_bucket_request) request.path = request.auth_path = '/?max-keys=0' request.params = {} expected = """GET / max-keys=0 host:awesome-bucket.s3-us-west-2.amazonaws.com user-agent:Boto x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 x-amz-date:20130605T193245Z host;user-agent;x-amz-content-sha256;x-amz-date e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855""" # Pre-mangle it. In practice, this happens as part of ``add_auth``, # but that's a side-effect that's hard to test. request = self.auth.mangle_path_and_params(request) authed_req = self.auth.canonical_request(request) self.assertEqual(authed_req, expected) def test_non_string_headers(self): self.awesome_bucket_request.headers['Content-Length'] = 8 # Headers in canonical order are alphabetized by key alone. # This test ensures we are not alphabetizing based on the header/value # separator as well: self.awesome_bucket_request.headers['x-amz-server-side-encryption-customer-key-md5'] = 2 self.awesome_bucket_request.headers['x-amz-server-side-encryption-customer-key'] = 1 canonical_headers = self.auth.canonical_headers( self.awesome_bucket_request.headers) self.assertEqual( canonical_headers, 'content-length:8\n' 'user-agent:Boto\n' 'x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae' '41e4649b934ca495991b7852b855\n' 'x-amz-date:20130605T193245Z\n' 'x-amz-server-side-encryption-customer-key:1\n' 'x-amz-server-side-encryption-customer-key-md5:2' ) class FakeS3Connection(object): def __init__(self, *args, **kwargs): self.host = kwargs.pop('host', None) self.anon = kwargs.pop('anon', None) @detect_potential_s3sigv4 def _required_auth_capability(self): if self.anon: return ['anon'] return ['nope'] def _mexe(self, *args, **kwargs): pass class FakeEC2Connection(object): def __init__(self, *args, **kwargs): self.region = kwargs.pop('region', None) @detect_potential_sigv4 def _required_auth_capability(self): return ['nope'] def _mexe(self, *args, **kwargs): pass def test_s3_sigv2_default(): sigv2_regions = [ 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-west-1', 'external-1', 'sa-east-1', 'us-east-1', 'us-gov-west-1', 'us-west-1', 'us-west-2' ] for region in sigv2_regions: _yield_all_region_tests(region, expected_signature_version='nope') def test_s3_sigv4_default(): sigv4_regions = [ 'ap-northeast-2', 'ap-south-1', 'ca-central-1', 'eu-central-1', 'eu-west-2', 'us-east-2' ] for region in sigv4_regions: _yield_all_region_tests(region) cn_regions = [ 'cn-north-1' ] for region in cn_regions: _yield_all_region_tests(region, dns_suffix='amazon.com.cn') # Unknown region _yield_all_region_tests('mars-west-1') def test_s3_special_domain_signature_version(): # Tests for specific domains, including the global host and custom domains. special_domains = [ 'storage.googleapis.com', 'mycustomdomain.example.com', 's3.amazonaws.com.example.com', 'mycustomdomain.example.com/amazonaws.com' ] for domain in special_domains: yield S3SignatureVersionTestCase(domain, 'nope').run def test_s3_default_signature_version(): # Default region. # s3.amazonaws.com is an alias to s2.us-east-1.amazonaws.com case = S3SignatureVersionTestCase('s3.amazonaws.com', 'hmac-v4-s3') yield case.run def test_s3_anon_signature_version(): # Anonymous case = S3SignatureVersionTestCase('s3.amazonaws.com', 'anon', anon=True) yield case.run def _yield_all_region_tests(region, expected_signature_version='hmac-v4-s3', dns_suffix='.amazonaws.com'): """Yield tests for every variation of a region's endpoints.""" # Standard endpoint host = 's3.' + region + dns_suffix case = S3SignatureVersionTestCase(host, expected_signature_version) yield case.run # Dashed endpoint host = 's3-' + region + dns_suffix case = S3SignatureVersionTestCase(host, expected_signature_version) yield case.run # Endpoint with host style addressing host = 'mybucket.s3-' + region + dns_suffix case = S3SignatureVersionTestCase(host, expected_signature_version) yield case.run class S3SignatureVersionTestCase(object): def __init__(self, host, expected_signture_version, anon=None): self.host = host self.connection = FakeS3Connection(host=host, anon=anon) self.expected_signature_version = expected_signture_version def run(self): auth = self.connection._required_auth_capability() message = ( "Expected signature version ['%s'] for host %s but found %s." % ( self.expected_signature_version, self.host, auth ) ) assert_equal(auth, [self.expected_signature_version], message) class TestS3SigV4OptInAndOut(MockServiceWithConfigTestCase): connection_class = FakeS3Connection def test_sigv4_opt_in_config(self): # Opt-in via the config. self.config = { 's3': { 'use-sigv4': 'true', }, } fake = FakeS3Connection() self.assertEqual(fake._required_auth_capability(), ['hmac-v4-s3']) def test_sigv4_opt_out_config(self): # Opt-in via the config. self.config = { 's3': { 'use-sigv4': 'False', }, } fake = FakeS3Connection() self.assertEqual(fake._required_auth_capability(), ['nope']) def test_sigv4_incorrect_config(self): """Test that default(sigv4) is chosen if incorrect value is present.""" self.config = { 's3': { 'use-sigv4': 'someval', }, } fake = FakeS3Connection(host='s3.amazonaws.com') self.assertEqual(fake._required_auth_capability(), ['hmac-v4-s3']) def test_sigv4_opt_in_env(self): # Opt-in via the ENV. self.environ['S3_USE_SIGV4'] = 'True' fake = FakeS3Connection(host='s3.amazonaws.com') self.assertEqual(fake._required_auth_capability(), ['hmac-v4-s3']) def test_sigv4_opt_out_env(self): # Opt-in via the ENV. self.environ['S3_USE_SIGV4'] = 'False' fake = FakeS3Connection(host='s3.amazonaws.com') self.assertEqual(fake._required_auth_capability(), ['nope']) class TestSigV4OptIn(MockServiceWithConfigTestCase): connection_class = FakeEC2Connection def setUp(self): super(TestSigV4OptIn, self).setUp() self.standard_region = RegionInfo( name='us-west-2', endpoint='ec2.us-west-2.amazonaws.com' ) self.sigv4_region = RegionInfo( name='cn-north-1', endpoint='ec2.cn-north-1.amazonaws.com.cn' ) def test_sigv4_opt_out(self): # Default is opt-out. fake = FakeEC2Connection(region=self.standard_region) self.assertEqual(fake._required_auth_capability(), ['nope']) def test_sigv4_non_optional(self): # Requires SigV4. fake = FakeEC2Connection(region=self.sigv4_region) self.assertEqual(fake._required_auth_capability(), ['hmac-v4']) def test_sigv4_opt_in_config(self): # Opt-in via the config. self.config = { 'ec2': { 'use-sigv4': True, }, } fake = FakeEC2Connection(region=self.standard_region) self.assertEqual(fake._required_auth_capability(), ['hmac-v4']) def test_sigv4_opt_in_env(self): # Opt-in via the ENV. self.environ['EC2_USE_SIGV4'] = True fake = FakeEC2Connection(region=self.standard_region) self.assertEqual(fake._required_auth_capability(), ['hmac-v4']) vendored/boto/tests/unit/auth/__init__.py 0000644 00000000000 15025013616 0014475 0 ustar 00 vendored/boto/tests/unit/machinelearning/test_machinelearning.py 0000644 00000004034 15025013616 0021317 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.compat import json from boto.machinelearning.layer1 import MachineLearningConnection from tests.unit import AWSMockServiceTestCase class TestMachineLearning(AWSMockServiceTestCase): connection_class = MachineLearningConnection def test_predict(self): ml_endpoint = 'mymlmodel.amazonaws.com' self.set_http_response(status_code=200, body=b'') self.service_connection.predict( ml_model_id='foo', record={'Foo': 'bar'}, predict_endpoint=ml_endpoint) self.assertEqual(self.actual_request.host, ml_endpoint) def test_predict_with_scheme_in_endpoint(self): ml_endpoint = 'mymlmodel.amazonaws.com' self.set_http_response(status_code=200, body=b'') self.service_connection.predict( ml_model_id='foo', record={'Foo': 'bar'}, predict_endpoint='https://' + ml_endpoint) self.assertEqual(self.actual_request.host, ml_endpoint) vendored/boto/tests/unit/machinelearning/__init__.py 0000644 00000002143 15025013616 0016672 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/unit/kms/test_kms.py 0000644 00000005114 15025013616 0014433 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.compat import json from boto.kms.layer1 import KMSConnection from tests.unit import AWSMockServiceTestCase class TestKinesis(AWSMockServiceTestCase): connection_class = KMSConnection def default_body(self): return b'{}' def test_binary_input(self): """ This test ensures that binary is base64 encoded when it is sent to the service. """ self.set_http_response(status_code=200) data = b'\x00\x01\x02\x03\x04\x05' self.service_connection.encrypt(key_id='foo', plaintext=data) body = json.loads(self.actual_request.body.decode('utf-8')) self.assertEqual(body['Plaintext'], 'AAECAwQF') def test_non_binary_input_for_blobs_fails(self): """ This test ensures that only binary is used for blob type parameters. """ self.set_http_response(status_code=200) data = u'\u00e9' with self.assertRaises(TypeError): self.service_connection.encrypt(key_id='foo', plaintext=data) def test_binary_ouput(self): """ This test ensures that the output is base64 decoded before it is returned to the user. """ content = {'Plaintext': 'AAECAwQF'} self.set_http_response(status_code=200, body=json.dumps(content).encode('utf-8')) response = self.service_connection.decrypt(b'some arbitrary value') self.assertEqual(response['Plaintext'], b'\x00\x01\x02\x03\x04\x05') vendored/boto/tests/unit/kms/__init__.py 0000644 00000002143 15025013616 0014340 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/unit/cloudformation/test_connection.py 0000644 00000073746 15025013616 0020253 0 ustar 00 #!/usr/bin/env python import unittest from datetime import datetime from mock import Mock from tests.unit import AWSMockServiceTestCase from boto.cloudformation.connection import CloudFormationConnection from boto.exception import BotoServerError from boto.compat import json SAMPLE_TEMPLATE = r""" { "AWSTemplateFormatVersion" : "2010-09-09", "Description" : "Sample template", "Parameters" : { "KeyName" : { "Description" : "key pair", "Type" : "String" } }, "Resources" : { "Ec2Instance" : { "Type" : "AWS::EC2::Instance", "Properties" : { "KeyName" : { "Ref" : "KeyName" }, "ImageId" : "ami-7f418316", "UserData" : { "Fn::Base64" : "80" } } } }, "Outputs" : { "InstanceId" : { "Description" : "InstanceId of the newly created EC2 instance", "Value" : { "Ref" : "Ec2Instance" } } } """ class CloudFormationConnectionBase(AWSMockServiceTestCase): connection_class = CloudFormationConnection def setUp(self): super(CloudFormationConnectionBase, self).setUp() self.stack_id = u'arn:aws:cloudformation:us-east-1:18:stack/Name/id' class TestCloudFormationCreateStack(CloudFormationConnectionBase): def default_body(self): return json.dumps( {u'CreateStackResponse': {u'CreateStackResult': {u'StackId': self.stack_id}, u'ResponseMetadata': {u'RequestId': u'1'}}}).encode('utf-8') def test_create_stack_has_correct_request_params(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_stack( 'stack_name', template_url='http://url', template_body=SAMPLE_TEMPLATE, parameters=[('KeyName', 'myKeyName')], tags={'TagKey': 'TagValue'}, notification_arns=['arn:notify1', 'arn:notify2'], disable_rollback=True, timeout_in_minutes=20, capabilities=['CAPABILITY_IAM'] ) self.assertEqual(api_response, self.stack_id) # These are the parameters that are actually sent to the CloudFormation # service. self.assert_request_parameters({ 'Action': 'CreateStack', 'Capabilities.member.1': 'CAPABILITY_IAM', 'ContentType': 'JSON', 'DisableRollback': 'true', 'NotificationARNs.member.1': 'arn:notify1', 'NotificationARNs.member.2': 'arn:notify2', 'Parameters.member.1.ParameterKey': 'KeyName', 'Parameters.member.1.ParameterValue': 'myKeyName', 'Tags.member.1.Key': 'TagKey', 'Tags.member.1.Value': 'TagValue', 'StackName': 'stack_name', 'Version': '2010-05-15', 'TimeoutInMinutes': 20, 'TemplateBody': SAMPLE_TEMPLATE, 'TemplateURL': 'http://url', }) # The test_create_stack_has_correct_request_params verified all of the # params needed when making a create_stack service call. The rest of the # tests for create_stack only verify specific parts of the params sent # to CloudFormation. def test_create_stack_with_minimum_args(self): # This will fail in practice, but the API docs only require stack_name. self.set_http_response(status_code=200) api_response = self.service_connection.create_stack('stack_name') self.assertEqual(api_response, self.stack_id) self.assert_request_parameters({ 'Action': 'CreateStack', 'ContentType': 'JSON', 'DisableRollback': 'false', 'StackName': 'stack_name', 'Version': '2010-05-15', }) def test_create_stack_fails(self): self.set_http_response(status_code=400, reason='Bad Request', body=b'{"Error": {"Code": 1, "Message": "Invalid arg."}}') with self.assertRaisesRegex(self.service_connection.ResponseError, 'Invalid arg.'): api_response = self.service_connection.create_stack( 'stack_name', template_body=SAMPLE_TEMPLATE, parameters=[('KeyName', 'myKeyName')]) def test_create_stack_fail_error(self): self.set_http_response(status_code=400, reason='Bad Request', body=b'{"RequestId": "abc", "Error": {"Code": 1, "Message": "Invalid arg."}}') try: api_response = self.service_connection.create_stack( 'stack_name', template_body=SAMPLE_TEMPLATE, parameters=[('KeyName', 'myKeyName')]) except BotoServerError as e: self.assertEqual('abc', e.request_id) self.assertEqual(1, e.error_code) self.assertEqual('Invalid arg.', e.message) class TestCloudFormationUpdateStack(CloudFormationConnectionBase): def default_body(self): return json.dumps( {u'UpdateStackResponse': {u'UpdateStackResult': {u'StackId': self.stack_id}, u'ResponseMetadata': {u'RequestId': u'1'}}}).encode('utf-8') def test_update_stack_all_args(self): self.set_http_response(status_code=200) api_response = self.service_connection.update_stack( 'stack_name', template_url='http://url', template_body=SAMPLE_TEMPLATE, parameters=[('KeyName', 'myKeyName'), ('KeyName2', "", True), ('KeyName3', "", False), ('KeyName4', None, True), ('KeyName5', "Ignore Me", True)], tags={'TagKey': 'TagValue'}, notification_arns=['arn:notify1', 'arn:notify2'], disable_rollback=True, timeout_in_minutes=20, use_previous_template=True ) self.assert_request_parameters({ 'Action': 'UpdateStack', 'ContentType': 'JSON', 'DisableRollback': 'true', 'NotificationARNs.member.1': 'arn:notify1', 'NotificationARNs.member.2': 'arn:notify2', 'Parameters.member.1.ParameterKey': 'KeyName', 'Parameters.member.1.ParameterValue': 'myKeyName', 'Parameters.member.2.ParameterKey': 'KeyName2', 'Parameters.member.2.UsePreviousValue': 'true', 'Parameters.member.3.ParameterKey': 'KeyName3', 'Parameters.member.3.ParameterValue': '', 'Parameters.member.4.UsePreviousValue': 'true', 'Parameters.member.4.ParameterKey': 'KeyName4', 'Parameters.member.5.UsePreviousValue': 'true', 'Parameters.member.5.ParameterKey': 'KeyName5', 'Tags.member.1.Key': 'TagKey', 'Tags.member.1.Value': 'TagValue', 'StackName': 'stack_name', 'Version': '2010-05-15', 'TimeoutInMinutes': 20, 'TemplateBody': SAMPLE_TEMPLATE, 'TemplateURL': 'http://url', 'UsePreviousTemplate': 'true', }) def test_update_stack_with_minimum_args(self): self.set_http_response(status_code=200) api_response = self.service_connection.update_stack('stack_name') self.assertEqual(api_response, self.stack_id) self.assert_request_parameters({ 'Action': 'UpdateStack', 'ContentType': 'JSON', 'DisableRollback': 'false', 'StackName': 'stack_name', 'Version': '2010-05-15', }) def test_update_stack_fails(self): self.set_http_response(status_code=400, reason='Bad Request', body=b'Invalid arg.') with self.assertRaises(self.service_connection.ResponseError): api_response = self.service_connection.update_stack( 'stack_name', template_body=SAMPLE_TEMPLATE, parameters=[('KeyName', 'myKeyName')]) class TestCloudFormationDeleteStack(CloudFormationConnectionBase): def default_body(self): return json.dumps( {u'DeleteStackResponse': {u'ResponseMetadata': {u'RequestId': u'1'}}}).encode('utf-8') def test_delete_stack(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_stack('stack_name') self.assertEqual(api_response, json.loads(self.default_body().decode('utf-8'))) self.assert_request_parameters({ 'Action': 'DeleteStack', 'ContentType': 'JSON', 'StackName': 'stack_name', 'Version': '2010-05-15', }) def test_delete_stack_fails(self): self.set_http_response(status_code=400) with self.assertRaises(self.service_connection.ResponseError): api_response = self.service_connection.delete_stack('stack_name') class TestCloudFormationDescribeStackResource(CloudFormationConnectionBase): def default_body(self): return json.dumps('fake server response').encode('utf-8') def test_describe_stack_resource(self): self.set_http_response(status_code=200) api_response = self.service_connection.describe_stack_resource( 'stack_name', 'resource_id') self.assertEqual(api_response, 'fake server response') self.assert_request_parameters({ 'Action': 'DescribeStackResource', 'ContentType': 'JSON', 'LogicalResourceId': 'resource_id', 'StackName': 'stack_name', 'Version': '2010-05-15', }) def test_describe_stack_resource_fails(self): self.set_http_response(status_code=400) with self.assertRaises(self.service_connection.ResponseError): api_response = self.service_connection.describe_stack_resource( 'stack_name', 'resource_id') class TestCloudFormationGetTemplate(CloudFormationConnectionBase): def default_body(self): return json.dumps('fake server response').encode('utf-8') def test_get_template(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_template('stack_name') self.assertEqual(api_response, 'fake server response') self.assert_request_parameters({ 'Action': 'GetTemplate', 'ContentType': 'JSON', 'StackName': 'stack_name', 'Version': '2010-05-15', }) def test_get_template_fails(self): self.set_http_response(status_code=400) with self.assertRaises(self.service_connection.ResponseError): api_response = self.service_connection.get_template('stack_name') class TestCloudFormationGetStackevents(CloudFormationConnectionBase): def default_body(self): return b""" <DescribeStackEventsResult> <StackEvents> <member> <EventId>Event-1-Id</EventId> <StackId>arn:aws:cfn:us-east-1:1:stack</StackId> <StackName>MyStack</StackName> <LogicalResourceId>MyStack</LogicalResourceId> <PhysicalResourceId>MyStack_One</PhysicalResourceId> <ResourceType>AWS::CloudFormation::Stack</ResourceType> <Timestamp>2010-07-27T22:26:28Z</Timestamp> <ResourceStatus>CREATE_IN_PROGRESS</ResourceStatus> <ResourceStatusReason>User initiated</ResourceStatusReason> </member> <member> <EventId>Event-2-Id</EventId> <StackId>arn:aws:cfn:us-east-1:1:stack</StackId> <StackName>MyStack</StackName> <LogicalResourceId>MySG1</LogicalResourceId> <PhysicalResourceId>MyStack_SG1</PhysicalResourceId> <ResourceType>AWS::SecurityGroup</ResourceType> <Timestamp>2010-07-27T22:28:28Z</Timestamp> <ResourceStatus>CREATE_COMPLETE</ResourceStatus> </member> </StackEvents> </DescribeStackEventsResult> """ def test_describe_stack_events(self): self.set_http_response(status_code=200) first, second = self.service_connection.describe_stack_events('stack_name', next_token='next_token') self.assertEqual(first.event_id, 'Event-1-Id') self.assertEqual(first.logical_resource_id, 'MyStack') self.assertEqual(first.physical_resource_id, 'MyStack_One') self.assertEqual(first.resource_properties, None) self.assertEqual(first.resource_status, 'CREATE_IN_PROGRESS') self.assertEqual(first.resource_status_reason, 'User initiated') self.assertEqual(first.resource_type, 'AWS::CloudFormation::Stack') self.assertEqual(first.stack_id, 'arn:aws:cfn:us-east-1:1:stack') self.assertEqual(first.stack_name, 'MyStack') self.assertIsNotNone(first.timestamp) self.assertEqual(second.event_id, 'Event-2-Id') self.assertEqual(second.logical_resource_id, 'MySG1') self.assertEqual(second.physical_resource_id, 'MyStack_SG1') self.assertEqual(second.resource_properties, None) self.assertEqual(second.resource_status, 'CREATE_COMPLETE') self.assertEqual(second.resource_status_reason, None) self.assertEqual(second.resource_type, 'AWS::SecurityGroup') self.assertEqual(second.stack_id, 'arn:aws:cfn:us-east-1:1:stack') self.assertEqual(second.stack_name, 'MyStack') self.assertIsNotNone(second.timestamp) self.assert_request_parameters({ 'Action': 'DescribeStackEvents', 'NextToken': 'next_token', 'StackName': 'stack_name', 'Version': '2010-05-15', }) class TestCloudFormationDescribeStackResources(CloudFormationConnectionBase): def default_body(self): return b""" <DescribeStackResourcesResult> <StackResources> <member> <StackId>arn:aws:cfn:us-east-1:1:stack</StackId> <StackName>MyStack</StackName> <LogicalResourceId>MyDBInstance</LogicalResourceId> <PhysicalResourceId>MyStack_DB1</PhysicalResourceId> <ResourceType>AWS::DBInstance</ResourceType> <Timestamp>2010-07-27T22:27:28Z</Timestamp> <ResourceStatus>CREATE_COMPLETE</ResourceStatus> </member> <member> <StackId>arn:aws:cfn:us-east-1:1:stack</StackId> <StackName>MyStack</StackName> <LogicalResourceId>MyAutoScalingGroup</LogicalResourceId> <PhysicalResourceId>MyStack_ASG1</PhysicalResourceId> <ResourceType>AWS::AutoScalingGroup</ResourceType> <Timestamp>2010-07-27T22:28:28Z</Timestamp> <ResourceStatus>CREATE_IN_PROGRESS</ResourceStatus> </member> </StackResources> </DescribeStackResourcesResult> """ def test_describe_stack_resources(self): self.set_http_response(status_code=200) first, second = self.service_connection.describe_stack_resources( 'stack_name', 'logical_resource_id', 'physical_resource_id') self.assertEqual(first.description, None) self.assertEqual(first.logical_resource_id, 'MyDBInstance') self.assertEqual(first.physical_resource_id, 'MyStack_DB1') self.assertEqual(first.resource_status, 'CREATE_COMPLETE') self.assertEqual(first.resource_status_reason, None) self.assertEqual(first.resource_type, 'AWS::DBInstance') self.assertEqual(first.stack_id, 'arn:aws:cfn:us-east-1:1:stack') self.assertEqual(first.stack_name, 'MyStack') self.assertIsNotNone(first.timestamp) self.assertEqual(second.description, None) self.assertEqual(second.logical_resource_id, 'MyAutoScalingGroup') self.assertEqual(second.physical_resource_id, 'MyStack_ASG1') self.assertEqual(second.resource_status, 'CREATE_IN_PROGRESS') self.assertEqual(second.resource_status_reason, None) self.assertEqual(second.resource_type, 'AWS::AutoScalingGroup') self.assertEqual(second.stack_id, 'arn:aws:cfn:us-east-1:1:stack') self.assertEqual(second.stack_name, 'MyStack') self.assertIsNotNone(second.timestamp) self.assert_request_parameters({ 'Action': 'DescribeStackResources', 'LogicalResourceId': 'logical_resource_id', 'PhysicalResourceId': 'physical_resource_id', 'StackName': 'stack_name', 'Version': '2010-05-15', }) class TestCloudFormationDescribeStacks(CloudFormationConnectionBase): def default_body(self): return b""" <DescribeStacksResponse> <DescribeStacksResult> <Stacks> <member> <StackId>arn:aws:cfn:us-east-1:1:stack</StackId> <StackStatus>CREATE_COMPLETE</StackStatus> <StackStatusReason>REASON</StackStatusReason> <StackName>MyStack</StackName> <Description>My Description</Description> <CreationTime>2012-05-16T22:55:31Z</CreationTime> <Capabilities> <member>CAPABILITY_IAM</member> </Capabilities> <NotificationARNs> <member>arn:aws:sns:region-name:account-name:topic-name</member> </NotificationARNs> <DisableRollback>false</DisableRollback> <Parameters> <member> <ParameterValue>MyValue</ParameterValue> <ParameterKey>MyKey</ParameterKey> </member> </Parameters> <Outputs> <member> <OutputValue>http://url/</OutputValue> <Description>Server URL</Description> <OutputKey>ServerURL</OutputKey> </member> </Outputs> <Tags> <member> <Key>MyTagKey</Key> <Value>MyTagValue</Value> </member> </Tags> </member> </Stacks> </DescribeStacksResult> <ResponseMetadata> <RequestId>12345</RequestId> </ResponseMetadata> </DescribeStacksResponse> """ def test_describe_stacks(self): self.set_http_response(status_code=200) stacks = self.service_connection.describe_stacks('MyStack') self.assertEqual(len(stacks), 1) stack = stacks[0] self.assertEqual(stack.creation_time, datetime(2012, 5, 16, 22, 55, 31)) self.assertEqual(stack.description, 'My Description') self.assertEqual(stack.disable_rollback, False) self.assertEqual(stack.stack_id, 'arn:aws:cfn:us-east-1:1:stack') self.assertEqual(stack.stack_status, 'CREATE_COMPLETE') self.assertEqual(stack.stack_name, 'MyStack') self.assertEqual(stack.stack_name_reason, 'REASON') self.assertEqual(stack.stack_status_reason, 'REASON') self.assertEqual(stack.timeout_in_minutes, None) self.assertEqual(len(stack.outputs), 1) self.assertEqual(stack.outputs[0].description, 'Server URL') self.assertEqual(stack.outputs[0].key, 'ServerURL') self.assertEqual(stack.outputs[0].value, 'http://url/') self.assertEqual(len(stack.parameters), 1) self.assertEqual(stack.parameters[0].key, 'MyKey') self.assertEqual(stack.parameters[0].value, 'MyValue') self.assertEqual(len(stack.capabilities), 1) self.assertEqual(stack.capabilities[0].value, 'CAPABILITY_IAM') self.assertEqual(len(stack.notification_arns), 1) self.assertEqual(stack.notification_arns[0].value, 'arn:aws:sns:region-name:account-name:topic-name') self.assertEqual(len(stack.tags), 1) self.assertEqual(stack.tags['MyTagKey'], 'MyTagValue') self.assert_request_parameters({ 'Action': 'DescribeStacks', 'StackName': 'MyStack', 'Version': '2010-05-15', }) class TestCloudFormationListStackResources(CloudFormationConnectionBase): def default_body(self): return b""" <ListStackResourcesResponse> <ListStackResourcesResult> <StackResourceSummaries> <member> <ResourceStatus>CREATE_COMPLETE</ResourceStatus> <LogicalResourceId>SampleDB</LogicalResourceId> <LastUpdatedTime>2011-06-21T20:25:57Z</LastUpdatedTime> <PhysicalResourceId>My-db-ycx</PhysicalResourceId> <ResourceType>AWS::RDS::DBInstance</ResourceType> </member> <member> <ResourceStatus>CREATE_COMPLETE</ResourceStatus> <LogicalResourceId>CPUAlarmHigh</LogicalResourceId> <LastUpdatedTime>2011-06-21T20:29:23Z</LastUpdatedTime> <PhysicalResourceId>MyStack-CPUH-PF</PhysicalResourceId> <ResourceType>AWS::CloudWatch::Alarm</ResourceType> </member> </StackResourceSummaries> </ListStackResourcesResult> <ResponseMetadata> <RequestId>2d06e36c-ac1d-11e0-a958-f9382b6eb86b</RequestId> </ResponseMetadata> </ListStackResourcesResponse> """ def test_list_stack_resources(self): self.set_http_response(status_code=200) resources = self.service_connection.list_stack_resources('MyStack', next_token='next_token') self.assertEqual(len(resources), 2) self.assertEqual(resources[0].last_updated_time, datetime(2011, 6, 21, 20, 25, 57)) self.assertEqual(resources[0].logical_resource_id, 'SampleDB') self.assertEqual(resources[0].physical_resource_id, 'My-db-ycx') self.assertEqual(resources[0].resource_status, 'CREATE_COMPLETE') self.assertEqual(resources[0].resource_status_reason, None) self.assertEqual(resources[0].resource_type, 'AWS::RDS::DBInstance') self.assertEqual(resources[1].last_updated_time, datetime(2011, 6, 21, 20, 29, 23)) self.assertEqual(resources[1].logical_resource_id, 'CPUAlarmHigh') self.assertEqual(resources[1].physical_resource_id, 'MyStack-CPUH-PF') self.assertEqual(resources[1].resource_status, 'CREATE_COMPLETE') self.assertEqual(resources[1].resource_status_reason, None) self.assertEqual(resources[1].resource_type, 'AWS::CloudWatch::Alarm') self.assert_request_parameters({ 'Action': 'ListStackResources', 'NextToken': 'next_token', 'StackName': 'MyStack', 'Version': '2010-05-15', }) class TestCloudFormationListStacks(CloudFormationConnectionBase): def default_body(self): return b""" <ListStacksResponse> <ListStacksResult> <StackSummaries> <member> <StackId>arn:aws:cfn:us-east-1:1:stack/Test1/aa</StackId> <StackStatus>CREATE_IN_PROGRESS</StackStatus> <StackName>vpc1</StackName> <CreationTime>2011-05-23T15:47:44Z</CreationTime> <TemplateDescription>My Description.</TemplateDescription> </member> </StackSummaries> </ListStacksResult> </ListStacksResponse> """ def test_list_stacks(self): self.set_http_response(status_code=200) stacks = self.service_connection.list_stacks(['CREATE_IN_PROGRESS'], next_token='next_token') self.assertEqual(len(stacks), 1) self.assertEqual(stacks[0].stack_id, 'arn:aws:cfn:us-east-1:1:stack/Test1/aa') self.assertEqual(stacks[0].stack_status, 'CREATE_IN_PROGRESS') self.assertEqual(stacks[0].stack_name, 'vpc1') self.assertEqual(stacks[0].creation_time, datetime(2011, 5, 23, 15, 47, 44)) self.assertEqual(stacks[0].deletion_time, None) self.assertEqual(stacks[0].template_description, 'My Description.') self.assert_request_parameters({ 'Action': 'ListStacks', 'NextToken': 'next_token', 'StackStatusFilter.member.1': 'CREATE_IN_PROGRESS', 'Version': '2010-05-15', }) class TestCloudFormationValidateTemplate(CloudFormationConnectionBase): def default_body(self): return b""" <ValidateTemplateResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/"> <ValidateTemplateResult> <Description>My Description.</Description> <Parameters> <member> <NoEcho>false</NoEcho> <ParameterKey>InstanceType</ParameterKey> <Description>Type of instance to launch</Description> <DefaultValue>m1.small</DefaultValue> </member> <member> <NoEcho>false</NoEcho> <ParameterKey>KeyName</ParameterKey> <Description>EC2 KeyPair</Description> </member> </Parameters> <CapabilitiesReason>Reason</CapabilitiesReason> <Capabilities> <member>CAPABILITY_IAM</member> </Capabilities> </ValidateTemplateResult> <ResponseMetadata> <RequestId>0be7b6e8-e4a0-11e0-a5bd-9f8d5a7dbc91</RequestId> </ResponseMetadata> </ValidateTemplateResponse> """ def test_validate_template(self): self.set_http_response(status_code=200) template = self.service_connection.validate_template(template_body=SAMPLE_TEMPLATE, template_url='http://url') self.assertEqual(template.description, 'My Description.') self.assertEqual(len(template.template_parameters), 2) param1, param2 = template.template_parameters self.assertEqual(param1.default_value, 'm1.small') self.assertEqual(param1.description, 'Type of instance to launch') self.assertEqual(param1.no_echo, True) self.assertEqual(param1.parameter_key, 'InstanceType') self.assertEqual(param2.default_value, None) self.assertEqual(param2.description, 'EC2 KeyPair') self.assertEqual(param2.no_echo, True) self.assertEqual(param2.parameter_key, 'KeyName') self.assertEqual(template.capabilities_reason, 'Reason') self.assertEqual(len(template.capabilities), 1) self.assertEqual(template.capabilities[0].value, 'CAPABILITY_IAM') self.assert_request_parameters({ 'Action': 'ValidateTemplate', 'TemplateBody': SAMPLE_TEMPLATE, 'TemplateURL': 'http://url', 'Version': '2010-05-15', }) class TestCloudFormationCancelUpdateStack(CloudFormationConnectionBase): def default_body(self): return b"""<CancelUpdateStackResult/>""" def test_cancel_update_stack(self): self.set_http_response(status_code=200) api_response = self.service_connection.cancel_update_stack('stack_name') self.assertEqual(api_response, True) self.assert_request_parameters({ 'Action': 'CancelUpdateStack', 'StackName': 'stack_name', 'Version': '2010-05-15', }) class TestCloudFormationEstimateTemplateCost(CloudFormationConnectionBase): def default_body(self): return b""" { "EstimateTemplateCostResponse": { "EstimateTemplateCostResult": { "Url": "http://calculator.s3.amazonaws.com/calc5.html?key=cf-2e351785-e821-450c-9d58-625e1e1ebfb6" } } } """ def test_estimate_template_cost(self): self.set_http_response(status_code=200) api_response = self.service_connection.estimate_template_cost( template_body='{}') self.assertEqual(api_response, 'http://calculator.s3.amazonaws.com/calc5.html?key=cf-2e351785-e821-450c-9d58-625e1e1ebfb6') self.assert_request_parameters({ 'Action': 'EstimateTemplateCost', 'ContentType': 'JSON', 'TemplateBody': '{}', 'Version': '2010-05-15', }) class TestCloudFormationGetStackPolicy(CloudFormationConnectionBase): def default_body(self): return b""" { "GetStackPolicyResponse": { "GetStackPolicyResult": { "StackPolicyBody": "{...}" } } } """ def test_get_stack_policy(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_stack_policy('stack-id') self.assertEqual(api_response, '{...}') self.assert_request_parameters({ 'Action': 'GetStackPolicy', 'ContentType': 'JSON', 'StackName': 'stack-id', 'Version': '2010-05-15', }) class TestCloudFormationSetStackPolicy(CloudFormationConnectionBase): def default_body(self): return b""" { "SetStackPolicyResponse": { "SetStackPolicyResult": { "Some": "content" } } } """ def test_set_stack_policy(self): self.set_http_response(status_code=200) api_response = self.service_connection.set_stack_policy('stack-id', stack_policy_body='{}') self.assertDictEqual(api_response, {'SetStackPolicyResult': {'Some': 'content'}}) self.assert_request_parameters({ 'Action': 'SetStackPolicy', 'ContentType': 'JSON', 'StackName': 'stack-id', 'StackPolicyBody': '{}', 'Version': '2010-05-15', }) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/cloudformation/test_stack.py 0000644 00000021720 15025013616 0017202 0 ustar 00 #!/usr/bin/env python import datetime import xml.sax import unittest import boto.handler import boto.resultset import boto.cloudformation SAMPLE_XML = b""" <DescribeStacksResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/"> <DescribeStacksResult> <Stacks> <member> <Tags> <member> <Value>value0</Value> <Key>key0</Key> </member> <member> <Key>key1</Key> <Value>value1</Value> </member> </Tags> <StackId>arn:aws:cloudformation:ap-southeast-1:100:stack/Name/id</StackId> <StackStatus>CREATE_COMPLETE</StackStatus> <StackName>Name</StackName> <StackStatusReason/> <Description/> <NotificationARNs> <member>arn:aws:sns:ap-southeast-1:100:name</member> </NotificationARNs> <CreationTime>2013-01-10T05:04:56Z</CreationTime> <DisableRollback>false</DisableRollback> <Outputs> <member> <OutputValue>value0</OutputValue> <Description>output0</Description> <OutputKey>key0</OutputKey> </member> <member> <OutputValue>value1</OutputValue> <Description>output1</Description> <OutputKey>key1</OutputKey> </member> </Outputs> </member> </Stacks> </DescribeStacksResult> <ResponseMetadata> <RequestId>1</RequestId> </ResponseMetadata> </DescribeStacksResponse> """ DESCRIBE_STACK_RESOURCE_XML = b""" <DescribeStackResourcesResult> <StackResources> <member> <StackId>arn:aws:cloudformation:us-east-1:123456789:stack/MyStack/aaf549a0-a413-11df-adb3-5081b3858e83</StackId> <StackName>MyStack</StackName> <LogicalResourceId>MyDBInstance</LogicalResourceId> <PhysicalResourceId>MyStack_DB1</PhysicalResourceId> <ResourceType>AWS::DBInstance</ResourceType> <Timestamp>2010-07-27T22:27:28Z</Timestamp> <ResourceStatus>CREATE_COMPLETE</ResourceStatus> </member> <member> <StackId>arn:aws:cloudformation:us-east-1:123456789:stack/MyStack/aaf549a0-a413-11df-adb3-5081b3858e83</StackId> <StackName>MyStack</StackName> <LogicalResourceId>MyAutoScalingGroup</LogicalResourceId> <PhysicalResourceId>MyStack_ASG1</PhysicalResourceId> <ResourceType>AWS::AutoScalingGroup</ResourceType> <Timestamp>2010-07-27T22:28:28.123456Z</Timestamp> <ResourceStatus>CREATE_IN_PROGRESS</ResourceStatus> </member> </StackResources> </DescribeStackResourcesResult> """ LIST_STACKS_XML = b""" <ListStacksResponse> <ListStacksResult> <StackSummaries> <member> <StackId> arn:aws:cloudformation:us-east-1:1234567:stack/TestCreate1/aaaaa </StackId> <StackStatus>CREATE_IN_PROGRESS</StackStatus> <StackName>vpc1</StackName> <CreationTime>2011-05-23T15:47:44Z</CreationTime> <TemplateDescription> Creates one EC2 instance and a load balancer. </TemplateDescription> </member> <member> <StackId> arn:aws:cloudformation:us-east-1:1234567:stack/TestDelete2/bbbbb </StackId> <StackStatus>DELETE_COMPLETE</StackStatus> <DeletionTime>2011-03-10T16:20:51.575757Z</DeletionTime> <StackName>WP1</StackName> <CreationTime>2011-03-05T19:57:58.161616Z</CreationTime> <TemplateDescription> A simple basic Cloudformation Template. </TemplateDescription> </member> </StackSummaries> </ListStacksResult> </ListStacksResponse> """ LIST_STACK_RESOURCES_XML = b""" <ListStackResourcesResponse> <ListStackResourcesResult> <StackResourceSummaries> <member> <ResourceStatus>CREATE_COMPLETE</ResourceStatus> <LogicalResourceId>DBSecurityGroup</LogicalResourceId> <LastUpdatedTime>2011-06-21T20:15:58Z</LastUpdatedTime> <PhysicalResourceId>gmarcteststack-dbsecuritygroup-1s5m0ez5lkk6w</PhysicalResourceId> <ResourceType>AWS::RDS::DBSecurityGroup</ResourceType> </member> <member> <ResourceStatus>CREATE_COMPLETE</ResourceStatus> <LogicalResourceId>SampleDB</LogicalResourceId> <LastUpdatedTime>2011-06-21T20:25:57.875643Z</LastUpdatedTime> <PhysicalResourceId>MyStack-sampledb-ycwhk1v830lx</PhysicalResourceId> <ResourceType>AWS::RDS::DBInstance</ResourceType> </member> </StackResourceSummaries> </ListStackResourcesResult> <ResponseMetadata> <RequestId>2d06e36c-ac1d-11e0-a958-f9382b6eb86b</RequestId> </ResponseMetadata> </ListStackResourcesResponse> """ class TestStackParse(unittest.TestCase): def test_parse_tags(self): rs = boto.resultset.ResultSet([ ('member', boto.cloudformation.stack.Stack) ]) h = boto.handler.XmlHandler(rs, None) xml.sax.parseString(SAMPLE_XML, h) tags = rs[0].tags self.assertEqual(tags, {u'key0': u'value0', u'key1': u'value1'}) def test_event_creation_time_with_millis(self): millis_xml = SAMPLE_XML.replace( b"<CreationTime>2013-01-10T05:04:56Z</CreationTime>", b"<CreationTime>2013-01-10T05:04:56.102342Z</CreationTime>" ) rs = boto.resultset.ResultSet([ ('member', boto.cloudformation.stack.Stack) ]) h = boto.handler.XmlHandler(rs, None) xml.sax.parseString(millis_xml, h) creation_time = rs[0].creation_time self.assertEqual( creation_time, datetime.datetime(2013, 1, 10, 5, 4, 56, 102342) ) def test_resource_time_with_millis(self): rs = boto.resultset.ResultSet([ ('member', boto.cloudformation.stack.StackResource) ]) h = boto.handler.XmlHandler(rs, None) xml.sax.parseString(DESCRIBE_STACK_RESOURCE_XML, h) timestamp_1 = rs[0].timestamp self.assertEqual( timestamp_1, datetime.datetime(2010, 7, 27, 22, 27, 28) ) timestamp_2 = rs[1].timestamp self.assertEqual( timestamp_2, datetime.datetime(2010, 7, 27, 22, 28, 28, 123456) ) def test_list_stacks_time_with_millis(self): rs = boto.resultset.ResultSet([ ('member', boto.cloudformation.stack.StackSummary) ]) h = boto.handler.XmlHandler(rs, None) xml.sax.parseString(LIST_STACKS_XML, h) timestamp_1 = rs[0].creation_time self.assertEqual( timestamp_1, datetime.datetime(2011, 5, 23, 15, 47, 44) ) timestamp_2 = rs[1].creation_time self.assertEqual( timestamp_2, datetime.datetime(2011, 3, 5, 19, 57, 58, 161616) ) timestamp_3 = rs[1].deletion_time self.assertEqual( timestamp_3, datetime.datetime(2011, 3, 10, 16, 20, 51, 575757) ) def test_list_stacks_time_with_millis_again(self): rs = boto.resultset.ResultSet([ ('member', boto.cloudformation.stack.StackResourceSummary) ]) h = boto.handler.XmlHandler(rs, None) xml.sax.parseString(LIST_STACK_RESOURCES_XML, h) timestamp_1 = rs[0].last_updated_time self.assertEqual( timestamp_1, datetime.datetime(2011, 6, 21, 20, 15, 58) ) timestamp_2 = rs[1].last_updated_time self.assertEqual( timestamp_2, datetime.datetime(2011, 6, 21, 20, 25, 57, 875643) ) def test_disable_rollback_false(self): # SAMPLE_XML defines DisableRollback=="false" rs = boto.resultset.ResultSet([('member', boto.cloudformation.stack.Stack)]) h = boto.handler.XmlHandler(rs, None) xml.sax.parseString(SAMPLE_XML, h) disable_rollback = rs[0].disable_rollback self.assertFalse(disable_rollback) def test_disable_rollback_false_upper(self): # Should also handle "False" rs = boto.resultset.ResultSet([('member', boto.cloudformation.stack.Stack)]) h = boto.handler.XmlHandler(rs, None) sample_xml_upper = SAMPLE_XML.replace(b'false', b'False') xml.sax.parseString(sample_xml_upper, h) disable_rollback = rs[0].disable_rollback self.assertFalse(disable_rollback) def test_disable_rollback_true(self): rs = boto.resultset.ResultSet([('member', boto.cloudformation.stack.Stack)]) h = boto.handler.XmlHandler(rs, None) sample_xml_upper = SAMPLE_XML.replace(b'false', b'true') xml.sax.parseString(sample_xml_upper, h) disable_rollback = rs[0].disable_rollback self.assertTrue(disable_rollback) def test_disable_rollback_true_upper(self): rs = boto.resultset.ResultSet([('member', boto.cloudformation.stack.Stack)]) h = boto.handler.XmlHandler(rs, None) sample_xml_upper = SAMPLE_XML.replace(b'false', b'True') xml.sax.parseString(sample_xml_upper, h) disable_rollback = rs[0].disable_rollback self.assertTrue(disable_rollback) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/cloudformation/__init__.py 0000644 00000000000 15025013616 0016561 0 ustar 00 vendored/boto/tests/unit/elasticache/test_api_interface.py 0000644 00000001263 15025013616 0020106 0 ustar 00 from boto.elasticache.layer1 import ElastiCacheConnection from tests.unit import AWSMockServiceTestCase class TestAPIInterface(AWSMockServiceTestCase): connection_class = ElastiCacheConnection def test_required_launch_params(self): """ Make sure only the AWS required params are required by boto """ name = 'test_cache_cluster' self.set_http_response(status_code=200, body=b'{}') self.service_connection.create_cache_cluster(name) self.assert_request_parameters({ 'Action': 'CreateCacheCluster', 'CacheClusterId': name, }, ignore_params_values=[ 'Version', 'ContentType', ]) vendored/boto/tests/unit/elasticache/__init__.py 0000644 00000000000 15025013616 0016001 0 ustar 00 vendored/boto/tests/unit/ecs/test_connection.py 0000644 00000005276 15025013616 0015771 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from boto.ecs import ECSConnection from tests.unit import AWSMockServiceTestCase class TestECSConnection(AWSMockServiceTestCase): connection_class = ECSConnection def default_body(self): return b""" <Items> <Request> <IsValid>True</IsValid> <ItemLookupRequest> <ItemId>B00008OE6I</ItemId> </ItemLookupRequest> </Request> <Item> <ASIN>B00008OE6I</ASIN> <ItemAttributes> <Manufacturer>Canon</Manufacturer> <ProductGroup>Photography</ProductGroup> <Title>Canon PowerShot S400 4MP Digital Camera w/ 3x Optical Zoom</Title> </ItemAttributes> </Item> </Items> """ def test_item_lookup(self): self.set_http_response(status_code=200) item_set = self.service_connection.item_lookup( ItemId='0316067938', ResponseGroup='Reviews' ) self.assert_request_parameters( {'ItemId': '0316067938', 'Operation': 'ItemLookup', 'ResponseGroup': 'Reviews', 'Service': 'AWSECommerceService'}, ignore_params_values=['Version', 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp']) items = list(item_set) self.assertEqual(len(items), 1) self.assertTrue(item_set.is_valid) self.assertEqual(items[0].ASIN, 'B00008OE6I') vendored/boto/tests/unit/ecs/__init__.py 0000644 00000000000 15025013616 0014306 0 ustar 00 vendored/boto/tests/unit/vpc/test_vpc.py 0000644 00000032660 15025013616 0014435 0 ustar 00 # -*- coding: UTF-8 -*- from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.vpc import VPCConnection, VPC from boto.ec2.securitygroup import SecurityGroup DESCRIBE_VPCS = b'''<?xml version="1.0" encoding="UTF-8"?> <DescribeVpcsResponse xmlns="http://ec2.amazonaws.com/doc/2013-02-01/"> <requestId>623040d1-b51c-40bc-8080-93486f38d03d</requestId> <vpcSet> <item> <vpcId>vpc-12345678</vpcId> <state>available</state> <cidrBlock>172.16.0.0/16</cidrBlock> <dhcpOptionsId>dopt-12345678</dhcpOptionsId> <instanceTenancy>default</instanceTenancy> <isDefault>false</isDefault> </item> </vpcSet> </DescribeVpcsResponse>''' class TestDescribeVPCs(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return DESCRIBE_VPCS def test_get_vpcs(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_vpcs() self.assertEqual(len(api_response), 1) vpc = api_response[0] self.assertFalse(vpc.is_default) self.assertEqual(vpc.instance_tenancy, 'default') class TestCreateVpc(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateVpcResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <vpc> <vpcId>vpc-1a2b3c4d</vpcId> <state>pending</state> <cidrBlock>10.0.0.0/16</cidrBlock> <dhcpOptionsId>dopt-1a2b3c4d2</dhcpOptionsId> <instanceTenancy>default</instanceTenancy> <tagSet/> </vpc> </CreateVpcResponse> """ def test_create_vpc(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_vpc('10.0.0.0/16', 'default') self.assert_request_parameters({ 'Action': 'CreateVpc', 'InstanceTenancy': 'default', 'CidrBlock': '10.0.0.0/16'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertIsInstance(api_response, VPC) self.assertEqual(api_response.id, 'vpc-1a2b3c4d') self.assertEqual(api_response.state, 'pending') self.assertEqual(api_response.cidr_block, '10.0.0.0/16') self.assertEqual(api_response.dhcp_options_id, 'dopt-1a2b3c4d2') self.assertEqual(api_response.instance_tenancy, 'default') class TestDeleteVpc(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteVpcResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </DeleteVpcResponse> """ def test_delete_vpc(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_vpc('vpc-1a2b3c4d') self.assert_request_parameters({ 'Action': 'DeleteVpc', 'VpcId': 'vpc-1a2b3c4d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestModifyVpcAttribute(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <ModifyVpcAttributeResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </ModifyVpcAttributeResponse> """ def test_modify_vpc_attribute_dns_support(self): self.set_http_response(status_code=200) api_response = self.service_connection.modify_vpc_attribute( 'vpc-1a2b3c4d', enable_dns_support=True) self.assert_request_parameters({ 'Action': 'ModifyVpcAttribute', 'VpcId': 'vpc-1a2b3c4d', 'EnableDnsSupport.Value': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) def test_modify_vpc_attribute_dns_hostnames(self): self.set_http_response(status_code=200) api_response = self.service_connection.modify_vpc_attribute( 'vpc-1a2b3c4d', enable_dns_hostnames=True) self.assert_request_parameters({ 'Action': 'ModifyVpcAttribute', 'VpcId': 'vpc-1a2b3c4d', 'EnableDnsHostnames.Value': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestGetAllClassicLinkVpc(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DescribeVpcClassicLinkResponse xmlns="http://ec2.amazonaws.com/doc/2014-09-01/"> <requestId>2484655d-d669-4950-bf55-7ba559805d36</requestId> <vpcSet> <item> <vpcId>vpc-6226ab07</vpcId> <classicLinkEnabled>false</classicLinkEnabled> <tagSet> <item> <key>Name</key> <value>hello</value>[ </item> </tagSet> </item> <item> <vpcId>vpc-9d24f8f8</vpcId> <classicLinkEnabled>true</classicLinkEnabled> <tagSet/> </item> </vpcSet> </DescribeVpcClassicLinkResponse> """ def test_get_all_classic_link_vpcs(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_classic_link_vpcs() self.assertEqual(len(response), 2) vpc = response[0] self.assertEqual(vpc.id, 'vpc-6226ab07') self.assertEqual(vpc.classic_link_enabled, 'false') self.assertEqual(vpc.tags, {'Name': 'hello'}) def test_get_all_classic_link_vpcs_params(self): self.set_http_response(status_code=200) self.service_connection.get_all_classic_link_vpcs( vpc_ids=['id1', 'id2'], filters={'GroupId': 'sg-9b4343fe'}, dry_run=True, ) self.assert_request_parameters({ 'Action': 'DescribeVpcClassicLink', 'VpcId.1': 'id1', 'VpcId.2': 'id2', 'Filter.1.Name': 'GroupId', 'Filter.1.Value.1': 'sg-9b4343fe', 'DryRun': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestVpcClassicLink(AWSMockServiceTestCase): connection_class = VPCConnection def setUp(self): super(TestVpcClassicLink, self).setUp() self.vpc = VPC(self.service_connection) self.vpc_id = 'myid' self.vpc.id = self.vpc_id class TestAttachClassicLinkVpc(TestVpcClassicLink): def default_body(self): return b""" <AttachClassicLinkVpcResponse xmlns="http://ec2.amazonaws.com/doc/2014-09-01/"> <requestId>88673bdf-cd16-40bf-87a1-6132fec47257</requestId> <return>true</return> </AttachClassicLinkVpcResponse> """ def test_attach_classic_link_instance_string_groups(self): groups = ['sg-foo', 'sg-bar'] self.set_http_response(status_code=200) response = self.vpc.attach_classic_instance( instance_id='my_instance_id', groups=groups, dry_run=True ) self.assertTrue(response) self.assert_request_parameters({ 'Action': 'AttachClassicLinkVpc', 'VpcId': self.vpc_id, 'InstanceId': 'my_instance_id', 'SecurityGroupId.1': 'sg-foo', 'SecurityGroupId.2': 'sg-bar', 'DryRun': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_attach_classic_link_instance_object_groups(self): sec_group_1 = SecurityGroup() sec_group_1.id = 'sg-foo' sec_group_2 = SecurityGroup() sec_group_2.id = 'sg-bar' groups = [sec_group_1, sec_group_2] self.set_http_response(status_code=200) response = self.vpc.attach_classic_instance( instance_id='my_instance_id', groups=groups, dry_run=True ) self.assertTrue(response) self.assert_request_parameters({ 'Action': 'AttachClassicLinkVpc', 'VpcId': self.vpc_id, 'InstanceId': 'my_instance_id', 'SecurityGroupId.1': 'sg-foo', 'SecurityGroupId.2': 'sg-bar', 'DryRun': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestDetachClassicLinkVpc(TestVpcClassicLink): def default_body(self): return b""" <DetachClassicLinkVpcResponse xmlns="http://ec2.amazonaws.com/doc/2014-09-01/"> <requestId>5565033d-1321-4eef-b121-6aa46f152ed7</requestId> <return>true</return> </DetachClassicLinkVpcResponse> """ def test_detach_classic_link_instance(self): self.set_http_response(status_code=200) response = self.vpc.detach_classic_instance( instance_id='my_instance_id', dry_run=True ) self.assertTrue(response) self.assert_request_parameters({ 'Action': 'DetachClassicLinkVpc', 'VpcId': self.vpc_id, 'InstanceId': 'my_instance_id', 'DryRun': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestEnableClassicLinkVpc(TestVpcClassicLink): def default_body(self): return b""" <EnableVpcClassicLinkResponse xmlns="http://ec2.amazonaws.com/doc/2014-09-01/"> <requestId>4ab2b2b3-a267-4366-a070-bab853b5927d</requestId> <return>true</return> </EnableVpcClassicLinkResponse> """ def test_enable_classic_link(self): self.set_http_response(status_code=200) response = self.vpc.enable_classic_link( dry_run=True ) self.assertTrue(response) self.assert_request_parameters({ 'Action': 'EnableVpcClassicLink', 'VpcId': self.vpc_id, 'DryRun': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestDisableClassicLinkVpc(TestVpcClassicLink): def default_body(self): return b""" <DisableVpcClassicLinkResponse xmlns="http://ec2.amazonaws.com/doc/2014-09-01/"> <requestId>4ab2b2b3-a267-4366-a070-bab853b5927d</requestId> <return>true</return> </DisableVpcClassicLinkResponse> """ def test_enable_classic_link(self): self.set_http_response(status_code=200) response = self.vpc.disable_classic_link( dry_run=True ) self.assertTrue(response) self.assert_request_parameters({ 'Action': 'DisableVpcClassicLink', 'VpcId': self.vpc_id, 'DryRun': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestUpdateClassicLinkVpc(TestVpcClassicLink): def default_body(self): return b""" <DescribeVpcClassicLinkResponse xmlns="http://ec2.amazonaws.com/doc/2014-09-01/"> <requestId>2484655d-d669-4950-bf55-7ba559805d36</requestId> <vpcSet> <item> <vpcId>myid</vpcId> <classicLinkEnabled>true</classicLinkEnabled> <tagSet/> </item> </vpcSet> </DescribeVpcClassicLinkResponse> """ def test_vpc_update_classic_link_enabled(self): self.vpc.classic_link_enabled = False self.set_http_response(status_code=200) self.vpc.update_classic_link_enabled( dry_run=True, validate=True ) self.assert_request_parameters({ 'Action': 'DescribeVpcClassicLink', 'VpcId.1': self.vpc_id, 'DryRun': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(self.vpc.classic_link_enabled, 'true') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/vpc/test_networkacl.py 0000644 00000050735 15025013616 0016021 0 ustar 00 from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.vpc import VPCConnection class TestDescribeNetworkAcls(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DescribeNetworkAclsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <networkAclSet> <item> <networkAclId>acl-5566953c</networkAclId> <vpcId>vpc-5266953b</vpcId> <default>true</default> <entrySet> <item> <ruleNumber>100</ruleNumber> <protocol>all</protocol> <ruleAction>allow</ruleAction> <egress>true</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> <item> <ruleNumber>32767</ruleNumber> <protocol>all</protocol> <ruleAction>deny</ruleAction> <egress>true</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> <item> <ruleNumber>100</ruleNumber> <protocol>all</protocol> <ruleAction>allow</ruleAction> <egress>false</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> <item> <ruleNumber>32767</ruleNumber> <protocol>all</protocol> <ruleAction>deny</ruleAction> <egress>false</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> </entrySet> <associationSet/> <tagSet/> </item> <item> <networkAclId>acl-5d659634</networkAclId> <vpcId>vpc-5266953b</vpcId> <default>false</default> <entrySet> <item> <ruleNumber>110</ruleNumber> <protocol>6</protocol> <ruleAction>allow</ruleAction> <egress>true</egress> <cidrBlock>0.0.0.0/0</cidrBlock> <portRange> <from>49152</from> <to>65535</to> </portRange> </item> <item> <ruleNumber>32767</ruleNumber> <protocol>all</protocol> <ruleAction>deny</ruleAction> <egress>true</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> <item> <ruleNumber>110</ruleNumber> <protocol>6</protocol> <ruleAction>allow</ruleAction> <egress>false</egress> <cidrBlock>0.0.0.0/0</cidrBlock> <portRange> <from>80</from> <to>80</to> </portRange> </item> <item> <ruleNumber>120</ruleNumber> <protocol>6</protocol> <ruleAction>allow</ruleAction> <egress>false</egress> <cidrBlock>0.0.0.0/0</cidrBlock> <portRange> <from>443</from> <to>443</to> </portRange> </item> <item> <ruleNumber>32767</ruleNumber> <protocol>all</protocol> <ruleAction>deny</ruleAction> <egress>false</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> </entrySet> <associationSet> <item> <networkAclAssociationId>aclassoc-5c659635</networkAclAssociationId> <networkAclId>acl-5d659634</networkAclId> <subnetId>subnet-ff669596</subnetId> </item> <item> <networkAclAssociationId>aclassoc-c26596ab</networkAclAssociationId> <networkAclId>acl-5d659634</networkAclId> <subnetId>subnet-f0669599</subnetId> </item> </associationSet> <tagSet/> </item> </networkAclSet> </DescribeNetworkAclsResponse> """ def test_get_all_network_acls(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_network_acls(['acl-5566953c', 'acl-5d659634'], [('vpc-id', 'vpc-5266953b')]) self.assert_request_parameters({ 'Action': 'DescribeNetworkAcls', 'NetworkAclId.1': 'acl-5566953c', 'NetworkAclId.2': 'acl-5d659634', 'Filter.1.Name': 'vpc-id', 'Filter.1.Value.1': 'vpc-5266953b'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(response), 2) class TestReplaceNetworkAclAssociation(AWSMockServiceTestCase): connection_class = VPCConnection get_all_network_acls_vpc_body = b""" <DescribeNetworkAclsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <networkAclSet> <item> <networkAclId>acl-5566953c</networkAclId> <vpcId>vpc-5266953b</vpcId> <default>true</default> <entrySet> <item> <ruleNumber>100</ruleNumber> <protocol>all</protocol> <ruleAction>allow</ruleAction> <egress>true</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> <item> <ruleNumber>32767</ruleNumber> <protocol>all</protocol> <ruleAction>deny</ruleAction> <egress>true</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> <item> <ruleNumber>100</ruleNumber> <protocol>all</protocol> <ruleAction>allow</ruleAction> <egress>false</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> <item> <ruleNumber>32767</ruleNumber> <protocol>all</protocol> <ruleAction>deny</ruleAction> <egress>false</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> </entrySet> <associationSet/> <tagSet/> </item> </networkAclSet> </DescribeNetworkAclsResponse> """ get_all_network_acls_subnet_body = b""" <DescribeNetworkAclsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <networkAclSet> <item> <networkAclId>acl-5d659634</networkAclId> <vpcId>vpc-5266953b</vpcId> <default>false</default> <entrySet> <item> <ruleNumber>110</ruleNumber> <protocol>6</protocol> <ruleAction>allow</ruleAction> <egress>true</egress> <cidrBlock>0.0.0.0/0</cidrBlock> <portRange> <from>49152</from> <to>65535</to> </portRange> </item> </entrySet> <associationSet> <item> <networkAclAssociationId>aclassoc-c26596ab</networkAclAssociationId> <networkAclId>acl-5d659634</networkAclId> <subnetId>subnet-f0669599</subnetId> </item> <item> <networkAclAssociationId>aclassoc-5c659635</networkAclAssociationId> <networkAclId>acl-5d659634</networkAclId> <subnetId>subnet-ff669596</subnetId> </item> </associationSet> <tagSet/> </item> </networkAclSet> </DescribeNetworkAclsResponse> """ def default_body(self): return b""" <ReplaceNetworkAclAssociationResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <newAssociationId>aclassoc-17b85d7e</newAssociationId> </ReplaceNetworkAclAssociationResponse> """ def test_associate_network_acl(self): self.https_connection.getresponse.side_effect = [ self.create_response(status_code=200, body=self.get_all_network_acls_subnet_body), self.create_response(status_code=200) ] response = self.service_connection.associate_network_acl('acl-5fb85d36', 'subnet-ff669596') # Note: Not testing proper call to get_all_network_acls! self.assert_request_parameters({ 'Action': 'ReplaceNetworkAclAssociation', 'NetworkAclId': 'acl-5fb85d36', 'AssociationId': 'aclassoc-5c659635'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(response, 'aclassoc-17b85d7e') def test_disassociate_network_acl(self): self.https_connection.getresponse.side_effect = [ self.create_response(status_code=200, body=self.get_all_network_acls_vpc_body), self.create_response(status_code=200, body=self.get_all_network_acls_subnet_body), self.create_response(status_code=200) ] response = self.service_connection.disassociate_network_acl('subnet-ff669596', 'vpc-5266953b') # Note: Not testing proper call to either call to get_all_network_acls! self.assert_request_parameters({ 'Action': 'ReplaceNetworkAclAssociation', 'NetworkAclId': 'acl-5566953c', 'AssociationId': 'aclassoc-5c659635'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(response, 'aclassoc-17b85d7e') class TestCreateNetworkAcl(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateNetworkAclResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <networkAcl> <networkAclId>acl-5fb85d36</networkAclId> <vpcId>vpc-11ad4878</vpcId> <default>false</default> <entrySet> <item> <ruleNumber>32767</ruleNumber> <protocol>all</protocol> <ruleAction>deny</ruleAction> <egress>true</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> <item> <ruleNumber>32767</ruleNumber> <protocol>all</protocol> <ruleAction>deny</ruleAction> <egress>false</egress> <cidrBlock>0.0.0.0/0</cidrBlock> </item> </entrySet> <associationSet/> <tagSet/> </networkAcl> </CreateNetworkAclResponse> """ def test_create_network_acl(self): self.set_http_response(status_code=200) response = self.service_connection.create_network_acl('vpc-11ad4878') self.assert_request_parameters({ 'Action': 'CreateNetworkAcl', 'VpcId': 'vpc-11ad4878'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(response.id, 'acl-5fb85d36') class DeleteCreateNetworkAcl(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteNetworkAclResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </DeleteNetworkAclResponse> """ def test_delete_network_acl(self): self.set_http_response(status_code=200) response = self.service_connection.delete_network_acl('acl-2cb85d45') self.assert_request_parameters({ 'Action': 'DeleteNetworkAcl', 'NetworkAclId': 'acl-2cb85d45'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(response, True) class TestCreateNetworkAclEntry(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateNetworkAclEntryResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </CreateNetworkAclEntryResponse> """ def test_create_network_acl(self): self.set_http_response(status_code=200) response = self.service_connection.create_network_acl_entry( 'acl-2cb85d45', 110, 'udp', 'allow', '0.0.0.0/0', egress=False, port_range_from=53, port_range_to=53) self.assert_request_parameters({ 'Action': 'CreateNetworkAclEntry', 'NetworkAclId': 'acl-2cb85d45', 'RuleNumber': 110, 'Protocol': 'udp', 'RuleAction': 'allow', 'Egress': 'false', 'CidrBlock': '0.0.0.0/0', 'PortRange.From': 53, 'PortRange.To': 53}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(response, True) def test_create_network_acl_icmp(self): self.set_http_response(status_code=200) response = self.service_connection.create_network_acl_entry( 'acl-2cb85d45', 110, 'udp', 'allow', '0.0.0.0/0', egress='true', icmp_code=-1, icmp_type=8) self.assert_request_parameters({ 'Action': 'CreateNetworkAclEntry', 'NetworkAclId': 'acl-2cb85d45', 'RuleNumber': 110, 'Protocol': 'udp', 'RuleAction': 'allow', 'Egress': 'true', 'CidrBlock': '0.0.0.0/0', 'Icmp.Code': -1, 'Icmp.Type': 8}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(response, True) class TestReplaceNetworkAclEntry(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <ReplaceNetworkAclEntryResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </ReplaceNetworkAclEntryResponse> """ def test_replace_network_acl(self): self.set_http_response(status_code=200) response = self.service_connection.replace_network_acl_entry( 'acl-2cb85d45', 110, 'tcp', 'deny', '0.0.0.0/0', egress=False, port_range_from=139, port_range_to=139) self.assert_request_parameters({ 'Action': 'ReplaceNetworkAclEntry', 'NetworkAclId': 'acl-2cb85d45', 'RuleNumber': 110, 'Protocol': 'tcp', 'RuleAction': 'deny', 'Egress': 'false', 'CidrBlock': '0.0.0.0/0', 'PortRange.From': 139, 'PortRange.To': 139}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(response, True) def test_replace_network_acl_icmp(self): self.set_http_response(status_code=200) response = self.service_connection.replace_network_acl_entry( 'acl-2cb85d45', 110, 'tcp', 'deny', '0.0.0.0/0', icmp_code=-1, icmp_type=8) self.assert_request_parameters({ 'Action': 'ReplaceNetworkAclEntry', 'NetworkAclId': 'acl-2cb85d45', 'RuleNumber': 110, 'Protocol': 'tcp', 'RuleAction': 'deny', 'CidrBlock': '0.0.0.0/0', 'Icmp.Code': -1, 'Icmp.Type': 8}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(response, True) class TestDeleteNetworkAclEntry(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteNetworkAclEntryResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </DeleteNetworkAclEntryResponse> """ def test_delete_network_acl(self): self.set_http_response(status_code=200) response = self.service_connection.delete_network_acl_entry('acl-2cb85d45', 100, egress=False) self.assert_request_parameters({ 'Action': 'DeleteNetworkAclEntry', 'NetworkAclId': 'acl-2cb85d45', 'RuleNumber': 100, 'Egress': 'false'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(response, True) class TestGetNetworkAclAssociations(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DescribeNetworkAclsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <networkAclSet> <item> <networkAclId>acl-5d659634</networkAclId> <vpcId>vpc-5266953b</vpcId> <default>false</default> <entrySet> <item> <ruleNumber>110</ruleNumber> <protocol>6</protocol> <ruleAction>allow</ruleAction> <egress>true</egress> <cidrBlock>0.0.0.0/0</cidrBlock> <portRange> <from>49152</from> <to>65535</to> </portRange> </item> </entrySet> <associationSet> <item> <networkAclAssociationId>aclassoc-c26596ab</networkAclAssociationId> <networkAclId>acl-5d659634</networkAclId> <subnetId>subnet-f0669599</subnetId> </item> </associationSet> <tagSet/> </item> </networkAclSet> </DescribeNetworkAclsResponse> """ def test_get_network_acl_associations(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_network_acls() association = api_response[0].associations[0] self.assertEqual(association.network_acl_id, 'acl-5d659634') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/vpc/test_dhcpoptions.py 0000644 00000021063 15025013616 0016172 0 ustar 00 from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.vpc import VPCConnection, DhcpOptions class TestDescribeDhcpOptions(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DescribeDhcpOptionsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <dhcpOptionsSet> <item> <dhcpOptionsId>dopt-7a8b9c2d</dhcpOptionsId> <dhcpConfigurationSet> <item> <key>domain-name</key> <valueSet> <item> <value>example.com</value> </item> </valueSet> </item> <item> <key>domain-name-servers</key> <valueSet> <item> <value>10.2.5.1</value> </item> </valueSet> </item> <item> <key>domain-name-servers</key> <valueSet> <item> <value>10.2.5.2</value> </item> </valueSet> </item> </dhcpConfigurationSet> <tagSet/> </item> </dhcpOptionsSet> </DescribeDhcpOptionsResponse> """ def test_get_all_dhcp_options(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_dhcp_options(['dopt-7a8b9c2d'], [('key', 'domain-name')]) self.assert_request_parameters({ 'Action': 'DescribeDhcpOptions', 'DhcpOptionsId.1': 'dopt-7a8b9c2d', 'Filter.1.Name': 'key', 'Filter.1.Value.1': 'domain-name'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(api_response), 1) self.assertIsInstance(api_response[0], DhcpOptions) self.assertEqual(api_response[0].id, 'dopt-7a8b9c2d') self.assertEqual(api_response[0].options['domain-name'], ['example.com']) self.assertEqual(api_response[0].options['domain-name-servers'], ['10.2.5.1', '10.2.5.2']) class TestCreateDhcpOptions(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateDhcpOptionsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <dhcpOptions> <dhcpOptionsId>dopt-7a8b9c2d</dhcpOptionsId> <dhcpConfigurationSet> <item> <key>domain-name</key> <valueSet> <item> <value>example.com</value> </item> </valueSet> </item> <item> <key>domain-name-servers</key> <valueSet> <item> <value>10.2.5.1</value> </item> <item> <value>10.2.5.2</value> </item> </valueSet> </item> <item> <key>ntp-servers</key> <valueSet> <item> <value>10.12.12.1</value> </item> <item> <value>10.12.12.2</value> </item> </valueSet> </item> <item> <key>netbios-name-servers</key> <valueSet> <item> <value>10.20.20.1</value> </item> </valueSet> </item> <item> <key>netbios-node-type</key> <valueSet> <item> <value>2</value> </item> </valueSet> </item> </dhcpConfigurationSet> <tagSet/> </dhcpOptions> </CreateDhcpOptionsResponse> """ def test_create_dhcp_options(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_dhcp_options( domain_name='example.com', domain_name_servers=['10.2.5.1', '10.2.5.2'], ntp_servers=('10.12.12.1', '10.12.12.2'), netbios_name_servers='10.20.20.1', netbios_node_type='2') self.assert_request_parameters({ 'Action': 'CreateDhcpOptions', 'DhcpConfiguration.1.Key': 'domain-name', 'DhcpConfiguration.1.Value.1': 'example.com', 'DhcpConfiguration.2.Key': 'domain-name-servers', 'DhcpConfiguration.2.Value.1': '10.2.5.1', 'DhcpConfiguration.2.Value.2': '10.2.5.2', 'DhcpConfiguration.3.Key': 'ntp-servers', 'DhcpConfiguration.3.Value.1': '10.12.12.1', 'DhcpConfiguration.3.Value.2': '10.12.12.2', 'DhcpConfiguration.4.Key': 'netbios-name-servers', 'DhcpConfiguration.4.Value.1': '10.20.20.1', 'DhcpConfiguration.5.Key': 'netbios-node-type', 'DhcpConfiguration.5.Value.1': '2'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertIsInstance(api_response, DhcpOptions) self.assertEqual(api_response.id, 'dopt-7a8b9c2d') self.assertEqual(api_response.options['domain-name'], ['example.com']) self.assertEqual(api_response.options['domain-name-servers'], ['10.2.5.1', '10.2.5.2']) self.assertEqual(api_response.options['ntp-servers'], ['10.12.12.1', '10.12.12.2']) self.assertEqual(api_response.options['netbios-name-servers'], ['10.20.20.1']) self.assertEqual(api_response.options['netbios-node-type'], ['2']) class TestDeleteDhcpOptions(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteDhcpOptionsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </DeleteDhcpOptionsResponse> """ def test_delete_dhcp_options(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_dhcp_options('dopt-7a8b9c2d') self.assert_request_parameters({ 'Action': 'DeleteDhcpOptions', 'DhcpOptionsId': 'dopt-7a8b9c2d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestAssociateDhcpOptions(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <AssociateDhcpOptionsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </AssociateDhcpOptionsResponse> """ def test_associate_dhcp_options(self): self.set_http_response(status_code=200) api_response = self.service_connection.associate_dhcp_options( 'dopt-7a8b9c2d', 'vpc-1a2b3c4d') self.assert_request_parameters({ 'Action': 'AssociateDhcpOptions', 'DhcpOptionsId': 'dopt-7a8b9c2d', 'VpcId': 'vpc-1a2b3c4d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/vpc/test_customergateway.py 0000644 00000010773 15025013616 0017071 0 ustar 00 from tests.compat import OrderedDict from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.vpc import VPCConnection, CustomerGateway class TestDescribeCustomerGateways(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DescribeCustomerGatewaysResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <customerGatewaySet> <item> <customerGatewayId>cgw-b4dc3961</customerGatewayId> <state>available</state> <type>ipsec.1</type> <ipAddress>12.1.2.3</ipAddress> <bgpAsn>65534</bgpAsn> <tagSet/> </item> </customerGatewaySet> </DescribeCustomerGatewaysResponse> """ def test_get_all_customer_gateways(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_customer_gateways( 'cgw-b4dc3961', filters=OrderedDict([('state', ['pending', 'available']), ('ip-address', '12.1.2.3')])) self.assert_request_parameters({ 'Action': 'DescribeCustomerGateways', 'CustomerGatewayId.1': 'cgw-b4dc3961', 'Filter.1.Name': 'state', 'Filter.1.Value.1': 'pending', 'Filter.1.Value.2': 'available', 'Filter.2.Name': 'ip-address', 'Filter.2.Value.1': '12.1.2.3'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(api_response), 1) self.assertIsInstance(api_response[0], CustomerGateway) self.assertEqual(api_response[0].id, 'cgw-b4dc3961') class TestCreateCustomerGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateCustomerGatewayResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <customerGateway> <customerGatewayId>cgw-b4dc3961</customerGatewayId> <state>pending</state> <type>ipsec.1</type> <ipAddress>12.1.2.3</ipAddress> <bgpAsn>65534</bgpAsn> <tagSet/> </customerGateway> </CreateCustomerGatewayResponse> """ def test_create_customer_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_customer_gateway( 'ipsec.1', '12.1.2.3', 65534) self.assert_request_parameters({ 'Action': 'CreateCustomerGateway', 'Type': 'ipsec.1', 'IpAddress': '12.1.2.3', 'BgpAsn': 65534}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertIsInstance(api_response, CustomerGateway) self.assertEqual(api_response.id, 'cgw-b4dc3961') self.assertEqual(api_response.state, 'pending') self.assertEqual(api_response.type, 'ipsec.1') self.assertEqual(api_response.ip_address, '12.1.2.3') self.assertEqual(api_response.bgp_asn, 65534) class TestDeleteCustomerGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteCustomerGatewayResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </DeleteCustomerGatewayResponse> """ def test_delete_customer_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_customer_gateway('cgw-b4dc3961') self.assert_request_parameters({ 'Action': 'DeleteCustomerGateway', 'CustomerGatewayId': 'cgw-b4dc3961'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/vpc/test_vpnconnection.py 0000644 00000024257 15025013616 0016533 0 ustar 00 # -*- coding: UTF-8 -*- from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.vpc import VPCConnection, VpnConnection DESCRIBE_VPNCONNECTIONS = b'''<?xml version="1.0" encoding="UTF-8"?> <DescribeVpnConnectionsResponse xmlns="http://ec2.amazonaws.com/doc/2013-02-01/"> <requestId>12345678-asdf-ghjk-zxcv-0987654321nb</requestId> <vpnConnectionSet> <item> <vpnConnectionId>vpn-12qw34er56ty</vpnConnectionId> <state>available</state> <customerGatewayConfiguration> <?xml version="1.0" encoding="UTF-8"?> </customerGatewayConfiguration> <type>ipsec.1</type> <customerGatewayId>cgw-1234qwe9</customerGatewayId> <vpnGatewayId>vgw-lkjh1234</vpnGatewayId> <tagSet> <item> <key>Name</key> <value>VPN 1</value> </item> </tagSet> <vgwTelemetry> <item> <outsideIpAddress>123.45.67.89</outsideIpAddress> <status>DOWN</status> <lastStatusChange>2013-03-19T19:20:34.000Z</lastStatusChange> <statusMessage/> <acceptedRouteCount>0</acceptedRouteCount> </item> <item> <outsideIpAddress>123.45.67.90</outsideIpAddress> <status>UP</status> <lastStatusChange>2013-03-20T08:00:14.000Z</lastStatusChange> <statusMessage/> <acceptedRouteCount>0</acceptedRouteCount> </item> </vgwTelemetry> <options> <staticRoutesOnly>true</staticRoutesOnly> </options> <routes> <item> <destinationCidrBlock>192.168.0.0/24</destinationCidrBlock> <source>static</source> <state>available</state> </item> </routes> </item> <item> <vpnConnectionId>vpn-qwerty12</vpnConnectionId> <state>pending</state> <customerGatewayConfiguration> <?xml version="1.0" encoding="UTF-8"?> </customerGatewayConfiguration> <type>ipsec.1</type> <customerGatewayId>cgw-01234567</customerGatewayId> <vpnGatewayId>vgw-asdfghjk</vpnGatewayId> <vgwTelemetry> <item> <outsideIpAddress>134.56.78.78</outsideIpAddress> <status>UP</status> <lastStatusChange>2013-03-20T01:46:30.000Z</lastStatusChange> <statusMessage/> <acceptedRouteCount>0</acceptedRouteCount> </item> <item> <outsideIpAddress>134.56.78.79</outsideIpAddress> <status>UP</status> <lastStatusChange>2013-03-19T19:23:59.000Z</lastStatusChange> <statusMessage/> <acceptedRouteCount>0</acceptedRouteCount> </item> </vgwTelemetry> <options> <staticRoutesOnly>true</staticRoutesOnly> </options> <routes> <item> <destinationCidrBlock>10.0.0.0/16</destinationCidrBlock> <source>static</source> <state>pending</state> </item> </routes> </item> </vpnConnectionSet> </DescribeVpnConnectionsResponse>''' class TestDescribeVPNConnections(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return DESCRIBE_VPNCONNECTIONS def test_get_vpcs(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_vpn_connections( ['vpn-12qw34er56ty', 'vpn-qwerty12'], filters=[('state', ['pending', 'available'])]) self.assert_request_parameters({ 'Action': 'DescribeVpnConnections', 'VpnConnectionId.1': 'vpn-12qw34er56ty', 'VpnConnectionId.2': 'vpn-qwerty12', 'Filter.1.Name': 'state', 'Filter.1.Value.1': 'pending', 'Filter.1.Value.2': 'available'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(api_response), 2) vpn0 = api_response[0] self.assertEqual(vpn0.type, 'ipsec.1') self.assertEqual(vpn0.customer_gateway_id, 'cgw-1234qwe9') self.assertEqual(vpn0.vpn_gateway_id, 'vgw-lkjh1234') self.assertEqual(len(vpn0.tunnels), 2) self.assertDictEqual(vpn0.tags, {'Name': 'VPN 1'}) vpn1 = api_response[1] self.assertEqual(vpn1.state, 'pending') self.assertEqual(len(vpn1.static_routes), 1) self.assertTrue(vpn1.options.static_routes_only) self.assertEqual(vpn1.tunnels[0].status, 'UP') self.assertEqual(vpn1.tunnels[1].status, 'UP') self.assertDictEqual(vpn1.tags, {}) self.assertEqual(vpn1.static_routes[0].source, 'static') self.assertEqual(vpn1.static_routes[0].state, 'pending') class TestCreateVPNConnection(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateVpnConnectionResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>5cc7891f-1f3b-4fc4-a626-bdea8f63ff5a</requestId> <vpnConnection> <vpnConnectionId>vpn-83ad48ea</vpnConnectionId> <state>pending</state> <customerGatewayConfiguration> <?xml version="1.0" encoding="UTF-8"?> </customerGatewayConfiguration> <customerGatewayId>cgw-b4dc3961</customerGatewayId> <vpnGatewayId>vgw-8db04f81</vpnGatewayId> <options> <staticRoutesOnly>true</staticRoutesOnly> </options> <routes/> </vpnConnection> </CreateVpnConnectionResponse> """ def test_create_vpn_connection(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_vpn_connection( 'ipsec.1', 'cgw-b4dc3961', 'vgw-8db04f81', static_routes_only=True) self.assert_request_parameters({ 'Action': 'CreateVpnConnection', 'Type': 'ipsec.1', 'CustomerGatewayId': 'cgw-b4dc3961', 'VpnGatewayId': 'vgw-8db04f81', 'Options.StaticRoutesOnly': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertIsInstance(api_response, VpnConnection) self.assertEqual(api_response.id, 'vpn-83ad48ea') self.assertEqual(api_response.customer_gateway_id, 'cgw-b4dc3961') self.assertEqual(api_response.options.static_routes_only, True) class TestDeleteVPNConnection(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteVpnConnectionResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </DeleteVpnConnectionResponse> """ def test_delete_vpn_connection(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_vpn_connection('vpn-44a8938f') self.assert_request_parameters({ 'Action': 'DeleteVpnConnection', 'VpnConnectionId': 'vpn-44a8938f'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestCreateVPNConnectionRoute(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateVpnConnectionRouteResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>4f35a1b2-c2c3-4093-b51f-abb9d7311990</requestId> <return>true</return> </CreateVpnConnectionRouteResponse> """ def test_create_vpn_connection_route(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_vpn_connection_route( '11.12.0.0/16', 'vpn-83ad48ea') self.assert_request_parameters({ 'Action': 'CreateVpnConnectionRoute', 'DestinationCidrBlock': '11.12.0.0/16', 'VpnConnectionId': 'vpn-83ad48ea'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestDeleteVPNConnectionRoute(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteVpnConnectionRouteResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>4f35a1b2-c2c3-4093-b51f-abb9d7311990</requestId> <return>true</return> </DeleteVpnConnectionRouteResponse> """ def test_delete_vpn_connection_route(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_vpn_connection_route( '11.12.0.0/16', 'vpn-83ad48ea') self.assert_request_parameters({ 'Action': 'DeleteVpnConnectionRoute', 'DestinationCidrBlock': '11.12.0.0/16', 'VpnConnectionId': 'vpn-83ad48ea'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/vpc/test_subnet.py 0000644 00000012545 15025013616 0015145 0 ustar 00 from tests.compat import OrderedDict from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.vpc import VPCConnection, Subnet class TestDescribeSubnets(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DescribeSubnetsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <subnetSet> <item> <subnetId>subnet-9d4a7b6c</subnetId> <state>available</state> <vpcId>vpc-1a2b3c4d</vpcId> <cidrBlock>10.0.1.0/24</cidrBlock> <availableIpAddressCount>251</availableIpAddressCount> <availabilityZone>us-east-1a</availabilityZone> <defaultForAz>false</defaultForAz> <mapPublicIpOnLaunch>false</mapPublicIpOnLaunch> <tagSet/> </item> <item> <subnetId>subnet-6e7f829e</subnetId> <state>available</state> <vpcId>vpc-1a2b3c4d</vpcId> <cidrBlock>10.0.0.0/24</cidrBlock> <availableIpAddressCount>251</availableIpAddressCount> <availabilityZone>us-east-1a</availabilityZone> <defaultForAz>false</defaultForAz> <mapPublicIpOnLaunch>false</mapPublicIpOnLaunch> <tagSet/> </item> </subnetSet> </DescribeSubnetsResponse> """ def test_get_all_subnets(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_subnets( ['subnet-9d4a7b6c', 'subnet-6e7f829e'], filters=OrderedDict([('state', 'available'), ('vpc-id', ['subnet-9d4a7b6c', 'subnet-6e7f829e'])])) self.assert_request_parameters({ 'Action': 'DescribeSubnets', 'SubnetId.1': 'subnet-9d4a7b6c', 'SubnetId.2': 'subnet-6e7f829e', 'Filter.1.Name': 'state', 'Filter.1.Value.1': 'available', 'Filter.2.Name': 'vpc-id', 'Filter.2.Value.1': 'subnet-9d4a7b6c', 'Filter.2.Value.2': 'subnet-6e7f829e'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(api_response), 2) self.assertIsInstance(api_response[0], Subnet) self.assertEqual(api_response[0].id, 'subnet-9d4a7b6c') self.assertEqual(api_response[1].id, 'subnet-6e7f829e') class TestCreateSubnet(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateSubnetResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <subnet> <subnetId>subnet-9d4a7b6c</subnetId> <state>pending</state> <vpcId>vpc-1a2b3c4d</vpcId> <cidrBlock>10.0.1.0/24</cidrBlock> <availableIpAddressCount>251</availableIpAddressCount> <availabilityZone>us-east-1a</availabilityZone> <tagSet/> </subnet> </CreateSubnetResponse> """ def test_create_subnet(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_subnet( 'vpc-1a2b3c4d', '10.0.1.0/24', 'us-east-1a') self.assert_request_parameters({ 'Action': 'CreateSubnet', 'VpcId': 'vpc-1a2b3c4d', 'CidrBlock': '10.0.1.0/24', 'AvailabilityZone': 'us-east-1a'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertIsInstance(api_response, Subnet) self.assertEqual(api_response.id, 'subnet-9d4a7b6c') self.assertEqual(api_response.state, 'pending') self.assertEqual(api_response.vpc_id, 'vpc-1a2b3c4d') self.assertEqual(api_response.cidr_block, '10.0.1.0/24') self.assertEqual(api_response.available_ip_address_count, 251) self.assertEqual(api_response.availability_zone, 'us-east-1a') class TestDeleteSubnet(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteSubnetResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </DeleteSubnetResponse> """ def test_delete_subnet(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_subnet('subnet-9d4a7b6c') self.assert_request_parameters({ 'Action': 'DeleteSubnet', 'SubnetId': 'subnet-9d4a7b6c'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/vpc/test_internetgateway.py 0000644 00000013674 15025013616 0017063 0 ustar 00 from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.vpc import VPCConnection, InternetGateway class TestDescribeInternetGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DescribeInternetGatewaysResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <internetGatewaySet> <item> <internetGatewayId>igw-eaad4883EXAMPLE</internetGatewayId> <attachmentSet> <item> <vpcId>vpc-11ad4878</vpcId> <state>available</state> </item> </attachmentSet> <tagSet/> </item> </internetGatewaySet> </DescribeInternetGatewaysResponse> """ def test_describe_internet_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_internet_gateways( 'igw-eaad4883EXAMPLE', filters=[('attachment.state', ['available', 'pending'])]) self.assert_request_parameters({ 'Action': 'DescribeInternetGateways', 'InternetGatewayId.1': 'igw-eaad4883EXAMPLE', 'Filter.1.Name': 'attachment.state', 'Filter.1.Value.1': 'available', 'Filter.1.Value.2': 'pending'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(api_response), 1) self.assertIsInstance(api_response[0], InternetGateway) self.assertEqual(api_response[0].id, 'igw-eaad4883EXAMPLE') class TestCreateInternetGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateInternetGatewayResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <internetGateway> <internetGatewayId>igw-eaad4883</internetGatewayId> <attachmentSet/> <tagSet/> </internetGateway> </CreateInternetGatewayResponse> """ def test_create_internet_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_internet_gateway() self.assert_request_parameters({ 'Action': 'CreateInternetGateway'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertIsInstance(api_response, InternetGateway) self.assertEqual(api_response.id, 'igw-eaad4883') class TestDeleteInternetGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteInternetGatewayResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </DeleteInternetGatewayResponse> """ def test_delete_internet_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_internet_gateway('igw-eaad4883') self.assert_request_parameters({ 'Action': 'DeleteInternetGateway', 'InternetGatewayId': 'igw-eaad4883'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestAttachInternetGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <AttachInternetGatewayResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </AttachInternetGatewayResponse> """ def test_attach_internet_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.attach_internet_gateway( 'igw-eaad4883', 'vpc-11ad4878') self.assert_request_parameters({ 'Action': 'AttachInternetGateway', 'InternetGatewayId': 'igw-eaad4883', 'VpcId': 'vpc-11ad4878'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestDetachInternetGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DetachInternetGatewayResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </DetachInternetGatewayResponse> """ def test_detach_internet_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.detach_internet_gateway( 'igw-eaad4883', 'vpc-11ad4878') self.assert_request_parameters({ 'Action': 'DetachInternetGateway', 'InternetGatewayId': 'igw-eaad4883', 'VpcId': 'vpc-11ad4878'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/vpc/test_vpngateway.py 0000644 00000021002 15025013616 0016016 0 ustar 00 # -*- coding: UTF-8 -*- from tests.compat import OrderedDict from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.vpc import VPCConnection, VpnGateway, Attachment class TestDescribeVpnGateways(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DescribeVpnGatewaysResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <vpnGatewaySet> <item> <vpnGatewayId>vgw-8db04f81</vpnGatewayId> <state>available</state> <type>ipsec.1</type> <availabilityZone>us-east-1a</availabilityZone> <attachments> <item> <vpcId>vpc-1a2b3c4d</vpcId> <state>attached</state> </item> </attachments> <tagSet/> </item> </vpnGatewaySet> </DescribeVpnGatewaysResponse> """ def test_get_all_vpn_gateways(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_vpn_gateways( 'vgw-8db04f81', filters=OrderedDict([('state', ['pending', 'available']), ('availability-zone', 'us-east-1a')])) self.assert_request_parameters({ 'Action': 'DescribeVpnGateways', 'VpnGatewayId.1': 'vgw-8db04f81', 'Filter.1.Name': 'state', 'Filter.1.Value.1': 'pending', 'Filter.1.Value.2': 'available', 'Filter.2.Name': 'availability-zone', 'Filter.2.Value.1': 'us-east-1a'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(api_response), 1) self.assertIsInstance(api_response[0], VpnGateway) self.assertEqual(api_response[0].id, 'vgw-8db04f81') class TestCreateVpnGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateVpnGatewayResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <vpnGateway> <vpnGatewayId>vgw-8db04f81</vpnGatewayId> <state>pending</state> <type>ipsec.1</type> <availabilityZone>us-east-1a</availabilityZone> <attachments/> <tagSet/> </vpnGateway> </CreateVpnGatewayResponse> """ def test_delete_vpn_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_vpn_gateway('ipsec.1', 'us-east-1a') self.assert_request_parameters({ 'Action': 'CreateVpnGateway', 'AvailabilityZone': 'us-east-1a', 'Type': 'ipsec.1'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertIsInstance(api_response, VpnGateway) self.assertEqual(api_response.id, 'vgw-8db04f81') class TestDeleteVpnGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteVpnGatewayResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </DeleteVpnGatewayResponse> """ def test_delete_vpn_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_vpn_gateway('vgw-8db04f81') self.assert_request_parameters({ 'Action': 'DeleteVpnGateway', 'VpnGatewayId': 'vgw-8db04f81'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestAttachVpnGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <AttachVpnGatewayResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <attachment> <vpcId>vpc-1a2b3c4d</vpcId> <state>attaching</state> </attachment> </AttachVpnGatewayResponse> """ def test_attach_vpn_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.attach_vpn_gateway('vgw-8db04f81', 'vpc-1a2b3c4d') self.assert_request_parameters({ 'Action': 'AttachVpnGateway', 'VpnGatewayId': 'vgw-8db04f81', 'VpcId': 'vpc-1a2b3c4d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertIsInstance(api_response, Attachment) self.assertEqual(api_response.vpc_id, 'vpc-1a2b3c4d') self.assertEqual(api_response.state, 'attaching') class TestDetachVpnGateway(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DetachVpnGatewayResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </DetachVpnGatewayResponse> """ def test_detach_vpn_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.detach_vpn_gateway('vgw-8db04f81', 'vpc-1a2b3c4d') self.assert_request_parameters({ 'Action': 'DetachVpnGateway', 'VpnGatewayId': 'vgw-8db04f81', 'VpcId': 'vpc-1a2b3c4d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestDisableVgwRoutePropagation(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DisableVgwRoutePropagationResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>4f35a1b2-c2c3-4093-b51f-abb9d7311990</requestId> <return>true</return> </DisableVgwRoutePropagationResponse> """ def test_disable_vgw_route_propagation(self): self.set_http_response(status_code=200) api_response = self.service_connection.disable_vgw_route_propagation( 'rtb-c98a35a0', 'vgw-d8e09e8a') self.assert_request_parameters({ 'Action': 'DisableVgwRoutePropagation', 'GatewayId': 'vgw-d8e09e8a', 'RouteTableId': 'rtb-c98a35a0'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestEnableVgwRoutePropagation(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DisableVgwRoutePropagationResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>4f35a1b2-c2c3-4093-b51f-abb9d7311990</requestId> <return>true</return> </DisableVgwRoutePropagationResponse> """ def test_enable_vgw_route_propagation(self): self.set_http_response(status_code=200) api_response = self.service_connection.enable_vgw_route_propagation( 'rtb-c98a35a0', 'vgw-d8e09e8a') self.assert_request_parameters({ 'Action': 'EnableVgwRoutePropagation', 'GatewayId': 'vgw-d8e09e8a', 'RouteTableId': 'rtb-c98a35a0'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/vpc/test_routetable.py 0000644 00000047752 15025013616 0016023 0 ustar 00 from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.vpc import VPCConnection, RouteTable class TestDescribeRouteTables(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DescribeRouteTablesResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>6f570b0b-9c18-4b07-bdec-73740dcf861a</requestId> <routeTableSet> <item> <routeTableId>rtb-13ad487a</routeTableId> <vpcId>vpc-11ad4878</vpcId> <routeSet> <item> <destinationCidrBlock>10.0.0.0/22</destinationCidrBlock> <gatewayId>local</gatewayId> <state>active</state> <origin>CreateRouteTable</origin> </item> </routeSet> <associationSet> <item> <routeTableAssociationId>rtbassoc-12ad487b</routeTableAssociationId> <routeTableId>rtb-13ad487a</routeTableId> <main>true</main> </item> </associationSet> <tagSet/> </item> <item> <routeTableId>rtb-f9ad4890</routeTableId> <vpcId>vpc-11ad4878</vpcId> <routeSet> <item> <destinationCidrBlock>10.0.0.0/22</destinationCidrBlock> <gatewayId>local</gatewayId> <state>active</state> <origin>CreateRouteTable</origin> </item> <item> <destinationCidrBlock>0.0.0.0/0</destinationCidrBlock> <gatewayId>igw-eaad4883</gatewayId> <state>active</state> <origin>CreateRoute</origin> </item> <item> <destinationCidrBlock>10.0.0.0/21</destinationCidrBlock> <networkInterfaceId>eni-884ec1d1</networkInterfaceId> <state>blackhole</state> <origin>CreateRoute</origin> </item> <item> <destinationCidrBlock>11.0.0.0/22</destinationCidrBlock> <vpcPeeringConnectionId>pcx-efc52b86</vpcPeeringConnectionId> <state>blackhole</state> <origin>CreateRoute</origin> </item> </routeSet> <associationSet> <item> <routeTableAssociationId>rtbassoc-faad4893</routeTableAssociationId> <routeTableId>rtb-f9ad4890</routeTableId> <subnetId>subnet-15ad487c</subnetId> </item> </associationSet> <tagSet/> </item> </routeTableSet> </DescribeRouteTablesResponse> """ def test_get_all_route_tables(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_route_tables( ['rtb-13ad487a', 'rtb-f9ad4890'], filters=[('route.state', 'active')]) self.assert_request_parameters({ 'Action': 'DescribeRouteTables', 'RouteTableId.1': 'rtb-13ad487a', 'RouteTableId.2': 'rtb-f9ad4890', 'Filter.1.Name': 'route.state', 'Filter.1.Value.1': 'active'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(api_response), 2) self.assertIsInstance(api_response[0], RouteTable) self.assertEqual(api_response[0].id, 'rtb-13ad487a') self.assertEqual(len(api_response[0].routes), 1) self.assertEqual(api_response[0].routes[0].destination_cidr_block, '10.0.0.0/22') self.assertEqual(api_response[0].routes[0].gateway_id, 'local') self.assertEqual(api_response[0].routes[0].state, 'active') self.assertEqual(api_response[0].routes[0].origin, 'CreateRouteTable') self.assertEqual(len(api_response[0].associations), 1) self.assertEqual(api_response[0].associations[0].id, 'rtbassoc-12ad487b') self.assertEqual(api_response[0].associations[0].route_table_id, 'rtb-13ad487a') self.assertIsNone(api_response[0].associations[0].subnet_id) self.assertEqual(api_response[0].associations[0].main, True) self.assertEqual(api_response[1].id, 'rtb-f9ad4890') self.assertEqual(len(api_response[1].routes), 4) self.assertEqual(api_response[1].routes[0].destination_cidr_block, '10.0.0.0/22') self.assertEqual(api_response[1].routes[0].gateway_id, 'local') self.assertEqual(api_response[1].routes[0].state, 'active') self.assertEqual(api_response[1].routes[0].origin, 'CreateRouteTable') self.assertEqual(api_response[1].routes[1].destination_cidr_block, '0.0.0.0/0') self.assertEqual(api_response[1].routes[1].gateway_id, 'igw-eaad4883') self.assertEqual(api_response[1].routes[1].state, 'active') self.assertEqual(api_response[1].routes[1].origin, 'CreateRoute') self.assertEqual(api_response[1].routes[2].destination_cidr_block, '10.0.0.0/21') self.assertEqual(api_response[1].routes[2].interface_id, 'eni-884ec1d1') self.assertEqual(api_response[1].routes[2].state, 'blackhole') self.assertEqual(api_response[1].routes[2].origin, 'CreateRoute') self.assertEqual(api_response[1].routes[3].destination_cidr_block, '11.0.0.0/22') self.assertEqual(api_response[1].routes[3].vpc_peering_connection_id, 'pcx-efc52b86') self.assertEqual(api_response[1].routes[3].state, 'blackhole') self.assertEqual(api_response[1].routes[3].origin, 'CreateRoute') self.assertEqual(len(api_response[1].associations), 1) self.assertEqual(api_response[1].associations[0].id, 'rtbassoc-faad4893') self.assertEqual(api_response[1].associations[0].route_table_id, 'rtb-f9ad4890') self.assertEqual(api_response[1].associations[0].subnet_id, 'subnet-15ad487c') self.assertEqual(api_response[1].associations[0].main, False) class TestAssociateRouteTable(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <AssociateRouteTableResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <associationId>rtbassoc-f8ad4891</associationId> </AssociateRouteTableResponse> """ def test_associate_route_table(self): self.set_http_response(status_code=200) api_response = self.service_connection.associate_route_table( 'rtb-e4ad488d', 'subnet-15ad487c') self.assert_request_parameters({ 'Action': 'AssociateRouteTable', 'RouteTableId': 'rtb-e4ad488d', 'SubnetId': 'subnet-15ad487c'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, 'rtbassoc-f8ad4891') class TestDisassociateRouteTable(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DisassociateRouteTableResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </DisassociateRouteTableResponse> """ def test_disassociate_route_table(self): self.set_http_response(status_code=200) api_response = self.service_connection.disassociate_route_table('rtbassoc-fdad4894') self.assert_request_parameters({ 'Action': 'DisassociateRouteTable', 'AssociationId': 'rtbassoc-fdad4894'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestCreateRouteTable(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateRouteTableResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <routeTable> <routeTableId>rtb-f9ad4890</routeTableId> <vpcId>vpc-11ad4878</vpcId> <routeSet> <item> <destinationCidrBlock>10.0.0.0/22</destinationCidrBlock> <gatewayId>local</gatewayId> <state>active</state> </item> </routeSet> <associationSet/> <tagSet/> </routeTable> </CreateRouteTableResponse> """ def test_create_route_table(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_route_table('vpc-11ad4878') self.assert_request_parameters({ 'Action': 'CreateRouteTable', 'VpcId': 'vpc-11ad4878'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertIsInstance(api_response, RouteTable) self.assertEqual(api_response.id, 'rtb-f9ad4890') self.assertEqual(len(api_response.routes), 1) self.assertEqual(api_response.routes[0].destination_cidr_block, '10.0.0.0/22') self.assertEqual(api_response.routes[0].gateway_id, 'local') self.assertEqual(api_response.routes[0].state, 'active') class TestDeleteRouteTable(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteRouteTableResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </DeleteRouteTableResponse> """ def test_delete_route_table(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_route_table('rtb-e4ad488d') self.assert_request_parameters({ 'Action': 'DeleteRouteTable', 'RouteTableId': 'rtb-e4ad488d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestReplaceRouteTableAssociation(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <ReplaceRouteTableAssociationResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <newAssociationId>rtbassoc-faad4893</newAssociationId> </ReplaceRouteTableAssociationResponse> """ def test_replace_route_table_assocation(self): self.set_http_response(status_code=200) api_response = self.service_connection.replace_route_table_assocation( 'rtbassoc-faad4893', 'rtb-f9ad4890') self.assert_request_parameters({ 'Action': 'ReplaceRouteTableAssociation', 'AssociationId': 'rtbassoc-faad4893', 'RouteTableId': 'rtb-f9ad4890'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) def test_replace_route_table_association_with_assoc(self): self.set_http_response(status_code=200) api_response = self.service_connection.replace_route_table_association_with_assoc( 'rtbassoc-faad4893', 'rtb-f9ad4890') self.assert_request_parameters({ 'Action': 'ReplaceRouteTableAssociation', 'AssociationId': 'rtbassoc-faad4893', 'RouteTableId': 'rtb-f9ad4890'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, 'rtbassoc-faad4893') class TestCreateRoute(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateRouteResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </CreateRouteResponse> """ def test_create_route_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_route( 'rtb-e4ad488d', '0.0.0.0/0', gateway_id='igw-eaad4883') self.assert_request_parameters({ 'Action': 'CreateRoute', 'RouteTableId': 'rtb-e4ad488d', 'DestinationCidrBlock': '0.0.0.0/0', 'GatewayId': 'igw-eaad4883'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) def test_create_route_instance(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_route( 'rtb-g8ff4ea2', '0.0.0.0/0', instance_id='i-1a2b3c4d') self.assert_request_parameters({ 'Action': 'CreateRoute', 'RouteTableId': 'rtb-g8ff4ea2', 'DestinationCidrBlock': '0.0.0.0/0', 'InstanceId': 'i-1a2b3c4d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) def test_create_route_interface(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_route( 'rtb-g8ff4ea2', '0.0.0.0/0', interface_id='eni-1a2b3c4d') self.assert_request_parameters({ 'Action': 'CreateRoute', 'RouteTableId': 'rtb-g8ff4ea2', 'DestinationCidrBlock': '0.0.0.0/0', 'NetworkInterfaceId': 'eni-1a2b3c4d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) def test_create_route_vpc_peering_connection(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_route( 'rtb-g8ff4ea2', '0.0.0.0/0', vpc_peering_connection_id='pcx-1a2b3c4d') self.assert_request_parameters({ 'Action': 'CreateRoute', 'RouteTableId': 'rtb-g8ff4ea2', 'DestinationCidrBlock': '0.0.0.0/0', 'VpcPeeringConnectionId': 'pcx-1a2b3c4d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestReplaceRoute(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <CreateRouteResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </CreateRouteResponse> """ def test_replace_route_gateway(self): self.set_http_response(status_code=200) api_response = self.service_connection.replace_route( 'rtb-e4ad488d', '0.0.0.0/0', gateway_id='igw-eaad4883') self.assert_request_parameters({ 'Action': 'ReplaceRoute', 'RouteTableId': 'rtb-e4ad488d', 'DestinationCidrBlock': '0.0.0.0/0', 'GatewayId': 'igw-eaad4883'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) def test_replace_route_instance(self): self.set_http_response(status_code=200) api_response = self.service_connection.replace_route( 'rtb-g8ff4ea2', '0.0.0.0/0', instance_id='i-1a2b3c4d') self.assert_request_parameters({ 'Action': 'ReplaceRoute', 'RouteTableId': 'rtb-g8ff4ea2', 'DestinationCidrBlock': '0.0.0.0/0', 'InstanceId': 'i-1a2b3c4d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) def test_replace_route_interface(self): self.set_http_response(status_code=200) api_response = self.service_connection.replace_route( 'rtb-g8ff4ea2', '0.0.0.0/0', interface_id='eni-1a2b3c4d') self.assert_request_parameters({ 'Action': 'ReplaceRoute', 'RouteTableId': 'rtb-g8ff4ea2', 'DestinationCidrBlock': '0.0.0.0/0', 'NetworkInterfaceId': 'eni-1a2b3c4d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) def test_replace_route_vpc_peering_connection(self): self.set_http_response(status_code=200) api_response = self.service_connection.replace_route( 'rtb-g8ff4ea2', '0.0.0.0/0', vpc_peering_connection_id='pcx-1a2b3c4d') self.assert_request_parameters({ 'Action': 'ReplaceRoute', 'RouteTableId': 'rtb-g8ff4ea2', 'DestinationCidrBlock': '0.0.0.0/0', 'VpcPeeringConnectionId': 'pcx-1a2b3c4d'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) class TestDeleteRoute(AWSMockServiceTestCase): connection_class = VPCConnection def default_body(self): return b""" <DeleteRouteTableResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> </DeleteRouteTableResponse> """ def test_delete_route(self): self.set_http_response(status_code=200) api_response = self.service_connection.delete_route('rtb-e4ad488d', '172.16.1.0/24') self.assert_request_parameters({ 'Action': 'DeleteRoute', 'RouteTableId': 'rtb-e4ad488d', 'DestinationCidrBlock': '172.16.1.0/24'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(api_response, True) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/vpc/test_vpc_peering_connection.py 0000644 00000031550 15025013616 0020362 0 ustar 00 # Copyright (c) 2014 Skytap http://skytap.com/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. from tests.unit import mock, unittest from tests.unit import AWSMockServiceTestCase from boto.vpc import VpcPeeringConnection, VPCConnection, Subnet class TestDescribeVpcPeeringConnections(AWSMockServiceTestCase): DESCRIBE_VPC_PEERING_CONNECTIONS= b"""<?xml version="1.0" encoding="UTF-8"?> <DescribeVpcPeeringConnectionsResponse xmlns="http://ec2.amazonaws.com/doc/2014-05-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <vpcPeeringConnectionSet> <item> <vpcPeeringConnectionId>pcx-111aaa22</vpcPeeringConnectionId> <requesterVpcInfo> <ownerId>777788889999</ownerId> <vpcId>vpc-1a2b3c4d</vpcId> <cidrBlock>172.31.0.0/16</cidrBlock> </requesterVpcInfo> <accepterVpcInfo> <ownerId>111122223333</ownerId> <vpcId>vpc-aa22cc33</vpcId> </accepterVpcInfo> <status> <code>pending-acceptance</code> <message>Pending Acceptance by 111122223333</message> </status> <expirationTime>2014-02-17T16:00:50.000Z</expirationTime> </item> <item> <vpcPeeringConnectionId>pcx-444bbb88</vpcPeeringConnectionId> <requesterVpcInfo> <ownerId>1237897234</ownerId> <vpcId>vpc-2398abcd</vpcId> <cidrBlock>172.30.0.0/16</cidrBlock> </requesterVpcInfo> <accepterVpcInfo> <ownerId>98654313</ownerId> <vpcId>vpc-0983bcda</vpcId> </accepterVpcInfo> <status> <code>pending-acceptance</code> <message>Pending Acceptance by 98654313</message> </status> <expirationTime>2015-02-17T16:00:50.000Z</expirationTime> </item> </vpcPeeringConnectionSet> </DescribeVpcPeeringConnectionsResponse>""" connection_class = VPCConnection def default_body(self): return self.DESCRIBE_VPC_PEERING_CONNECTIONS def test_get_vpc_peering_connections(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_vpc_peering_connections( ['pcx-111aaa22', 'pcx-444bbb88'], filters=[('status-code', ['pending-acceptance'])]) self.assertEqual(len(api_response), 2) for vpc_peering_connection in api_response: if vpc_peering_connection.id == 'pcx-111aaa22': self.assertEqual(vpc_peering_connection.id, 'pcx-111aaa22') self.assertEqual(vpc_peering_connection.status_code, 'pending-acceptance') self.assertEqual(vpc_peering_connection.status_message, 'Pending Acceptance by 111122223333') self.assertEqual(vpc_peering_connection.requester_vpc_info.owner_id, '777788889999') self.assertEqual(vpc_peering_connection.requester_vpc_info.vpc_id, 'vpc-1a2b3c4d') self.assertEqual(vpc_peering_connection.requester_vpc_info.cidr_block, '172.31.0.0/16') self.assertEqual(vpc_peering_connection.accepter_vpc_info.owner_id, '111122223333') self.assertEqual(vpc_peering_connection.accepter_vpc_info.vpc_id, 'vpc-aa22cc33') self.assertEqual(vpc_peering_connection.expiration_time, '2014-02-17T16:00:50.000Z') else: self.assertEqual(vpc_peering_connection.id, 'pcx-444bbb88') self.assertEqual(vpc_peering_connection.status_code, 'pending-acceptance') self.assertEqual(vpc_peering_connection.status_message, 'Pending Acceptance by 98654313') self.assertEqual(vpc_peering_connection.requester_vpc_info.owner_id, '1237897234') self.assertEqual(vpc_peering_connection.requester_vpc_info.vpc_id, 'vpc-2398abcd') self.assertEqual(vpc_peering_connection.requester_vpc_info.cidr_block, '172.30.0.0/16') self.assertEqual(vpc_peering_connection.accepter_vpc_info.owner_id, '98654313') self.assertEqual(vpc_peering_connection.accepter_vpc_info.vpc_id, 'vpc-0983bcda') self.assertEqual(vpc_peering_connection.expiration_time, '2015-02-17T16:00:50.000Z') class TestCreateVpcPeeringConnection(AWSMockServiceTestCase): CREATE_VPC_PEERING_CONNECTION= b"""<CreateVpcPeeringConnectionResponse xmlns="http://ec2.amazonaws.com/doc/2014-05-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <vpcPeeringConnection> <vpcPeeringConnectionId>pcx-73a5401a</vpcPeeringConnectionId> <requesterVpcInfo> <ownerId>777788889999</ownerId> <vpcId>vpc-1a2b3c4d</vpcId> <cidrBlock>10.0.0.0/28</cidrBlock> </requesterVpcInfo> <accepterVpcInfo> <ownerId>123456789012</ownerId> <vpcId>vpc-a1b2c3d4</vpcId> </accepterVpcInfo> <status> <code>initiating-request</code> <message>Initiating Request to 123456789012</message> </status> <expirationTime>2014-02-18T14:37:25.000Z</expirationTime> <tagSet/> </vpcPeeringConnection> </CreateVpcPeeringConnectionResponse>""" connection_class = VPCConnection def default_body(self): return self.CREATE_VPC_PEERING_CONNECTION def test_create_vpc_peering_connection(self): self.set_http_response(status_code=200) vpc_peering_connection = self.service_connection.create_vpc_peering_connection('vpc-1a2b3c4d', 'vpc-a1b2c3d4', '123456789012') self.assertEqual(vpc_peering_connection.id, 'pcx-73a5401a') self.assertEqual(vpc_peering_connection.status_code, 'initiating-request') self.assertEqual(vpc_peering_connection.status_message, 'Initiating Request to 123456789012') self.assertEqual(vpc_peering_connection.requester_vpc_info.owner_id, '777788889999') self.assertEqual(vpc_peering_connection.requester_vpc_info.vpc_id, 'vpc-1a2b3c4d') self.assertEqual(vpc_peering_connection.requester_vpc_info.cidr_block, '10.0.0.0/28') self.assertEqual(vpc_peering_connection.accepter_vpc_info.owner_id, '123456789012') self.assertEqual(vpc_peering_connection.accepter_vpc_info.vpc_id, 'vpc-a1b2c3d4') self.assertEqual(vpc_peering_connection.expiration_time, '2014-02-18T14:37:25.000Z') class TestDeleteVpcPeeringConnection(AWSMockServiceTestCase): DELETE_VPC_PEERING_CONNECTION= b"""<DeleteVpcPeeringConnectionResponse xmlns="http://ec2.amazonaws.com/doc/2014-05-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </DeleteVpcPeeringConnectionResponse>""" connection_class = VPCConnection def default_body(self): return self.DELETE_VPC_PEERING_CONNECTION def test_delete_vpc_peering_connection(self): self.set_http_response(status_code=200) self.assertEqual(self.service_connection.delete_vpc_peering_connection('pcx-12345678'), True) class TestDeleteVpcPeeringConnectionShortForm(unittest.TestCase): DESCRIBE_VPC_PEERING_CONNECTIONS= b"""<?xml version="1.0" encoding="UTF-8"?> <DescribeVpcPeeringConnectionsResponse xmlns="http://ec2.amazonaws.com/doc/2014-05-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <vpcPeeringConnectionSet> <item> <vpcPeeringConnectionId>pcx-111aaa22</vpcPeeringConnectionId> <requesterVpcInfo> <ownerId>777788889999</ownerId> <vpcId>vpc-1a2b3c4d</vpcId> <cidrBlock>172.31.0.0/16</cidrBlock> </requesterVpcInfo> <accepterVpcInfo> <ownerId>111122223333</ownerId> <vpcId>vpc-aa22cc33</vpcId> </accepterVpcInfo> <status> <code>pending-acceptance</code> <message>Pending Acceptance by 111122223333</message> </status> <expirationTime>2014-02-17T16:00:50.000Z</expirationTime> </item> </vpcPeeringConnectionSet> </DescribeVpcPeeringConnectionsResponse>""" DELETE_VPC_PEERING_CONNECTION= b"""<DeleteVpcPeeringConnectionResponse xmlns="http://ec2.amazonaws.com/doc/2014-05-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </DeleteVpcPeeringConnectionResponse>""" def test_delete_vpc_peering_connection(self): vpc_conn = VPCConnection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') mock_response = mock.Mock() mock_response.read.return_value = self.DESCRIBE_VPC_PEERING_CONNECTIONS mock_response.status = 200 vpc_conn.make_request = mock.Mock(return_value=mock_response) vpc_peering_connections = vpc_conn.get_all_vpc_peering_connections() self.assertEqual(1, len(vpc_peering_connections)) vpc_peering_connection = vpc_peering_connections[0] mock_response = mock.Mock() mock_response.read.return_value = self.DELETE_VPC_PEERING_CONNECTION mock_response.status = 200 vpc_conn.make_request = mock.Mock(return_value=mock_response) self.assertEqual(True, vpc_peering_connection.delete()) self.assertIn('DeleteVpcPeeringConnection', vpc_conn.make_request.call_args_list[0][0]) self.assertNotIn('DeleteVpc', vpc_conn.make_request.call_args_list[0][0]) class TestRejectVpcPeeringConnection(AWSMockServiceTestCase): REJECT_VPC_PEERING_CONNECTION= b"""<RejectVpcPeeringConnectionResponse xmlns="http://ec2.amazonaws.com/doc/2014-05-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </RejectVpcPeeringConnectionResponse>""" connection_class = VPCConnection def default_body(self): return self.REJECT_VPC_PEERING_CONNECTION def test_reject_vpc_peering_connection(self): self.set_http_response(status_code=200) self.assertEqual(self.service_connection.reject_vpc_peering_connection('pcx-12345678'), True) class TestAcceptVpcPeeringConnection(AWSMockServiceTestCase): ACCEPT_VPC_PEERING_CONNECTION= b"""<AcceptVpcPeeringConnectionResponse xmlns="http://ec2.amazonaws.com/doc/2014-05-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <vpcPeeringConnection> <vpcPeeringConnectionId>pcx-1a2b3c4d</vpcPeeringConnectionId> <requesterVpcInfo> <ownerId>123456789012</ownerId> <vpcId>vpc-1a2b3c4d</vpcId> <cidrBlock>10.0.0.0/28</cidrBlock> </requesterVpcInfo> <accepterVpcInfo> <ownerId>777788889999</ownerId> <vpcId>vpc-111aaa22</vpcId> <cidrBlock>10.0.1.0/28</cidrBlock> </accepterVpcInfo> <status> <code>active</code> <message>Active</message> </status> <tagSet/> </vpcPeeringConnection> </AcceptVpcPeeringConnectionResponse>""" connection_class = VPCConnection def default_body(self): return self.ACCEPT_VPC_PEERING_CONNECTION def test_accept_vpc_peering_connection(self): self.set_http_response(status_code=200) vpc_peering_connection = self.service_connection.accept_vpc_peering_connection('pcx-1a2b3c4d') self.assertEqual(vpc_peering_connection.id, 'pcx-1a2b3c4d') self.assertEqual(vpc_peering_connection.status_code, 'active') self.assertEqual(vpc_peering_connection.status_message, 'Active') self.assertEqual(vpc_peering_connection.requester_vpc_info.owner_id, '123456789012') self.assertEqual(vpc_peering_connection.requester_vpc_info.vpc_id, 'vpc-1a2b3c4d') self.assertEqual(vpc_peering_connection.requester_vpc_info.cidr_block, '10.0.0.0/28') self.assertEqual(vpc_peering_connection.accepter_vpc_info.owner_id, '777788889999') self.assertEqual(vpc_peering_connection.accepter_vpc_info.vpc_id, 'vpc-111aaa22') self.assertEqual(vpc_peering_connection.accepter_vpc_info.cidr_block, '10.0.1.0/28') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/vpc/__init__.py 0000644 00000000035 15025013616 0014334 0 ustar 00 """ Test package for VPC """ vendored/boto/tests/unit/ses/test_identity.py 0000644 00000016635 15025013616 0015504 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.jsonresponse import ListElement from boto.ses.connection import SESConnection class TestSESIdentity(AWSMockServiceTestCase): connection_class = SESConnection def setUp(self): super(TestSESIdentity, self).setUp() def default_body(self): return b"""<GetIdentityDkimAttributesResponse \ xmlns="http://ses.amazonaws.com/doc/2010-12-01/"> <GetIdentityDkimAttributesResult> <DkimAttributes> <entry> <key>test@amazon.com</key> <value> <DkimEnabled>true</DkimEnabled> <DkimVerificationStatus>Success</DkimVerificationStatus> <DkimTokens> <member>vvjuipp74whm76gqoni7qmwwn4w4qusjiainivf6f</member> <member>3frqe7jn4obpuxjpwpolz6ipb3k5nvt2nhjpik2oy</member> <member>wrqplteh7oodxnad7hsl4mixg2uavzneazxv5sxi2</member> </DkimTokens> </value> </entry> <entry> <key>secondtest@amazon.com</key> <value> <DkimEnabled>false</DkimEnabled> <DkimVerificationStatus>NotStarted</DkimVerificationStatus> </value> </entry> </DkimAttributes> </GetIdentityDkimAttributesResult> <ResponseMetadata> <RequestId>bb5a105d-c468-11e1-82eb-dff885ccc06a</RequestId> </ResponseMetadata> </GetIdentityDkimAttributesResponse>""" def test_ses_get_identity_dkim_list(self): self.set_http_response(status_code=200) response = self.service_connection\ .get_identity_dkim_attributes(['test@amazon.com', 'secondtest@amazon.com']) response = response['GetIdentityDkimAttributesResponse'] result = response['GetIdentityDkimAttributesResult'] first_entry = result['DkimAttributes'][0] entry_key = first_entry['key'] attributes = first_entry['value'] tokens = attributes['DkimTokens'] self.assertEqual(entry_key, 'test@amazon.com') self.assertEqual(ListElement, type(tokens)) self.assertEqual(3, len(tokens)) self.assertEqual('vvjuipp74whm76gqoni7qmwwn4w4qusjiainivf6f', tokens[0]) self.assertEqual('3frqe7jn4obpuxjpwpolz6ipb3k5nvt2nhjpik2oy', tokens[1]) self.assertEqual('wrqplteh7oodxnad7hsl4mixg2uavzneazxv5sxi2', tokens[2]) second_entry = result['DkimAttributes'][1] entry_key = second_entry['key'] attributes = second_entry['value'] dkim_enabled = attributes['DkimEnabled'] dkim_verification_status = attributes['DkimVerificationStatus'] self.assertEqual(entry_key, 'secondtest@amazon.com') self.assertEqual(dkim_enabled, 'false') self.assertEqual(dkim_verification_status, 'NotStarted') class TestSESSetIdentityNotificationTopic(AWSMockServiceTestCase): connection_class = SESConnection def setUp(self): super(TestSESSetIdentityNotificationTopic, self).setUp() def default_body(self): return b"""<SetIdentityNotificationTopicResponse \ xmlns="http://ses.amazonaws.com/doc/2010-12-01/"> <SetIdentityNotificationTopicResult/> <ResponseMetadata> <RequestId>299f4af4-b72a-11e1-901f-1fbd90e8104f</RequestId> </ResponseMetadata> </SetIdentityNotificationTopicResponse>""" def test_ses_set_identity_notification_topic_bounce(self): self.set_http_response(status_code=200) response = self.service_connection\ .set_identity_notification_topic( identity='user@example.com', notification_type='Bounce', sns_topic='arn:aws:sns:us-east-1:123456789012:example') response = response['SetIdentityNotificationTopicResponse'] result = response['SetIdentityNotificationTopicResult'] self.assertEqual(2, len(response)) self.assertEqual(0, len(result)) def test_ses_set_identity_notification_topic_complaint(self): self.set_http_response(status_code=200) response = self.service_connection\ .set_identity_notification_topic( identity='user@example.com', notification_type='Complaint', sns_topic='arn:aws:sns:us-east-1:123456789012:example') response = response['SetIdentityNotificationTopicResponse'] result = response['SetIdentityNotificationTopicResult'] self.assertEqual(2, len(response)) self.assertEqual(0, len(result)) class TestSESSetIdentityFeedbackForwardingEnabled(AWSMockServiceTestCase): connection_class = SESConnection def setUp(self): super(TestSESSetIdentityFeedbackForwardingEnabled, self).setUp() def default_body(self): return b"""<SetIdentityFeedbackForwardingEnabledResponse \ xmlns="http://ses.amazonaws.com/doc/2010-12-01/"> <SetIdentityFeedbackForwardingEnabledResult/> <ResponseMetadata> <RequestId>299f4af4-b72a-11e1-901f-1fbd90e8104f</RequestId> </ResponseMetadata> </SetIdentityFeedbackForwardingEnabledResponse>""" def test_ses_set_identity_feedback_forwarding_enabled_true(self): self.set_http_response(status_code=200) response = self.service_connection\ .set_identity_feedback_forwarding_enabled( identity='user@example.com', forwarding_enabled=True) response = response['SetIdentityFeedbackForwardingEnabledResponse'] result = response['SetIdentityFeedbackForwardingEnabledResult'] self.assertEqual(2, len(response)) self.assertEqual(0, len(result)) def test_ses_set_identity_notification_topic_enabled_false(self): self.set_http_response(status_code=200) response = self.service_connection\ .set_identity_feedback_forwarding_enabled( identity='user@example.com', forwarding_enabled=False) response = response['SetIdentityFeedbackForwardingEnabledResponse'] result = response['SetIdentityFeedbackForwardingEnabledResult'] self.assertEqual(2, len(response)) self.assertEqual(0, len(result)) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ses/__init__.py 0000644 00000000000 15025013616 0014326 0 ustar 00 vendored/boto/tests/unit/sns/test_connection.py 0000644 00000025666 15025013616 0016027 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import json from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from mock import Mock from boto.sns.connection import SNSConnection QUEUE_POLICY = { u'Policy': (u'{"Version":"2008-10-17","Id":"arn:aws:sqs:us-east-1:' 'idnum:testqueuepolicy/SQSDefaultPolicy","Statement":' '[{"Sid":"sidnum","Effect":"Allow","Principal":{"AWS":"*"},' '"Action":"SQS:GetQueueUrl","Resource":' '"arn:aws:sqs:us-east-1:idnum:testqueuepolicy"}]}')} class TestSNSConnection(AWSMockServiceTestCase): connection_class = SNSConnection def setUp(self): super(TestSNSConnection, self).setUp() def default_body(self): return b"{}" def test_sqs_with_existing_policy(self): self.set_http_response(status_code=200) queue = Mock() queue.get_attributes.return_value = QUEUE_POLICY queue.arn = 'arn:aws:sqs:us-east-1:idnum:queuename' self.service_connection.subscribe_sqs_queue('topic_arn', queue) self.assert_request_parameters({ 'Action': 'Subscribe', 'ContentType': 'JSON', 'Endpoint': 'arn:aws:sqs:us-east-1:idnum:queuename', 'Protocol': 'sqs', 'TopicArn': 'topic_arn', 'Version': '2010-03-31', }, ignore_params_values=[]) # Verify that the queue policy was properly updated. actual_policy = json.loads(queue.set_attribute.call_args[0][1]) self.assertEqual(actual_policy['Version'], '2008-10-17') # A new statement should be appended to the end of the statement list. self.assertEqual(len(actual_policy['Statement']), 2) self.assertEqual(actual_policy['Statement'][1]['Action'], 'SQS:SendMessage') def test_sqs_with_no_previous_policy(self): self.set_http_response(status_code=200) queue = Mock() queue.get_attributes.return_value = {} queue.arn = 'arn:aws:sqs:us-east-1:idnum:queuename' self.service_connection.subscribe_sqs_queue('topic_arn', queue) self.assert_request_parameters({ 'Action': 'Subscribe', 'ContentType': 'JSON', 'Endpoint': 'arn:aws:sqs:us-east-1:idnum:queuename', 'Protocol': 'sqs', 'TopicArn': 'topic_arn', 'Version': '2010-03-31', }, ignore_params_values=[]) actual_policy = json.loads(queue.set_attribute.call_args[0][1]) # Only a single statement should be part of the policy. self.assertEqual(len(actual_policy['Statement']), 1) def test_publish_with_positional_args(self): self.set_http_response(status_code=200) self.service_connection.publish('topic', 'message', 'subject') self.assert_request_parameters({ 'Action': 'Publish', 'TopicArn': 'topic', 'Subject': 'subject', 'Message': 'message', }, ignore_params_values=['Version', 'ContentType']) def test_publish_with_kwargs(self): self.set_http_response(status_code=200) self.service_connection.publish(topic='topic', message='message', subject='subject') self.assert_request_parameters({ 'Action': 'Publish', 'TopicArn': 'topic', 'Subject': 'subject', 'Message': 'message', }, ignore_params_values=['Version', 'ContentType']) def test_publish_with_target_arn(self): self.set_http_response(status_code=200) self.service_connection.publish(target_arn='target_arn', message='message', subject='subject') self.assert_request_parameters({ 'Action': 'Publish', 'TargetArn': 'target_arn', 'Subject': 'subject', 'Message': 'message', }, ignore_params_values=['Version', 'ContentType']) def test_create_platform_application(self): self.set_http_response(status_code=200) self.service_connection.create_platform_application( name='MyApp', platform='APNS', attributes={ 'PlatformPrincipal': 'a ssl certificate', 'PlatformCredential': 'a private key' } ) self.assert_request_parameters({ 'Action': 'CreatePlatformApplication', 'Name': 'MyApp', 'Platform': 'APNS', 'Attributes.entry.1.key': 'PlatformCredential', 'Attributes.entry.1.value': 'a private key', 'Attributes.entry.2.key': 'PlatformPrincipal', 'Attributes.entry.2.value': 'a ssl certificate', }, ignore_params_values=['Version', 'ContentType']) def test_set_platform_application_attributes(self): self.set_http_response(status_code=200) self.service_connection.set_platform_application_attributes( platform_application_arn='arn:myapp', attributes={'PlatformPrincipal': 'a ssl certificate', 'PlatformCredential': 'a private key'}) self.assert_request_parameters({ 'Action': 'SetPlatformApplicationAttributes', 'PlatformApplicationArn': 'arn:myapp', 'Attributes.entry.1.key': 'PlatformCredential', 'Attributes.entry.1.value': 'a private key', 'Attributes.entry.2.key': 'PlatformPrincipal', 'Attributes.entry.2.value': 'a ssl certificate', }, ignore_params_values=['Version', 'ContentType']) def test_create_platform_endpoint(self): self.set_http_response(status_code=200) self.service_connection.create_platform_endpoint( platform_application_arn='arn:myapp', token='abcde12345', custom_user_data='john', attributes={'Enabled': False}) self.assert_request_parameters({ 'Action': 'CreatePlatformEndpoint', 'PlatformApplicationArn': 'arn:myapp', 'Token': 'abcde12345', 'CustomUserData': 'john', 'Attributes.entry.1.key': 'Enabled', 'Attributes.entry.1.value': False, }, ignore_params_values=['Version', 'ContentType']) def test_set_endpoint_attributes(self): self.set_http_response(status_code=200) self.service_connection.set_endpoint_attributes( endpoint_arn='arn:myendpoint', attributes={'CustomUserData': 'john', 'Enabled': False}) self.assert_request_parameters({ 'Action': 'SetEndpointAttributes', 'EndpointArn': 'arn:myendpoint', 'Attributes.entry.1.key': 'CustomUserData', 'Attributes.entry.1.value': 'john', 'Attributes.entry.2.key': 'Enabled', 'Attributes.entry.2.value': False, }, ignore_params_values=['Version', 'ContentType']) def test_message_is_required(self): self.set_http_response(status_code=200) with self.assertRaises(TypeError): self.service_connection.publish(topic='topic', subject='subject') def test_publish_with_json(self): self.set_http_response(status_code=200) self.service_connection.publish( message=json.dumps({ 'default': 'Ignored.', 'GCM': { 'data': 'goes here', } }), message_structure='json', subject='subject', target_arn='target_arn' ) self.assert_request_parameters({ 'Action': 'Publish', 'TargetArn': 'target_arn', 'Subject': 'subject', 'MessageStructure': 'json', }, ignore_params_values=['Version', 'ContentType', 'Message']) self.assertDictEqual( json.loads(self.actual_request.params["Message"]), {"default": "Ignored.", "GCM": {"data": "goes here"}}) def test_publish_with_utf8_message(self): self.set_http_response(status_code=200) subject = message = u'We \u2665 utf-8'.encode('utf-8') self.service_connection.publish('topic', message, subject) self.assert_request_parameters({ 'Action': 'Publish', 'TopicArn': 'topic', 'Subject': subject, 'Message': message, }, ignore_params_values=['Version', 'ContentType']) def test_publish_with_attributes(self): self.set_http_response(status_code=200) self.service_connection.publish( message=json.dumps({ 'default': 'Ignored.', 'GCM': { 'data': 'goes here', } }, sort_keys=True), message_structure='json', subject='subject', target_arn='target_arn', message_attributes={ 'name1': { 'data_type': 'Number', 'string_value': '42' }, 'name2': { 'data_type': 'String', 'string_value': 'Bob' }, }, ) self.assert_request_parameters({ 'Action': 'Publish', 'TargetArn': 'target_arn', 'Subject': 'subject', 'Message': '{"GCM": {"data": "goes here"}, "default": "Ignored."}', 'MessageStructure': 'json', 'MessageAttributes.entry.1.Name': 'name1', 'MessageAttributes.entry.1.Value.DataType': 'Number', 'MessageAttributes.entry.1.Value.StringValue': '42', 'MessageAttributes.entry.2.Name': 'name2', 'MessageAttributes.entry.2.Value.DataType': 'String', 'MessageAttributes.entry.2.Value.StringValue': 'Bob', }, ignore_params_values=['Version', 'ContentType']) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/sns/__init__.py 0000644 00000000000 15025013616 0014337 0 ustar 00 vendored/boto/tests/unit/pyami/test_config.py 0000644 00000005060 15025013616 0015433 0 ustar 00 # Copyright (c) 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.compat import mock, unittest from boto.pyami import config from boto.compat import StringIO class TestCanLoadConfigFile(unittest.TestCase): maxDiff = None def setUp(self): self.file_contents = StringIO() file_contents = StringIO( '[Boto]\n' 'https_validate_certificates = true\n' 'other = false\n' 'http_socket_timeout = 1\n' '[Credentials]\n' 'aws_access_key_id=foo\n' 'aws_secret_access_key=bar\n' ) self.config = config.Config(fp=file_contents) def test_can_get_bool(self): self.assertTrue( self.config.getbool('Boto', 'https_validate_certificates')) self.assertFalse(self.config.getbool('Boto', 'other')) self.assertFalse(self.config.getbool('Boto', 'does-not-exist')) def test_can_get_int(self): self.assertEqual(self.config.getint('Boto', 'http_socket_timeout'), 1) self.assertEqual(self.config.getint('Boto', 'does-not-exist'), 0) self.assertEqual( self.config.getint('Boto', 'does-not-exist', default=20), 20) def test_can_get_strings(self): self.assertEqual( self.config.get('Credentials', 'aws_access_key_id'), 'foo') self.assertIsNone( self.config.get('Credentials', 'no-exist')) self.assertEqual( self.config.get('Credentials', 'no-exist', 'default-value'), 'default-value') vendored/boto/tests/unit/pyami/__init__.py 0000644 00000000000 15025013616 0014653 0 ustar 00 vendored/boto/tests/unit/ec2/test_instancetype.py 0000644 00000012145 15025013616 0016230 0 ustar 00 #!/usr/bin/env python from tests.unit import unittest from tests.unit import AWSMockServiceTestCase import boto.ec2 from boto.ec2.connection import EC2Connection class TestEC2ConnectionBase(AWSMockServiceTestCase): connection_class = EC2Connection def setUp(self): super(TestEC2ConnectionBase, self).setUp() self.ec2 = self.service_connection class TestReservedInstanceOfferings(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeInstanceTypesResponseType xmlns="http://ec2.amazonaws.com/doc/2013-02-01/"> <VmTypeMessage/> <instanceTypeDetails> <item> <name>m1.small</name><cpu>1</cpu><disk>5</disk><memory>256</memory> <availability/><ephemeralDisk/> </item> <item> <name>t1.micro</name><cpu>1</cpu><disk>5</disk><memory>256</memory> <availability/><ephemeralDisk/> </item> <item> <name>m1.medium</name><cpu>1</cpu><disk>10</disk><memory>512</memory> <availability/><ephemeralDisk/> </item> <item> <name>c1.medium</name><cpu>2</cpu><disk>10</disk><memory>512</memory> <availability/><ephemeralDisk/> </item> <item> <name>m1.large</name><cpu>2</cpu><disk>10</disk><memory>512</memory> <availability/><ephemeralDisk/> </item> <item> <name>m1.xlarge</name><cpu>2</cpu><disk>10</disk><memory>1024</memory> <availability/><ephemeralDisk/> </item> <item> <name>c1.xlarge</name><cpu>2</cpu><disk>10</disk><memory>2048</memory> <availability/><ephemeralDisk/> </item> <item> <name>m2.xlarge</name><cpu>2</cpu><disk>10</disk><memory>2048</memory> <availability/><ephemeralDisk/> </item> <item> <name>m3.xlarge</name><cpu>4</cpu><disk>15</disk><memory>2048</memory> <availability/><ephemeralDisk/> </item> <item> <name>m2.2xlarge</name><cpu>2</cpu><disk>30</disk><memory>4096</memory> <availability/><ephemeralDisk/> </item> <item> <name>m3.2xlarge</name><cpu>4</cpu><disk>30</disk><memory>4096</memory> <availability/><ephemeralDisk/> </item> <item> <name>cc1.4xlarge</name><cpu>8</cpu><disk>60</disk><memory>3072</memory> <availability/><ephemeralDisk/> </item> <item> <name>m2.4xlarge</name><cpu>8</cpu><disk>60</disk><memory>4096</memory> <availability/><ephemeralDisk/> </item> <item> <name>hi1.4xlarge</name><cpu>8</cpu><disk>120</disk><memory>6144</memory> <availability/><ephemeralDisk/> </item> <item> <name>cc2.8xlarge</name><cpu>16</cpu><disk>120</disk><memory>6144</memory> <availability/><ephemeralDisk/> </item> <item> <name>cg1.4xlarge</name><cpu>16</cpu><disk>200</disk><memory>12288</memory> <availability/><ephemeralDisk/> </item> <item> <name>cr1.8xlarge</name><cpu>16</cpu><disk>240</disk><memory>16384</memory> <availability/><ephemeralDisk/> </item> <item> <name>hs1.8xlarge</name><cpu>48</cpu><disk>24000</disk><memory>119808</memory> <availability/><ephemeralDisk/> </item> </instanceTypeDetails> </DescribeInstanceTypesResponseType> """ def test_get_instance_types(self): self.set_http_response(status_code=200) response = self.ec2.get_all_instance_types() self.assertEqual(len(response), 18) instance_type = response[0] self.assertEqual(instance_type.name, 'm1.small') self.assertEqual(instance_type.cores, '1') self.assertEqual(instance_type.disk, '5') self.assertEqual(instance_type.memory, '256') instance_type = response[17] self.assertEqual(instance_type.name, 'hs1.8xlarge') self.assertEqual(instance_type.cores, '48') self.assertEqual(instance_type.disk, '24000') self.assertEqual(instance_type.memory, '119808') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/test_networkinterface.py 0000644 00000030436 15025013616 0017077 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.compat import mock, unittest from boto.exception import BotoClientError from boto.ec2.networkinterface import NetworkInterfaceCollection from boto.ec2.networkinterface import NetworkInterfaceSpecification from boto.ec2.networkinterface import PrivateIPAddress from boto.ec2.networkinterface import Attachment, NetworkInterface class NetworkInterfaceTests(unittest.TestCase): def setUp(self): self.attachment = Attachment() self.attachment.id = 'eni-attach-1' self.attachment.instance_id = 10 self.attachment.status = "some status" self.attachment.device_index = 100 self.eni_one = NetworkInterface() self.eni_one.id = 'eni-1' self.eni_one.status = "one_status" self.eni_one.attachment = self.attachment self.eni_two = NetworkInterface() self.eni_two.connection = mock.Mock() self.eni_two.id = 'eni-2' self.eni_two.status = "two_status" self.eni_two.attachment = None def test_update_with_validate_true_raises_value_error(self): self.eni_one.connection = mock.Mock() self.eni_one.connection.get_all_network_interfaces.return_value = [] with self.assertRaisesRegex(ValueError, "^eni-1 is not a valid ENI ID$"): self.eni_one.update(True) def test_update_with_result_set_greater_than_0_updates_dict(self): self.eni_two.connection.get_all_network_interfaces.return_value = [self.eni_one] self.eni_two.update() assert all([self.eni_two.status == "one_status", self.eni_two.id == 'eni-1', self.eni_two.attachment == self.attachment]) def test_update_returns_status(self): self.eni_one.connection = mock.Mock() self.eni_one.connection.get_all_network_interfaces.return_value = [self.eni_two] retval = self.eni_one.update() self.assertEqual(retval, "two_status") def test_attach_calls_attach_eni(self): self.eni_one.connection = mock.Mock() self.eni_one.attach("instance_id", 11) self.eni_one.connection.attach_network_interface.assert_called_with( 'eni-1', "instance_id", 11, dry_run=False ) def test_detach_calls_detach_network_interface(self): self.eni_one.connection = mock.Mock() self.eni_one.detach() self.eni_one.connection.detach_network_interface.assert_called_with( 'eni-attach-1', False, dry_run=False ) def test_detach_with_no_attach_data(self): self.eni_two.connection = mock.Mock() self.eni_two.detach() self.eni_two.connection.detach_network_interface.assert_called_with( None, False, dry_run=False) def test_detach_with_force_calls_detach_network_interface_with_force(self): self.eni_one.connection = mock.Mock() self.eni_one.detach(True) self.eni_one.connection.detach_network_interface.assert_called_with( 'eni-attach-1', True, dry_run=False) class TestNetworkInterfaceCollection(unittest.TestCase): maxDiff = None def setUp(self): self.private_ip_address1 = PrivateIPAddress( private_ip_address='10.0.0.10', primary=False) self.private_ip_address2 = PrivateIPAddress( private_ip_address='10.0.0.11', primary=False) self.network_interfaces_spec1 = NetworkInterfaceSpecification( device_index=1, subnet_id='subnet_id', description='description1', private_ip_address='10.0.0.54', delete_on_termination=False, private_ip_addresses=[self.private_ip_address1, self.private_ip_address2] ) self.private_ip_address3 = PrivateIPAddress( private_ip_address='10.0.1.10', primary=False) self.private_ip_address4 = PrivateIPAddress( private_ip_address='10.0.1.11', primary=False) self.network_interfaces_spec2 = NetworkInterfaceSpecification( device_index=2, subnet_id='subnet_id2', description='description2', groups=['group_id1', 'group_id2'], private_ip_address='10.0.1.54', delete_on_termination=False, private_ip_addresses=[self.private_ip_address3, self.private_ip_address4] ) self.network_interfaces_spec3 = NetworkInterfaceSpecification( device_index=0, subnet_id='subnet_id2', description='description2', groups=['group_id1', 'group_id2'], private_ip_address='10.0.1.54', delete_on_termination=False, private_ip_addresses=[self.private_ip_address3, self.private_ip_address4], associate_public_ip_address=True ) def test_param_serialization(self): collection = NetworkInterfaceCollection(self.network_interfaces_spec1, self.network_interfaces_spec2) params = {} collection.build_list_params(params) self.assertDictEqual(params, { 'NetworkInterface.0.DeviceIndex': '1', 'NetworkInterface.0.DeleteOnTermination': 'false', 'NetworkInterface.0.Description': 'description1', 'NetworkInterface.0.PrivateIpAddress': '10.0.0.54', 'NetworkInterface.0.SubnetId': 'subnet_id', 'NetworkInterface.0.PrivateIpAddresses.0.Primary': 'false', 'NetworkInterface.0.PrivateIpAddresses.0.PrivateIpAddress': '10.0.0.10', 'NetworkInterface.0.PrivateIpAddresses.1.Primary': 'false', 'NetworkInterface.0.PrivateIpAddresses.1.PrivateIpAddress': '10.0.0.11', 'NetworkInterface.1.DeviceIndex': '2', 'NetworkInterface.1.Description': 'description2', 'NetworkInterface.1.DeleteOnTermination': 'false', 'NetworkInterface.1.PrivateIpAddress': '10.0.1.54', 'NetworkInterface.1.SubnetId': 'subnet_id2', 'NetworkInterface.1.SecurityGroupId.0': 'group_id1', 'NetworkInterface.1.SecurityGroupId.1': 'group_id2', 'NetworkInterface.1.PrivateIpAddresses.0.Primary': 'false', 'NetworkInterface.1.PrivateIpAddresses.0.PrivateIpAddress': '10.0.1.10', 'NetworkInterface.1.PrivateIpAddresses.1.Primary': 'false', 'NetworkInterface.1.PrivateIpAddresses.1.PrivateIpAddress': '10.0.1.11', }) def test_add_prefix_to_serialization(self): collection = NetworkInterfaceCollection(self.network_interfaces_spec1, self.network_interfaces_spec2) params = {} collection.build_list_params(params, prefix='LaunchSpecification.') # We already tested the actual serialization previously, so # we're just checking a few keys to make sure we get the proper # prefix. self.assertDictEqual(params, { 'LaunchSpecification.NetworkInterface.0.DeviceIndex': '1', 'LaunchSpecification.NetworkInterface.0.DeleteOnTermination': 'false', 'LaunchSpecification.NetworkInterface.0.Description': 'description1', 'LaunchSpecification.NetworkInterface.0.PrivateIpAddress': '10.0.0.54', 'LaunchSpecification.NetworkInterface.0.SubnetId': 'subnet_id', 'LaunchSpecification.NetworkInterface.0.PrivateIpAddresses.0.Primary': 'false', 'LaunchSpecification.NetworkInterface.0.PrivateIpAddresses.0.PrivateIpAddress': '10.0.0.10', 'LaunchSpecification.NetworkInterface.0.PrivateIpAddresses.1.Primary': 'false', 'LaunchSpecification.NetworkInterface.0.PrivateIpAddresses.1.PrivateIpAddress': '10.0.0.11', 'LaunchSpecification.NetworkInterface.1.DeviceIndex': '2', 'LaunchSpecification.NetworkInterface.1.Description': 'description2', 'LaunchSpecification.NetworkInterface.1.DeleteOnTermination': 'false', 'LaunchSpecification.NetworkInterface.1.PrivateIpAddress': '10.0.1.54', 'LaunchSpecification.NetworkInterface.1.SubnetId': 'subnet_id2', 'LaunchSpecification.NetworkInterface.1.SecurityGroupId.0': 'group_id1', 'LaunchSpecification.NetworkInterface.1.SecurityGroupId.1': 'group_id2', 'LaunchSpecification.NetworkInterface.1.PrivateIpAddresses.0.Primary': 'false', 'LaunchSpecification.NetworkInterface.1.PrivateIpAddresses.0.PrivateIpAddress': '10.0.1.10', 'LaunchSpecification.NetworkInterface.1.PrivateIpAddresses.1.Primary': 'false', 'LaunchSpecification.NetworkInterface.1.PrivateIpAddresses.1.PrivateIpAddress': '10.0.1.11', }) def test_cant_use_public_ip(self): collection = NetworkInterfaceCollection(self.network_interfaces_spec3, self.network_interfaces_spec1) params = {} # First, verify we can't incorrectly create multiple interfaces with # on having a public IP. with self.assertRaises(BotoClientError): collection.build_list_params(params, prefix='LaunchSpecification.') # Next, ensure it can't be on device index 1. self.network_interfaces_spec3.device_index = 1 collection = NetworkInterfaceCollection(self.network_interfaces_spec3) params = {} with self.assertRaises(BotoClientError): collection.build_list_params(params, prefix='LaunchSpecification.') def test_public_ip(self): # With public IP. collection = NetworkInterfaceCollection(self.network_interfaces_spec3) params = {} collection.build_list_params(params, prefix='LaunchSpecification.') self.assertDictEqual(params, { 'LaunchSpecification.NetworkInterface.0.AssociatePublicIpAddress': 'true', 'LaunchSpecification.NetworkInterface.0.DeviceIndex': '0', 'LaunchSpecification.NetworkInterface.0.DeleteOnTermination': 'false', 'LaunchSpecification.NetworkInterface.0.Description': 'description2', 'LaunchSpecification.NetworkInterface.0.PrivateIpAddress': '10.0.1.54', 'LaunchSpecification.NetworkInterface.0.SubnetId': 'subnet_id2', 'LaunchSpecification.NetworkInterface.0.PrivateIpAddresses.0.Primary': 'false', 'LaunchSpecification.NetworkInterface.0.PrivateIpAddresses.0.PrivateIpAddress': '10.0.1.10', 'LaunchSpecification.NetworkInterface.0.PrivateIpAddresses.1.Primary': 'false', 'LaunchSpecification.NetworkInterface.0.PrivateIpAddresses.1.PrivateIpAddress': '10.0.1.11', 'LaunchSpecification.NetworkInterface.0.SecurityGroupId.0': 'group_id1', 'LaunchSpecification.NetworkInterface.0.SecurityGroupId.1': 'group_id2', }) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/cloudwatch/test_connection.py 0000644 00000007313 15025013616 0020017 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import datetime from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.ec2.cloudwatch import CloudWatchConnection class TestCloudWatchConnection(AWSMockServiceTestCase): connection_class = CloudWatchConnection def test_build_put_params_multiple_everything(self): # This dictionary gets modified by the method call. # Check to make sure all updates happen appropriately. params = {} # Again, these are rubbish parameters. Pay them no mind, we care more # about the functionality of the method name = ['whatever', 'goeshere'] value = None timestamp = [ datetime.datetime(2013, 5, 13, 9, 2, 35), datetime.datetime(2013, 5, 12, 9, 2, 35), ] unit = ['lbs', 'ft'] dimensions = None statistics = [ { 'maximum': 5, 'minimum': 1, 'samplecount': 3, 'sum': 7, }, { 'maximum': 6, 'minimum': 2, 'samplecount': 4, 'sum': 5, }, ] # The important part is that this shouldn't generate a warning (due # to overwriting a variable) & should have the correct number of # Metrics (2). self.service_connection.build_put_params( params, name=name, value=value, timestamp=timestamp, unit=unit, dimensions=dimensions, statistics=statistics ) self.assertEqual(params, { 'MetricData.member.1.MetricName': 'whatever', 'MetricData.member.1.StatisticValues.Maximum': 5, 'MetricData.member.1.StatisticValues.Minimum': 1, 'MetricData.member.1.StatisticValues.SampleCount': 3, 'MetricData.member.1.StatisticValues.Sum': 7, 'MetricData.member.1.Timestamp': '2013-05-13T09:02:35', 'MetricData.member.1.Unit': 'lbs', 'MetricData.member.2.MetricName': 'goeshere', 'MetricData.member.2.StatisticValues.Maximum': 6, 'MetricData.member.2.StatisticValues.Minimum': 2, 'MetricData.member.2.StatisticValues.SampleCount': 4, 'MetricData.member.2.StatisticValues.Sum': 5, 'MetricData.member.2.Timestamp': '2013-05-12T09:02:35', # If needed, comment this next line to cause a test failure & see # the logging warning. 'MetricData.member.2.Unit': 'ft', }) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/cloudwatch/__init__.py 0000644 00000000000 15025013616 0016342 0 ustar 00 vendored/boto/tests/unit/ec2/test_reservedinstance.py 0000644 00000003202 15025013616 0017060 0 ustar 00 from tests.unit import AWSMockServiceTestCase from boto.ec2.connection import EC2Connection from boto.ec2.reservedinstance import ReservedInstance class TestReservedInstancesSet(AWSMockServiceTestCase): connection_class = EC2Connection def default_body(self): return b""" <reservedInstancesSet> <item> <reservedInstancesId>ididididid</reservedInstancesId> <instanceType>t1.micro</instanceType> <start>2014-05-03T14:10:10.944Z</start> <end>2014-05-03T14:10:11.000Z</end> <duration>64800000</duration> <fixedPrice>62.5</fixedPrice> <usagePrice>0.0</usagePrice> <instanceCount>5</instanceCount> <productDescription>Linux/UNIX</productDescription> <state>retired</state> <instanceTenancy>default</instanceTenancy> <currencyCode>USD</currencyCode> <offeringType>Heavy Utilization</offeringType> <recurringCharges> <item> <frequency>Hourly</frequency> <amount>0.005</amount> </item> </recurringCharges> </item> </reservedInstancesSet>""" def test_get_all_reserved_instaces(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_reserved_instances() self.assertEqual(len(response), 1) self.assertTrue(isinstance(response[0], ReservedInstance)) self.assertEqual(response[0].id, 'ididididid') self.assertEqual(response[0].instance_count, 5) self.assertEqual(response[0].start, '2014-05-03T14:10:10.944Z') self.assertEqual(response[0].end, '2014-05-03T14:10:11.000Z') vendored/boto/tests/unit/ec2/test_address.py 0000644 00000013476 15025013616 0015157 0 ustar 00 from tests.compat import mock, unittest from boto.ec2.address import Address class AddressTest(unittest.TestCase): def setUp(self): self.address = Address() self.address.connection = mock.Mock() self.address.public_ip = "192.168.1.1" def check_that_attribute_has_been_set(self, name, value, attribute): self.address.endElement(name, value, None) self.assertEqual(getattr(self.address, attribute), value) def test_endElement_sets_correct_attributes_with_values(self): for arguments in [("publicIp", "192.168.1.1", "public_ip"), ("instanceId", 1, "instance_id"), ("domain", "some domain", "domain"), ("allocationId", 1, "allocation_id"), ("associationId", 1, "association_id"), ("somethingRandom", "somethingRandom", "somethingRandom")]: self.check_that_attribute_has_been_set(arguments[0], arguments[1], arguments[2]) def test_release_calls_connection_release_address_with_correct_args(self): self.address.release() self.address.connection.release_address.assert_called_with( public_ip="192.168.1.1", dry_run=False ) def test_associate_calls_connection_associate_address_with_correct_args(self): self.address.associate(1) self.address.connection.associate_address.assert_called_with( instance_id=1, public_ip="192.168.1.1", allow_reassociation=False, network_interface_id=None, private_ip_address=None, dry_run=False ) def test_disassociate_calls_connection_disassociate_address_with_correct_args(self): self.address.disassociate() self.address.connection.disassociate_address.assert_called_with( public_ip="192.168.1.1", dry_run=False ) class AddressWithAllocationTest(unittest.TestCase): def setUp(self): self.address = Address() self.address.connection = mock.Mock() self.address.public_ip = "192.168.1.1" self.address.allocation_id = "aid1" def check_that_attribute_has_been_set(self, name, value, attribute): self.address.endElement(name, value, None) self.assertEqual(getattr(self.address, attribute), value) def test_endElement_sets_correct_attributes_with_values(self): for arguments in [("publicIp", "192.168.1.1", "public_ip"), ("instanceId", 1, "instance_id"), ("domain", "some domain", "domain"), ("allocationId", 1, "allocation_id"), ("associationId", 1, "association_id"), ("somethingRandom", "somethingRandom", "somethingRandom")]: self.check_that_attribute_has_been_set(arguments[0], arguments[1], arguments[2]) def test_release_calls_connection_release_address_with_correct_args(self): self.address.release() self.address.connection.release_address.assert_called_with( allocation_id="aid1", dry_run=False ) def test_associate_calls_connection_associate_address_with_correct_args(self): self.address.associate(1) self.address.connection.associate_address.assert_called_with( instance_id=1, public_ip="192.168.1.1", allocation_id="aid1", network_interface_id=None, private_ip_address=None, allow_reassociation=False, dry_run=False ) def test_disassociate_calls_connection_disassociate_address_with_correct_args(self): self.address.disassociate() self.address.connection.disassociate_address.assert_called_with( public_ip="192.168.1.1", dry_run=False ) class AddressWithNetworkInterfaceTest(unittest.TestCase): def setUp(self): self.address = Address() self.address.connection = mock.Mock() self.address.public_ip = "192.168.1.1" self.address.allocation_id = "aid1" def check_that_attribute_has_been_set(self, name, value, attribute): self.address.endElement(name, value, None) self.assertEqual(getattr(self.address, attribute), value) def test_endElement_sets_correct_attributes_with_values(self): for arguments in [("publicIp", "192.168.1.1", "public_ip"), ("instanceId", 1, "instance_id"), ("domain", "some domain", "domain"), ("allocationId", 1, "allocation_id"), ("associationId", 1, "association_id"), ("somethingRandom", "somethingRandom", "somethingRandom")]: self.check_that_attribute_has_been_set(arguments[0], arguments[1], arguments[2]) def test_release_calls_connection_release_address_with_correct_args(self): self.address.release() self.address.connection.release_address.assert_called_with( allocation_id="aid1", dry_run=False ) def test_associate_calls_connection_associate_address_with_correct_args(self): self.address.associate(network_interface_id=1) self.address.connection.associate_address.assert_called_with( instance_id=None, public_ip="192.168.1.1", network_interface_id=1, private_ip_address=None, allocation_id="aid1", allow_reassociation=False, dry_run=False ) def test_disassociate_calls_connection_disassociate_address_with_correct_args(self): self.address.disassociate() self.address.connection.disassociate_address.assert_called_with( public_ip="192.168.1.1", dry_run=False ) if __name__ == "__main__": unittest.main() vendored/boto/tests/unit/ec2/test_connection.py 0000644 00000225046 15025013616 0015667 0 ustar 00 #!/usr/bin/env python from datetime import datetime, timedelta from mock import MagicMock, Mock from tests.unit import unittest from tests.unit import AWSMockServiceTestCase import boto.ec2 from boto.regioninfo import RegionInfo from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping from boto.ec2.connection import EC2Connection from boto.ec2.snapshot import Snapshot from boto.ec2.reservedinstance import ReservedInstancesConfiguration from boto.compat import http_client class TestEC2ConnectionBase(AWSMockServiceTestCase): connection_class = EC2Connection def setUp(self): super(TestEC2ConnectionBase, self).setUp() self.ec2 = self.service_connection class TestReservedInstanceOfferings(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeReservedInstancesOfferingsResponse> <requestId>d3253568-edcf-4897-9a3d-fb28e0b3fa38</requestId> <reservedInstancesOfferingsSet> <item> <reservedInstancesOfferingId>2964d1bf71d8</reservedInstancesOfferingId> <instanceType>c1.medium</instanceType> <availabilityZone>us-east-1c</availabilityZone> <duration>94608000</duration> <fixedPrice>775.0</fixedPrice> <usagePrice>0.0</usagePrice> <productDescription>product description</productDescription> <instanceTenancy>default</instanceTenancy> <currencyCode>USD</currencyCode> <offeringType>Heavy Utilization</offeringType> <recurringCharges> <item> <frequency>Hourly</frequency> <amount>0.095</amount> </item> </recurringCharges> <marketplace>false</marketplace> <pricingDetailsSet> <item> <price>0.045</price> <count>1</count> </item> </pricingDetailsSet> </item> <item> <reservedInstancesOfferingId>2dce26e46889</reservedInstancesOfferingId> <instanceType>c1.medium</instanceType> <availabilityZone>us-east-1c</availabilityZone> <duration>94608000</duration> <fixedPrice>775.0</fixedPrice> <usagePrice>0.0</usagePrice> <productDescription>Linux/UNIX</productDescription> <instanceTenancy>default</instanceTenancy> <currencyCode>USD</currencyCode> <offeringType>Heavy Utilization</offeringType> <recurringCharges> <item> <frequency>Hourly</frequency> <amount>0.035</amount> </item> </recurringCharges> <marketplace>false</marketplace> <pricingDetailsSet/> </item> </reservedInstancesOfferingsSet> <nextToken>next_token</nextToken> </DescribeReservedInstancesOfferingsResponse> """ def test_get_reserved_instance_offerings(self): self.set_http_response(status_code=200) response = self.ec2.get_all_reserved_instances_offerings() self.assertEqual(len(response), 2) instance = response[0] self.assertEqual(instance.id, '2964d1bf71d8') self.assertEqual(instance.instance_type, 'c1.medium') self.assertEqual(instance.availability_zone, 'us-east-1c') self.assertEqual(instance.duration, 94608000) self.assertEqual(instance.fixed_price, '775.0') self.assertEqual(instance.usage_price, '0.0') self.assertEqual(instance.description, 'product description') self.assertEqual(instance.instance_tenancy, 'default') self.assertEqual(instance.currency_code, 'USD') self.assertEqual(instance.offering_type, 'Heavy Utilization') self.assertEqual(len(instance.recurring_charges), 1) self.assertEqual(instance.recurring_charges[0].frequency, 'Hourly') self.assertEqual(instance.recurring_charges[0].amount, '0.095') self.assertEqual(len(instance.pricing_details), 1) self.assertEqual(instance.pricing_details[0].price, '0.045') self.assertEqual(instance.pricing_details[0].count, '1') def test_get_reserved_instance_offerings_params(self): self.set_http_response(status_code=200) self.ec2.get_all_reserved_instances_offerings( reserved_instances_offering_ids=['id1', 'id2'], instance_type='t1.micro', availability_zone='us-east-1', product_description='description', instance_tenancy='dedicated', offering_type='offering_type', include_marketplace=False, min_duration=100, max_duration=1000, max_instance_count=1, next_token='next_token', max_results=10 ) self.assert_request_parameters({ 'Action': 'DescribeReservedInstancesOfferings', 'ReservedInstancesOfferingId.1': 'id1', 'ReservedInstancesOfferingId.2': 'id2', 'InstanceType': 't1.micro', 'AvailabilityZone': 'us-east-1', 'ProductDescription': 'description', 'InstanceTenancy': 'dedicated', 'OfferingType': 'offering_type', 'IncludeMarketplace': 'false', 'MinDuration': '100', 'MaxDuration': '1000', 'MaxInstanceCount': '1', 'NextToken': 'next_token', 'MaxResults': '10', }, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestPurchaseReservedInstanceOffering(TestEC2ConnectionBase): def default_body(self): return b"""<PurchaseReservedInstancesOffering />""" def test_serialized_api_args(self): self.set_http_response(status_code=200) response = self.ec2.purchase_reserved_instance_offering( 'offering_id', 1, (100.0, 'USD')) self.assert_request_parameters({ 'Action': 'PurchaseReservedInstancesOffering', 'InstanceCount': 1, 'ReservedInstancesOfferingId': 'offering_id', 'LimitPrice.Amount': '100.0', 'LimitPrice.CurrencyCode': 'USD', }, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestCreateImage(TestEC2ConnectionBase): def default_body(self): return b"""<CreateImageResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <imageId>ami-4fa54026</imageId> </CreateImageResponse>""" def test_minimal(self): self.set_http_response(status_code=200) response = self.ec2.create_image( 'instance_id', 'name') self.assert_request_parameters({ 'Action': 'CreateImage', 'InstanceId': 'instance_id', 'Name': 'name'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_block_device_mapping(self): self.set_http_response(status_code=200) bdm = BlockDeviceMapping() bdm['test'] = BlockDeviceType() response = self.ec2.create_image( 'instance_id', 'name', block_device_mapping=bdm) self.assert_request_parameters({ 'Action': 'CreateImage', 'InstanceId': 'instance_id', 'Name': 'name', 'BlockDeviceMapping.1.DeviceName': 'test', 'BlockDeviceMapping.1.Ebs.DeleteOnTermination': 'false'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestCancelReservedInstancesListing(TestEC2ConnectionBase): def default_body(self): return b""" <CancelReservedInstancesListingResponse> <requestId>request_id</requestId> <reservedInstancesListingsSet> <item> <reservedInstancesListingId>listing_id</reservedInstancesListingId> <reservedInstancesId>instance_id</reservedInstancesId> <createDate>2012-07-12T16:55:28.000Z</createDate> <updateDate>2012-07-12T16:55:28.000Z</updateDate> <status>cancelled</status> <statusMessage>CANCELLED</statusMessage> <instanceCounts> <item> <state>Available</state> <instanceCount>0</instanceCount> </item> <item> <state>Sold</state> <instanceCount>0</instanceCount> </item> <item> <state>Cancelled</state> <instanceCount>1</instanceCount> </item> <item> <state>Pending</state> <instanceCount>0</instanceCount> </item> </instanceCounts> <priceSchedules> <item> <term>5</term> <price>166.64</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>4</term> <price>133.32</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>3</term> <price>99.99</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>2</term> <price>66.66</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>1</term> <price>33.33</price> <currencyCode>USD</currencyCode> <active>false</active> </item> </priceSchedules> <tagSet/> <clientToken>XqJIt1342112125076</clientToken> </item> </reservedInstancesListingsSet> </CancelReservedInstancesListingResponse> """ def test_reserved_instances_listing(self): self.set_http_response(status_code=200) response = self.ec2.cancel_reserved_instances_listing() self.assertEqual(len(response), 1) cancellation = response[0] self.assertEqual(cancellation.status, 'cancelled') self.assertEqual(cancellation.status_message, 'CANCELLED') self.assertEqual(len(cancellation.instance_counts), 4) first = cancellation.instance_counts[0] self.assertEqual(first.state, 'Available') self.assertEqual(first.instance_count, 0) self.assertEqual(len(cancellation.price_schedules), 5) schedule = cancellation.price_schedules[0] self.assertEqual(schedule.term, 5) self.assertEqual(schedule.price, '166.64') self.assertEqual(schedule.currency_code, 'USD') self.assertEqual(schedule.active, False) class TestCreateReservedInstancesListing(TestEC2ConnectionBase): def default_body(self): return b""" <CreateReservedInstancesListingResponse> <requestId>request_id</requestId> <reservedInstancesListingsSet> <item> <reservedInstancesListingId>listing_id</reservedInstancesListingId> <reservedInstancesId>instance_id</reservedInstancesId> <createDate>2012-07-17T17:11:09.449Z</createDate> <updateDate>2012-07-17T17:11:09.468Z</updateDate> <status>active</status> <statusMessage>ACTIVE</statusMessage> <instanceCounts> <item> <state>Available</state> <instanceCount>1</instanceCount> </item> <item> <state>Sold</state> <instanceCount>0</instanceCount> </item> <item> <state>Cancelled</state> <instanceCount>0</instanceCount> </item> <item> <state>Pending</state> <instanceCount>0</instanceCount> </item> </instanceCounts> <priceSchedules> <item> <term>11</term> <price>2.5</price> <currencyCode>USD</currencyCode> <active>true</active> </item> <item> <term>10</term> <price>2.5</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>9</term> <price>2.5</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>8</term> <price>2.0</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>7</term> <price>2.0</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>6</term> <price>2.0</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>5</term> <price>1.5</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>4</term> <price>1.5</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>3</term> <price>0.7</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>2</term> <price>0.7</price> <currencyCode>USD</currencyCode> <active>false</active> </item> <item> <term>1</term> <price>0.1</price> <currencyCode>USD</currencyCode> <active>false</active> </item> </priceSchedules> <tagSet/> <clientToken>myIdempToken1</clientToken> </item> </reservedInstancesListingsSet> </CreateReservedInstancesListingResponse> """ def test_create_reserved_instances_listing(self): self.set_http_response(status_code=200) response = self.ec2.create_reserved_instances_listing( 'instance_id', 1, [('2.5', 11), ('2.0', 8)], 'client_token') self.assertEqual(len(response), 1) cancellation = response[0] self.assertEqual(cancellation.status, 'active') self.assertEqual(cancellation.status_message, 'ACTIVE') self.assertEqual(len(cancellation.instance_counts), 4) first = cancellation.instance_counts[0] self.assertEqual(first.state, 'Available') self.assertEqual(first.instance_count, 1) self.assertEqual(len(cancellation.price_schedules), 11) schedule = cancellation.price_schedules[0] self.assertEqual(schedule.term, 11) self.assertEqual(schedule.price, '2.5') self.assertEqual(schedule.currency_code, 'USD') self.assertEqual(schedule.active, True) self.assert_request_parameters({ 'Action': 'CreateReservedInstancesListing', 'ReservedInstancesId': 'instance_id', 'InstanceCount': '1', 'ClientToken': 'client_token', 'PriceSchedules.0.Price': '2.5', 'PriceSchedules.0.Term': '11', 'PriceSchedules.1.Price': '2.0', 'PriceSchedules.1.Term': '8', }, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestDescribeSpotInstanceRequests(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeSpotInstanceRequestsResponse> <requestId>requestid</requestId> <spotInstanceRequestSet> <item> <spotInstanceRequestId>sir-id</spotInstanceRequestId> <spotPrice>0.003000</spotPrice> <type>one-time</type> <state>active</state> <status> <code>fulfilled</code> <updateTime>2012-10-19T18:09:26.000Z</updateTime> <message>Your Spot request is fulfilled.</message> </status> <launchGroup>mylaunchgroup</launchGroup> <launchSpecification> <imageId>ami-id</imageId> <keyName>mykeypair</keyName> <groupSet> <item> <groupId>sg-id</groupId> <groupName>groupname</groupName> </item> </groupSet> <instanceType>t1.micro</instanceType> <monitoring> <enabled>false</enabled> </monitoring> </launchSpecification> <instanceId>i-id</instanceId> <createTime>2012-10-19T18:07:05.000Z</createTime> <productDescription>Linux/UNIX</productDescription> <launchedAvailabilityZone>us-east-1d</launchedAvailabilityZone> </item> </spotInstanceRequestSet> </DescribeSpotInstanceRequestsResponse> """ def test_describe_spot_instance_requets(self): self.set_http_response(status_code=200) response = self.ec2.get_all_spot_instance_requests() self.assertEqual(len(response), 1) spotrequest = response[0] self.assertEqual(spotrequest.id, 'sir-id') self.assertEqual(spotrequest.price, 0.003) self.assertEqual(spotrequest.type, 'one-time') self.assertEqual(spotrequest.state, 'active') self.assertEqual(spotrequest.fault, None) self.assertEqual(spotrequest.valid_from, None) self.assertEqual(spotrequest.valid_until, None) self.assertEqual(spotrequest.launch_group, 'mylaunchgroup') self.assertEqual(spotrequest.launched_availability_zone, 'us-east-1d') self.assertEqual(spotrequest.product_description, 'Linux/UNIX') self.assertEqual(spotrequest.availability_zone_group, None) self.assertEqual(spotrequest.create_time, '2012-10-19T18:07:05.000Z') self.assertEqual(spotrequest.instance_id, 'i-id') launch_spec = spotrequest.launch_specification self.assertEqual(launch_spec.key_name, 'mykeypair') self.assertEqual(launch_spec.instance_type, 't1.micro') self.assertEqual(launch_spec.image_id, 'ami-id') self.assertEqual(launch_spec.placement, None) self.assertEqual(launch_spec.kernel, None) self.assertEqual(launch_spec.ramdisk, None) self.assertEqual(launch_spec.monitored, False) self.assertEqual(launch_spec.subnet_id, None) self.assertEqual(launch_spec.block_device_mapping, None) self.assertEqual(launch_spec.instance_profile, None) self.assertEqual(launch_spec.ebs_optimized, False) status = spotrequest.status self.assertEqual(status.code, 'fulfilled') self.assertEqual(status.update_time, '2012-10-19T18:09:26.000Z') self.assertEqual(status.message, 'Your Spot request is fulfilled.') class TestCopySnapshot(TestEC2ConnectionBase): def default_body(self): return b""" <CopySnapshotResponse xmlns="http://ec2.amazonaws.com/doc/2012-12-01/"> <requestId>request_id</requestId> <snapshotId>snap-copied-id</snapshotId> </CopySnapshotResponse> """ def test_copy_snapshot(self): self.set_http_response(status_code=200) snapshot_id = self.ec2.copy_snapshot('us-west-2', 'snap-id', 'description') self.assertEqual(snapshot_id, 'snap-copied-id') self.assert_request_parameters({ 'Action': 'CopySnapshot', 'Description': 'description', 'SourceRegion': 'us-west-2', 'SourceSnapshotId': 'snap-id'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestCopyImage(TestEC2ConnectionBase): def default_body(self): return b""" <CopyImageResponse xmlns="http://ec2.amazonaws.com/doc/2013-07-15/"> <requestId>request_id</requestId> <imageId>ami-copied-id</imageId> </CopyImageResponse> """ def test_copy_image_required_params(self): self.set_http_response(status_code=200) copied_ami = self.ec2.copy_image('us-west-2', 'ami-id') self.assertEqual(copied_ami.image_id, 'ami-copied-id') self.assert_request_parameters({ 'Action': 'CopyImage', 'SourceRegion': 'us-west-2', 'SourceImageId': 'ami-id' }, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_copy_image_name_and_description(self): self.set_http_response(status_code=200) copied_ami = self.ec2.copy_image('us-west-2', 'ami-id', 'name', 'description') self.assertEqual(copied_ami.image_id, 'ami-copied-id') self.assert_request_parameters({ 'Action': 'CopyImage', 'SourceRegion': 'us-west-2', 'SourceImageId': 'ami-id', 'Name': 'name', 'Description': 'description' }, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_copy_image_client_token(self): self.set_http_response(status_code=200) copied_ami = self.ec2.copy_image('us-west-2', 'ami-id', client_token='client-token') self.assertEqual(copied_ami.image_id, 'ami-copied-id') self.assert_request_parameters({ 'Action': 'CopyImage', 'SourceRegion': 'us-west-2', 'SourceImageId': 'ami-id', 'ClientToken': 'client-token' }, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_copy_image_encrypted(self): self.set_http_response(status_code=200) copied_ami = self.ec2.copy_image('us-west-2', 'ami-id', encrypted=True) self.assertEqual(copied_ami.image_id, 'ami-copied-id') self.assert_request_parameters({ 'Action': 'CopyImage', 'SourceRegion': 'us-west-2', 'SourceImageId': 'ami-id', 'Encrypted': 'true' }, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_copy_image_not_encrypted(self): self.set_http_response(status_code=200) copied_ami = self.ec2.copy_image('us-west-2', 'ami-id', encrypted=False) self.assertEqual(copied_ami.image_id, 'ami-copied-id') self.assert_request_parameters({ 'Action': 'CopyImage', 'SourceRegion': 'us-west-2', 'SourceImageId': 'ami-id', 'Encrypted': 'false' }, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_copy_image_encrypted_with_kms_key(self): self.set_http_response(status_code=200) copied_ami = self.ec2.copy_image('us-west-2', 'ami-id', encrypted=False, kms_key_id='kms-key') self.assertEqual(copied_ami.image_id, 'ami-copied-id') self.assert_request_parameters({ 'Action': 'CopyImage', 'SourceRegion': 'us-west-2', 'SourceImageId': 'ami-id', 'Encrypted': 'false', 'KmsKeyId': 'kms-key' }, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestAccountAttributes(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeAccountAttributesResponse xmlns="http://ec2.amazonaws.com/doc/2012-12-01/"> <requestId>6d042e8a-4bc3-43e8-8265-3cbc54753f14</requestId> <accountAttributeSet> <item> <attributeName>vpc-max-security-groups-per-interface</attributeName> <attributeValueSet> <item> <attributeValue>5</attributeValue> </item> </attributeValueSet> </item> <item> <attributeName>max-instances</attributeName> <attributeValueSet> <item> <attributeValue>50</attributeValue> </item> </attributeValueSet> </item> <item> <attributeName>supported-platforms</attributeName> <attributeValueSet> <item> <attributeValue>EC2</attributeValue> </item> <item> <attributeValue>VPC</attributeValue> </item> </attributeValueSet> </item> <item> <attributeName>default-vpc</attributeName> <attributeValueSet> <item> <attributeValue>none</attributeValue> </item> </attributeValueSet> </item> </accountAttributeSet> </DescribeAccountAttributesResponse> """ def test_describe_account_attributes(self): self.set_http_response(status_code=200) parsed = self.ec2.describe_account_attributes() self.assertEqual(len(parsed), 4) self.assertEqual(parsed[0].attribute_name, 'vpc-max-security-groups-per-interface') self.assertEqual(parsed[0].attribute_values, ['5']) self.assertEqual(parsed[-1].attribute_name, 'default-vpc') self.assertEqual(parsed[-1].attribute_values, ['none']) class TestDescribeVPCAttribute(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeVpcAttributeResponse xmlns="http://ec2.amazonaws.com/doc/2013-02-01/"> <requestId>request_id</requestId> <vpcId>vpc-id</vpcId> <enableDnsHostnames> <value>false</value> </enableDnsHostnames> </DescribeVpcAttributeResponse> """ def test_describe_vpc_attribute(self): self.set_http_response(status_code=200) parsed = self.ec2.describe_vpc_attribute('vpc-id', 'enableDnsHostnames') self.assertEqual(parsed.vpc_id, 'vpc-id') self.assertFalse(parsed.enable_dns_hostnames) self.assert_request_parameters({ 'Action': 'DescribeVpcAttribute', 'VpcId': 'vpc-id', 'Attribute': 'enableDnsHostnames', }, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestGetAllNetworkInterfaces(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeNetworkInterfacesResponse xmlns="http://ec2.amazonaws.com/\ doc/2013-06-15/"> <requestId>fc45294c-006b-457b-bab9-012f5b3b0e40</requestId> <networkInterfaceSet> <item> <networkInterfaceId>eni-0f62d866</networkInterfaceId> <subnetId>subnet-c53c87ac</subnetId> <vpcId>vpc-cc3c87a5</vpcId> <availabilityZone>ap-southeast-1b</availabilityZone> <description/> <ownerId>053230519467</ownerId> <requesterManaged>false</requesterManaged> <status>in-use</status> <macAddress>02:81:60:cb:27:37</macAddress> <privateIpAddress>10.0.0.146</privateIpAddress> <sourceDestCheck>true</sourceDestCheck> <groupSet> <item> <groupId>sg-3f4b5653</groupId> <groupName>default</groupName> </item> </groupSet> <attachment> <attachmentId>eni-attach-6537fc0c</attachmentId> <instanceId>i-22197876</instanceId> <instanceOwnerId>053230519467</instanceOwnerId> <deviceIndex>5</deviceIndex> <status>attached</status> <attachTime>2012-07-01T21:45:27.000Z</attachTime> <deleteOnTermination>true</deleteOnTermination> </attachment> <tagSet/> <privateIpAddressesSet> <item> <privateIpAddress>10.0.0.146</privateIpAddress> <primary>true</primary> </item> <item> <privateIpAddress>10.0.0.148</privateIpAddress> <primary>false</primary> </item> <item> <privateIpAddress>10.0.0.150</privateIpAddress> <primary>false</primary> </item> </privateIpAddressesSet> </item> </networkInterfaceSet> </DescribeNetworkInterfacesResponse>""" def test_get_all_network_interfaces(self): self.set_http_response(status_code=200) result = self.ec2.get_all_network_interfaces(network_interface_ids=['eni-0f62d866']) self.assert_request_parameters({ 'Action': 'DescribeNetworkInterfaces', 'NetworkInterfaceId.1': 'eni-0f62d866'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(result), 1) self.assertEqual(result[0].id, 'eni-0f62d866') def test_attachment_has_device_index(self): self.set_http_response(status_code=200) parsed = self.ec2.get_all_network_interfaces() self.assertEqual(5, parsed[0].attachment.device_index) class TestGetAllImages(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeImagesResponse xmlns="http://ec2.amazonaws.com/doc/2013-02-01/"> <requestId>e32375e8-4ac3-4099-a8bf-3ec902b9023e</requestId> <imagesSet> <item> <imageId>ami-abcd1234</imageId> <imageLocation>111111111111/windows2008r2-hvm-i386-20130702</imageLocation> <imageState>available</imageState> <imageOwnerId>111111111111</imageOwnerId> <isPublic>false</isPublic> <architecture>i386</architecture> <imageType>machine</imageType> <platform>windows</platform> <viridianEnabled>true</viridianEnabled> <name>Windows Test</name> <description>Windows Test Description</description> <billingProducts> <item> <billingProduct>bp-6ba54002</billingProduct> </item> </billingProducts> <rootDeviceType>ebs</rootDeviceType> <rootDeviceName>/dev/sda1</rootDeviceName> <blockDeviceMapping> <item> <deviceName>/dev/sda1</deviceName> <ebs> <snapshotId>snap-abcd1234</snapshotId> <volumeSize>30</volumeSize> <deleteOnTermination>true</deleteOnTermination> <volumeType>standard</volumeType> </ebs> </item> <item> <deviceName>xvdb</deviceName> <virtualName>ephemeral0</virtualName> </item> <item> <deviceName>xvdc</deviceName> <virtualName>ephemeral1</virtualName> </item> <item> <deviceName>xvdd</deviceName> <virtualName>ephemeral2</virtualName> </item> <item> <deviceName>xvde</deviceName> <virtualName>ephemeral3</virtualName> </item> </blockDeviceMapping> <virtualizationType>hvm</virtualizationType> <hypervisor>xen</hypervisor> </item> </imagesSet> </DescribeImagesResponse>""" def test_get_all_images(self): self.set_http_response(status_code=200) parsed = self.ec2.get_all_images() self.assertEqual(1, len(parsed)) self.assertEqual("ami-abcd1234", parsed[0].id) self.assertEqual("111111111111/windows2008r2-hvm-i386-20130702", parsed[0].location) self.assertEqual("available", parsed[0].state) self.assertEqual("111111111111", parsed[0].ownerId) self.assertEqual("111111111111", parsed[0].owner_id) self.assertEqual(False, parsed[0].is_public) self.assertEqual("i386", parsed[0].architecture) self.assertEqual("machine", parsed[0].type) self.assertEqual(None, parsed[0].kernel_id) self.assertEqual(None, parsed[0].ramdisk_id) self.assertEqual(None, parsed[0].owner_alias) self.assertEqual("windows", parsed[0].platform) self.assertEqual("Windows Test", parsed[0].name) self.assertEqual("Windows Test Description", parsed[0].description) self.assertEqual("ebs", parsed[0].root_device_type) self.assertEqual("/dev/sda1", parsed[0].root_device_name) self.assertEqual("hvm", parsed[0].virtualization_type) self.assertEqual("xen", parsed[0].hypervisor) self.assertEqual(None, parsed[0].instance_lifecycle) # 1 billing product parsed into a list self.assertEqual(1, len(parsed[0].billing_products)) self.assertEqual("bp-6ba54002", parsed[0].billing_products[0]) # Just verify length, there is already a block_device_mapping test self.assertEqual(5, len(parsed[0].block_device_mapping)) # TODO: No tests for product codes? class TestModifyInterfaceAttribute(TestEC2ConnectionBase): def default_body(self): return b""" <ModifyNetworkInterfaceAttributeResponse \ xmlns="http://ec2.amazonaws.com/doc/2013-06-15/"> <requestId>657a4623-5620-4232-b03b-427e852d71cf</requestId> <return>true</return> </ModifyNetworkInterfaceAttributeResponse> """ def test_modify_description(self): self.set_http_response(status_code=200) self.ec2.modify_network_interface_attribute('id', 'description', 'foo') self.assert_request_parameters({ 'Action': 'ModifyNetworkInterfaceAttribute', 'NetworkInterfaceId': 'id', 'Description.Value': 'foo'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_modify_source_dest_check_bool(self): self.set_http_response(status_code=200) self.ec2.modify_network_interface_attribute('id', 'sourceDestCheck', True) self.assert_request_parameters({ 'Action': 'ModifyNetworkInterfaceAttribute', 'NetworkInterfaceId': 'id', 'SourceDestCheck.Value': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_modify_source_dest_check_str(self): self.set_http_response(status_code=200) self.ec2.modify_network_interface_attribute('id', 'sourceDestCheck', 'true') self.assert_request_parameters({ 'Action': 'ModifyNetworkInterfaceAttribute', 'NetworkInterfaceId': 'id', 'SourceDestCheck.Value': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_modify_source_dest_check_invalid(self): self.set_http_response(status_code=200) with self.assertRaises(ValueError): self.ec2.modify_network_interface_attribute('id', 'sourceDestCheck', 123) def test_modify_delete_on_termination_str(self): self.set_http_response(status_code=200) self.ec2.modify_network_interface_attribute('id', 'deleteOnTermination', True, attachment_id='bar') self.assert_request_parameters({ 'Action': 'ModifyNetworkInterfaceAttribute', 'NetworkInterfaceId': 'id', 'Attachment.AttachmentId': 'bar', 'Attachment.DeleteOnTermination': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_modify_delete_on_termination_bool(self): self.set_http_response(status_code=200) self.ec2.modify_network_interface_attribute('id', 'deleteOnTermination', 'false', attachment_id='bar') self.assert_request_parameters({ 'Action': 'ModifyNetworkInterfaceAttribute', 'NetworkInterfaceId': 'id', 'Attachment.AttachmentId': 'bar', 'Attachment.DeleteOnTermination': 'false'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_modify_delete_on_termination_invalid(self): self.set_http_response(status_code=200) with self.assertRaises(ValueError): self.ec2.modify_network_interface_attribute('id', 'deleteOnTermination', 123, attachment_id='bar') def test_modify_group_set_list(self): self.set_http_response(status_code=200) self.ec2.modify_network_interface_attribute('id', 'groupSet', ['sg-1', 'sg-2']) self.assert_request_parameters({ 'Action': 'ModifyNetworkInterfaceAttribute', 'NetworkInterfaceId': 'id', 'SecurityGroupId.1': 'sg-1', 'SecurityGroupId.2': 'sg-2'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_modify_group_set_invalid(self): self.set_http_response(status_code=200) with self.assertRaisesRegex(TypeError, 'iterable'): self.ec2.modify_network_interface_attribute('id', 'groupSet', False) def test_modify_attr_invalid(self): self.set_http_response(status_code=200) with self.assertRaisesRegex(ValueError, 'Unknown attribute'): self.ec2.modify_network_interface_attribute('id', 'invalid', 0) class TestConnectToRegion(unittest.TestCase): def setUp(self): self.https_connection = Mock(spec=http_client.HTTPSConnection) self.https_connection_factory = ( Mock(return_value=self.https_connection), ()) def test_aws_region(self): region = list(boto.ec2.RegionData.keys())[0] self.ec2 = boto.ec2.connect_to_region( region, https_connection_factory=self.https_connection_factory, aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key' ) self.assertEqual(boto.ec2.RegionData[region], self.ec2.host) def test_non_aws_region(self): self.ec2 = boto.ec2.connect_to_region( 'foo', https_connection_factory=self.https_connection_factory, aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key', region=RegionInfo(name='foo', endpoint='https://foo.com/bar') ) self.assertEqual('https://foo.com/bar', self.ec2.host) def test_missing_region(self): self.ec2 = boto.ec2.connect_to_region( 'foo', https_connection_factory=self.https_connection_factory, aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key' ) self.assertEqual(None, self.ec2) class TestTrimSnapshots(TestEC2ConnectionBase): """ Test snapshot trimming functionality by ensuring that expected calls are made when given a known set of volume snapshots. """ def _get_snapshots(self): """ Generate a list of fake snapshots with names and dates. """ snaps = [] # Generate some dates offset by days, weeks, months. # This is to validate the various types of snapshot logic handled by # ``trim_snapshots``. now = datetime.now() dates = [ now, now - timedelta(days=1), now - timedelta(days=2), now - timedelta(days=7), now - timedelta(days=14), # We want to simulate 30/60/90-day snapshots, but February is # short (only 28 days), so we decrease the delta by 2 days apiece. # This prevents the ``delete_snapshot`` code below from being # called, since they don't fall outside the allowed timeframes # for the snapshots. datetime(now.year, now.month, 1) - timedelta(days=28), datetime(now.year, now.month, 1) - timedelta(days=58), datetime(now.year, now.month, 1) - timedelta(days=88) ] for date in dates: # Create a fake snapshot for each date snap = Snapshot(self.ec2) snap.tags['Name'] = 'foo' # Times are expected to be ISO8601 strings snap.start_time = date.strftime('%Y-%m-%dT%H:%M:%S.000Z') snaps.append(snap) return snaps def test_trim_defaults(self): """ Test trimming snapshots with the default arguments, which should keep all monthly backups forever. The result of this test should be that nothing is deleted. """ # Setup mocks orig = { 'get_all_snapshots': self.ec2.get_all_snapshots, 'delete_snapshot': self.ec2.delete_snapshot } snaps = self._get_snapshots() self.ec2.get_all_snapshots = MagicMock(return_value=snaps) self.ec2.delete_snapshot = MagicMock() # Call the tested method self.ec2.trim_snapshots() # Assertions self.assertEqual(True, self.ec2.get_all_snapshots.called) self.assertEqual(False, self.ec2.delete_snapshot.called) # Restore self.ec2.get_all_snapshots = orig['get_all_snapshots'] self.ec2.delete_snapshot = orig['delete_snapshot'] def test_trim_months(self): """ Test trimming monthly snapshots and ensure that older months get deleted properly. The result of this test should be that the two oldest snapshots get deleted. """ # Setup mocks orig = { 'get_all_snapshots': self.ec2.get_all_snapshots, 'delete_snapshot': self.ec2.delete_snapshot } snaps = self._get_snapshots() self.ec2.get_all_snapshots = MagicMock(return_value=snaps) self.ec2.delete_snapshot = MagicMock() # Call the tested method self.ec2.trim_snapshots(monthly_backups=1) # Assertions self.assertEqual(True, self.ec2.get_all_snapshots.called) self.assertEqual(2, self.ec2.delete_snapshot.call_count) # Restore self.ec2.get_all_snapshots = orig['get_all_snapshots'] self.ec2.delete_snapshot = orig['delete_snapshot'] class TestModifyReservedInstances(TestEC2ConnectionBase): def default_body(self): return b"""<ModifyReservedInstancesResponse xmlns='http://ec2.amazonaws.com/doc/2013-08-15/'> <requestId>bef729b6-0731-4489-8881-2258746ae163</requestId> <reservedInstancesModificationId>rimod-3aae219d-3d63-47a9-a7e9-e764example</reservedInstancesModificationId> </ModifyReservedInstancesResponse>""" def test_serialized_api_args(self): self.set_http_response(status_code=200) response = self.ec2.modify_reserved_instances( 'a-token-goes-here', reserved_instance_ids=[ '2567o137-8a55-48d6-82fb-7258506bb497', ], target_configurations=[ ReservedInstancesConfiguration( availability_zone='us-west-2c', platform='EC2-VPC', instance_count=3, instance_type='c3.large' ), ] ) self.assert_request_parameters({ 'Action': 'ModifyReservedInstances', 'ClientToken': 'a-token-goes-here', 'ReservedInstancesConfigurationSetItemType.0.AvailabilityZone': 'us-west-2c', 'ReservedInstancesConfigurationSetItemType.0.InstanceCount': 3, 'ReservedInstancesConfigurationSetItemType.0.Platform': 'EC2-VPC', 'ReservedInstancesConfigurationSetItemType.0.InstanceType': 'c3.large', 'ReservedInstancesId.1': '2567o137-8a55-48d6-82fb-7258506bb497' }, ignore_params_values=[ 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version' ]) self.assertEqual(response, 'rimod-3aae219d-3d63-47a9-a7e9-e764example') def test_none_token(self): """Ensures that if the token is set to None, nothing is serialized.""" self.set_http_response(status_code=200) response = self.ec2.modify_reserved_instances( None, reserved_instance_ids=[ '2567o137-8a55-48d6-82fb-7258506bb497', ], target_configurations=[ ReservedInstancesConfiguration( availability_zone='us-west-2c', platform='EC2-VPC', instance_count=3, instance_type='c3.large' ), ] ) self.assert_request_parameters({ 'Action': 'ModifyReservedInstances', 'ReservedInstancesConfigurationSetItemType.0.AvailabilityZone': 'us-west-2c', 'ReservedInstancesConfigurationSetItemType.0.InstanceCount': 3, 'ReservedInstancesConfigurationSetItemType.0.Platform': 'EC2-VPC', 'ReservedInstancesConfigurationSetItemType.0.InstanceType': 'c3.large', 'ReservedInstancesId.1': '2567o137-8a55-48d6-82fb-7258506bb497' }, ignore_params_values=[ 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version' ]) self.assertEqual(response, 'rimod-3aae219d-3d63-47a9-a7e9-e764example') class TestDescribeReservedInstancesModifications(TestEC2ConnectionBase): def default_body(self): return b"""<DescribeReservedInstancesModificationsResponse xmlns='http://ec2.amazonaws.com/doc/2013-08-15/'> <requestId>eb4a6e3c-3689-445c-b536-19e38df35898</requestId> <reservedInstancesModificationsSet> <item> <reservedInstancesModificationId>rimod-49b9433e-fdc7-464a-a6e5-9dabcexample</reservedInstancesModificationId> <reservedInstancesSet> <item> <reservedInstancesId>2567o137-8a55-48d6-82fb-7258506bb497</reservedInstancesId> </item> </reservedInstancesSet> <modificationResultSet> <item> <reservedInstancesId>9d5cb137-5d65-4479-b4ac-8c337example</reservedInstancesId> <targetConfiguration> <availabilityZone>us-east-1b</availabilityZone> <platform>EC2-VPC</platform> <instanceCount>1</instanceCount> </targetConfiguration> </item> </modificationResultSet> <createDate>2013-09-02T21:20:19.637Z</createDate> <updateDate>2013-09-02T21:38:24.143Z</updateDate> <effectiveDate>2013-09-02T21:00:00.000Z</effectiveDate> <status>fulfilled</status> <clientToken>token-f5b56c05-09b0-4d17-8d8c-c75d8a67b806</clientToken> </item> </reservedInstancesModificationsSet> </DescribeReservedInstancesModificationsResponse>""" def test_serialized_api_args(self): self.set_http_response(status_code=200) response = self.ec2.describe_reserved_instances_modifications( reserved_instances_modification_ids=[ '2567o137-8a55-48d6-82fb-7258506bb497' ], filters={ 'status': 'processing', } ) self.assert_request_parameters({ 'Action': 'DescribeReservedInstancesModifications', 'Filter.1.Name': 'status', 'Filter.1.Value.1': 'processing', 'ReservedInstancesModificationId.1': '2567o137-8a55-48d6-82fb-7258506bb497' }, ignore_params_values=[ 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version' ]) # Make sure the response was parsed correctly. self.assertEqual( response[0].modification_id, 'rimod-49b9433e-fdc7-464a-a6e5-9dabcexample' ) self.assertEqual( response[0].create_date, datetime(2013, 9, 2, 21, 20, 19, 637000) ) self.assertEqual( response[0].update_date, datetime(2013, 9, 2, 21, 38, 24, 143000) ) self.assertEqual( response[0].effective_date, datetime(2013, 9, 2, 21, 0, 0, 0) ) self.assertEqual( response[0].status, 'fulfilled' ) self.assertEqual( response[0].status_message, None ) self.assertEqual( response[0].client_token, 'token-f5b56c05-09b0-4d17-8d8c-c75d8a67b806' ) self.assertEqual( response[0].reserved_instances[0].id, '2567o137-8a55-48d6-82fb-7258506bb497' ) self.assertEqual( response[0].modification_results[0].availability_zone, 'us-east-1b' ) self.assertEqual( response[0].modification_results[0].platform, 'EC2-VPC' ) self.assertEqual( response[0].modification_results[0].instance_count, 1 ) self.assertEqual(len(response), 1) class TestRegisterImage(TestEC2ConnectionBase): def default_body(self): return b""" <RegisterImageResponse xmlns="http://ec2.amazonaws.com/doc/2013-08-15/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <imageId>ami-1a2b3c4d</imageId> </RegisterImageResponse> """ def test_vm_type_default(self): self.set_http_response(status_code=200) self.ec2.register_image('name', 'description', image_location='s3://foo') self.assert_request_parameters({ 'Action': 'RegisterImage', 'ImageLocation': 's3://foo', 'Name': 'name', 'Description': 'description', }, ignore_params_values=[ 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version' ]) def test_vm_type_hvm(self): self.set_http_response(status_code=200) self.ec2.register_image('name', 'description', image_location='s3://foo', virtualization_type='hvm') self.assert_request_parameters({ 'Action': 'RegisterImage', 'ImageLocation': 's3://foo', 'Name': 'name', 'Description': 'description', 'VirtualizationType': 'hvm' }, ignore_params_values=[ 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version' ]) def test_sriov_net_support_simple(self): self.set_http_response(status_code=200) self.ec2.register_image('name', 'description', image_location='s3://foo', sriov_net_support='simple') self.assert_request_parameters({ 'Action': 'RegisterImage', 'ImageLocation': 's3://foo', 'Name': 'name', 'Description': 'description', 'SriovNetSupport': 'simple' }, ignore_params_values=[ 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version' ]) def test_volume_delete_on_termination_on(self): self.set_http_response(status_code=200) self.ec2.register_image('name', 'description', snapshot_id='snap-12345678', delete_root_volume_on_termination=True) self.assert_request_parameters({ 'Action': 'RegisterImage', 'Name': 'name', 'Description': 'description', 'BlockDeviceMapping.1.DeviceName': None, 'BlockDeviceMapping.1.Ebs.DeleteOnTermination': 'true', 'BlockDeviceMapping.1.Ebs.SnapshotId': 'snap-12345678', }, ignore_params_values=[ 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version' ]) def test_volume_delete_on_termination_default(self): self.set_http_response(status_code=200) self.ec2.register_image('name', 'description', snapshot_id='snap-12345678') self.assert_request_parameters({ 'Action': 'RegisterImage', 'Name': 'name', 'Description': 'description', 'BlockDeviceMapping.1.DeviceName': None, 'BlockDeviceMapping.1.Ebs.DeleteOnTermination': 'false', 'BlockDeviceMapping.1.Ebs.SnapshotId': 'snap-12345678', }, ignore_params_values=[ 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version' ]) class TestTerminateInstances(TestEC2ConnectionBase): def default_body(self): return b"""<?xml version="1.0" ?> <TerminateInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2013-07-15/"> <requestId>req-59a9ad52-0434-470c-ad48-4f89ded3a03e</requestId> <instancesSet> <item> <instanceId>i-000043a2</instanceId> <shutdownState> <code>16</code> <name>running</name> </shutdownState> <previousState> <code>16</code> <name>running</name> </previousState> </item> </instancesSet> </TerminateInstancesResponse> """ def test_terminate_bad_response(self): self.set_http_response(status_code=200) self.ec2.terminate_instances('foo') class TestDescribeInstances(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeInstancesResponse> </DescribeInstancesResponse> """ def test_default_behavior(self): self.set_http_response(status_code=200) self.ec2.get_all_instances() self.assert_request_parameters({ 'Action': 'DescribeInstances'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.ec2.get_all_reservations() self.assert_request_parameters({ 'Action': 'DescribeInstances'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.ec2.get_only_instances() self.assert_request_parameters({ 'Action': 'DescribeInstances'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_max_results(self): self.set_http_response(status_code=200) self.ec2.get_all_instances( max_results=10 ) self.assert_request_parameters({ 'Action': 'DescribeInstances', 'MaxResults': 10}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_next_token(self): self.set_http_response(status_code=200) self.ec2.get_all_reservations( next_token='abcdefgh', ) self.assert_request_parameters({ 'Action': 'DescribeInstances', 'NextToken': 'abcdefgh'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestDescribeTags(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeTagsResponse> </DescribeTagsResponse> """ def test_default_behavior(self): self.set_http_response(status_code=200) self.ec2.get_all_tags() self.assert_request_parameters({ 'Action': 'DescribeTags'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) def test_max_results(self): self.set_http_response(status_code=200) self.ec2.get_all_tags( max_results=10 ) self.assert_request_parameters({ 'Action': 'DescribeTags', 'MaxResults': 10}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) class TestSignatureAlteration(TestEC2ConnectionBase): def test_unchanged(self): self.assertEqual( self.service_connection._required_auth_capability(), ['hmac-v4'] ) def test_switched(self): region = RegionInfo( name='cn-north-1', endpoint='ec2.cn-north-1.amazonaws.com.cn', connection_cls=EC2Connection ) conn = self.connection_class( aws_access_key_id='less', aws_secret_access_key='more', region=region ) self.assertEqual( conn._required_auth_capability(), ['hmac-v4'] ) class TestAssociateAddress(TestEC2ConnectionBase): def default_body(self): return b""" <AssociateAddressResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-15/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <return>true</return> <associationId>eipassoc-fc5ca095</associationId> </AssociateAddressResponse> """ def test_associate_address(self): self.set_http_response(status_code=200) result = self.ec2.associate_address(instance_id='i-1234', public_ip='192.0.2.1') self.assertEqual(True, result) def test_associate_address_object(self): self.set_http_response(status_code=200) result = self.ec2.associate_address_object(instance_id='i-1234', public_ip='192.0.2.1') self.assertEqual('eipassoc-fc5ca095', result.association_id) class TestAssociateAddressFail(TestEC2ConnectionBase): def default_body(self): return b""" <Response> <Errors> <Error> <Code>InvalidInstanceID.NotFound</Code> <Message>The instance ID 'i-4cbc822a' does not exist</Message> </Error> </Errors> <RequestID>ea966190-f9aa-478e-9ede-cb5432daacc0</RequestID> <StatusCode>Failure</StatusCode> </Response> """ def test_associate_address(self): self.set_http_response(status_code=200) result = self.ec2.associate_address(instance_id='i-1234', public_ip='192.0.2.1') self.assertEqual(False, result) class TestDescribeVolumes(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeVolumesResponse xmlns="http://ec2.amazonaws.com/doc/2014-02-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <volumeSet> <item> <volumeId>vol-1a2b3c4d</volumeId> <size>80</size> <snapshotId/> <availabilityZone>us-east-1a</availabilityZone> <status>in-use</status> <createTime>YYYY-MM-DDTHH:MM:SS.SSSZ</createTime> <attachmentSet> <item> <volumeId>vol-1a2b3c4d</volumeId> <instanceId>i-1a2b3c4d</instanceId> <device>/dev/sdh</device> <status>attached</status> <attachTime>YYYY-MM-DDTHH:MM:SS.SSSZ</attachTime> <deleteOnTermination>false</deleteOnTermination> </item> </attachmentSet> <volumeType>standard</volumeType> <encrypted>true</encrypted> </item> <item> <volumeId>vol-5e6f7a8b</volumeId> <size>80</size> <snapshotId/> <availabilityZone>us-east-1a</availabilityZone> <status>in-use</status> <createTime>YYYY-MM-DDTHH:MM:SS.SSSZ</createTime> <attachmentSet> <item> <volumeId>vol-5e6f7a8b</volumeId> <instanceId>i-5e6f7a8b</instanceId> <device>/dev/sdz</device> <status>attached</status> <attachTime>YYYY-MM-DDTHH:MM:SS.SSSZ</attachTime> <deleteOnTermination>false</deleteOnTermination> </item> </attachmentSet> <volumeType>standard</volumeType> <encrypted>false</encrypted> </item> </volumeSet> </DescribeVolumesResponse> """ def test_get_all_volumes(self): self.set_http_response(status_code=200) result = self.ec2.get_all_volumes(volume_ids=['vol-1a2b3c4d', 'vol-5e6f7a8b']) self.assert_request_parameters({ 'Action': 'DescribeVolumes', 'VolumeId.1': 'vol-1a2b3c4d', 'VolumeId.2': 'vol-5e6f7a8b'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(result), 2) self.assertEqual(result[0].id, 'vol-1a2b3c4d') self.assertTrue(result[0].encrypted) self.assertEqual(result[1].id, 'vol-5e6f7a8b') self.assertFalse(result[1].encrypted) class TestDescribeSnapshots(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeSnapshotsResponse xmlns="http://ec2.amazonaws.com/doc/2014-02-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <snapshotSet> <item> <snapshotId>snap-1a2b3c4d</snapshotId> <volumeId>vol-1a2b3c4d</volumeId> <status>pending</status> <startTime>YYYY-MM-DDTHH:MM:SS.SSSZ</startTime> <progress>80%</progress> <ownerId>111122223333</ownerId> <volumeSize>15</volumeSize> <description>Daily Backup</description> <tagSet/> <encrypted>true</encrypted> </item> </snapshotSet> <snapshotSet> <item> <snapshotId>snap-5e6f7a8b</snapshotId> <volumeId>vol-5e6f7a8b</volumeId> <status>completed</status> <startTime>YYYY-MM-DDTHH:MM:SS.SSSZ</startTime> <progress>100%</progress> <ownerId>111122223333</ownerId> <volumeSize>15</volumeSize> <description>Daily Backup</description> <tagSet/> <encrypted>false</encrypted> </item> </snapshotSet> </DescribeSnapshotsResponse> """ def test_get_all_snapshots(self): self.set_http_response(status_code=200) result = self.ec2.get_all_snapshots(snapshot_ids=['snap-1a2b3c4d', 'snap-5e6f7a8b']) self.assert_request_parameters({ 'Action': 'DescribeSnapshots', 'SnapshotId.1': 'snap-1a2b3c4d', 'SnapshotId.2': 'snap-5e6f7a8b'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(result), 2) self.assertEqual(result[0].id, 'snap-1a2b3c4d') self.assertTrue(result[0].encrypted) self.assertEqual(result[1].id, 'snap-5e6f7a8b') self.assertFalse(result[1].encrypted) class TestCreateVolume(TestEC2ConnectionBase): def default_body(self): return b""" <CreateVolumeResponse xmlns="http://ec2.amazonaws.com/doc/2014-05-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <volumeId>vol-1a2b3c4d</volumeId> <size>80</size> <snapshotId/> <availabilityZone>us-east-1a</availabilityZone> <status>creating</status> <createTime>YYYY-MM-DDTHH:MM:SS.000Z</createTime> <volumeType>standard</volumeType> <encrypted>true</encrypted> </CreateVolumeResponse> """ def test_create_volume(self): self.set_http_response(status_code=200) result = self.ec2.create_volume(80, 'us-east-1e', snapshot='snap-1a2b3c4d', encrypted=True) self.assert_request_parameters({ 'Action': 'CreateVolume', 'AvailabilityZone': 'us-east-1e', 'Size': 80, 'SnapshotId': 'snap-1a2b3c4d', 'Encrypted': 'true'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(result.id, 'vol-1a2b3c4d') self.assertTrue(result.encrypted) def test_create_volume_with_specify_kms(self): self.set_http_response(status_code=200) result = self.ec2.create_volume(80, 'us-east-1e', snapshot='snap-1a2b3c4d', encrypted=True,kms_key_id='arn:aws:kms:us-east-1:012345678910:key/abcd1234-a123-456a-a12b-a123b4cd56ef') self.assert_request_parameters({ 'Action': 'CreateVolume', 'AvailabilityZone': 'us-east-1e', 'Size': 80, 'SnapshotId': 'snap-1a2b3c4d', 'Encrypted': 'true', 'KmsKeyId': 'arn:aws:kms:us-east-1:012345678910:key/abcd1234-a123-456a-a12b-a123b4cd56ef'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(result.id, 'vol-1a2b3c4d') self.assertTrue(result.encrypted) class TestGetClassicLinkInstances(TestEC2ConnectionBase): def default_body(self): return b""" <DescribeClassicLinkInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2014-09-01/"> <requestId>f4bf0cc6-5967-4687-9355-90ce48394bd3</requestId> <instancesSet> <item> <instanceId>i-31489bd8</instanceId> <vpcId>vpc-9d24f8f8</vpcId> <groupSet> <item> <groupId>sg-9b4343fe</groupId> </item> </groupSet> <tagSet> <item> <key>Name</key> <value>hello</value> </item> </tagSet> </item> </instancesSet> </DescribeClassicLinkInstancesResponse> """ def test_get_classic_link_instances(self): self.set_http_response(status_code=200) response = self.ec2.get_all_classic_link_instances() self.assertEqual(len(response), 1) instance = response[0] self.assertEqual(instance.id, 'i-31489bd8') self.assertEqual(instance.vpc_id, 'vpc-9d24f8f8') self.assertEqual(len(instance.groups), 1) self.assertEqual(instance.groups[0].id, 'sg-9b4343fe') self.assertEqual(instance.tags, {'Name': 'hello'}) def test_get_classic_link_instances_params(self): self.set_http_response(status_code=200) self.ec2.get_all_classic_link_instances( instance_ids=['id1', 'id2'], filters={'GroupId': 'sg-9b4343fe'}, dry_run=True, next_token='next_token', max_results=10 ) self.assert_request_parameters({ 'Action': 'DescribeClassicLinkInstances', 'InstanceId.1': 'id1', 'InstanceId.2': 'id2', 'Filter.1.Name': 'GroupId', 'Filter.1.Value.1': 'sg-9b4343fe', 'DryRun': 'true', 'NextToken': 'next_token', 'MaxResults': 10}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/test_blockdevicemapping.py 0000644 00000015123 15025013616 0017347 0 ustar 00 from tests.compat import unittest from boto.ec2.connection import EC2Connection from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping from tests.compat import OrderedDict from tests.unit import AWSMockServiceTestCase class BlockDeviceTypeTests(unittest.TestCase): def setUp(self): self.block_device_type = BlockDeviceType() def check_that_attribute_has_been_set(self, name, value, attribute): self.block_device_type.endElement(name, value, None) self.assertEqual(getattr(self.block_device_type, attribute), value) def test_endElement_sets_correct_attributes_with_values(self): for arguments in [("volumeId", 1, "volume_id"), ("virtualName", "some name", "ephemeral_name"), ("snapshotId", 1, "snapshot_id"), ("volumeSize", 1, "size"), ("status", "some status", "status"), ("attachTime", 1, "attach_time"), ("somethingRandom", "somethingRandom", "somethingRandom")]: self.check_that_attribute_has_been_set(arguments[0], arguments[1], arguments[2]) def test_endElement_with_name_NoDevice_value_true(self): self.block_device_type.endElement("NoDevice", 'true', None) self.assertEqual(self.block_device_type.no_device, True) def test_endElement_with_name_NoDevice_value_other(self): self.block_device_type.endElement("NoDevice", 'something else', None) self.assertEqual(self.block_device_type.no_device, False) def test_endElement_with_name_deleteOnTermination_value_true(self): self.block_device_type.endElement("deleteOnTermination", "true", None) self.assertEqual(self.block_device_type.delete_on_termination, True) def test_endElement_with_name_deleteOnTermination_value_other(self): self.block_device_type.endElement("deleteOnTermination", 'something else', None) self.assertEqual(self.block_device_type.delete_on_termination, False) def test_endElement_with_name_encrypted_value_true(self): self.block_device_type.endElement("Encrypted", "true", None) self.assertEqual(self.block_device_type.encrypted, True) def test_endElement_with_name_Encrypted_value_other(self): self.block_device_type.endElement("Encrypted", 'something else', None) self.assertEqual(self.block_device_type.encrypted, False) class BlockDeviceMappingTests(unittest.TestCase): def setUp(self): self.block_device_mapping = BlockDeviceMapping() def block_device_type_eq(self, b1, b2): if isinstance(b1, BlockDeviceType) and isinstance(b2, BlockDeviceType): return all([b1.connection == b2.connection, b1.ephemeral_name == b2.ephemeral_name, b1.no_device == b2.no_device, b1.volume_id == b2.volume_id, b1.snapshot_id == b2.snapshot_id, b1.status == b2.status, b1.attach_time == b2.attach_time, b1.delete_on_termination == b2.delete_on_termination, b1.size == b2.size, b1.encrypted == b2.encrypted]) def test_startElement_with_name_ebs_sets_and_returns_current_value(self): retval = self.block_device_mapping.startElement("ebs", None, None) assert self.block_device_type_eq(retval, BlockDeviceType(self.block_device_mapping)) def test_startElement_with_name_virtualName_sets_and_returns_current_value(self): retval = self.block_device_mapping.startElement("virtualName", None, None) assert self.block_device_type_eq(retval, BlockDeviceType(self.block_device_mapping)) def test_endElement_with_name_device_sets_current_name_dev_null(self): self.block_device_mapping.endElement("device", "/dev/null", None) self.assertEqual(self.block_device_mapping.current_name, "/dev/null") def test_endElement_with_name_device_sets_current_name(self): self.block_device_mapping.endElement("deviceName", "some device name", None) self.assertEqual(self.block_device_mapping.current_name, "some device name") def test_endElement_with_name_item_sets_current_name_key_to_current_value(self): self.block_device_mapping.current_name = "some name" self.block_device_mapping.current_value = "some value" self.block_device_mapping.endElement("item", "some item", None) self.assertEqual(self.block_device_mapping["some name"], "some value") class TestLaunchConfiguration(AWSMockServiceTestCase): connection_class = EC2Connection def default_body(self): # This is a dummy response return b""" <DescribeLaunchConfigurationsResponse> </DescribeLaunchConfigurationsResponse> """ def test_run_instances_block_device_mapping(self): # Same as the test in ``unit/ec2/autoscale/test_group.py:TestLaunchConfiguration``, # but with modified request parameters (due to a mismatch between EC2 & # Autoscaling). self.set_http_response(status_code=200) dev_sdf = BlockDeviceType(snapshot_id='snap-12345') dev_sdg = BlockDeviceType(snapshot_id='snap-12346', delete_on_termination=True, encrypted=True) class OrderedBlockDeviceMapping(OrderedDict, BlockDeviceMapping): pass bdm = OrderedBlockDeviceMapping() bdm.update(OrderedDict((('/dev/sdf', dev_sdf), ('/dev/sdg', dev_sdg)))) response = self.service_connection.run_instances( image_id='123456', instance_type='m1.large', security_groups=['group1', 'group2'], block_device_map=bdm ) self.assert_request_parameters({ 'Action': 'RunInstances', 'BlockDeviceMapping.1.DeviceName': '/dev/sdf', 'BlockDeviceMapping.1.Ebs.DeleteOnTermination': 'false', 'BlockDeviceMapping.1.Ebs.SnapshotId': 'snap-12345', 'BlockDeviceMapping.2.DeviceName': '/dev/sdg', 'BlockDeviceMapping.2.Ebs.DeleteOnTermination': 'true', 'BlockDeviceMapping.2.Ebs.SnapshotId': 'snap-12346', 'BlockDeviceMapping.2.Ebs.Encrypted': 'true', 'ImageId': '123456', 'InstanceType': 'm1.large', 'MaxCount': 1, 'MinCount': 1, 'SecurityGroup.1': 'group1', 'SecurityGroup.2': 'group2', }, ignore_params_values=[ 'Version', 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp' ]) if __name__ == "__main__": unittest.main() vendored/boto/tests/unit/ec2/autoscale/test_group.py 0000644 00000107362 15025013616 0016644 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import base64 from datetime import datetime from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.ec2.autoscale import AutoScaleConnection from boto.ec2.autoscale.group import AutoScalingGroup from boto.ec2.autoscale.policy import ScalingPolicy from boto.ec2.autoscale.tag import Tag from boto.ec2.blockdevicemapping import EBSBlockDeviceType, BlockDeviceMapping from boto.ec2.autoscale import launchconfig, LaunchConfiguration class TestAutoScaleGroup(AWSMockServiceTestCase): connection_class = AutoScaleConnection def setUp(self): super(TestAutoScaleGroup, self).setUp() def default_body(self): return b""" <CreateLaunchConfigurationResponse> <ResponseMetadata> <RequestId>requestid</RequestId> </ResponseMetadata> </CreateLaunchConfigurationResponse> """ def test_autoscaling_group_with_termination_policies(self): self.set_http_response(status_code=200) autoscale = AutoScalingGroup( name='foo', launch_config='lauch_config', min_size=1, max_size=2, termination_policies=['OldestInstance', 'OldestLaunchConfiguration'], instance_id='test-id') self.service_connection.create_auto_scaling_group(autoscale) self.assert_request_parameters({ 'Action': 'CreateAutoScalingGroup', 'AutoScalingGroupName': 'foo', 'LaunchConfigurationName': 'lauch_config', 'MaxSize': 2, 'MinSize': 1, 'TerminationPolicies.member.1': 'OldestInstance', 'TerminationPolicies.member.2': 'OldestLaunchConfiguration', 'InstanceId': 'test-id', }, ignore_params_values=['Version']) def test_autoscaling_group_single_vpc_zone_identifier(self): self.set_http_response(status_code=200) autoscale = AutoScalingGroup( name='foo', vpc_zone_identifier='vpc_zone_1') self.service_connection.create_auto_scaling_group(autoscale) self.assert_request_parameters({ 'Action': 'CreateAutoScalingGroup', 'AutoScalingGroupName': 'foo', 'VPCZoneIdentifier': 'vpc_zone_1', }, ignore_params_values=['MaxSize', 'MinSize', 'LaunchConfigurationName', 'Version']) def test_autoscaling_group_vpc_zone_identifier_list(self): self.set_http_response(status_code=200) autoscale = AutoScalingGroup( name='foo', vpc_zone_identifier=['vpc_zone_1', 'vpc_zone_2']) self.service_connection.create_auto_scaling_group(autoscale) self.assert_request_parameters({ 'Action': 'CreateAutoScalingGroup', 'AutoScalingGroupName': 'foo', 'VPCZoneIdentifier': 'vpc_zone_1,vpc_zone_2', }, ignore_params_values=['MaxSize', 'MinSize', 'LaunchConfigurationName', 'Version']) def test_autoscaling_group_vpc_zone_identifier_multi(self): self.set_http_response(status_code=200) autoscale = AutoScalingGroup( name='foo', vpc_zone_identifier='vpc_zone_1,vpc_zone_2') self.service_connection.create_auto_scaling_group(autoscale) self.assert_request_parameters({ 'Action': 'CreateAutoScalingGroup', 'AutoScalingGroupName': 'foo', 'VPCZoneIdentifier': 'vpc_zone_1,vpc_zone_2', }, ignore_params_values=['MaxSize', 'MinSize', 'LaunchConfigurationName', 'Version']) class TestAutoScaleGroupHonorCooldown(AWSMockServiceTestCase): connection_class = AutoScaleConnection def default_body(self): return b""" <SetDesiredCapacityResponse> <ResponseMetadata> <RequestId>9fb7e2db-6998-11e2-a985-57c82EXAMPLE</RequestId> </ResponseMetadata> </SetDesiredCapacityResponse> """ def test_honor_cooldown(self): self.set_http_response(status_code=200) self.service_connection.set_desired_capacity('foo', 10, True) self.assert_request_parameters({ 'Action': 'SetDesiredCapacity', 'AutoScalingGroupName': 'foo', 'DesiredCapacity': 10, 'HonorCooldown': 'true', }, ignore_params_values=['Version']) class TestScheduledGroup(AWSMockServiceTestCase): connection_class = AutoScaleConnection def setUp(self): super(TestScheduledGroup, self).setUp() def default_body(self): return b""" <PutScheduledUpdateGroupActionResponse> <ResponseMetadata> <RequestId>requestid</RequestId> </ResponseMetadata> </PutScheduledUpdateGroupActionResponse> """ def test_scheduled_group_creation(self): self.set_http_response(status_code=200) self.service_connection.create_scheduled_group_action('foo', 'scheduled-foo', desired_capacity=1, start_time=datetime(2013, 1, 1, 22, 55, 31), end_time=datetime(2013, 2, 1, 22, 55, 31), min_size=1, max_size=2, recurrence='0 10 * * *') self.assert_request_parameters({ 'Action': 'PutScheduledUpdateGroupAction', 'AutoScalingGroupName': 'foo', 'ScheduledActionName': 'scheduled-foo', 'MaxSize': 2, 'MinSize': 1, 'DesiredCapacity': 1, 'EndTime': '2013-02-01T22:55:31', 'StartTime': '2013-01-01T22:55:31', 'Recurrence': '0 10 * * *', }, ignore_params_values=['Version']) class TestParseAutoScaleGroupResponse(AWSMockServiceTestCase): connection_class = AutoScaleConnection def default_body(self): return b""" <DescribeAutoScalingGroupsResult> <AutoScalingGroups> <member> <Tags/> <SuspendedProcesses/> <AutoScalingGroupName>test_group</AutoScalingGroupName> <HealthCheckType>EC2</HealthCheckType> <CreatedTime>2012-09-27T20:19:47.082Z</CreatedTime> <EnabledMetrics/> <LaunchConfigurationName>test_launchconfig</LaunchConfigurationName> <Instances> <member> <HealthStatus>Healthy</HealthStatus> <AvailabilityZone>us-east-1a</AvailabilityZone> <InstanceId>i-z118d054</InstanceId> <LaunchConfigurationName>test_launchconfig</LaunchConfigurationName> <LifecycleState>InService</LifecycleState> </member> </Instances> <DesiredCapacity>1</DesiredCapacity> <AvailabilityZones> <member>us-east-1c</member> <member>us-east-1a</member> </AvailabilityZones> <LoadBalancerNames/> <MinSize>1</MinSize> <VPCZoneIdentifier/> <HealthCheckGracePeriod>0</HealthCheckGracePeriod> <DefaultCooldown>300</DefaultCooldown> <AutoScalingGroupARN>myarn</AutoScalingGroupARN> <TerminationPolicies> <member>OldestInstance</member> <member>OldestLaunchConfiguration</member> </TerminationPolicies> <MaxSize>2</MaxSize> <InstanceId>Something</InstanceId> </member> </AutoScalingGroups> </DescribeAutoScalingGroupsResult> """ def test_get_all_groups_is_parsed_correctly(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_groups(names=['test_group']) self.assertEqual(len(response), 1, response) as_group = response[0] self.assertEqual(as_group.availability_zones, ['us-east-1c', 'us-east-1a']) self.assertEqual(as_group.default_cooldown, 300) self.assertEqual(as_group.desired_capacity, 1) self.assertEqual(as_group.enabled_metrics, []) self.assertEqual(as_group.health_check_period, 0) self.assertEqual(as_group.health_check_type, 'EC2') self.assertEqual(as_group.launch_config_name, 'test_launchconfig') self.assertEqual(as_group.load_balancers, []) self.assertEqual(as_group.min_size, 1) self.assertEqual(as_group.max_size, 2) self.assertEqual(as_group.name, 'test_group') self.assertEqual(as_group.suspended_processes, []) self.assertEqual(as_group.tags, []) self.assertEqual(as_group.termination_policies, ['OldestInstance', 'OldestLaunchConfiguration']) self.assertEqual(as_group.instance_id, 'Something') class TestDescribeTerminationPolicies(AWSMockServiceTestCase): connection_class = AutoScaleConnection def default_body(self): return b""" <DescribeTerminationPolicyTypesResponse> <DescribeTerminationPolicyTypesResult> <TerminationPolicyTypes> <member>ClosestToNextInstanceHour</member> <member>Default</member> <member>NewestInstance</member> <member>OldestInstance</member> <member>OldestLaunchConfiguration</member> </TerminationPolicyTypes> </DescribeTerminationPolicyTypesResult> <ResponseMetadata> <RequestId>requestid</RequestId> </ResponseMetadata> </DescribeTerminationPolicyTypesResponse> """ def test_autoscaling_group_with_termination_policies(self): self.set_http_response(status_code=200) response = self.service_connection.get_termination_policies() self.assertListEqual( response, ['ClosestToNextInstanceHour', 'Default', 'NewestInstance', 'OldestInstance', 'OldestLaunchConfiguration']) class TestLaunchConfigurationDescribe(AWSMockServiceTestCase): connection_class = AutoScaleConnection def default_body(self): # This is a dummy response return b""" <DescribeLaunchConfigurationsResponse> <DescribeLaunchConfigurationsResult> <LaunchConfigurations> <member> <AssociatePublicIpAddress>true</AssociatePublicIpAddress> <SecurityGroups/> <CreatedTime>2013-01-21T23:04:42.200Z</CreatedTime> <KernelId/> <LaunchConfigurationName>my-test-lc</LaunchConfigurationName> <UserData/> <InstanceType>m1.small</InstanceType> <LaunchConfigurationARN>arn:aws:autoscaling:us-east-1:803981987763:launchConfiguration:9dbbbf87-6141-428a-a409-0752edbe6cad:launchConfigurationName/my-test-lc</LaunchConfigurationARN> <BlockDeviceMappings/> <ImageId>ami-514ac838</ImageId> <KeyName/> <RamdiskId/> <InstanceMonitoring> <Enabled>true</Enabled> </InstanceMonitoring> <EbsOptimized>false</EbsOptimized> <ClassicLinkVPCId>vpc-12345</ClassicLinkVPCId> <ClassicLinkVPCSecurityGroups> <member>sg-1234</member> </ClassicLinkVPCSecurityGroups> </member> </LaunchConfigurations> </DescribeLaunchConfigurationsResult> <ResponseMetadata> <RequestId>d05a22f8-b690-11e2-bf8e-2113fEXAMPLE</RequestId> </ResponseMetadata> </DescribeLaunchConfigurationsResponse> """ def test_get_all_launch_configurations(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_launch_configurations() self.assertTrue(isinstance(response, list)) self.assertEqual(len(response), 1) self.assertTrue(isinstance(response[0], LaunchConfiguration)) self.assertEqual(response[0].associate_public_ip_address, True) self.assertEqual(response[0].name, "my-test-lc") self.assertEqual(response[0].instance_type, "m1.small") self.assertEqual(response[0].launch_configuration_arn, "arn:aws:autoscaling:us-east-1:803981987763:launchConfiguration:9dbbbf87-6141-428a-a409-0752edbe6cad:launchConfigurationName/my-test-lc") self.assertEqual(response[0].image_id, "ami-514ac838") self.assertTrue(isinstance(response[0].instance_monitoring, launchconfig.InstanceMonitoring)) self.assertEqual(response[0].instance_monitoring.enabled, 'true') self.assertEqual(response[0].ebs_optimized, False) self.assertEqual(response[0].block_device_mappings, []) self.assertEqual(response[0].classic_link_vpc_id, 'vpc-12345') self.assertEqual(response[0].classic_link_vpc_security_groups, ['sg-1234']) self.assert_request_parameters({ 'Action': 'DescribeLaunchConfigurations', }, ignore_params_values=['Version']) def test_get_all_configuration_limited(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_launch_configurations(max_records=10, names=["my-test1", "my-test2"]) self.assert_request_parameters({ 'Action': 'DescribeLaunchConfigurations', 'MaxRecords': 10, 'LaunchConfigurationNames.member.1': 'my-test1', 'LaunchConfigurationNames.member.2': 'my-test2' }, ignore_params_values=['Version']) class TestLaunchConfiguration(AWSMockServiceTestCase): connection_class = AutoScaleConnection def default_body(self): # This is a dummy response return b""" <DescribeLaunchConfigurationsResponse> </DescribeLaunchConfigurationsResponse> """ def test_launch_config(self): # This unit test is based on #753 and #1343 self.set_http_response(status_code=200) dev_sdf = EBSBlockDeviceType(snapshot_id='snap-12345') bdm = BlockDeviceMapping() bdm['/dev/sdf'] = dev_sdf lc = launchconfig.LaunchConfiguration( connection=self.service_connection, name='launch_config', image_id='123456', instance_type='m1.large', user_data='#!/bin/bash', security_groups=['group1'], spot_price='price', block_device_mappings=[bdm], associate_public_ip_address=True, volume_type='atype', delete_on_termination=False, iops=3000, classic_link_vpc_id='vpc-1234', classic_link_vpc_security_groups=['classic_link_group'] ) response = self.service_connection.create_launch_configuration(lc) self.assert_request_parameters({ 'Action': 'CreateLaunchConfiguration', 'BlockDeviceMappings.member.1.DeviceName': '/dev/sdf', 'BlockDeviceMappings.member.1.Ebs.DeleteOnTermination': 'false', 'BlockDeviceMappings.member.1.Ebs.SnapshotId': 'snap-12345', 'EbsOptimized': 'false', 'LaunchConfigurationName': 'launch_config', 'ImageId': '123456', 'UserData': base64.b64encode(b'#!/bin/bash').decode('utf-8'), 'InstanceMonitoring.Enabled': 'false', 'InstanceType': 'm1.large', 'SecurityGroups.member.1': 'group1', 'SpotPrice': 'price', 'AssociatePublicIpAddress': 'true', 'VolumeType': 'atype', 'DeleteOnTermination': 'false', 'Iops': 3000, 'ClassicLinkVPCId': 'vpc-1234', 'ClassicLinkVPCSecurityGroups.member.1': 'classic_link_group' }, ignore_params_values=['Version']) class TestCreateAutoScalePolicy(AWSMockServiceTestCase): connection_class = AutoScaleConnection def setUp(self): super(TestCreateAutoScalePolicy, self).setUp() def default_body(self): return b""" <PutScalingPolicyResponse xmlns="http://autoscaling.amazonaws.com\ /doc/2011-01-01/"> <PutScalingPolicyResult> <PolicyARN>arn:aws:autoscaling:us-east-1:803981987763:scaling\ Policy:b0dcf5e8 -02e6-4e31-9719-0675d0dc31ae:autoScalingGroupName/my-test-asg:\ policyName/my-scal eout-policy</PolicyARN> </PutScalingPolicyResult> <ResponseMetadata> <RequestId>3cfc6fef-c08b-11e2-a697-2922EXAMPLE</RequestId> </ResponseMetadata> </PutScalingPolicyResponse> """ def test_scaling_policy_with_min_adjustment_step(self): self.set_http_response(status_code=200) policy = ScalingPolicy( name='foo', as_name='bar', adjustment_type='PercentChangeInCapacity', scaling_adjustment=50, min_adjustment_step=30) self.service_connection.create_scaling_policy(policy) self.assert_request_parameters({ 'Action': 'PutScalingPolicy', 'PolicyName': 'foo', 'AutoScalingGroupName': 'bar', 'AdjustmentType': 'PercentChangeInCapacity', 'ScalingAdjustment': 50, 'MinAdjustmentStep': 30 }, ignore_params_values=['Version']) def test_scaling_policy_with_wrong_adjustment_type(self): self.set_http_response(status_code=200) policy = ScalingPolicy( name='foo', as_name='bar', adjustment_type='ChangeInCapacity', scaling_adjustment=50, min_adjustment_step=30) self.service_connection.create_scaling_policy(policy) self.assert_request_parameters({ 'Action': 'PutScalingPolicy', 'PolicyName': 'foo', 'AutoScalingGroupName': 'bar', 'AdjustmentType': 'ChangeInCapacity', 'ScalingAdjustment': 50 }, ignore_params_values=['Version']) def test_scaling_policy_without_min_adjustment_step(self): self.set_http_response(status_code=200) policy = ScalingPolicy( name='foo', as_name='bar', adjustment_type='PercentChangeInCapacity', scaling_adjustment=50) self.service_connection.create_scaling_policy(policy) self.assert_request_parameters({ 'Action': 'PutScalingPolicy', 'PolicyName': 'foo', 'AutoScalingGroupName': 'bar', 'AdjustmentType': 'PercentChangeInCapacity', 'ScalingAdjustment': 50 }, ignore_params_values=['Version']) class TestPutNotificationConfiguration(AWSMockServiceTestCase): connection_class = AutoScaleConnection def setUp(self): super(TestPutNotificationConfiguration, self).setUp() def default_body(self): return b""" <PutNotificationConfigurationResponse> <ResponseMetadata> <RequestId>requestid</RequestId> </ResponseMetadata> </PutNotificationConfigurationResponse> """ def test_autoscaling_group_put_notification_configuration(self): self.set_http_response(status_code=200) autoscale = AutoScalingGroup( name='ana', launch_config='lauch_config', min_size=1, max_size=2, termination_policies=['OldestInstance', 'OldestLaunchConfiguration']) self.service_connection.put_notification_configuration(autoscale, 'arn:aws:sns:us-east-1:19890506:AutoScaling-Up', ['autoscaling:EC2_INSTANCE_LAUNCH']) self.assert_request_parameters({ 'Action': 'PutNotificationConfiguration', 'AutoScalingGroupName': 'ana', 'NotificationTypes.member.1': 'autoscaling:EC2_INSTANCE_LAUNCH', 'TopicARN': 'arn:aws:sns:us-east-1:19890506:AutoScaling-Up', }, ignore_params_values=['Version']) class TestDeleteNotificationConfiguration(AWSMockServiceTestCase): connection_class = AutoScaleConnection def setUp(self): super(TestDeleteNotificationConfiguration, self).setUp() def default_body(self): return b""" <DeleteNotificationConfigurationResponse> <ResponseMetadata> <RequestId>requestid</RequestId> </ResponseMetadata> </DeleteNotificationConfigurationResponse> """ def test_autoscaling_group_put_notification_configuration(self): self.set_http_response(status_code=200) autoscale = AutoScalingGroup( name='ana', launch_config='lauch_config', min_size=1, max_size=2, termination_policies=['OldestInstance', 'OldestLaunchConfiguration']) self.service_connection.delete_notification_configuration(autoscale, 'arn:aws:sns:us-east-1:19890506:AutoScaling-Up') self.assert_request_parameters({ 'Action': 'DeleteNotificationConfiguration', 'AutoScalingGroupName': 'ana', 'TopicARN': 'arn:aws:sns:us-east-1:19890506:AutoScaling-Up', }, ignore_params_values=['Version']) class TestAutoScalingTag(AWSMockServiceTestCase): connection_class = AutoScaleConnection def default_body(self): return b""" <CreateOrUpdateTagsResponse> <ResponseMetadata> <RequestId>requestId</RequestId> </ResponseMetadata> </CreateOrUpdateTagsResponse> """ def test_create_or_update_tags(self): self.set_http_response(status_code=200) tags = [ Tag( connection=self.service_connection, key='alpha', value='tango', resource_id='sg-00000000', resource_type='auto-scaling-group', propagate_at_launch=True ), Tag( connection=self.service_connection, key='bravo', value='sierra', resource_id='sg-00000000', resource_type='auto-scaling-group', propagate_at_launch=False )] response = self.service_connection.create_or_update_tags(tags) self.assert_request_parameters({ 'Action': 'CreateOrUpdateTags', 'Tags.member.1.ResourceType': 'auto-scaling-group', 'Tags.member.1.ResourceId': 'sg-00000000', 'Tags.member.1.Key': 'alpha', 'Tags.member.1.Value': 'tango', 'Tags.member.1.PropagateAtLaunch': 'true', 'Tags.member.2.ResourceType': 'auto-scaling-group', 'Tags.member.2.ResourceId': 'sg-00000000', 'Tags.member.2.Key': 'bravo', 'Tags.member.2.Value': 'sierra', 'Tags.member.2.PropagateAtLaunch': 'false' }, ignore_params_values=['Version']) def test_endElement(self): for i in [ ('Key', 'mykey', 'key'), ('Value', 'myvalue', 'value'), ('ResourceType', 'auto-scaling-group', 'resource_type'), ('ResourceId', 'sg-01234567', 'resource_id'), ('PropagateAtLaunch', 'true', 'propagate_at_launch')]: self.check_tag_attributes_set(i[0], i[1], i[2]) def check_tag_attributes_set(self, name, value, attr): tag = Tag() tag.endElement(name, value, None) if value == 'true': self.assertEqual(getattr(tag, attr), True) else: self.assertEqual(getattr(tag, attr), value) class TestAttachInstances(AWSMockServiceTestCase): connection_class = AutoScaleConnection def setUp(self): super(TestAttachInstances, self).setUp() def default_body(self): return b""" <AttachInstancesResponse> <ResponseMetadata> <RequestId>requestid</RequestId> </ResponseMetadata> </AttachInstancesResponse> """ def test_attach_instances(self): self.set_http_response(status_code=200) self.service_connection.attach_instances( 'autoscale', ['inst2', 'inst1', 'inst4'] ) self.assert_request_parameters({ 'Action': 'AttachInstances', 'AutoScalingGroupName': 'autoscale', 'InstanceIds.member.1': 'inst2', 'InstanceIds.member.2': 'inst1', 'InstanceIds.member.3': 'inst4', }, ignore_params_values=['Version']) class TestDetachInstances(AWSMockServiceTestCase): connection_class = AutoScaleConnection def setUp(self): super(TestDetachInstances, self).setUp() def default_body(self): return b""" <DetachInstancesResponse> <ResponseMetadata> <RequestId>requestid</RequestId> </ResponseMetadata> </DetachInstancesResponse> """ def test_detach_instances(self): self.set_http_response(status_code=200) self.service_connection.detach_instances( 'autoscale', ['inst2', 'inst1', 'inst4'] ) self.assert_request_parameters({ 'Action': 'DetachInstances', 'AutoScalingGroupName': 'autoscale', 'InstanceIds.member.1': 'inst2', 'InstanceIds.member.2': 'inst1', 'InstanceIds.member.3': 'inst4', 'ShouldDecrementDesiredCapacity': 'true', }, ignore_params_values=['Version']) def test_detach_instances_with_decrement_desired_capacity(self): self.set_http_response(status_code=200) self.service_connection.detach_instances( 'autoscale', ['inst2', 'inst1', 'inst4'], True ) self.assert_request_parameters({ 'Action': 'DetachInstances', 'AutoScalingGroupName': 'autoscale', 'InstanceIds.member.1': 'inst2', 'InstanceIds.member.2': 'inst1', 'InstanceIds.member.3': 'inst4', 'ShouldDecrementDesiredCapacity': 'true', }, ignore_params_values=['Version']) def test_detach_instances_without_decrement_desired_capacity(self): self.set_http_response(status_code=200) self.service_connection.detach_instances( 'autoscale', ['inst2', 'inst1', 'inst4'], False ) self.assert_request_parameters({ 'Action': 'DetachInstances', 'AutoScalingGroupName': 'autoscale', 'InstanceIds.member.1': 'inst2', 'InstanceIds.member.2': 'inst1', 'InstanceIds.member.3': 'inst4', 'ShouldDecrementDesiredCapacity': 'false', }, ignore_params_values=['Version']) class TestGetAccountLimits(AWSMockServiceTestCase): connection_class = AutoScaleConnection def setUp(self): super(TestGetAccountLimits, self).setUp() def default_body(self): return b""" <DescribeAccountLimitsAnswer> <MaxNumberOfAutoScalingGroups>6</MaxNumberOfAutoScalingGroups> <MaxNumberOfLaunchConfigurations>3</MaxNumberOfLaunchConfigurations> <ResponseMetadata> <RequestId>requestid</RequestId> </ResponseMetadata> </DescribeAccountLimitsAnswer> """ def test_autoscaling_group_put_notification_configuration(self): self.set_http_response(status_code=200) limits = self.service_connection.get_account_limits() self.assert_request_parameters({ 'Action': 'DescribeAccountLimits', }, ignore_params_values=['Version']) self.assertEqual(limits.max_autoscaling_groups, 6) self.assertEqual(limits.max_launch_configurations, 3) class TestGetAdjustmentTypes(AWSMockServiceTestCase): connection_class = AutoScaleConnection def setUp(self): super(TestGetAdjustmentTypes, self).setUp() def default_body(self): return b""" <DescribeAdjustmentTypesResponse xmlns="http://autoscaling.amazonaws.com/doc/201-01-01/"> <DescribeAdjustmentTypesResult> <AdjustmentTypes> <member> <AdjustmentType>ChangeInCapacity</AdjustmentType> </member> <member> <AdjustmentType>ExactCapacity</AdjustmentType> </member> <member> <AdjustmentType>PercentChangeInCapacity</AdjustmentType> </member> </AdjustmentTypes> </DescribeAdjustmentTypesResult> <ResponseMetadata> <RequestId>requestId</RequestId> </ResponseMetadata> </DescribeAdjustmentTypesResponse> """ def test_autoscaling_adjustment_types(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_adjustment_types() self.assert_request_parameters({ 'Action': 'DescribeAdjustmentTypes' }, ignore_params_values=['Version']) self.assertTrue(isinstance(response, list)) self.assertEqual(response[0].adjustment_type, "ChangeInCapacity") self.assertEqual(response[1].adjustment_type, "ExactCapacity") self.assertEqual(response[2].adjustment_type, "PercentChangeInCapacity") class TestLaunchConfigurationDescribeWithBlockDeviceTypes(AWSMockServiceTestCase): connection_class = AutoScaleConnection def default_body(self): # This is a dummy response return b""" <DescribeLaunchConfigurationsResponse> <DescribeLaunchConfigurationsResult> <LaunchConfigurations> <member> <AssociatePublicIpAddress>true</AssociatePublicIpAddress> <SecurityGroups/> <CreatedTime>2013-01-21T23:04:42.200Z</CreatedTime> <KernelId/> <LaunchConfigurationName>my-test-lc</LaunchConfigurationName> <UserData/> <InstanceType>m1.small</InstanceType> <LaunchConfigurationARN>arn:aws:autoscaling:us-east-1:803981987763:launchConfiguration:9dbbbf87-6141-428a-a409-0752edbe6cad:launchConfigurationName/my-test-lc</LaunchConfigurationARN> <BlockDeviceMappings> <member> <DeviceName>/dev/xvdp</DeviceName> <Ebs> <SnapshotId>snap-1234abcd</SnapshotId> <Iops>1000</Iops> <DeleteOnTermination>true</DeleteOnTermination> <VolumeType>io1</VolumeType> <VolumeSize>100</VolumeSize> </Ebs> </member> <member> <VirtualName>ephemeral1</VirtualName> <DeviceName>/dev/xvdc</DeviceName> </member> <member> <VirtualName>ephemeral0</VirtualName> <DeviceName>/dev/xvdb</DeviceName> </member> <member> <DeviceName>/dev/xvdh</DeviceName> <Ebs> <Iops>2000</Iops> <DeleteOnTermination>false</DeleteOnTermination> <VolumeType>io1</VolumeType> <VolumeSize>200</VolumeSize> </Ebs> </member> </BlockDeviceMappings> <ImageId>ami-514ac838</ImageId> <KeyName/> <RamdiskId/> <InstanceMonitoring> <Enabled>true</Enabled> </InstanceMonitoring> <EbsOptimized>false</EbsOptimized> </member> </LaunchConfigurations> </DescribeLaunchConfigurationsResult> <ResponseMetadata> <RequestId>d05a22f8-b690-11e2-bf8e-2113fEXAMPLE</RequestId> </ResponseMetadata> </DescribeLaunchConfigurationsResponse> """ def test_get_all_launch_configurations_with_block_device_types(self): self.set_http_response(status_code=200) self.service_connection.use_block_device_types = True response = self.service_connection.get_all_launch_configurations() self.assertTrue(isinstance(response, list)) self.assertEqual(len(response), 1) self.assertTrue(isinstance(response[0], LaunchConfiguration)) self.assertEqual(response[0].associate_public_ip_address, True) self.assertEqual(response[0].name, "my-test-lc") self.assertEqual(response[0].instance_type, "m1.small") self.assertEqual(response[0].launch_configuration_arn, "arn:aws:autoscaling:us-east-1:803981987763:launchConfiguration:9dbbbf87-6141-428a-a409-0752edbe6cad:launchConfigurationName/my-test-lc") self.assertEqual(response[0].image_id, "ami-514ac838") self.assertTrue(isinstance(response[0].instance_monitoring, launchconfig.InstanceMonitoring)) self.assertEqual(response[0].instance_monitoring.enabled, 'true') self.assertEqual(response[0].ebs_optimized, False) self.assertEqual(response[0].block_device_mappings['/dev/xvdb'].ephemeral_name, 'ephemeral0') self.assertEqual(response[0].block_device_mappings['/dev/xvdc'].ephemeral_name, 'ephemeral1') self.assertEqual(response[0].block_device_mappings['/dev/xvdp'].snapshot_id, 'snap-1234abcd') self.assertEqual(response[0].block_device_mappings['/dev/xvdp'].delete_on_termination, True) self.assertEqual(response[0].block_device_mappings['/dev/xvdp'].iops, 1000) self.assertEqual(response[0].block_device_mappings['/dev/xvdp'].size, 100) self.assertEqual(response[0].block_device_mappings['/dev/xvdp'].volume_type, 'io1') self.assertEqual(response[0].block_device_mappings['/dev/xvdh'].delete_on_termination, False) self.assertEqual(response[0].block_device_mappings['/dev/xvdh'].iops, 2000) self.assertEqual(response[0].block_device_mappings['/dev/xvdh'].size, 200) self.assertEqual(response[0].block_device_mappings['/dev/xvdh'].volume_type, 'io1') self.assert_request_parameters({ 'Action': 'DescribeLaunchConfigurations', }, ignore_params_values=['Version']) def test_get_all_configuration_limited(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_launch_configurations(max_records=10, names=["my-test1", "my-test2"]) self.assert_request_parameters({ 'Action': 'DescribeLaunchConfigurations', 'MaxRecords': 10, 'LaunchConfigurationNames.member.1': 'my-test1', 'LaunchConfigurationNames.member.2': 'my-test2' }, ignore_params_values=['Version']) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/autoscale/__init__.py 0000644 00000000000 15025013616 0016165 0 ustar 00 vendored/boto/tests/unit/ec2/elb/test_attribute.py 0000644 00000020172 15025013616 0016266 0 ustar 00 from tests.unit import unittest from tests.compat import mock from boto.ec2.elb import ELBConnection from boto.ec2.elb import LoadBalancer from boto.ec2.elb.attributes import LbAttributes ATTRIBUTE_GET_TRUE_CZL_RESPONSE = b"""<?xml version="1.0" encoding="UTF-8"?> <DescribeLoadBalancerAttributesResponse xmlns="http://elasticloadbalancing.amazonaws.com/doc/2012-06-01/"> <DescribeLoadBalancerAttributesResult> <LoadBalancerAttributes> <CrossZoneLoadBalancing> <Enabled>true</Enabled> </CrossZoneLoadBalancing> </LoadBalancerAttributes> </DescribeLoadBalancerAttributesResult> <ResponseMetadata> <RequestId>83c88b9d-12b7-11e3-8b82-87b12EXAMPLE</RequestId> </ResponseMetadata> </DescribeLoadBalancerAttributesResponse> """ ATTRIBUTE_GET_FALSE_CZL_RESPONSE = b"""<?xml version="1.0" encoding="UTF-8"?> <DescribeLoadBalancerAttributesResponse xmlns="http://elasticloadbalancing.amazonaws.com/doc/2012-06-01/"> <DescribeLoadBalancerAttributesResult> <LoadBalancerAttributes> <CrossZoneLoadBalancing> <Enabled>false</Enabled> </CrossZoneLoadBalancing> </LoadBalancerAttributes> </DescribeLoadBalancerAttributesResult> <ResponseMetadata> <RequestId>83c88b9d-12b7-11e3-8b82-87b12EXAMPLE</RequestId> </ResponseMetadata> </DescribeLoadBalancerAttributesResponse> """ ATTRIBUTE_GET_CS_RESPONSE = b"""<?xml version="1.0" encoding="UTF-8"?> <DescribeLoadBalancerAttributesResponse xmlns="http://elasticloadbalancing.amazonaws.com/doc/2012-06-01/"> <DescribeLoadBalancerAttributesResult> <LoadBalancerAttributes> <ConnectionSettings> <IdleTimeout>30</IdleTimeout> </ConnectionSettings> </LoadBalancerAttributes> </DescribeLoadBalancerAttributesResult> <ResponseMetadata> <RequestId>83c88b9d-12b7-11e3-8b82-87b12EXAMPLE</RequestId> </ResponseMetadata> </DescribeLoadBalancerAttributesResponse> """ ATTRIBUTE_SET_RESPONSE = b"""<?xml version="1.0" encoding="UTF-8"?> <ModifyLoadBalancerAttributesResponse xmlns="http://elasticloadbalancing.amazonaws.com/doc/2012-06-01/"> <ModifyLoadBalancerAttributesResult/> <ResponseMetadata> <RequestId>83c88b9d-12b7-11e3-8b82-87b12EXAMPLE</RequestId> </ResponseMetadata> </ModifyLoadBalancerAttributesResponse> """ # make_request arguments for setting attributes. # Format: (API_COMMAND, API_PARAMS, API_PATH, API_METHOD) ATTRIBUTE_SET_CZL_TRUE_REQUEST = ( 'ModifyLoadBalancerAttributes', {'LoadBalancerAttributes.CrossZoneLoadBalancing.Enabled': 'true', 'LoadBalancerName': 'test_elb'}, mock.ANY, mock.ANY) ATTRIBUTE_SET_CZL_FALSE_REQUEST = ( 'ModifyLoadBalancerAttributes', {'LoadBalancerAttributes.CrossZoneLoadBalancing.Enabled': 'false', 'LoadBalancerName': 'test_elb'}, mock.ANY, mock.ANY) # Tests to be run on an LbAttributes # Format: # (EC2_RESPONSE_STRING, list( (string_of_attribute_to_test, value) ) ) ATTRIBUTE_TESTS = [ (ATTRIBUTE_GET_TRUE_CZL_RESPONSE, [('cross_zone_load_balancing.enabled', True)]), (ATTRIBUTE_GET_FALSE_CZL_RESPONSE, [('cross_zone_load_balancing.enabled', False)]), (ATTRIBUTE_GET_CS_RESPONSE, [('connecting_settings.idle_timeout', 30)]), ] class TestLbAttributes(unittest.TestCase): """Tests LB Attributes.""" def _setup_mock(self): """Sets up a mock elb request. Returns: response, elb connection and LoadBalancer """ mock_response = mock.Mock() mock_response.status = 200 elb = ELBConnection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') elb.make_request = mock.Mock(return_value=mock_response) return mock_response, elb, LoadBalancer(elb, 'test_elb') def _verify_attributes(self, attributes, attr_tests): """Verifies an LbAttributes object.""" for attr, result in attr_tests: attr_result = attributes for sub_attr in attr.split('.'): attr_result = getattr(attr_result, sub_attr, None) self.assertEqual(attr_result, result) def test_get_all_lb_attributes(self): """Tests getting the LbAttributes from the elb.connection.""" mock_response, elb, _ = self._setup_mock() for response, attr_tests in ATTRIBUTE_TESTS: mock_response.read.return_value = response attributes = elb.get_all_lb_attributes('test_elb') self.assertTrue(isinstance(attributes, LbAttributes)) self._verify_attributes(attributes, attr_tests) def test_get_lb_attribute(self): """Tests getting a single attribute from elb.connection.""" mock_response, elb, _ = self._setup_mock() tests = [ ('crossZoneLoadBalancing', True, ATTRIBUTE_GET_TRUE_CZL_RESPONSE), ('crossZoneLoadBalancing', False, ATTRIBUTE_GET_FALSE_CZL_RESPONSE), ] for attr, value, response in tests: mock_response.read.return_value = response status = elb.get_lb_attribute('test_elb', attr) self.assertEqual(status, value) def test_modify_lb_attribute(self): """Tests setting the attributes from elb.connection.""" mock_response, elb, _ = self._setup_mock() tests = [ ('crossZoneLoadBalancing', True, ATTRIBUTE_SET_CZL_TRUE_REQUEST), ('crossZoneLoadBalancing', False, ATTRIBUTE_SET_CZL_FALSE_REQUEST), ] for attr, value, args in tests: mock_response.read.return_value = ATTRIBUTE_SET_RESPONSE result = elb.modify_lb_attribute('test_elb', attr, value) self.assertTrue(result) elb.make_request.assert_called_with(*args) def test_lb_get_attributes(self): """Tests the LbAttributes from the ELB object.""" mock_response, _, lb = self._setup_mock() for response, attr_tests in ATTRIBUTE_TESTS: mock_response.read.return_value = response attributes = lb.get_attributes(force=True) self.assertTrue(isinstance(attributes, LbAttributes)) self._verify_attributes(attributes, attr_tests) def test_lb_is_cross_zone_load_balancing(self): """Tests checking is_cross_zone_load_balancing.""" mock_response, _, lb = self._setup_mock() tests = [ # Format: (method, args, result, response) # Gets a true result. (lb.is_cross_zone_load_balancing, [], True, ATTRIBUTE_GET_TRUE_CZL_RESPONSE), # Returns the previous calls cached value. (lb.is_cross_zone_load_balancing, [], True, ATTRIBUTE_GET_FALSE_CZL_RESPONSE), # Gets a false result. (lb.is_cross_zone_load_balancing, [True], False, ATTRIBUTE_GET_FALSE_CZL_RESPONSE), ] for method, args, result, response in tests: mock_response.read.return_value = response self.assertEqual(method(*args), result) def test_lb_enable_cross_zone_load_balancing(self): """Tests enabling cross zone balancing from LoadBalancer.""" mock_response, elb, lb = self._setup_mock() mock_response.read.return_value = ATTRIBUTE_SET_RESPONSE self.assertTrue(lb.enable_cross_zone_load_balancing()) elb.make_request.assert_called_with(*ATTRIBUTE_SET_CZL_TRUE_REQUEST) def test_lb_disable_cross_zone_load_balancing(self): """Tests disabling cross zone balancing from LoadBalancer.""" mock_response, elb, lb = self._setup_mock() mock_response.read.return_value = ATTRIBUTE_SET_RESPONSE self.assertTrue(lb.disable_cross_zone_load_balancing()) elb.make_request.assert_called_with(*ATTRIBUTE_SET_CZL_FALSE_REQUEST) def test_lb_get_connection_settings(self): """Tests checking connectionSettings attribute""" mock_response, elb, _ = self._setup_mock() attrs = [('idle_timeout', 30), ] mock_response.read.return_value = ATTRIBUTE_GET_CS_RESPONSE attributes = elb.get_all_lb_attributes('test_elb') self.assertTrue(isinstance(attributes, LbAttributes)) for attr, value in attrs: self.assertEqual(getattr(attributes.connecting_settings, attr), value) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/elb/test_loadbalancer.py 0000644 00000013034 15025013616 0016671 0 ustar 00 #!/usr/bin/env python from tests.unit import unittest from tests.compat import mock from boto.ec2.elb import ELBConnection from boto.ec2.elb import LoadBalancer DISABLE_RESPONSE = b"""<?xml version="1.0" encoding="UTF-8"?> <DisableAvailabilityZonesForLoadBalancerResult xmlns="http://ec2.amazonaws.com/doc/2013-02-01/"> <requestId>3be1508e-c444-4fef-89cc-0b1223c4f02fEXAMPLE</requestId> <AvailabilityZones> <member>sample-zone</member> </AvailabilityZones> </DisableAvailabilityZonesForLoadBalancerResult> """ class TestInstanceStatusResponseParsing(unittest.TestCase): def test_next_token(self): elb = ELBConnection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') mock_response = mock.Mock() mock_response.read.return_value = DISABLE_RESPONSE mock_response.status = 200 elb.make_request = mock.Mock(return_value=mock_response) disabled = elb.disable_availability_zones('mine', ['sample-zone']) self.assertEqual(disabled, ['sample-zone']) DESCRIBE_RESPONSE = b"""<?xml version="1.0" encoding="UTF-8"?> <DescribeLoadBalancersResponse xmlns="http://elasticloadbalancing.amazonaws.com/doc/2012-06-01/"> <DescribeLoadBalancersResult> <LoadBalancerDescriptions> <member> <SecurityGroups/> <CreatedTime>2013-07-09T19:18:00.520Z</CreatedTime> <LoadBalancerName>elb-boto-unit-test</LoadBalancerName> <HealthCheck/> <ListenerDescriptions> <member> <PolicyNames/> <Listener/> </member> </ListenerDescriptions> <Instances/> <Policies> <AppCookieStickinessPolicies/> <OtherPolicies> <member>AWSConsole-SSLNegotiationPolicy-my-test-loadbalancer</member> <member>EnableProxyProtocol</member> </OtherPolicies> <LBCookieStickinessPolicies/> </Policies> <AvailabilityZones> <member>us-east-1a</member> </AvailabilityZones> <CanonicalHostedZoneName>elb-boto-unit-test-408121642.us-east-1.elb.amazonaws.com</CanonicalHostedZoneName> <CanonicalHostedZoneNameID>Z3DZXE0Q79N41H</CanonicalHostedZoneNameID> <Scheme>internet-facing</Scheme> <SourceSecurityGroup> <OwnerAlias>amazon-elb</OwnerAlias> <GroupName>amazon-elb-sg</GroupName> </SourceSecurityGroup> <DNSName>elb-boto-unit-test-408121642.us-east-1.elb.amazonaws.com</DNSName> <BackendServerDescriptions> <member> <PolicyNames> <member>EnableProxyProtocol</member> </PolicyNames> <InstancePort>80</InstancePort> </member> </BackendServerDescriptions> <Subnets/> </member> </LoadBalancerDescriptions> <Marker>1234</Marker> </DescribeLoadBalancersResult> <ResponseMetadata> <RequestId>5763d932-e8cc-11e2-a940-11136cceffb8</RequestId> </ResponseMetadata> </DescribeLoadBalancersResponse> """ class TestDescribeLoadBalancers(unittest.TestCase): def test_other_policy(self): elb = ELBConnection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') mock_response = mock.Mock() mock_response.read.return_value = DESCRIBE_RESPONSE mock_response.status = 200 elb.make_request = mock.Mock(return_value=mock_response) load_balancers = elb.get_all_load_balancers() self.assertEqual(len(load_balancers), 1) lb = load_balancers[0] self.assertEqual(len(lb.policies.other_policies), 2) self.assertEqual(lb.policies.other_policies[0].policy_name, 'AWSConsole-SSLNegotiationPolicy-my-test-loadbalancer') self.assertEqual(lb.policies.other_policies[1].policy_name, 'EnableProxyProtocol') self.assertEqual(len(lb.backends), 1) self.assertEqual(len(lb.backends[0].policies), 1) self.assertEqual(lb.backends[0].policies[0].policy_name, 'EnableProxyProtocol') self.assertEqual(lb.backends[0].instance_port, 80) def test_request_with_marker(self): elb = ELBConnection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') mock_response = mock.Mock() mock_response.read.return_value = DESCRIBE_RESPONSE mock_response.status = 200 elb.make_request = mock.Mock(return_value=mock_response) load_balancers1 = elb.get_all_load_balancers() self.assertEqual('1234', load_balancers1.marker) load_balancers2 = elb.get_all_load_balancers(marker=load_balancers1.marker) self.assertEqual(len(load_balancers2), 1) DETACH_RESPONSE = r"""<?xml version="1.0" encoding="UTF-8"?> <DetachLoadBalancerFromSubnets xmlns="http://ec2.amazonaws.com/doc/2013-02-01/"> <requestId>3be1508e-c444-4fef-89cc-0b1223c4f02fEXAMPLE</requestId> </DetachLoadBalancerFromSubnets> """ class TestDetachSubnets(unittest.TestCase): def test_detach_subnets(self): elb = ELBConnection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') lb = LoadBalancer(elb, "mylb") mock_response = mock.Mock() mock_response.read.return_value = DETACH_RESPONSE mock_response.status = 200 elb.make_request = mock.Mock(return_value=mock_response) lb.detach_subnets("s-xxx") if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/elb/test_listener.py 0000644 00000010576 15025013616 0016117 0 ustar 00 #!/usr/bin/env python import xml.sax from tests.unit import unittest import boto.resultset from boto.ec2.elb.loadbalancer import LoadBalancer from boto.ec2.elb.listener import Listener LISTENERS_RESPONSE = b"""<?xml version="1.0" encoding="UTF-8"?> <DescribeLoadBalancersResponse xmlns="http://elasticloadbalancing.amazonaws.com/doc/2012-06-01/"> <DescribeLoadBalancersResult> <LoadBalancerDescriptions> <member> <SecurityGroups/> <CreatedTime>2013-07-09T19:18:00.520Z</CreatedTime> <LoadBalancerName>elb-boto-unit-test</LoadBalancerName> <HealthCheck> <Interval>30</Interval> <Target>TCP:8000</Target> <HealthyThreshold>10</HealthyThreshold> <Timeout>5</Timeout> <UnhealthyThreshold>2</UnhealthyThreshold> </HealthCheck> <ListenerDescriptions> <member> <PolicyNames/> <Listener> <Protocol>HTTP</Protocol> <LoadBalancerPort>80</LoadBalancerPort> <InstanceProtocol>HTTP</InstanceProtocol> <InstancePort>8000</InstancePort> </Listener> </member> <member> <PolicyNames/> <Listener> <Protocol>HTTP</Protocol> <LoadBalancerPort>8080</LoadBalancerPort> <InstanceProtocol>HTTP</InstanceProtocol> <InstancePort>80</InstancePort> </Listener> </member> <member> <PolicyNames/> <Listener> <Protocol>TCP</Protocol> <LoadBalancerPort>2525</LoadBalancerPort> <InstanceProtocol>TCP</InstanceProtocol> <InstancePort>25</InstancePort> </Listener> </member> </ListenerDescriptions> <Instances/> <Policies> <AppCookieStickinessPolicies/> <OtherPolicies/> <LBCookieStickinessPolicies/> </Policies> <AvailabilityZones> <member>us-east-1a</member> </AvailabilityZones> <CanonicalHostedZoneName>elb-boto-unit-test-408121642.us-east-1.elb.amazonaws.com</CanonicalHostedZoneName> <CanonicalHostedZoneNameID>Z3DZXE0Q79N41H</CanonicalHostedZoneNameID> <Scheme>internet-facing</Scheme> <SourceSecurityGroup> <OwnerAlias>amazon-elb</OwnerAlias> <GroupName>amazon-elb-sg</GroupName> </SourceSecurityGroup> <DNSName>elb-boto-unit-test-408121642.us-east-1.elb.amazonaws.com</DNSName> <BackendServerDescriptions/> <Subnets/> </member> </LoadBalancerDescriptions> </DescribeLoadBalancersResult> <ResponseMetadata> <RequestId>5763d932-e8cc-11e2-a940-11136cceffb8</RequestId> </ResponseMetadata> </DescribeLoadBalancersResponse> """ class TestListenerResponseParsing(unittest.TestCase): def test_parse_complex(self): rs = boto.resultset.ResultSet([ ('member', LoadBalancer) ]) h = boto.handler.XmlHandler(rs, None) xml.sax.parseString(LISTENERS_RESPONSE, h) listeners = rs[0].listeners self.assertEqual( sorted([l.get_complex_tuple() for l in listeners]), [ (80, 8000, 'HTTP', 'HTTP'), (2525, 25, 'TCP', 'TCP'), (8080, 80, 'HTTP', 'HTTP'), ] ) class TestListenerGetItem(unittest.TestCase): def test_getitem_for_http_listener(self): listener = Listener(load_balancer_port=80, instance_port=80, protocol='HTTP', instance_protocol='HTTP') self.assertEqual(listener[0], 80) self.assertEqual(listener[1], 80) self.assertEqual(listener[2], 'HTTP') self.assertEqual(listener[3], 'HTTP') def test_getitem_for_https_listener(self): listener = Listener(load_balancer_port=443, instance_port=80, protocol='HTTPS', instance_protocol='HTTP', ssl_certificate_id='look_at_me_im_an_arn') self.assertEqual(listener[0], 443) self.assertEqual(listener[1], 80) self.assertEqual(listener[2], 'HTTPS') self.assertEqual(listener[3], 'HTTP') self.assertEqual(listener[4], 'look_at_me_im_an_arn') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/elb/__init__.py 0000644 00000000000 15025013616 0014747 0 ustar 00 vendored/boto/tests/unit/ec2/test_snapshot.py 0000644 00000005255 15025013616 0015365 0 ustar 00 from tests.compat import OrderedDict from tests.unit import AWSMockServiceTestCase from boto.ec2.connection import EC2Connection from boto.ec2.snapshot import Snapshot class TestDescribeSnapshots(AWSMockServiceTestCase): connection_class = EC2Connection def default_body(self): return b""" <DescribeSnapshotsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <snapshotSet> <item> <snapshotId>snap-1a2b3c4d</snapshotId> <volumeId>vol-1a2b3c4d</volumeId> <status>pending</status> <startTime>YYYY-MM-DDTHH:MM:SS.SSSZ</startTime> <progress>30%</progress> <ownerId>111122223333</ownerId> <volumeSize>15</volumeSize> <description>Daily Backup</description> <tagSet> <item> <key>Purpose</key> <value>demo_db_14_backup</value> </item> </tagSet> <encrypted>false</encrypted> </item> </snapshotSet> </DescribeSnapshotsResponse> """ def test_describe_snapshots(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_snapshots(['snap-1a2b3c4d', 'snap-9f8e7d6c'], owner=['self', '111122223333'], restorable_by='999988887777', filters=OrderedDict((('status', 'pending'), ('tag-value', '*db_*')))) self.assert_request_parameters({ 'Action': 'DescribeSnapshots', 'SnapshotId.1': 'snap-1a2b3c4d', 'SnapshotId.2': 'snap-9f8e7d6c', 'Owner.1': 'self', 'Owner.2': '111122223333', 'RestorableBy.1': '999988887777', 'Filter.1.Name': 'status', 'Filter.1.Value.1': 'pending', 'Filter.2.Name': 'tag-value', 'Filter.2.Value.1': '*db_*'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(response), 1) self.assertIsInstance(response[0], Snapshot) self.assertEqual(response[0].id, 'snap-1a2b3c4d') vendored/boto/tests/unit/ec2/test_instance.py 0000644 00000025452 15025013616 0015333 0 ustar 00 #!/usr/bin/env python import base64 from tests.compat import unittest, mock from tests.unit import AWSMockServiceTestCase from boto.ec2.connection import EC2Connection DESCRIBE_INSTANCE_VPC = br"""<?xml version="1.0" encoding="UTF-8"?> <DescribeInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2012-10-01/"> <requestId>c6132c74-b524-4884-87f5-0f4bde4a9760</requestId> <reservationSet> <item> <reservationId>r-72ef4a0a</reservationId> <ownerId>184906166255</ownerId> <groupSet/> <instancesSet> <item> <instanceId>i-instance</instanceId> <imageId>ami-1624987f</imageId> <instanceState> <code>16</code> <name>running</name> </instanceState> <privateDnsName/> <dnsName/> <reason/> <keyName>mykeypair</keyName> <amiLaunchIndex>0</amiLaunchIndex> <productCodes/> <instanceType>m1.small</instanceType> <launchTime>2012-12-14T23:48:37.000Z</launchTime> <placement> <availabilityZone>us-east-1d</availabilityZone> <groupName/> <tenancy>default</tenancy> </placement> <kernelId>aki-88aa75e1</kernelId> <monitoring> <state>disabled</state> </monitoring> <subnetId>subnet-0dc60667</subnetId> <vpcId>vpc-id</vpcId> <privateIpAddress>10.0.0.67</privateIpAddress> <sourceDestCheck>true</sourceDestCheck> <groupSet> <item> <groupId>sg-id</groupId> <groupName>WebServerSG</groupName> </item> </groupSet> <architecture>x86_64</architecture> <rootDeviceType>ebs</rootDeviceType> <rootDeviceName>/dev/sda1</rootDeviceName> <blockDeviceMapping> <item> <deviceName>/dev/sda1</deviceName> <ebs> <volumeId>vol-id</volumeId> <status>attached</status> <attachTime>2012-12-14T23:48:43.000Z</attachTime> <deleteOnTermination>true</deleteOnTermination> </ebs> </item> </blockDeviceMapping> <virtualizationType>paravirtual</virtualizationType> <clientToken>foo</clientToken> <tagSet> <item> <key>Name</key> <value/> </item> </tagSet> <hypervisor>xen</hypervisor> <networkInterfaceSet> <item> <networkInterfaceId>eni-id</networkInterfaceId> <subnetId>subnet-id</subnetId> <vpcId>vpc-id</vpcId> <description>Primary network interface</description> <ownerId>ownerid</ownerId> <status>in-use</status> <privateIpAddress>10.0.0.67</privateIpAddress> <sourceDestCheck>true</sourceDestCheck> <groupSet> <item> <groupId>sg-id</groupId> <groupName>WebServerSG</groupName> </item> </groupSet> <attachment> <attachmentId>eni-attach-id</attachmentId> <deviceIndex>0</deviceIndex> <status>attached</status> <attachTime>2012-12-14T23:48:37.000Z</attachTime> <deleteOnTermination>true</deleteOnTermination> </attachment> <privateIpAddressesSet> <item> <privateIpAddress>10.0.0.67</privateIpAddress> <primary>true</primary> </item> <item> <privateIpAddress>10.0.0.54</privateIpAddress> <primary>false</primary> </item> <item> <privateIpAddress>10.0.0.55</privateIpAddress> <primary>false</primary> </item> </privateIpAddressesSet> </item> </networkInterfaceSet> <ebsOptimized>false</ebsOptimized> </item> </instancesSet> </item> </reservationSet> </DescribeInstancesResponse> """ RUN_INSTANCE_RESPONSE = br""" <RunInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2012-06-01/"> <requestId>ad4b83c2-f606-4c39-90c6-5dcc5be823e1</requestId> <reservationId>r-c5cef7a7</reservationId> <ownerId>ownerid</ownerId> <groupSet> <item> <groupId>sg-id</groupId> <groupName>SSH</groupName> </item> </groupSet> <instancesSet> <item> <instanceId>i-ff0f1299</instanceId> <imageId>ami-ed65ba84</imageId> <instanceState> <code>0</code> <name>pending</name> </instanceState> <privateDnsName/> <dnsName/> <reason/> <keyName>awskeypair</keyName> <amiLaunchIndex>0</amiLaunchIndex> <productCodes/> <instanceType>t1.micro</instanceType> <launchTime>2012-05-30T19:21:18.000Z</launchTime> <placement> <availabilityZone>us-east-1a</availabilityZone> <groupName/> <tenancy>default</tenancy> </placement> <kernelId>aki-b6aa75df</kernelId> <monitoring> <state>disabled</state> </monitoring> <groupSet> <item> <groupId>sg-99a710f1</groupId> <groupName>SSH</groupName> </item> </groupSet> <stateReason> <code>pending</code> <message>pending</message> </stateReason> <architecture>i386</architecture> <rootDeviceType>ebs</rootDeviceType> <rootDeviceName>/dev/sda1</rootDeviceName> <blockDeviceMapping/> <virtualizationType>paravirtual</virtualizationType> <clientToken/> <hypervisor>xen</hypervisor> <networkInterfaceSet/> <iamInstanceProfile> <arn>arn:aws:iam::ownerid:instance-profile/myinstanceprofile</arn> <id>iamid</id> </iamInstanceProfile> </item> </instancesSet> </RunInstancesResponse> """ class TestRunInstanceResponseParsing(unittest.TestCase): def testIAMInstanceProfileParsedCorrectly(self): ec2 = EC2Connection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') mock_response = mock.Mock() mock_response.read.return_value = RUN_INSTANCE_RESPONSE mock_response.status = 200 ec2.make_request = mock.Mock(return_value=mock_response) reservation = ec2.run_instances(image_id='ami-12345') self.assertEqual(len(reservation.instances), 1) instance = reservation.instances[0] self.assertEqual(instance.image_id, 'ami-ed65ba84') # iamInstanceProfile has an ID element, so we want to make sure # that this does not map to instance.id (which should be the # id of the ec2 instance). self.assertEqual(instance.id, 'i-ff0f1299') self.assertDictEqual( instance.instance_profile, {'arn': ('arn:aws:iam::ownerid:' 'instance-profile/myinstanceprofile'), 'id': 'iamid'}) class TestRunInstances(AWSMockServiceTestCase): connection_class = EC2Connection def default_body(self): # This is a dummy response return b""" <DescribeLaunchConfigurationsResponse> </DescribeLaunchConfigurationsResponse> """ def test_run_instances_user_data(self): self.set_http_response(status_code=200) response = self.service_connection.run_instances( image_id='123456', instance_type='m1.large', security_groups=['group1', 'group2'], user_data='#!/bin/bash' ) self.assert_request_parameters({ 'Action': 'RunInstances', 'ImageId': '123456', 'InstanceType': 'm1.large', 'UserData': base64.b64encode(b'#!/bin/bash').decode('utf-8'), 'MaxCount': 1, 'MinCount': 1, 'SecurityGroup.1': 'group1', 'SecurityGroup.2': 'group2', }, ignore_params_values=[ 'Version', 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp' ]) class TestDescribeInstances(AWSMockServiceTestCase): connection_class = EC2Connection def default_body(self): return DESCRIBE_INSTANCE_VPC def test_multiple_private_ip_addresses(self): self.set_http_response(status_code=200) api_response = self.service_connection.get_all_reservations() self.assertEqual(len(api_response), 1) instances = api_response[0].instances self.assertEqual(len(instances), 1) instance = instances[0] self.assertEqual(len(instance.interfaces), 1) interface = instance.interfaces[0] self.assertEqual(len(interface.private_ip_addresses), 3) addresses = interface.private_ip_addresses self.assertEqual(addresses[0].private_ip_address, '10.0.0.67') self.assertTrue(addresses[0].primary) self.assertEqual(addresses[1].private_ip_address, '10.0.0.54') self.assertFalse(addresses[1].primary) self.assertEqual(addresses[2].private_ip_address, '10.0.0.55') self.assertFalse(addresses[2].primary) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/test_securitygroup.py 0000644 00000020554 15025013616 0016451 0 ustar 00 #!/usr/bin/env python from tests.compat import unittest from tests.unit import AWSMockServiceTestCase from boto.ec2.connection import EC2Connection from boto.ec2.securitygroup import SecurityGroup DESCRIBE_SECURITY_GROUP = br"""<?xml version="1.0" encoding="UTF-8"?> <DescribeSecurityGroupsResponse xmlns="http://ec2.amazonaws.com/doc/2013-06-15/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <securityGroupInfo> <item> <ownerId>111122223333</ownerId> <groupId>sg-1a2b3c4d</groupId> <groupName>WebServers</groupName> <groupDescription>Web Servers</groupDescription> <vpcId/> <ipPermissions> <item> <ipProtocol>tcp</ipProtocol> <fromPort>80</fromPort> <toPort>80</toPort> <groups/> <ipRanges> <item> <cidrIp>0.0.0.0/0</cidrIp> </item> </ipRanges> </item> </ipPermissions> <ipPermissionsEgress/> </item> <item> <ownerId>111122223333</ownerId> <groupId>sg-2a2b3c4d</groupId> <groupName>RangedPortsBySource</groupName> <groupDescription>Group A</groupDescription> <ipPermissions> <item> <ipProtocol>tcp</ipProtocol> <fromPort>6000</fromPort> <toPort>7000</toPort> <groups> <item> <userId>111122223333</userId> <groupId>sg-3a2b3c4d</groupId> <groupName>Group B</groupName> </item> </groups> <ipRanges/> </item> </ipPermissions> <ipPermissionsEgress/> </item> </securityGroupInfo> </DescribeSecurityGroupsResponse>""" DESCRIBE_INSTANCES = br"""<?xml version="1.0" encoding="UTF-8"?> <DescribeInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2012-10-01/"> <requestId>c6132c74-b524-4884-87f5-0f4bde4a9760</requestId> <reservationSet> <item> <reservationId>r-72ef4a0a</reservationId> <ownerId>184906166255</ownerId> <groupSet/> <instancesSet> <item> <instanceId>i-instance</instanceId> <imageId>ami-1624987f</imageId> <instanceState> <code>16</code> <name>running</name> </instanceState> <privateDnsName/> <dnsName/> <reason/> <keyName>mykeypair</keyName> <amiLaunchIndex>0</amiLaunchIndex> <productCodes/> <instanceType>m1.small</instanceType> <launchTime>2012-12-14T23:48:37.000Z</launchTime> <placement> <availabilityZone>us-east-1d</availabilityZone> <groupName/> <tenancy>default</tenancy> </placement> <kernelId>aki-88aa75e1</kernelId> <monitoring> <state>disabled</state> </monitoring> <subnetId>subnet-0dc60667</subnetId> <vpcId>vpc-id</vpcId> <privateIpAddress>10.0.0.67</privateIpAddress> <sourceDestCheck>true</sourceDestCheck> <groupSet> <item> <groupId>sg-1a2b3c4d</groupId> <groupName>WebServerSG</groupName> </item> </groupSet> <architecture>x86_64</architecture> <rootDeviceType>ebs</rootDeviceType> <rootDeviceName>/dev/sda1</rootDeviceName> <blockDeviceMapping> <item> <deviceName>/dev/sda1</deviceName> <ebs> <volumeId>vol-id</volumeId> <status>attached</status> <attachTime>2012-12-14T23:48:43.000Z</attachTime> <deleteOnTermination>true</deleteOnTermination> </ebs> </item> </blockDeviceMapping> <virtualizationType>paravirtual</virtualizationType> <clientToken>foo</clientToken> <tagSet> <item> <key>Name</key> <value/> </item> </tagSet> <hypervisor>xen</hypervisor> <networkInterfaceSet> <item> <networkInterfaceId>eni-id</networkInterfaceId> <subnetId>subnet-id</subnetId> <vpcId>vpc-id</vpcId> <description>Primary network interface</description> <ownerId>ownerid</ownerId> <status>in-use</status> <privateIpAddress>10.0.0.67</privateIpAddress> <sourceDestCheck>true</sourceDestCheck> <groupSet> <item> <groupId>sg-id</groupId> <groupName>WebServerSG</groupName> </item> </groupSet> <attachment> <attachmentId>eni-attach-id</attachmentId> <deviceIndex>0</deviceIndex> <status>attached</status> <attachTime>2012-12-14T23:48:37.000Z</attachTime> <deleteOnTermination>true</deleteOnTermination> </attachment> <privateIpAddressesSet> <item> <privateIpAddress>10.0.0.67</privateIpAddress> <primary>true</primary> </item> <item> <privateIpAddress>10.0.0.54</privateIpAddress> <primary>false</primary> </item> <item> <privateIpAddress>10.0.0.55</privateIpAddress> <primary>false</primary> </item> </privateIpAddressesSet> </item> </networkInterfaceSet> <ebsOptimized>false</ebsOptimized> </item> </instancesSet> </item> </reservationSet> </DescribeInstancesResponse> """ class TestDescribeSecurityGroups(AWSMockServiceTestCase): connection_class = EC2Connection def test_get_instances(self): self.set_http_response(status_code=200, body=DESCRIBE_SECURITY_GROUP) groups = self.service_connection.get_all_security_groups() self.set_http_response(status_code=200, body=DESCRIBE_INSTANCES) instances = groups[0].instances() self.assertEqual(1, len(instances)) self.assertEqual(groups[0].id, instances[0].groups[0].id) class SecurityGroupTest(unittest.TestCase): def test_add_rule(self): sg = SecurityGroup() self.assertEqual(len(sg.rules), 0) # Regression: ``dry_run`` was being passed (but unhandled) before. sg.add_rule( ip_protocol='http', from_port='80', to_port='8080', src_group_name='groupy', src_group_owner_id='12345', cidr_ip='10.0.0.1', src_group_group_id='54321', dry_run=False ) self.assertEqual(len(sg.rules), 1) def test_remove_rule_on_empty_group(self): # Remove a rule from a group with no rules sg = SecurityGroup() with self.assertRaises(ValueError): sg.remove_rule('ip', 80, 80, None, None, None, None) vendored/boto/tests/unit/ec2/test_ec2object.py 0000644 00000017132 15025013616 0015363 0 ustar 00 #!/usr/bin/env python from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.ec2.connection import EC2Connection from boto.ec2.ec2object import TaggedEC2Object CREATE_TAGS_RESPONSE = br"""<?xml version="1.0" encoding="UTF-8"?> <CreateTagsResponse xmlns="http://ec2.amazonaws.com/doc/2014-05-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </CreateTagsResponse> """ DELETE_TAGS_RESPONSE = br"""<?xml version="1.0" encoding="UTF-8"?> <DeleteTagsResponse xmlns="http://ec2.amazonaws.com/doc/2014-05-01/"> <requestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</requestId> <return>true</return> </DeleteTagsResponse> """ class TestAddTags(AWSMockServiceTestCase): connection_class = EC2Connection def default_body(self): return CREATE_TAGS_RESPONSE def test_add_tag(self): self.set_http_response(status_code=200) taggedEC2Object = TaggedEC2Object(self.service_connection) taggedEC2Object.id = "i-abcd1234" taggedEC2Object.tags["already_present_key"] = "already_present_value" taggedEC2Object.add_tag("new_key", "new_value") self.assert_request_parameters({ 'ResourceId.1': 'i-abcd1234', 'Action': 'CreateTags', 'Tag.1.Key': 'new_key', 'Tag.1.Value': 'new_value'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(taggedEC2Object.tags, { "already_present_key": "already_present_value", "new_key": "new_value"}) def test_add_tags(self): self.set_http_response(status_code=200) taggedEC2Object = TaggedEC2Object(self.service_connection) taggedEC2Object.id = "i-abcd1234" taggedEC2Object.tags["already_present_key"] = "already_present_value" taggedEC2Object.add_tags({"key1": "value1", "key2": "value2"}) self.assert_request_parameters({ 'ResourceId.1': 'i-abcd1234', 'Action': 'CreateTags', 'Tag.1.Key': 'key1', 'Tag.1.Value': 'value1', 'Tag.2.Key': 'key2', 'Tag.2.Value': 'value2'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(taggedEC2Object.tags, { "already_present_key": "already_present_value", "key1": "value1", "key2": "value2"}) class TestRemoveTags(AWSMockServiceTestCase): connection_class = EC2Connection def default_body(self): return DELETE_TAGS_RESPONSE def test_remove_tag(self): self.set_http_response(status_code=200) taggedEC2Object = TaggedEC2Object(self.service_connection) taggedEC2Object.id = "i-abcd1234" taggedEC2Object.tags["key1"] = "value1" taggedEC2Object.tags["key2"] = "value2" taggedEC2Object.remove_tag("key1", "value1") self.assert_request_parameters({ 'ResourceId.1': 'i-abcd1234', 'Action': 'DeleteTags', 'Tag.1.Key': 'key1', 'Tag.1.Value': 'value1'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(taggedEC2Object.tags, {"key2": "value2"}) def test_remove_tag_no_value(self): self.set_http_response(status_code=200) taggedEC2Object = TaggedEC2Object(self.service_connection) taggedEC2Object.id = "i-abcd1234" taggedEC2Object.tags["key1"] = "value1" taggedEC2Object.tags["key2"] = "value2" taggedEC2Object.remove_tag("key1") self.assert_request_parameters({ 'ResourceId.1': 'i-abcd1234', 'Action': 'DeleteTags', 'Tag.1.Key': 'key1'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(taggedEC2Object.tags, {"key2": "value2"}) def test_remove_tag_empty_value(self): self.set_http_response(status_code=200) taggedEC2Object = TaggedEC2Object(self.service_connection) taggedEC2Object.id = "i-abcd1234" taggedEC2Object.tags["key1"] = "value1" taggedEC2Object.tags["key2"] = "value2" taggedEC2Object.remove_tag("key1", "") self.assert_request_parameters({ 'ResourceId.1': 'i-abcd1234', 'Action': 'DeleteTags', 'Tag.1.Key': 'key1', 'Tag.1.Value': ''}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(taggedEC2Object.tags, {"key1": "value1", "key2": "value2"}) def test_remove_tags(self): self.set_http_response(status_code=200) taggedEC2Object = TaggedEC2Object(self.service_connection) taggedEC2Object.id = "i-abcd1234" taggedEC2Object.tags["key1"] = "value1" taggedEC2Object.tags["key2"] = "value2" taggedEC2Object.remove_tags({"key1": "value1", "key2": "value2"}) self.assert_request_parameters({ 'ResourceId.1': 'i-abcd1234', 'Action': 'DeleteTags', 'Tag.1.Key': 'key1', 'Tag.1.Value': 'value1', 'Tag.2.Key': 'key2', 'Tag.2.Value': 'value2'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(taggedEC2Object.tags, {}) def test_remove_tags_wrong_values(self): self.set_http_response(status_code=200) taggedEC2Object = TaggedEC2Object(self.service_connection) taggedEC2Object.id = "i-abcd1234" taggedEC2Object.tags["key1"] = "value1" taggedEC2Object.tags["key2"] = "value2" taggedEC2Object.remove_tags({"key1": "value1", "key2": "value3"}) self.assert_request_parameters({ 'ResourceId.1': 'i-abcd1234', 'Action': 'DeleteTags', 'Tag.1.Key': 'key1', 'Tag.1.Value': 'value1', 'Tag.2.Key': 'key2', 'Tag.2.Value': 'value3'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(taggedEC2Object.tags, {"key2": "value2"}) def test_remove_tags_none_values(self): self.set_http_response(status_code=200) taggedEC2Object = TaggedEC2Object(self.service_connection) taggedEC2Object.id = "i-abcd1234" taggedEC2Object.tags["key1"] = "value1" taggedEC2Object.tags["key2"] = "value2" taggedEC2Object.remove_tags({"key1": "value1", "key2": None}) self.assert_request_parameters({ 'ResourceId.1': 'i-abcd1234', 'Action': 'DeleteTags', 'Tag.1.Key': 'key1', 'Tag.1.Value': 'value1', 'Tag.2.Key': 'key2'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(taggedEC2Object.tags, {}) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/__init__.py 0000644 00000000000 15025013616 0014205 0 ustar 00 vendored/boto/tests/unit/ec2/test_spotinstance.py 0000644 00000011755 15025013616 0016242 0 ustar 00 from tests.unit import AWSMockServiceTestCase from boto.ec2.connection import EC2Connection class TestCancelSpotInstanceRequests(AWSMockServiceTestCase): connection_class = EC2Connection def default_body(self): return b""" <CancelSpotInstanceRequestsResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-01/"> <requestId>59dbff89-35bd-4eac-99ed-be587EXAMPLE</requestId> <spotInstanceRequestSet> <item> <spotInstanceRequestId>sir-1a2b3c4d</spotInstanceRequestId> <state>cancelled</state> </item> <item> <spotInstanceRequestId>sir-9f8e7d6c</spotInstanceRequestId> <state>cancelled</state> </item> </spotInstanceRequestSet> </CancelSpotInstanceRequestsResponse> """ def test_cancel_spot_instance_requests(self): self.set_http_response(status_code=200) response = self.service_connection.cancel_spot_instance_requests(['sir-1a2b3c4d', 'sir-9f8e7d6c']) self.assert_request_parameters({ 'Action': 'CancelSpotInstanceRequests', 'SpotInstanceRequestId.1': 'sir-1a2b3c4d', 'SpotInstanceRequestId.2': 'sir-9f8e7d6c'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(response), 2) self.assertEqual(response[0].id, 'sir-1a2b3c4d') self.assertEqual(response[0].state, 'cancelled') self.assertEqual(response[1].id, 'sir-9f8e7d6c') self.assertEqual(response[1].state, 'cancelled') class TestGetSpotPriceHistory(AWSMockServiceTestCase): connection_class = EC2Connection def default_body(self): return b""" <DescribeSpotPriceHistoryResponse xmlns="http://ec2.amazonaws.com/doc/2013-10-15/"> <requestId>b6c6978c-bd13-4ad7-9bc8-6f0ac9d32bcc</requestId> <spotPriceHistorySet> <item> <instanceType>c3.large</instanceType> <productDescription>Linux/UNIX</productDescription> <spotPrice>0.032000</spotPrice> <timestamp>2013-12-28T12:17:43.000Z</timestamp> <availabilityZone>us-west-2c</availabilityZone> </item> <item> <instanceType>c3.large</instanceType> <productDescription>Windows (Amazon VPC)</productDescription> <spotPrice>0.104000</spotPrice> <timestamp>2013-12-28T07:49:40.000Z</timestamp> <availabilityZone>us-west-2b</availabilityZone> </item> </spotPriceHistorySet> <nextToken>q5GwEl5bMGjKq6YmhpDLJ7hEwyWU54jJC2GQ93n61vZV4s1+fzZ674xzvUlTihrl</nextToken> </DescribeSpotPriceHistoryResponse> """ def test_get_spot_price_history(self): self.set_http_response(status_code=200) response = self.service_connection.get_spot_price_history( instance_type='c3.large') self.assert_request_parameters({ 'Action': 'DescribeSpotPriceHistory', 'InstanceType': 'c3.large'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) self.assertEqual(len(response), 2) self.assertEqual(response.next_token, 'q5GwEl5bMGjKq6YmhpDLJ7hEwyWU54jJC2GQ93n61vZV4s1+fzZ674xzvUlTihrl') self.assertEqual(response.nextToken, 'q5GwEl5bMGjKq6YmhpDLJ7hEwyWU54jJC2GQ93n61vZV4s1+fzZ674xzvUlTihrl') self.assertEqual(response[0].instance_type, 'c3.large') self.assertEqual(response[0].availability_zone, 'us-west-2c') self.assertEqual(response[1].instance_type, 'c3.large') self.assertEqual(response[1].availability_zone, 'us-west-2b') response = self.service_connection.get_spot_price_history( filters={'instance-type': 'c3.large'}) self.assert_request_parameters({ 'Action': 'DescribeSpotPriceHistory', 'Filter.1.Name': 'instance-type', 'Filter.1.Value.1': 'c3.large'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) response = self.service_connection.get_spot_price_history( next_token='foobar') self.assert_request_parameters({ 'Action': 'DescribeSpotPriceHistory', 'NextToken': 'foobar'}, ignore_params_values=['AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Timestamp', 'Version']) vendored/boto/tests/unit/ec2/test_instancestatus.py 0000644 00000003423 15025013616 0016571 0 ustar 00 #!/usr/bin/env python from tests.compat import mock, unittest from boto.ec2.connection import EC2Connection INSTANCE_STATUS_RESPONSE = br"""<?xml version="1.0" encoding="UTF-8"?> <DescribeInstanceStatusResponse xmlns="http://ec2.amazonaws.com/doc/2013-02-01/"> <requestId>3be1508e-c444-4fef-89cc-0b1223c4f02fEXAMPLE</requestId> <nextToken>page-2</nextToken> <instanceStatusSet /> </DescribeInstanceStatusResponse> """ class TestInstanceStatusResponseParsing(unittest.TestCase): def test_next_token(self): ec2 = EC2Connection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') mock_response = mock.Mock() mock_response.read.return_value = INSTANCE_STATUS_RESPONSE mock_response.status = 200 ec2.make_request = mock.Mock(return_value=mock_response) all_statuses = ec2.get_all_instance_status() self.assertNotIn('IncludeAllInstances', ec2.make_request.call_args[0][1]) self.assertEqual(all_statuses.next_token, 'page-2') def test_include_all_instances(self): ec2 = EC2Connection(aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') mock_response = mock.Mock() mock_response.read.return_value = INSTANCE_STATUS_RESPONSE mock_response.status = 200 ec2.make_request = mock.Mock(return_value=mock_response) all_statuses = ec2.get_all_instance_status(include_all_instances=True) self.assertIn('IncludeAllInstances', ec2.make_request.call_args[0][1]) self.assertEqual('true', ec2.make_request.call_args[0][1]['IncludeAllInstances']) self.assertEqual(all_statuses.next_token, 'page-2') if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/ec2/test_volume.py 0000644 00000026260 15025013616 0015034 0 ustar 00 from tests.compat import mock, unittest from boto.ec2.snapshot import Snapshot from boto.ec2.tag import Tag, TagSet from boto.ec2.volume import Volume, AttachmentSet, VolumeAttribute class VolumeTests(unittest.TestCase): def setUp(self): self.attach_data = AttachmentSet() self.attach_data.id = 1 self.attach_data.instance_id = 2 self.attach_data.status = "some status" self.attach_data.attach_time = 5 self.attach_data.device = "/dev/null" self.volume_one = Volume() self.volume_one.id = 1 self.volume_one.create_time = 5 self.volume_one.status = "one_status" self.volume_one.size = "one_size" self.volume_one.snapshot_id = 1 self.volume_one.attach_data = self.attach_data self.volume_one.zone = "one_zone" self.volume_two = Volume() self.volume_two.connection = mock.Mock() self.volume_two.id = 1 self.volume_two.create_time = 6 self.volume_two.status = "two_status" self.volume_two.size = "two_size" self.volume_two.snapshot_id = 2 self.volume_two.attach_data = None self.volume_two.zone = "two_zone" @mock.patch("boto.ec2.volume.TaggedEC2Object.startElement") def test_startElement_calls_TaggedEC2Object_startElement_with_correct_args(self, startElement): volume = Volume() volume.startElement("some name", "some attrs", None) startElement.assert_called_with( "some name", "some attrs", None ) @mock.patch("boto.ec2.volume.TaggedEC2Object.startElement") def test_startElement_retval_not_None_returns_correct_thing(self, startElement): tag_set = mock.Mock(TagSet) startElement.return_value = tag_set volume = Volume() retval = volume.startElement(None, None, None) self.assertEqual(retval, tag_set) @mock.patch("boto.ec2.volume.TaggedEC2Object.startElement") @mock.patch("boto.resultset.ResultSet") def test_startElement_with_name_tagSet_calls_ResultSet(self, ResultSet, startElement): startElement.return_value = None result_set = mock.Mock(ResultSet([("item", Tag)])) volume = Volume() volume.tags = result_set retval = volume.startElement("tagSet", None, None) self.assertEqual(retval, volume.tags) @mock.patch("boto.ec2.volume.TaggedEC2Object.startElement") def test_startElement_with_name_attachmentSet_returns_AttachmentSet(self, startElement): startElement.return_value = None attach_data = AttachmentSet() volume = Volume() volume.attach_data = attach_data retval = volume.startElement("attachmentSet", None, None) self.assertEqual(retval, volume.attach_data) @mock.patch("boto.ec2.volume.TaggedEC2Object.startElement") def test_startElement_else_returns_None(self, startElement): startElement.return_value = None volume = Volume() retval = volume.startElement("not tagSet or attachmentSet", None, None) self.assertEqual(retval, None) def check_that_attribute_has_been_set(self, name, value, attribute, obj_value=None): volume = Volume() volume.endElement(name, value, None) expected_value = obj_value if obj_value is not None else value self.assertEqual(getattr(volume, attribute), expected_value) def test_endElement_sets_correct_attributes_with_values(self): for arguments in [("volumeId", "some value", "id"), ("createTime", "some time", "create_time"), ("status", "some status", "status"), ("size", 5, "size"), ("snapshotId", 1, "snapshot_id"), ("availabilityZone", "some zone", "zone"), ("someName", "some value", "someName"), ("encrypted", "true", "encrypted", True)]: self.check_that_attribute_has_been_set(*arguments) def test_endElement_with_name_status_and_empty_string_value_doesnt_set_status(self): volume = Volume() volume.endElement("status", "", None) self.assertNotEqual(volume.status, "") def test_update_with_result_set_greater_than_0_updates_dict(self): self.volume_two.connection.get_all_volumes.return_value = [self.volume_one] self.volume_two.update() assert all([self.volume_two.create_time == 5, self.volume_two.status == "one_status", self.volume_two.size == "one_size", self.volume_two.snapshot_id == 1, self.volume_two.attach_data == self.attach_data, self.volume_two.zone == "one_zone"]) def test_update_with_validate_true_raises_value_error(self): self.volume_one.connection = mock.Mock() self.volume_one.connection.get_all_volumes.return_value = [] with self.assertRaisesRegex(ValueError, "^1 is not a valid Volume ID$"): self.volume_one.update(True) def test_update_returns_status(self): self.volume_one.connection = mock.Mock() self.volume_one.connection.get_all_volumes.return_value = [self.volume_two] retval = self.volume_one.update() self.assertEqual(retval, "two_status") def test_delete_calls_delete_volume(self): self.volume_one.connection = mock.Mock() self.volume_one.delete() self.volume_one.connection.delete_volume.assert_called_with( 1, dry_run=False ) def test_attach_calls_attach_volume(self): self.volume_one.connection = mock.Mock() self.volume_one.attach("instance_id", "/dev/null") self.volume_one.connection.attach_volume.assert_called_with( 1, "instance_id", "/dev/null", dry_run=False ) def test_detach_calls_detach_volume(self): self.volume_one.connection = mock.Mock() self.volume_one.detach() self.volume_one.connection.detach_volume.assert_called_with( 1, 2, "/dev/null", False, dry_run=False) def test_detach_with_no_attach_data(self): self.volume_two.connection = mock.Mock() self.volume_two.detach() self.volume_two.connection.detach_volume.assert_called_with( 1, None, None, False, dry_run=False) def test_detach_with_force_calls_detach_volume_with_force(self): self.volume_one.connection = mock.Mock() self.volume_one.detach(True) self.volume_one.connection.detach_volume.assert_called_with( 1, 2, "/dev/null", True, dry_run=False) def test_create_snapshot_calls_connection_create_snapshot(self): self.volume_one.connection = mock.Mock() self.volume_one.create_snapshot() self.volume_one.connection.create_snapshot.assert_called_with( 1, None, dry_run=False ) def test_create_snapshot_with_description(self): self.volume_one.connection = mock.Mock() self.volume_one.create_snapshot("some description") self.volume_one.connection.create_snapshot.assert_called_with( 1, "some description", dry_run=False ) def test_volume_state_returns_status(self): retval = self.volume_one.volume_state() self.assertEqual(retval, "one_status") def test_attachment_state_returns_state(self): retval = self.volume_one.attachment_state() self.assertEqual(retval, "some status") def test_attachment_state_no_attach_data_returns_None(self): retval = self.volume_two.attachment_state() self.assertEqual(retval, None) def test_snapshots_returns_snapshots(self): snapshot_one = Snapshot() snapshot_one.volume_id = 1 snapshot_two = Snapshot() snapshot_two.volume_id = 2 self.volume_one.connection = mock.Mock() self.volume_one.connection.get_all_snapshots.return_value = [snapshot_one, snapshot_two] retval = self.volume_one.snapshots() self.assertEqual(retval, [snapshot_one]) def test_snapshots__with_owner_and_restorable_by(self): self.volume_one.connection = mock.Mock() self.volume_one.connection.get_all_snapshots.return_value = [] self.volume_one.snapshots("owner", "restorable_by") self.volume_one.connection.get_all_snapshots.assert_called_with( owner="owner", restorable_by="restorable_by", dry_run=False) class AttachmentSetTests(unittest.TestCase): def check_that_attribute_has_been_set(self, name, value, attribute): attachment_set = AttachmentSet() attachment_set.endElement(name, value, None) self.assertEqual(getattr(attachment_set, attribute), value) def test_endElement_with_name_volumeId_sets_id(self): return self.check_that_attribute_has_been_set("volumeId", "some value", "id") def test_endElement_with_name_instanceId_sets_instance_id(self): return self.check_that_attribute_has_been_set("instanceId", 1, "instance_id") def test_endElement_with_name_status_sets_status(self): return self.check_that_attribute_has_been_set("status", "some value", "status") def test_endElement_with_name_attachTime_sets_attach_time(self): return self.check_that_attribute_has_been_set("attachTime", 5, "attach_time") def test_endElement_with_name_device_sets_device(self): return self.check_that_attribute_has_been_set("device", "/dev/null", "device") def test_endElement_with_other_name_sets_other_name_attribute(self): return self.check_that_attribute_has_been_set("someName", "some value", "someName") class VolumeAttributeTests(unittest.TestCase): def setUp(self): self.volume_attribute = VolumeAttribute() self.volume_attribute._key_name = "key_name" self.volume_attribute.attrs = {"key_name": False} def test_startElement_with_name_autoEnableIO_sets_key_name(self): self.volume_attribute.startElement("autoEnableIO", None, None) self.assertEqual(self.volume_attribute._key_name, "autoEnableIO") def test_startElement_without_name_autoEnableIO_returns_None(self): retval = self.volume_attribute.startElement("some name", None, None) self.assertEqual(retval, None) def test_endElement_with_name_value_and_value_true_sets_attrs_key_name_True(self): self.volume_attribute.endElement("value", "true", None) self.assertEqual(self.volume_attribute.attrs['key_name'], True) def test_endElement_with_name_value_and_value_false_sets_attrs_key_name_False(self): self.volume_attribute._key_name = "other_key_name" self.volume_attribute.endElement("value", "false", None) self.assertEqual(self.volume_attribute.attrs['other_key_name'], False) def test_endElement_with_name_volumeId_sets_id(self): self.volume_attribute.endElement("volumeId", "some_value", None) self.assertEqual(self.volume_attribute.id, "some_value") def test_endElement_with_other_name_sets_other_name_attribute(self): self.volume_attribute.endElement("someName", "some value", None) self.assertEqual(self.volume_attribute.someName, "some value") if __name__ == "__main__": unittest.main() vendored/boto/tests/unit/cloudtrail/test_layer1.py 0000644 00000005253 15025013616 0016412 0 ustar 00 #!/usr/bin/env python import json from boto.cloudtrail.layer1 import CloudTrailConnection from tests.unit import AWSMockServiceTestCase class TestDescribeTrails(AWSMockServiceTestCase): connection_class = CloudTrailConnection def default_body(self): return b''' {"trailList": [ { "IncludeGlobalServiceEvents": false, "Name": "test", "SnsTopicName": "cloudtrail-1", "S3BucketName": "cloudtrail-1" } ] }''' def test_describe(self): self.set_http_response(status_code=200) api_response = self.service_connection.describe_trails() self.assertEqual(1, len(api_response['trailList'])) self.assertEqual('test', api_response['trailList'][0]['Name']) self.assert_request_parameters({}) target = self.actual_request.headers['X-Amz-Target'] self.assertTrue('DescribeTrails' in target) def test_describe_name_list(self): self.set_http_response(status_code=200) api_response = self.service_connection.describe_trails( trail_name_list=['test']) self.assertEqual(1, len(api_response['trailList'])) self.assertEqual('test', api_response['trailList'][0]['Name']) self.assertEqual(json.dumps({ 'trailNameList': ['test'] }), self.actual_request.body.decode('utf-8')) target = self.actual_request.headers['X-Amz-Target'] self.assertTrue('DescribeTrails' in target) class TestCreateTrail(AWSMockServiceTestCase): connection_class = CloudTrailConnection def default_body(self): return b''' {"trail": { "IncludeGlobalServiceEvents": false, "Name": "test", "SnsTopicName": "cloudtrail-1", "S3BucketName": "cloudtrail-1" } }''' def test_create(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_trail( 'test', 'cloudtrail-1', sns_topic_name='cloudtrail-1', include_global_service_events=False) self.assertEqual('test', api_response['trail']['Name']) self.assertEqual('cloudtrail-1', api_response['trail']['S3BucketName']) self.assertEqual('cloudtrail-1', api_response['trail']['SnsTopicName']) self.assertEqual(False, api_response['trail']['IncludeGlobalServiceEvents']) target = self.actual_request.headers['X-Amz-Target'] self.assertTrue('CreateTrail' in target) vendored/boto/tests/unit/cloudtrail/__init__.py 0000644 00000000000 15025013616 0015676 0 ustar 00 vendored/boto/tests/unit/awslambda/test_awslambda.py 0000644 00000010317 15025013616 0016736 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import tempfile import shutil import os import socket from boto.compat import json from boto.awslambda.layer1 import AWSLambdaConnection from tests.unit import AWSMockServiceTestCase from tests.compat import mock class TestAWSLambda(AWSMockServiceTestCase): connection_class = AWSLambdaConnection def default_body(self): return b'{}' def test_upload_function_binary(self): self.set_http_response(status_code=201) function_data = b'This is my file' self.service_connection.upload_function( function_name='my-function', function_zip=function_data, role='myrole', handler='myhandler', mode='event', runtime='nodejs' ) self.assertEqual(self.actual_request.body, function_data) self.assertEqual( self.actual_request.headers['Content-Length'], str(len(function_data)) ) self.assertEqual( self.actual_request.path, '/2014-11-13/functions/my-function?Handler=myhandler&Mode' '=event&Role=myrole&Runtime=nodejs' ) def test_upload_function_file(self): self.set_http_response(status_code=201) rootdir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, rootdir) filename = 'test_file' function_data = b'This is my file' full_path = os.path.join(rootdir, filename) with open(full_path, 'wb') as f: f.write(function_data) with open(full_path, 'rb') as f: self.service_connection.upload_function( function_name='my-function', function_zip=f, role='myrole', handler='myhandler', mode='event', runtime='nodejs' ) self.assertEqual(self.actual_request.body.read(), function_data) self.assertEqual( self.actual_request.headers['Content-Length'], str(len(function_data)) ) self.assertEqual( self.actual_request.path, '/2014-11-13/functions/my-function?Handler=myhandler&Mode' '=event&Role=myrole&Runtime=nodejs' ) def test_upload_function_unseekable_file_no_tell(self): sock = socket.socket() with self.assertRaises(TypeError): self.service_connection.upload_function( function_name='my-function', function_zip=sock, role='myrole', handler='myhandler', mode='event', runtime='nodejs' ) def test_upload_function_unseekable_file_cannot_tell(self): mock_file = mock.Mock() mock_file.tell.side_effect = IOError with self.assertRaises(TypeError): self.service_connection.upload_function( function_name='my-function', function_zip=mock_file, role='myrole', handler='myhandler', mode='event', runtime='nodejs' ) vendored/boto/tests/unit/awslambda/__init__.py 0000644 00000002143 15025013616 0015501 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/unit/cloudsearch2/test_document.py 0000644 00000027443 15025013616 0017254 0 ustar 00 #!/usr/bin env python from boto.cloudsearch2.domain import Domain from boto.cloudsearch2.layer1 import CloudSearchConnection from tests.unit import unittest, AWSMockServiceTestCase from httpretty import HTTPretty from mock import MagicMock import json from boto.cloudsearch2.document import DocumentServiceConnection from boto.cloudsearch2.document import CommitMismatchError, EncodingError, \ ContentTooLongError, DocumentServiceConnection import boto from tests.unit.cloudsearch2 import DEMO_DOMAIN_DATA class CloudSearchDocumentConnectionTest(AWSMockServiceTestCase): connection_class = CloudSearchConnection def test_proxy(self): conn = self.service_connection conn.proxy = "127.0.0.1" conn.proxy_user = "john.doe" conn.proxy_pass="p4ssw0rd" conn.proxy_port="8180" conn.use_proxy = True domain = Domain(conn, DEMO_DOMAIN_DATA) service = DocumentServiceConnection(domain=domain) self.assertEqual(service.proxy, {'http': 'http://john.doe:p4ssw0rd@127.0.0.1:8180'}) class CloudSearchDocumentTest(unittest.TestCase): def setUp(self): HTTPretty.enable() HTTPretty.register_uri( HTTPretty.POST, ("http://doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com/" "2013-01-01/documents/batch"), body=json.dumps(self.response).encode('utf-8'), content_type="application/json") def tearDown(self): HTTPretty.disable() class CloudSearchDocumentSingleTest(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 1, 'deletes': 0, } def test_cloudsearch_add_basics(self): """ Check that a simple add document actually sends an add document request to AWS. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8'))[0] self.assertEqual(args['type'], 'add') def test_cloudsearch_add_single_basic(self): """ Check that a simple add document sends correct document metadata to AWS. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8'))[0] self.assertEqual(args['id'], '1234') self.assertEqual(args['type'], 'add') def test_cloudsearch_add_single_fields(self): """ Check that a simple add document sends the actual document to AWS. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8'))[0] self.assertEqual(args['fields']['category'], ['cat_a', 'cat_b', 'cat_c']) self.assertEqual(args['fields']['id'], '1234') self.assertEqual(args['fields']['title'], 'Title 1') def test_cloudsearch_add_single_result(self): """ Check that the reply from adding a single document is correctly parsed. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) doc = document.commit() self.assertEqual(doc.status, 'success') self.assertEqual(doc.adds, 1) self.assertEqual(doc.deletes, 0) self.assertEqual(doc.doc_service, document) class CloudSearchDocumentMultipleAddTest(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 3, 'deletes': 0, } objs = { '1234': { 'fields': {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}}, '1235': { 'fields': {"id": "1235", "title": "Title 2", "category": ["cat_b", "cat_c", "cat_d"]}}, '1236': { 'fields': {"id": "1236", "title": "Title 3", "category": ["cat_e", "cat_f", "cat_g"]}}, } def test_cloudsearch_add_basics(self): """Check that multiple documents are added correctly to AWS""" document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") for (key, obj) in self.objs.items(): document.add(key, obj['fields']) document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8')) for arg in args: self.assertTrue(arg['id'] in self.objs) self.assertEqual(arg['fields']['id'], self.objs[arg['id']]['fields']['id']) self.assertEqual(arg['fields']['title'], self.objs[arg['id']]['fields']['title']) self.assertEqual(arg['fields']['category'], self.objs[arg['id']]['fields']['category']) def test_cloudsearch_add_results(self): """ Check that the result from adding multiple documents is parsed correctly. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") for (key, obj) in self.objs.items(): document.add(key, obj['fields']) doc = document.commit() self.assertEqual(doc.status, 'success') self.assertEqual(doc.adds, len(self.objs)) self.assertEqual(doc.deletes, 0) class CloudSearchDocumentDelete(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 0, 'deletes': 1, } def test_cloudsearch_delete(self): """ Test that the request for a single document deletion is done properly. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.delete("5") document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8'))[0] self.assertEqual(args['type'], 'delete') self.assertEqual(args['id'], '5') def test_cloudsearch_delete_results(self): """ Check that the result of a single document deletion is parsed properly. """ document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.delete("5") doc = document.commit() self.assertEqual(doc.status, 'success') self.assertEqual(doc.adds, 0) self.assertEqual(doc.deletes, 1) class CloudSearchDocumentDeleteMultiple(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 0, 'deletes': 2, } def test_cloudsearch_delete_multiples(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.delete("5") document.delete("6") document.commit() args = json.loads(HTTPretty.last_request.body.decode('utf-8')) self.assertEqual(len(args), 2) for arg in args: self.assertEqual(arg['type'], 'delete') class CloudSearchSDFManipulation(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 1, 'deletes': 0, } def test_cloudsearch_initial_sdf_is_blank(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") self.assertEqual(document.get_sdf(), '[]') def test_cloudsearch_single_document_sdf(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) self.assertNotEqual(document.get_sdf(), '[]') document.clear_sdf() self.assertEqual(document.get_sdf(), '[]') class CloudSearchBadSDFTesting(CloudSearchDocumentTest): response = { 'status': 'success', 'adds': 1, 'deletes': 0, } def test_cloudsearch_erroneous_sdf(self): original = boto.log.error boto.log.error = MagicMock() document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", {"id": "1234", "title": None, "category": ["cat_a", "cat_b", "cat_c"]}) document.commit() self.assertNotEqual(len(boto.log.error.call_args_list), 1) boto.log.error = original class CloudSearchDocumentErrorBadUnicode(CloudSearchDocumentTest): response = { 'status': 'error', 'adds': 0, 'deletes': 0, 'errors': [{'message': 'Illegal Unicode character in document'}] } def test_fake_bad_unicode(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) self.assertRaises(EncodingError, document.commit) class CloudSearchDocumentErrorDocsTooBig(CloudSearchDocumentTest): response = { 'status': 'error', 'adds': 0, 'deletes': 0, 'errors': [{'message': 'The Content-Length is too long'}] } def test_fake_docs_too_big(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) self.assertRaises(ContentTooLongError, document.commit) class CloudSearchDocumentErrorMismatch(CloudSearchDocumentTest): response = { 'status': 'error', 'adds': 0, 'deletes': 0, 'errors': [{'message': 'Something went wrong'}] } def test_fake_failure(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com") document.add("1234", {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) self.assertRaises(CommitMismatchError, document.commit) def test_attached_errors_list(self): document = DocumentServiceConnection( endpoint="doc-demo-userdomain.us-east-1.cloudsearch.amazonaws.com" ) document.add("1234", {"id": "1234", "title": "Title 1", "category": ["cat_a", "cat_b", "cat_c"]}) try: document.commit() #If we get here that is a problem #Working around the assertRaises not giving me exception instance. self.assertTrue(False) except CommitMismatchError as e: self.assertTrue(hasattr(e, 'errors')) self.assertIsInstance(e.errors, list) self.assertEqual(e.errors[0], self.response['errors'][0].get('message')) vendored/boto/tests/unit/cloudsearch2/test_exceptions.py 0000644 00000002673 15025013616 0017615 0 ustar 00 from boto.compat import json from tests.compat import mock, unittest from tests.unit.cloudsearch2.test_search import HOSTNAME, \ CloudSearchSearchBaseTest from boto.cloudsearch2.search import SearchConnection, SearchServiceException def fake_loads_value_error(content, *args, **kwargs): """Callable to generate a fake ValueError""" raise ValueError("HAHAHA! Totally not simplejson & you gave me bad JSON.") def fake_loads_json_error(content, *args, **kwargs): """Callable to generate a fake JSONDecodeError""" raise json.JSONDecodeError('Using simplejson & you gave me bad JSON.', '', 0) class CloudSearchJSONExceptionTest(CloudSearchSearchBaseTest): response = b'{}' def test_no_simplejson_value_error(self): with mock.patch.object(json, 'loads', fake_loads_value_error): search = SearchConnection(endpoint=HOSTNAME) with self.assertRaisesRegex(SearchServiceException, 'non-json'): search.search(q='test') @unittest.skipUnless(hasattr(json, 'JSONDecodeError'), 'requires simplejson') def test_simplejson_jsondecodeerror(self): with mock.patch.object(json, 'loads', fake_loads_json_error): search = SearchConnection(endpoint=HOSTNAME) with self.assertRaisesRegex(SearchServiceException, 'non-json'): search.search(q='test') vendored/boto/tests/unit/cloudsearch2/test_connection.py 0000644 00000020126 15025013616 0017564 0 ustar 00 #!/usr/bin env python from tests.unit import AWSMockServiceTestCase from boto.cloudsearch2.domain import Domain from boto.cloudsearch2.layer1 import CloudSearchConnection class TestCloudSearchCreateDomain(AWSMockServiceTestCase): connection_class = CloudSearchConnection def default_body(self): return b""" { "CreateDomainResponse": { "CreateDomainResult": { "DomainStatus": { "SearchInstanceType": null, "DomainId": "1234567890/demo", "DomainName": "demo", "Deleted": false, "SearchInstanceCount": 0, "Created": true, "SearchService": { "Endpoint": "search-demo.us-east-1.cloudsearch.amazonaws.com" }, "RequiresIndexDocuments": false, "Processing": false, "DocService": { "Endpoint": "doc-demo.us-east-1.cloudsearch.amazonaws.com" }, "ARN": "arn:aws:cs:us-east-1:1234567890:domain/demo", "SearchPartitionCount": 0 } }, "ResponseMetadata": { "RequestId": "00000000-0000-0000-0000-000000000000" } } } """ def test_create_domain(self): self.set_http_response(status_code=200) self.service_connection.create_domain('demo') self.assert_request_parameters({ 'Action': 'CreateDomain', 'ContentType': 'JSON', 'DomainName': 'demo', 'Version': '2013-01-01', }) def test_cloudsearch_connect_result_endpoints(self): """Check that endpoints & ARNs are correctly returned from AWS""" self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') domain = Domain(self, api_response['CreateDomainResponse'] ['CreateDomainResult'] ['DomainStatus']) self.assertEqual( domain.doc_service_endpoint, "doc-demo.us-east-1.cloudsearch.amazonaws.com") self.assertEqual(domain.service_arn, "arn:aws:cs:us-east-1:1234567890:domain/demo") self.assertEqual( domain.search_service_endpoint, "search-demo.us-east-1.cloudsearch.amazonaws.com") def test_cloudsearch_connect_result_statuses(self): """Check that domain statuses are correctly returned from AWS""" self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') domain = Domain(self, api_response['CreateDomainResponse'] ['CreateDomainResult'] ['DomainStatus']) self.assertEqual(domain.created, True) self.assertEqual(domain.processing, False) self.assertEqual(domain.requires_index_documents, False) self.assertEqual(domain.deleted, False) def test_cloudsearch_connect_result_details(self): """Check that the domain information is correctly returned from AWS""" self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') domain = Domain(self, api_response['CreateDomainResponse'] ['CreateDomainResult'] ['DomainStatus']) self.assertEqual(domain.id, "1234567890/demo") self.assertEqual(domain.name, "demo") def test_cloudsearch_documentservice_creation(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') domain = Domain(self, api_response['CreateDomainResponse'] ['CreateDomainResult'] ['DomainStatus']) document = domain.get_document_service() self.assertEqual( document.endpoint, "doc-demo.us-east-1.cloudsearch.amazonaws.com") def test_cloudsearch_searchservice_creation(self): self.set_http_response(status_code=200) api_response = self.service_connection.create_domain('demo') domain = Domain(self, api_response['CreateDomainResponse'] ['CreateDomainResult'] ['DomainStatus']) search = domain.get_search_service() self.assertEqual( search.endpoint, "search-demo.us-east-1.cloudsearch.amazonaws.com") class CloudSearchConnectionDeletionTest(AWSMockServiceTestCase): connection_class = CloudSearchConnection def default_body(self): return b""" { "DeleteDomainResponse": { "DeleteDomainResult": { "DomainStatus": { "SearchInstanceType": null, "DomainId": "1234567890/demo", "DomainName": "test", "Deleted": true, "SearchInstanceCount": 0, "Created": true, "SearchService": { "Endpoint": null }, "RequiresIndexDocuments": false, "Processing": false, "DocService": { "Endpoint": null }, "ARN": "arn:aws:cs:us-east-1:1234567890:domain/demo", "SearchPartitionCount": 0 } }, "ResponseMetadata": { "RequestId": "00000000-0000-0000-0000-000000000000" } } } """ def test_cloudsearch_deletion(self): """ Check that the correct arguments are sent to AWS when creating a cloudsearch connection. """ self.set_http_response(status_code=200) self.service_connection.delete_domain('demo') self.assert_request_parameters({ 'Action': 'DeleteDomain', 'ContentType': 'JSON', 'DomainName': 'demo', 'Version': '2013-01-01', }) class CloudSearchConnectionIndexDocumentTest(AWSMockServiceTestCase): connection_class = CloudSearchConnection def default_body(self): return b""" { "IndexDocumentsResponse": { "IndexDocumentsResult": { "FieldNames": [ "average_score", "brand_id", "colors", "context", "context_owner", "created_at", "creator_id", "description", "file_size", "format", "has_logo", "has_messaging", "height", "image_id", "ingested_from", "is_advertising", "is_photo", "is_reviewed", "modified_at", "subject_date", "tags", "title", "width" ] }, "ResponseMetadata": { "RequestId": "42e618d9-c4d9-11e3-8242-c32da3041159" } } } """ def test_cloudsearch_index_documents(self): """ Check that the correct arguments are sent to AWS when indexing a domain. """ self.set_http_response(status_code=200) self.service_connection.index_documents('demo') self.assert_request_parameters({ 'Action': 'IndexDocuments', 'ContentType': 'JSON', 'DomainName': 'demo', 'Version': '2013-01-01', }) def test_cloudsearch_index_documents_resp(self): """ Check that the AWS response is being parsed correctly when indexing a domain. """ self.set_http_response(status_code=200) api_response = self.service_connection.index_documents('demo') fields = (api_response['IndexDocumentsResponse'] ['IndexDocumentsResult'] ['FieldNames']) self.assertEqual(fields, ['average_score', 'brand_id', 'colors', 'context', 'context_owner', 'created_at', 'creator_id', 'description', 'file_size', 'format', 'has_logo', 'has_messaging', 'height', 'image_id', 'ingested_from', 'is_advertising', 'is_photo', 'is_reviewed', 'modified_at', 'subject_date', 'tags', 'title', 'width']) vendored/boto/tests/unit/cloudsearch2/test_search.py 0000644 00000030050 15025013616 0016667 0 ustar 00 #!/usr/bin env python from boto.cloudsearch2.domain import Domain from boto.cloudsearch2.layer1 import CloudSearchConnection from tests.compat import mock, unittest from httpretty import HTTPretty import json from boto.cloudsearch2.search import SearchConnection, SearchServiceException from boto.compat import six, map from tests.unit import AWSMockServiceTestCase from tests.unit.cloudsearch2 import DEMO_DOMAIN_DATA from tests.unit.cloudsearch2.test_connection import TestCloudSearchCreateDomain HOSTNAME = "search-demo-userdomain.us-east-1.cloudsearch.amazonaws.com" FULL_URL = 'http://%s/2013-01-01/search' % HOSTNAME class CloudSearchSearchBaseTest(unittest.TestCase): hits = [ { 'id': '12341', 'fields': { 'title': 'Document 1', 'rank': 1 } }, { 'id': '12342', 'fields': { 'title': 'Document 2', 'rank': 2 } }, { 'id': '12343', 'fields': { 'title': 'Document 3', 'rank': 3 } }, { 'id': '12344', 'fields': { 'title': 'Document 4', 'rank': 4 } }, { 'id': '12345', 'fields': { 'title': 'Document 5', 'rank': 5 } }, { 'id': '12346', 'fields': { 'title': 'Document 6', 'rank': 6 } }, { 'id': '12347', 'fields': { 'title': 'Document 7', 'rank': 7 } }, ] content_type = "text/xml" response_status = 200 def get_args(self, requestline): (_, request, _) = requestline.split(b" ") (_, request) = request.split(b"?", 1) args = six.moves.urllib.parse.parse_qs(request) return args def setUp(self): HTTPretty.enable() body = self.response if not isinstance(body, bytes): body = json.dumps(body).encode('utf-8') HTTPretty.register_uri(HTTPretty.GET, FULL_URL, body=body, content_type=self.content_type, status=self.response_status) def tearDown(self): HTTPretty.disable() class CloudSearchSearchTest(CloudSearchSearchBaseTest): response = { 'rank': '-text_relevance', 'match-expr': "Test", 'hits': { 'found': 30, 'start': 0, 'hit': CloudSearchSearchBaseTest.hits }, 'status': { 'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08', 'time-ms': 2, 'cpu-time-ms': 0 } } def test_cloudsearch_qsearch(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', options='TestOptions') args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'q'], [b"Test"]) self.assertEqual(args[b'q.options'], [b"TestOptions"]) self.assertEqual(args[b'start'], [b"0"]) self.assertEqual(args[b'size'], [b"10"]) def test_cloudsearch_search_details(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', size=50, start=20) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'q'], [b"Test"]) self.assertEqual(args[b'size'], [b"50"]) self.assertEqual(args[b'start'], [b"20"]) def test_cloudsearch_facet_constraint_single(self): search = SearchConnection(endpoint=HOSTNAME) search.search( q='Test', facet={'author': "'John Smith','Mark Smith'"}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet.author'], [b"'John Smith','Mark Smith'"]) def test_cloudsearch_facet_constraint_multiple(self): search = SearchConnection(endpoint=HOSTNAME) search.search( q='Test', facet={'author': "'John Smith','Mark Smith'", 'category': "'News','Reviews'"}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet.author'], [b"'John Smith','Mark Smith'"]) self.assertEqual(args[b'facet.category'], [b"'News','Reviews'"]) def test_cloudsearch_facet_sort_single(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', facet={'author': {'sort': 'alpha'}}) args = self.get_args(HTTPretty.last_request.raw_requestline) print(args) self.assertEqual(args[b'facet.author'], [b'{"sort": "alpha"}']) def test_cloudsearch_facet_sort_multiple(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', facet={'author': {'sort': 'alpha'}, 'cat': {'sort': 'count'}}) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'facet.author'], [b'{"sort": "alpha"}']) self.assertEqual(args[b'facet.cat'], [b'{"sort": "count"}']) def test_cloudsearch_result_fields_single(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', return_fields=['author']) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'return'], [b'author']) def test_cloudsearch_result_fields_multiple(self): search = SearchConnection(endpoint=HOSTNAME) search.search(q='Test', return_fields=['author', 'title']) args = self.get_args(HTTPretty.last_request.raw_requestline) self.assertEqual(args[b'return'], [b'author,title']) def test_cloudsearch_results_meta(self): """Check returned metadata is parsed correctly""" search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test') # These rely on the default response which is fed into HTTPretty self.assertEqual(results.hits, 30) self.assertEqual(results.docs[0]['fields']['rank'], 1) def test_cloudsearch_results_info(self): """Check num_pages_needed is calculated correctly""" search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test') # This relies on the default response which is fed into HTTPretty self.assertEqual(results.num_pages_needed, 3.0) def test_cloudsearch_results_matched(self): """ Check that information objects are passed back through the API correctly. """ search = SearchConnection(endpoint=HOSTNAME) query = search.build_query(q='Test') results = search(query) self.assertEqual(results.search_service, search) self.assertEqual(results.query, query) def test_cloudsearch_results_hits(self): """Check that documents are parsed properly from AWS""" search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test') hits = list(map(lambda x: x['id'], results.docs)) # This relies on the default response which is fed into HTTPretty self.assertEqual( hits, ["12341", "12342", "12343", "12344", "12345", "12346", "12347"]) def test_cloudsearch_results_iterator(self): """Check the results iterator""" search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test') results_correct = iter(["12341", "12342", "12343", "12344", "12345", "12346", "12347"]) for x in results: self.assertEqual(x['id'], next(results_correct)) def test_cloudsearch_results_internal_consistancy(self): """Check the documents length matches the iterator details""" search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test') self.assertEqual(len(results), len(results.docs)) def test_cloudsearch_search_nextpage(self): """Check next page query is correct""" search = SearchConnection(endpoint=HOSTNAME) query1 = search.build_query(q='Test') query2 = search.build_query(q='Test') results = search(query2) self.assertEqual(results.next_page().query.start, query1.start + query1.size) self.assertEqual(query1.q, query2.q) class CloudSearchSearchFacetTest(CloudSearchSearchBaseTest): response = { 'rank': '-text_relevance', 'match-expr': "Test", 'hits': { 'found': 30, 'start': 0, 'hit': CloudSearchSearchBaseTest.hits }, 'status': { 'rid': 'b7c167f6c2da6d93531b9a7b314ad030b3a74803b4b7797edb905ba5a6a08', 'time-ms': 2, 'cpu-time-ms': 0 }, 'facets': { 'tags': {}, 'animals': {'buckets': [{'count': '2', 'value': 'fish'}, {'count': '1', 'value': 'lions'}]}, } } def test_cloudsearch_search_facets(self): #self.response['facets'] = {'tags': {}} search = SearchConnection(endpoint=HOSTNAME) results = search.search(q='Test', facet={'tags': {}}) self.assertTrue('tags' not in results.facets) self.assertEqual(results.facets['animals'], {u'lions': u'1', u'fish': u'2'}) class CloudSearchNonJsonTest(CloudSearchSearchBaseTest): response = b'<html><body><h1>500 Internal Server Error</h1></body></html>' response_status = 500 content_type = 'text/xml' def test_response(self): search = SearchConnection(endpoint=HOSTNAME) with self.assertRaises(SearchServiceException): search.search(q='Test') class CloudSearchUnauthorizedTest(CloudSearchSearchBaseTest): response = b'<html><body><h1>403 Forbidden</h1>foo bar baz</body></html>' response_status = 403 content_type = 'text/html' def test_response(self): search = SearchConnection(endpoint=HOSTNAME) with self.assertRaisesRegex(SearchServiceException, 'foo bar baz'): search.search(q='Test') class FakeResponse(object): status_code = 405 content = b'' class CloudSearchConnectionTest(AWSMockServiceTestCase): cloudsearch = True connection_class = CloudSearchConnection def setUp(self): super(CloudSearchConnectionTest, self).setUp() self.conn = SearchConnection( endpoint='test-domain.cloudsearch.amazonaws.com' ) def test_expose_additional_error_info(self): mpo = mock.patch.object fake = FakeResponse() fake.content = b'Nopenopenope' # First, in the case of a non-JSON, non-403 error. with mpo(self.conn.session, 'get', return_value=fake) as mock_request: with self.assertRaises(SearchServiceException) as cm: self.conn.search(q='not_gonna_happen') self.assertTrue('non-json response' in str(cm.exception)) self.assertTrue('Nopenopenope' in str(cm.exception)) # Then with JSON & an 'error' key within. fake.content = json.dumps({ 'error': "Something went wrong. Oops." }).encode('utf-8') with mpo(self.conn.session, 'get', return_value=fake) as mock_request: with self.assertRaises(SearchServiceException) as cm: self.conn.search(q='no_luck_here') self.assertTrue('Unknown error' in str(cm.exception)) self.assertTrue('went wrong. Oops' in str(cm.exception)) def test_proxy(self): conn = self.service_connection conn.proxy = "127.0.0.1" conn.proxy_user = "john.doe" conn.proxy_pass="p4ssw0rd" conn.proxy_port="8180" conn.use_proxy = True domain = Domain(conn, DEMO_DOMAIN_DATA) search = SearchConnection(domain=domain) self.assertEqual(search.session.proxies, {'http': 'http://john.doe:p4ssw0rd@127.0.0.1:8180'}) vendored/boto/tests/unit/cloudsearch2/__init__.py 0000644 00000001025 15025013616 0016122 0 ustar 00 DEMO_DOMAIN_DATA = { "SearchInstanceType": None, "DomainId": "1234567890/demo", "DomainName": "demo", "Deleted": False, "SearchInstanceCount": 0, "Created": True, "SearchService": { "Endpoint": "search-demo.us-east-1.cloudsearch.amazonaws.com" }, "RequiresIndexDocuments": False, "Processing": False, "DocService": { "Endpoint": "doc-demo.us-east-1.cloudsearch.amazonaws.com" }, "ARN": "arn:aws:cs:us-east-1:1234567890:domain/demo", "SearchPartitionCount": 0 } vendored/boto/tests/unit/data/new_endpoints.json 0000644 00000101136 15025013616 0016117 0 ustar 00 { "partitions": [ { "defaults": { "hostname": "{service}.{region}.{dnsSuffix}", "protocols": [ "https" ], "signatureVersions": [ "v4" ] }, "dnsSuffix": "amazonaws.com", "partition": "aws", "partitionName": "AWS Standard", "regionRegex": "^(us|eu|ap|sa|ca)\\-\\w+\\-\\d+$", "regions": { "ap-northeast-1": { "description": "Asia Pacific (Tokyo)" }, "ap-northeast-2": { "description": "Asia Pacific (Seoul)" }, "ap-south-1": { "description": "Asia Pacific (Mumbai)" }, "ap-southeast-1": { "description": "Asia Pacific (Singapore)" }, "ap-southeast-2": { "description": "Asia Pacific (Sydney)" }, "ca-central-1": { "description": "Canada (Central)" }, "eu-central-1": { "description": "EU (Frankfurt)" }, "eu-west-1": { "description": "EU (Ireland)" }, "eu-west-2": { "description": "EU (London)" }, "sa-east-1": { "description": "South America (Sao Paulo)" }, "us-east-1": { "description": "US East (N. Virginia)" }, "us-east-2": { "description": "US East (Ohio)" }, "us-west-1": { "description": "US West (N. California)" }, "us-west-2": { "description": "US West (Oregon)" } }, "services": { "autoscaling": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "cloudformation": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "cloudfront": { "endpoints": { "aws-global": { "credentialScope": { "region": "us-east-1" }, "hostname": "cloudfront.amazonaws.com", "protocols": [ "http", "https" ] } }, "isRegionalized": false, "partitionEndpoint": "aws-global" }, "cloudhsm": { "endpoints": { "ap-northeast-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "cloudsearch": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "eu-central-1": {}, "eu-west-1": {}, "sa-east-1": {}, "us-east-1": {}, "us-west-1": {}, "us-west-2": {} } }, "cloudtrail": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "codedeploy": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "cognito-identity": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-southeast-2": {}, "eu-central-1": {}, "eu-west-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-2": {} } }, "cognito-sync": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-southeast-2": {}, "eu-central-1": {}, "eu-west-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-2": {} } }, "config": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "datapipeline": { "endpoints": { "ap-northeast-1": {}, "ap-southeast-2": {}, "eu-west-1": {}, "us-east-1": {}, "us-west-2": {} } }, "directconnect": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "dynamodb": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "local": { "credentialScope": { "region": "us-east-1" }, "hostname": "localhost:8000", "protocols": [ "http" ] }, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "ec2": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "ecs": { "endpoints": { "ap-northeast-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "elasticache": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "elasticbeanstalk": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "elasticloadbalancing": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "elasticmapreduce": { "defaults": { "protocols": [ "http", "https" ], "sslCommonName": "{region}.{service}.{dnsSuffix}" }, "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": { "sslCommonName": "{service}.{region}.{dnsSuffix}" }, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": { "sslCommonName": "{service}.{region}.{dnsSuffix}" }, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "elastictranscoder": { "endpoints": { "ap-northeast-1": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "eu-west-1": {}, "us-east-1": {}, "us-west-1": {}, "us-west-2": {} } }, "email": { "endpoints": { "eu-west-1": {}, "us-east-1": {}, "us-west-2": {} } }, "glacier": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "iam": { "endpoints": { "aws-global": { "credentialScope": { "region": "us-east-1" }, "hostname": "iam.amazonaws.com" } }, "isRegionalized": false, "partitionEndpoint": "aws-global" }, "importexport": { "endpoints": { "aws-global": { "credentialScope": { "region": "us-east-1", "service": "IngestionService" }, "hostname": "importexport.amazonaws.com", "signatureVersions": [ "v2", "v4" ] } }, "isRegionalized": false, "partitionEndpoint": "aws-global" }, "kinesis": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "kms": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "lambda": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "eu-central-1": {}, "eu-west-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "logs": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "machinelearning": { "endpoints": { "eu-west-1": {}, "us-east-1": {} } }, "monitoring": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "opsworks": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "rds": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": { "sslCommonName": "{service}.{dnsSuffix}" }, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "redshift": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "route53": { "endpoints": { "aws-global": { "credentialScope": { "region": "us-east-1" }, "hostname": "route53.amazonaws.com" } }, "isRegionalized": false, "partitionEndpoint": "aws-global" }, "route53domains": { "endpoints": { "us-east-1": {} } }, "s3": { "defaults": { "protocols": [ "http", "https" ], "signatureVersions": [ "s3v4" ] }, "endpoints": { "ap-northeast-1": { "hostname": "s3-ap-northeast-1.amazonaws.com", "signatureVersions": [ "s3", "s3v4" ] }, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": { "hostname": "s3-ap-southeast-1.amazonaws.com", "signatureVersions": [ "s3", "s3v4" ] }, "ap-southeast-2": { "hostname": "s3-ap-southeast-2.amazonaws.com", "signatureVersions": [ "s3", "s3v4" ] }, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": { "hostname": "s3-eu-west-1.amazonaws.com", "signatureVersions": [ "s3", "s3v4" ] }, "eu-west-2": {}, "s3-external-1": { "credentialScope": { "region": "us-east-1" }, "hostname": "s3-external-1.amazonaws.com", "signatureVersions": [ "s3", "s3v4" ] }, "sa-east-1": { "hostname": "s3-sa-east-1.amazonaws.com", "signatureVersions": [ "s3", "s3v4" ] }, "us-east-1": { "hostname": "s3.amazonaws.com", "signatureVersions": [ "s3", "s3v4" ] }, "us-east-2": {}, "us-west-1": { "hostname": "s3-us-west-1.amazonaws.com", "signatureVersions": [ "s3", "s3v4" ] }, "us-west-2": { "hostname": "s3-us-west-2.amazonaws.com", "signatureVersions": [ "s3", "s3v4" ] } }, "isRegionalized": true, "partitionEndpoint": "us-east-1" }, "sdb": { "defaults": { "protocols": [ "http", "https" ], "signatureVersions": [ "v2" ] }, "endpoints": { "ap-northeast-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "eu-west-1": {}, "sa-east-1": {}, "us-east-1": { "hostname": "sdb.amazonaws.com" }, "us-west-1": {}, "us-west-2": {} } }, "sns": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "sqs": { "defaults": { "protocols": [ "http", "https" ], "sslCommonName": "{region}.queue.{dnsSuffix}" }, "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": { "sslCommonName": "queue.{dnsSuffix}" }, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "storagegateway": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } }, "sts": { "defaults": { "credentialScope": { "region": "us-east-1" }, "hostname": "sts.amazonaws.com" }, "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": { "credentialScope": { "region": "ap-northeast-2" }, "hostname": "sts.ap-northeast-2.amazonaws.com" }, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "aws-global": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} }, "partitionEndpoint": "aws-global" }, "support": { "endpoints": { "us-east-1": {} } }, "swf": { "endpoints": { "ap-northeast-1": {}, "ap-northeast-2": {}, "ap-south-1": {}, "ap-southeast-1": {}, "ap-southeast-2": {}, "ca-central-1": {}, "eu-central-1": {}, "eu-west-1": {}, "eu-west-2": {}, "sa-east-1": {}, "us-east-1": {}, "us-east-2": {}, "us-west-1": {}, "us-west-2": {} } } } }, { "defaults": { "hostname": "{service}.{region}.{dnsSuffix}", "protocols": [ "https" ], "signatureVersions": [ "v4" ] }, "dnsSuffix": "amazonaws.com.cn", "partition": "aws-cn", "partitionName": "AWS China", "regionRegex": "^cn\\-\\w+\\-\\d+$", "regions": { "cn-north-1": { "description": "China (Beijing)" } }, "services": { "autoscaling": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "cn-north-1": {} } }, "cloudformation": { "endpoints": { "cn-north-1": {} } }, "cloudtrail": { "endpoints": { "cn-north-1": {} } }, "config": { "endpoints": { "cn-north-1": {} } }, "directconnect": { "endpoints": { "cn-north-1": {} } }, "dynamodb": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "cn-north-1": {} } }, "ec2": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "cn-north-1": {} } }, "elasticache": { "endpoints": { "cn-north-1": {} } }, "elasticbeanstalk": { "endpoints": { "cn-north-1": {} } }, "elasticloadbalancing": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "cn-north-1": {} } }, "elasticmapreduce": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "cn-north-1": {} } }, "glacier": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "cn-north-1": {} } }, "iam": { "endpoints": { "aws-cn-global": { "credentialScope": { "region": "cn-north-1" }, "hostname": "iam.cn-north-1.amazonaws.com.cn" } }, "isRegionalized": false, "partitionEndpoint": "aws-cn-global" }, "kinesis": { "endpoints": { "cn-north-1": {} } }, "logs": { "endpoints": { "cn-north-1": {} } }, "monitoring": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "cn-north-1": {} } }, "rds": { "endpoints": { "cn-north-1": {} } }, "redshift": { "endpoints": { "cn-north-1": {} } }, "s3": { "defaults": { "protocols": [ "http", "https" ], "signatureVersions": [ "s3v4" ] }, "endpoints": { "cn-north-1": {} } }, "sns": { "defaults": { "protocols": [ "http", "https" ] }, "endpoints": { "cn-north-1": {} } }, "sqs": { "defaults": { "protocols": [ "http", "https" ], "sslCommonName": "{region}.queue.{dnsSuffix}" }, "endpoints": { "cn-north-1": {} } }, "storagegateway": { "endpoints": { "cn-north-1": {} } }, "sts": { "endpoints": { "cn-north-1": {} } }, "swf": { "endpoints": { "cn-north-1": {} } } } }, { "defaults": { "hostname": "{service}.{region}.{dnsSuffix}", "protocols": [ "https" ], "signatureVersions": [ "v4" ] }, "dnsSuffix": "amazonaws.com", "partition": "aws-us-gov", "partitionName": "AWS GovCloud (US)", "regionRegex": "^us\\-gov\\-\\w+\\-\\d+$", "regions": { "us-gov-west-1": { "description": "AWS GovCloud (US)" } }, "services": { "autoscaling": { "endpoints": { "us-gov-west-1": { "protocols": [ "http", "https" ] } } }, "cloudformation": { "endpoints": { "us-gov-west-1": {} } }, "cloudhsm": { "endpoints": { "us-gov-west-1": {} } }, "cloudtrail": { "endpoints": { "us-gov-west-1": {} } }, "config": { "endpoints": { "us-gov-west-1": {} } }, "directconnect": { "endpoints": { "us-gov-west-1": {} } }, "dynamodb": { "endpoints": { "us-gov-west-1": {} } }, "ec2": { "endpoints": { "us-gov-west-1": {} } }, "elasticache": { "endpoints": { "us-gov-west-1": {} } }, "elasticloadbalancing": { "endpoints": { "us-gov-west-1": { "protocols": [ "http", "https" ] } } }, "elasticmapreduce": { "endpoints": { "us-gov-west-1": { "protocols": [ "http", "https" ] } } }, "glacier": { "endpoints": { "us-gov-west-1": { "protocols": [ "http", "https" ] } } }, "iam": { "endpoints": { "aws-us-gov-global": { "credentialScope": { "region": "us-gov-west-1" }, "hostname": "iam.us-gov.amazonaws.com" } }, "isRegionalized": false, "partitionEndpoint": "aws-us-gov-global" }, "kms": { "endpoints": { "us-gov-west-1": {} } }, "logs": { "endpoints": { "us-gov-west-1": {} } }, "monitoring": { "endpoints": { "us-gov-west-1": {} } }, "rds": { "endpoints": { "us-gov-west-1": {} } }, "redshift": { "endpoints": { "us-gov-west-1": {} } }, "s3": { "defaults": { "signatureVersions": [ "s3", "s3v4" ] }, "endpoints": { "fips-us-gov-west-1": { "credentialScope": { "region": "us-gov-west-1" }, "hostname": "s3-fips-us-gov-west-1.amazonaws.com" }, "us-gov-west-1": { "hostname": "s3-us-gov-west-1.amazonaws.com", "protocols": [ "http", "https" ] } } }, "sns": { "endpoints": { "us-gov-west-1": { "protocols": [ "http", "https" ] } } }, "sqs": { "endpoints": { "us-gov-west-1": { "protocols": [ "http", "https" ], "sslCommonName": "{region}.queue.{dnsSuffix}" } } }, "sts": { "endpoints": { "us-gov-west-1": {} } }, "swf": { "endpoints": { "us-gov-west-1": {} } } } } ], "version": 3 } vendored/boto/tests/unit/data/old_endpoints.json 0000644 00000103477 15025013616 0016116 0 ustar 00 { "autoscaling": { "ap-northeast-1": "autoscaling.ap-northeast-1.amazonaws.com", "ap-northeast-2": "autoscaling.ap-northeast-2.amazonaws.com", "ap-south-1": "autoscaling.ap-south-1.amazonaws.com", "ap-southeast-1": "autoscaling.ap-southeast-1.amazonaws.com", "ap-southeast-2": "autoscaling.ap-southeast-2.amazonaws.com", "ca-central-1": "autoscaling.ca-central-1.amazonaws.com", "cn-north-1": "autoscaling.cn-north-1.amazonaws.com.cn", "eu-central-1": "autoscaling.eu-central-1.amazonaws.com", "eu-west-1": "autoscaling.eu-west-1.amazonaws.com", "eu-west-2": "autoscaling.eu-west-2.amazonaws.com", "sa-east-1": "autoscaling.sa-east-1.amazonaws.com", "us-east-1": "autoscaling.us-east-1.amazonaws.com", "us-east-2": "autoscaling.us-east-2.amazonaws.com", "us-gov-west-1": "autoscaling.us-gov-west-1.amazonaws.com", "us-west-1": "autoscaling.us-west-1.amazonaws.com", "us-west-2": "autoscaling.us-west-2.amazonaws.com" }, "awslambda": { "ap-northeast-1": "lambda.ap-northeast-1.amazonaws.com", "ap-northeast-2": "lambda.ap-northeast-2.amazonaws.com", "ap-southeast-1": "lambda.ap-southeast-1.amazonaws.com", "ap-southeast-2": "lambda.ap-southeast-2.amazonaws.com", "eu-central-1": "lambda.eu-central-1.amazonaws.com", "eu-west-1": "lambda.eu-west-1.amazonaws.com", "us-east-1": "lambda.us-east-1.amazonaws.com", "us-east-2": "lambda.us-east-2.amazonaws.com", "us-west-1": "lambda.us-west-1.amazonaws.com", "us-west-2": "lambda.us-west-2.amazonaws.com" }, "cloudformation": { "ap-northeast-1": "cloudformation.ap-northeast-1.amazonaws.com", "ap-northeast-2": "cloudformation.ap-northeast-2.amazonaws.com", "ap-south-1": "cloudformation.ap-south-1.amazonaws.com", "ap-southeast-1": "cloudformation.ap-southeast-1.amazonaws.com", "ap-southeast-2": "cloudformation.ap-southeast-2.amazonaws.com", "ca-central-1": "cloudformation.ca-central-1.amazonaws.com", "cn-north-1": "cloudformation.cn-north-1.amazonaws.com.cn", "eu-central-1": "cloudformation.eu-central-1.amazonaws.com", "eu-west-1": "cloudformation.eu-west-1.amazonaws.com", "eu-west-2": "cloudformation.eu-west-2.amazonaws.com", "sa-east-1": "cloudformation.sa-east-1.amazonaws.com", "us-east-1": "cloudformation.us-east-1.amazonaws.com", "us-east-2": "cloudformation.us-east-2.amazonaws.com", "us-gov-west-1": "cloudformation.us-gov-west-1.amazonaws.com", "us-west-1": "cloudformation.us-west-1.amazonaws.com", "us-west-2": "cloudformation.us-west-2.amazonaws.com" }, "cloudfront": { "ap-northeast-1": "cloudfront.amazonaws.com", "ap-northeast-2": "cloudfront.amazonaws.com", "ap-south-1": "cloudfront.amazonaws.com", "ap-southeast-1": "cloudfront.amazonaws.com", "ap-southeast-2": "cloudfront.amazonaws.com", "ca-central-1": "cloudfront.amazonaws.com", "eu-central-1": "cloudfront.amazonaws.com", "eu-west-1": "cloudfront.amazonaws.com", "eu-west-2": "cloudfront.amazonaws.com", "sa-east-1": "cloudfront.amazonaws.com", "us-east-1": "cloudfront.amazonaws.com", "us-east-2": "cloudfront.amazonaws.com", "us-west-1": "cloudfront.amazonaws.com", "us-west-2": "cloudfront.amazonaws.com" }, "cloudhsm": { "ap-northeast-1": "cloudhsm.ap-northeast-1.amazonaws.com", "ap-southeast-1": "cloudhsm.ap-southeast-1.amazonaws.com", "ap-southeast-2": "cloudhsm.ap-southeast-2.amazonaws.com", "ca-central-1": "cloudhsm.ca-central-1.amazonaws.com", "eu-central-1": "cloudhsm.eu-central-1.amazonaws.com", "eu-west-1": "cloudhsm.eu-west-1.amazonaws.com", "us-east-1": "cloudhsm.us-east-1.amazonaws.com", "us-east-2": "cloudhsm.us-east-2.amazonaws.com", "us-gov-west-1": "cloudhsm.us-gov-west-1.amazonaws.com", "us-west-1": "cloudhsm.us-west-1.amazonaws.com", "us-west-2": "cloudhsm.us-west-2.amazonaws.com" }, "cloudsearch": { "ap-northeast-1": "cloudsearch.ap-northeast-1.amazonaws.com", "ap-northeast-2": "cloudsearch.ap-northeast-2.amazonaws.com", "ap-southeast-1": "cloudsearch.ap-southeast-1.amazonaws.com", "ap-southeast-2": "cloudsearch.ap-southeast-2.amazonaws.com", "eu-central-1": "cloudsearch.eu-central-1.amazonaws.com", "eu-west-1": "cloudsearch.eu-west-1.amazonaws.com", "sa-east-1": "cloudsearch.sa-east-1.amazonaws.com", "us-east-1": "cloudsearch.us-east-1.amazonaws.com", "us-west-1": "cloudsearch.us-west-1.amazonaws.com", "us-west-2": "cloudsearch.us-west-2.amazonaws.com" }, "cloudsearchdomain": { "ap-northeast-1": "cloudsearch.ap-northeast-1.amazonaws.com", "ap-northeast-2": "cloudsearch.ap-northeast-2.amazonaws.com", "ap-southeast-1": "cloudsearch.ap-southeast-1.amazonaws.com", "ap-southeast-2": "cloudsearch.ap-southeast-2.amazonaws.com", "eu-central-1": "cloudsearch.eu-central-1.amazonaws.com", "eu-west-1": "cloudsearch.eu-west-1.amazonaws.com", "sa-east-1": "cloudsearch.sa-east-1.amazonaws.com", "us-east-1": "cloudsearch.us-east-1.amazonaws.com", "us-west-1": "cloudsearch.us-west-1.amazonaws.com", "us-west-2": "cloudsearch.us-west-2.amazonaws.com" }, "cloudtrail": { "ap-northeast-1": "cloudtrail.ap-northeast-1.amazonaws.com", "ap-northeast-2": "cloudtrail.ap-northeast-2.amazonaws.com", "ap-south-1": "cloudtrail.ap-south-1.amazonaws.com", "ap-southeast-1": "cloudtrail.ap-southeast-1.amazonaws.com", "ap-southeast-2": "cloudtrail.ap-southeast-2.amazonaws.com", "ca-central-1": "cloudtrail.ca-central-1.amazonaws.com", "cn-north-1": "cloudtrail.cn-north-1.amazonaws.com.cn", "eu-central-1": "cloudtrail.eu-central-1.amazonaws.com", "eu-west-1": "cloudtrail.eu-west-1.amazonaws.com", "eu-west-2": "cloudtrail.eu-west-2.amazonaws.com", "sa-east-1": "cloudtrail.sa-east-1.amazonaws.com", "us-east-1": "cloudtrail.us-east-1.amazonaws.com", "us-east-2": "cloudtrail.us-east-2.amazonaws.com", "us-gov-west-1": "cloudtrail.us-gov-west-1.amazonaws.com", "us-west-1": "cloudtrail.us-west-1.amazonaws.com", "us-west-2": "cloudtrail.us-west-2.amazonaws.com" }, "cloudwatch": { "ap-northeast-1": "monitoring.ap-northeast-1.amazonaws.com", "ap-northeast-2": "monitoring.ap-northeast-2.amazonaws.com", "ap-south-1": "monitoring.ap-south-1.amazonaws.com", "ap-southeast-1": "monitoring.ap-southeast-1.amazonaws.com", "ap-southeast-2": "monitoring.ap-southeast-2.amazonaws.com", "ca-central-1": "monitoring.ca-central-1.amazonaws.com", "cn-north-1": "monitoring.cn-north-1.amazonaws.com.cn", "eu-central-1": "monitoring.eu-central-1.amazonaws.com", "eu-west-1": "monitoring.eu-west-1.amazonaws.com", "eu-west-2": "monitoring.eu-west-2.amazonaws.com", "sa-east-1": "monitoring.sa-east-1.amazonaws.com", "us-east-1": "monitoring.us-east-1.amazonaws.com", "us-east-2": "monitoring.us-east-2.amazonaws.com", "us-gov-west-1": "monitoring.us-gov-west-1.amazonaws.com", "us-west-1": "monitoring.us-west-1.amazonaws.com", "us-west-2": "monitoring.us-west-2.amazonaws.com" }, "codedeploy": { "ap-northeast-1": "codedeploy.ap-northeast-1.amazonaws.com", "ap-northeast-2": "codedeploy.ap-northeast-2.amazonaws.com", "ap-south-1": "codedeploy.ap-south-1.amazonaws.com", "ap-southeast-1": "codedeploy.ap-southeast-1.amazonaws.com", "ap-southeast-2": "codedeploy.ap-southeast-2.amazonaws.com", "ca-central-1": "codedeploy.ca-central-1.amazonaws.com", "eu-central-1": "codedeploy.eu-central-1.amazonaws.com", "eu-west-1": "codedeploy.eu-west-1.amazonaws.com", "eu-west-2": "codedeploy.eu-west-2.amazonaws.com", "sa-east-1": "codedeploy.sa-east-1.amazonaws.com", "us-east-1": "codedeploy.us-east-1.amazonaws.com", "us-east-2": "codedeploy.us-east-2.amazonaws.com", "us-west-1": "codedeploy.us-west-1.amazonaws.com", "us-west-2": "codedeploy.us-west-2.amazonaws.com" }, "cognito-identity": { "ap-northeast-1": "cognito-identity.ap-northeast-1.amazonaws.com", "ap-northeast-2": "cognito-identity.ap-northeast-2.amazonaws.com", "ap-southeast-2": "cognito-identity.ap-southeast-2.amazonaws.com", "eu-central-1": "cognito-identity.eu-central-1.amazonaws.com", "eu-west-1": "cognito-identity.eu-west-1.amazonaws.com", "us-east-1": "cognito-identity.us-east-1.amazonaws.com", "us-east-2": "cognito-identity.us-east-2.amazonaws.com", "us-west-2": "cognito-identity.us-west-2.amazonaws.com" }, "cognito-sync": { "ap-northeast-1": "cognito-sync.ap-northeast-1.amazonaws.com", "ap-northeast-2": "cognito-sync.ap-northeast-2.amazonaws.com", "ap-southeast-2": "cognito-sync.ap-southeast-2.amazonaws.com", "eu-central-1": "cognito-sync.eu-central-1.amazonaws.com", "eu-west-1": "cognito-sync.eu-west-1.amazonaws.com", "us-east-1": "cognito-sync.us-east-1.amazonaws.com", "us-east-2": "cognito-sync.us-east-2.amazonaws.com", "us-west-2": "cognito-sync.us-west-2.amazonaws.com" }, "configservice": { "ap-northeast-1": "config.ap-northeast-1.amazonaws.com", "ap-northeast-2": "config.ap-northeast-2.amazonaws.com", "ap-south-1": "config.ap-south-1.amazonaws.com", "ap-southeast-1": "config.ap-southeast-1.amazonaws.com", "ap-southeast-2": "config.ap-southeast-2.amazonaws.com", "ca-central-1": "config.ca-central-1.amazonaws.com", "cn-north-1": "config.cn-north-1.amazonaws.com.cn", "eu-central-1": "config.eu-central-1.amazonaws.com", "eu-west-1": "config.eu-west-1.amazonaws.com", "eu-west-2": "config.eu-west-2.amazonaws.com", "sa-east-1": "config.sa-east-1.amazonaws.com", "us-east-1": "config.us-east-1.amazonaws.com", "us-east-2": "config.us-east-2.amazonaws.com", "us-gov-west-1": "config.us-gov-west-1.amazonaws.com", "us-west-1": "config.us-west-1.amazonaws.com", "us-west-2": "config.us-west-2.amazonaws.com" }, "datapipeline": { "ap-northeast-1": "datapipeline.ap-northeast-1.amazonaws.com", "ap-southeast-2": "datapipeline.ap-southeast-2.amazonaws.com", "eu-west-1": "datapipeline.eu-west-1.amazonaws.com", "us-east-1": "datapipeline.us-east-1.amazonaws.com", "us-west-2": "datapipeline.us-west-2.amazonaws.com" }, "directconnect": { "ap-northeast-1": "directconnect.ap-northeast-1.amazonaws.com", "ap-northeast-2": "directconnect.ap-northeast-2.amazonaws.com", "ap-south-1": "directconnect.ap-south-1.amazonaws.com", "ap-southeast-1": "directconnect.ap-southeast-1.amazonaws.com", "ap-southeast-2": "directconnect.ap-southeast-2.amazonaws.com", "ca-central-1": "directconnect.ca-central-1.amazonaws.com", "cn-north-1": "directconnect.cn-north-1.amazonaws.com.cn", "eu-central-1": "directconnect.eu-central-1.amazonaws.com", "eu-west-1": "directconnect.eu-west-1.amazonaws.com", "eu-west-2": "directconnect.eu-west-2.amazonaws.com", "sa-east-1": "directconnect.sa-east-1.amazonaws.com", "us-east-1": "directconnect.us-east-1.amazonaws.com", "us-east-2": "directconnect.us-east-2.amazonaws.com", "us-gov-west-1": "directconnect.us-gov-west-1.amazonaws.com", "us-west-1": "directconnect.us-west-1.amazonaws.com", "us-west-2": "directconnect.us-west-2.amazonaws.com" }, "dynamodb": { "ap-northeast-1": "dynamodb.ap-northeast-1.amazonaws.com", "ap-northeast-2": "dynamodb.ap-northeast-2.amazonaws.com", "ap-south-1": "dynamodb.ap-south-1.amazonaws.com", "ap-southeast-1": "dynamodb.ap-southeast-1.amazonaws.com", "ap-southeast-2": "dynamodb.ap-southeast-2.amazonaws.com", "ca-central-1": "dynamodb.ca-central-1.amazonaws.com", "cn-north-1": "dynamodb.cn-north-1.amazonaws.com.cn", "eu-central-1": "dynamodb.eu-central-1.amazonaws.com", "eu-west-1": "dynamodb.eu-west-1.amazonaws.com", "eu-west-2": "dynamodb.eu-west-2.amazonaws.com", "sa-east-1": "dynamodb.sa-east-1.amazonaws.com", "us-east-1": "dynamodb.us-east-1.amazonaws.com", "us-east-2": "dynamodb.us-east-2.amazonaws.com", "us-gov-west-1": "dynamodb.us-gov-west-1.amazonaws.com", "us-west-1": "dynamodb.us-west-1.amazonaws.com", "us-west-2": "dynamodb.us-west-2.amazonaws.com" }, "ec2": { "ap-northeast-1": "ec2.ap-northeast-1.amazonaws.com", "ap-northeast-2": "ec2.ap-northeast-2.amazonaws.com", "ap-south-1": "ec2.ap-south-1.amazonaws.com", "ap-southeast-1": "ec2.ap-southeast-1.amazonaws.com", "ap-southeast-2": "ec2.ap-southeast-2.amazonaws.com", "ca-central-1": "ec2.ca-central-1.amazonaws.com", "cn-north-1": "ec2.cn-north-1.amazonaws.com.cn", "eu-central-1": "ec2.eu-central-1.amazonaws.com", "eu-west-1": "ec2.eu-west-1.amazonaws.com", "eu-west-2": "ec2.eu-west-2.amazonaws.com", "sa-east-1": "ec2.sa-east-1.amazonaws.com", "us-east-1": "ec2.us-east-1.amazonaws.com", "us-east-2": "ec2.us-east-2.amazonaws.com", "us-gov-west-1": "ec2.us-gov-west-1.amazonaws.com", "us-west-1": "ec2.us-west-1.amazonaws.com", "us-west-2": "ec2.us-west-2.amazonaws.com" }, "ec2containerservice": { "ap-northeast-1": "ecs.ap-northeast-1.amazonaws.com", "ap-southeast-1": "ecs.ap-southeast-1.amazonaws.com", "ap-southeast-2": "ecs.ap-southeast-2.amazonaws.com", "ca-central-1": "ecs.ca-central-1.amazonaws.com", "eu-central-1": "ecs.eu-central-1.amazonaws.com", "eu-west-1": "ecs.eu-west-1.amazonaws.com", "eu-west-2": "ecs.eu-west-2.amazonaws.com", "us-east-1": "ecs.us-east-1.amazonaws.com", "us-east-2": "ecs.us-east-2.amazonaws.com", "us-west-1": "ecs.us-west-1.amazonaws.com", "us-west-2": "ecs.us-west-2.amazonaws.com" }, "elasticache": { "ap-northeast-1": "elasticache.ap-northeast-1.amazonaws.com", "ap-northeast-2": "elasticache.ap-northeast-2.amazonaws.com", "ap-south-1": "elasticache.ap-south-1.amazonaws.com", "ap-southeast-1": "elasticache.ap-southeast-1.amazonaws.com", "ap-southeast-2": "elasticache.ap-southeast-2.amazonaws.com", "ca-central-1": "elasticache.ca-central-1.amazonaws.com", "cn-north-1": "elasticache.cn-north-1.amazonaws.com.cn", "eu-central-1": "elasticache.eu-central-1.amazonaws.com", "eu-west-1": "elasticache.eu-west-1.amazonaws.com", "eu-west-2": "elasticache.eu-west-2.amazonaws.com", "sa-east-1": "elasticache.sa-east-1.amazonaws.com", "us-east-1": "elasticache.us-east-1.amazonaws.com", "us-east-2": "elasticache.us-east-2.amazonaws.com", "us-gov-west-1": "elasticache.us-gov-west-1.amazonaws.com", "us-west-1": "elasticache.us-west-1.amazonaws.com", "us-west-2": "elasticache.us-west-2.amazonaws.com" }, "elasticbeanstalk": { "ap-northeast-1": "elasticbeanstalk.ap-northeast-1.amazonaws.com", "ap-northeast-2": "elasticbeanstalk.ap-northeast-2.amazonaws.com", "ap-south-1": "elasticbeanstalk.ap-south-1.amazonaws.com", "ap-southeast-1": "elasticbeanstalk.ap-southeast-1.amazonaws.com", "ap-southeast-2": "elasticbeanstalk.ap-southeast-2.amazonaws.com", "ca-central-1": "elasticbeanstalk.ca-central-1.amazonaws.com", "cn-north-1": "elasticbeanstalk.cn-north-1.amazonaws.com.cn", "eu-central-1": "elasticbeanstalk.eu-central-1.amazonaws.com", "eu-west-1": "elasticbeanstalk.eu-west-1.amazonaws.com", "eu-west-2": "elasticbeanstalk.eu-west-2.amazonaws.com", "sa-east-1": "elasticbeanstalk.sa-east-1.amazonaws.com", "us-east-1": "elasticbeanstalk.us-east-1.amazonaws.com", "us-east-2": "elasticbeanstalk.us-east-2.amazonaws.com", "us-west-1": "elasticbeanstalk.us-west-1.amazonaws.com", "us-west-2": "elasticbeanstalk.us-west-2.amazonaws.com" }, "elasticloadbalancing": { "ap-northeast-1": "elasticloadbalancing.ap-northeast-1.amazonaws.com", "ap-northeast-2": "elasticloadbalancing.ap-northeast-2.amazonaws.com", "ap-south-1": "elasticloadbalancing.ap-south-1.amazonaws.com", "ap-southeast-1": "elasticloadbalancing.ap-southeast-1.amazonaws.com", "ap-southeast-2": "elasticloadbalancing.ap-southeast-2.amazonaws.com", "ca-central-1": "elasticloadbalancing.ca-central-1.amazonaws.com", "cn-north-1": "elasticloadbalancing.cn-north-1.amazonaws.com.cn", "eu-central-1": "elasticloadbalancing.eu-central-1.amazonaws.com", "eu-west-1": "elasticloadbalancing.eu-west-1.amazonaws.com", "eu-west-2": "elasticloadbalancing.eu-west-2.amazonaws.com", "sa-east-1": "elasticloadbalancing.sa-east-1.amazonaws.com", "us-east-1": "elasticloadbalancing.us-east-1.amazonaws.com", "us-east-2": "elasticloadbalancing.us-east-2.amazonaws.com", "us-gov-west-1": "elasticloadbalancing.us-gov-west-1.amazonaws.com", "us-west-1": "elasticloadbalancing.us-west-1.amazonaws.com", "us-west-2": "elasticloadbalancing.us-west-2.amazonaws.com" }, "elasticmapreduce": { "ap-northeast-1": "ap-northeast-1.elasticmapreduce.amazonaws.com", "ap-northeast-2": "ap-northeast-2.elasticmapreduce.amazonaws.com", "ap-south-1": "ap-south-1.elasticmapreduce.amazonaws.com", "ap-southeast-1": "ap-southeast-1.elasticmapreduce.amazonaws.com", "ap-southeast-2": "ap-southeast-2.elasticmapreduce.amazonaws.com", "ca-central-1": "ca-central-1.elasticmapreduce.amazonaws.com", "cn-north-1": "elasticmapreduce.cn-north-1.amazonaws.com.cn", "eu-central-1": "elasticmapreduce.eu-central-1.amazonaws.com", "eu-west-1": "eu-west-1.elasticmapreduce.amazonaws.com", "eu-west-2": "eu-west-2.elasticmapreduce.amazonaws.com", "sa-east-1": "sa-east-1.elasticmapreduce.amazonaws.com", "us-east-1": "elasticmapreduce.us-east-1.amazonaws.com", "us-east-2": "us-east-2.elasticmapreduce.amazonaws.com", "us-gov-west-1": "elasticmapreduce.us-gov-west-1.amazonaws.com", "us-west-1": "us-west-1.elasticmapreduce.amazonaws.com", "us-west-2": "us-west-2.elasticmapreduce.amazonaws.com" }, "elastictranscoder": { "ap-northeast-1": "elastictranscoder.ap-northeast-1.amazonaws.com", "ap-south-1": "elastictranscoder.ap-south-1.amazonaws.com", "ap-southeast-1": "elastictranscoder.ap-southeast-1.amazonaws.com", "ap-southeast-2": "elastictranscoder.ap-southeast-2.amazonaws.com", "eu-west-1": "elastictranscoder.eu-west-1.amazonaws.com", "us-east-1": "elastictranscoder.us-east-1.amazonaws.com", "us-west-1": "elastictranscoder.us-west-1.amazonaws.com", "us-west-2": "elastictranscoder.us-west-2.amazonaws.com" }, "glacier": { "ap-northeast-1": "glacier.ap-northeast-1.amazonaws.com", "ap-northeast-2": "glacier.ap-northeast-2.amazonaws.com", "ap-south-1": "glacier.ap-south-1.amazonaws.com", "ap-southeast-2": "glacier.ap-southeast-2.amazonaws.com", "ca-central-1": "glacier.ca-central-1.amazonaws.com", "cn-north-1": "glacier.cn-north-1.amazonaws.com.cn", "eu-central-1": "glacier.eu-central-1.amazonaws.com", "eu-west-1": "glacier.eu-west-1.amazonaws.com", "eu-west-2": "glacier.eu-west-2.amazonaws.com", "us-east-1": "glacier.us-east-1.amazonaws.com", "us-east-2": "glacier.us-east-2.amazonaws.com", "us-gov-west-1": "glacier.us-gov-west-1.amazonaws.com", "us-west-1": "glacier.us-west-1.amazonaws.com", "us-west-2": "glacier.us-west-2.amazonaws.com" }, "iam": { "ap-northeast-1": "iam.amazonaws.com", "ap-northeast-2": "iam.amazonaws.com", "ap-south-1": "iam.amazonaws.com", "ap-southeast-1": "iam.amazonaws.com", "ap-southeast-2": "iam.amazonaws.com", "ca-central-1": "iam.amazonaws.com", "cn-north-1": "iam.cn-north-1.amazonaws.com.cn", "eu-central-1": "iam.amazonaws.com", "eu-west-1": "iam.amazonaws.com", "eu-west-2": "iam.amazonaws.com", "sa-east-1": "iam.amazonaws.com", "us-east-1": "iam.amazonaws.com", "us-east-2": "iam.amazonaws.com", "us-gov-west-1": "iam.us-gov.amazonaws.com", "us-west-1": "iam.amazonaws.com", "us-west-2": "iam.amazonaws.com" }, "importexport": { "ap-northeast-1": "importexport.amazonaws.com", "ap-northeast-2": "importexport.amazonaws.com", "ap-south-1": "importexport.amazonaws.com", "ap-southeast-1": "importexport.amazonaws.com", "ap-southeast-2": "importexport.amazonaws.com", "ca-central-1": "importexport.amazonaws.com", "eu-central-1": "importexport.amazonaws.com", "eu-west-1": "importexport.amazonaws.com", "eu-west-2": "importexport.amazonaws.com", "sa-east-1": "importexport.amazonaws.com", "us-east-1": "importexport.amazonaws.com", "us-east-2": "importexport.amazonaws.com", "us-west-1": "importexport.amazonaws.com", "us-west-2": "importexport.amazonaws.com" }, "kinesis": { "ap-northeast-1": "kinesis.ap-northeast-1.amazonaws.com", "ap-northeast-2": "kinesis.ap-northeast-2.amazonaws.com", "ap-south-1": "kinesis.ap-south-1.amazonaws.com", "ap-southeast-1": "kinesis.ap-southeast-1.amazonaws.com", "ap-southeast-2": "kinesis.ap-southeast-2.amazonaws.com", "ca-central-1": "kinesis.ca-central-1.amazonaws.com", "cn-north-1": "kinesis.cn-north-1.amazonaws.com.cn", "eu-central-1": "kinesis.eu-central-1.amazonaws.com", "eu-west-1": "kinesis.eu-west-1.amazonaws.com", "eu-west-2": "kinesis.eu-west-2.amazonaws.com", "sa-east-1": "kinesis.sa-east-1.amazonaws.com", "us-east-1": "kinesis.us-east-1.amazonaws.com", "us-east-2": "kinesis.us-east-2.amazonaws.com", "us-west-1": "kinesis.us-west-1.amazonaws.com", "us-west-2": "kinesis.us-west-2.amazonaws.com" }, "kms": { "ap-northeast-1": "kms.ap-northeast-1.amazonaws.com", "ap-northeast-2": "kms.ap-northeast-2.amazonaws.com", "ap-south-1": "kms.ap-south-1.amazonaws.com", "ap-southeast-1": "kms.ap-southeast-1.amazonaws.com", "ap-southeast-2": "kms.ap-southeast-2.amazonaws.com", "ca-central-1": "kms.ca-central-1.amazonaws.com", "eu-central-1": "kms.eu-central-1.amazonaws.com", "eu-west-1": "kms.eu-west-1.amazonaws.com", "eu-west-2": "kms.eu-west-2.amazonaws.com", "sa-east-1": "kms.sa-east-1.amazonaws.com", "us-east-1": "kms.us-east-1.amazonaws.com", "us-east-2": "kms.us-east-2.amazonaws.com", "us-gov-west-1": "kms.us-gov-west-1.amazonaws.com", "us-west-1": "kms.us-west-1.amazonaws.com", "us-west-2": "kms.us-west-2.amazonaws.com" }, "logs": { "ap-northeast-1": "logs.ap-northeast-1.amazonaws.com", "ap-northeast-2": "logs.ap-northeast-2.amazonaws.com", "ap-south-1": "logs.ap-south-1.amazonaws.com", "ap-southeast-1": "logs.ap-southeast-1.amazonaws.com", "ap-southeast-2": "logs.ap-southeast-2.amazonaws.com", "ca-central-1": "logs.ca-central-1.amazonaws.com", "cn-north-1": "logs.cn-north-1.amazonaws.com.cn", "eu-central-1": "logs.eu-central-1.amazonaws.com", "eu-west-1": "logs.eu-west-1.amazonaws.com", "eu-west-2": "logs.eu-west-2.amazonaws.com", "sa-east-1": "logs.sa-east-1.amazonaws.com", "us-east-1": "logs.us-east-1.amazonaws.com", "us-east-2": "logs.us-east-2.amazonaws.com", "us-gov-west-1": "logs.us-gov-west-1.amazonaws.com", "us-west-1": "logs.us-west-1.amazonaws.com", "us-west-2": "logs.us-west-2.amazonaws.com" }, "machinelearning": { "eu-west-1": "machinelearning.eu-west-1.amazonaws.com", "us-east-1": "machinelearning.us-east-1.amazonaws.com" }, "opsworks": { "ap-northeast-1": "opsworks.ap-northeast-1.amazonaws.com", "ap-northeast-2": "opsworks.ap-northeast-2.amazonaws.com", "ap-south-1": "opsworks.ap-south-1.amazonaws.com", "ap-southeast-1": "opsworks.ap-southeast-1.amazonaws.com", "ap-southeast-2": "opsworks.ap-southeast-2.amazonaws.com", "eu-central-1": "opsworks.eu-central-1.amazonaws.com", "eu-west-1": "opsworks.eu-west-1.amazonaws.com", "eu-west-2": "opsworks.eu-west-2.amazonaws.com", "sa-east-1": "opsworks.sa-east-1.amazonaws.com", "us-east-1": "opsworks.us-east-1.amazonaws.com", "us-east-2": "opsworks.us-east-2.amazonaws.com", "us-west-1": "opsworks.us-west-1.amazonaws.com", "us-west-2": "opsworks.us-west-2.amazonaws.com" }, "rds": { "ap-northeast-1": "rds.ap-northeast-1.amazonaws.com", "ap-northeast-2": "rds.ap-northeast-2.amazonaws.com", "ap-south-1": "rds.ap-south-1.amazonaws.com", "ap-southeast-1": "rds.ap-southeast-1.amazonaws.com", "ap-southeast-2": "rds.ap-southeast-2.amazonaws.com", "ca-central-1": "rds.ca-central-1.amazonaws.com", "cn-north-1": "rds.cn-north-1.amazonaws.com.cn", "eu-central-1": "rds.eu-central-1.amazonaws.com", "eu-west-1": "rds.eu-west-1.amazonaws.com", "eu-west-2": "rds.eu-west-2.amazonaws.com", "sa-east-1": "rds.sa-east-1.amazonaws.com", "us-east-1": "rds.amazonaws.com", "us-east-2": "rds.us-east-2.amazonaws.com", "us-gov-west-1": "rds.us-gov-west-1.amazonaws.com", "us-west-1": "rds.us-west-1.amazonaws.com", "us-west-2": "rds.us-west-2.amazonaws.com" }, "redshift": { "ap-northeast-1": "redshift.ap-northeast-1.amazonaws.com", "ap-northeast-2": "redshift.ap-northeast-2.amazonaws.com", "ap-south-1": "redshift.ap-south-1.amazonaws.com", "ap-southeast-1": "redshift.ap-southeast-1.amazonaws.com", "ap-southeast-2": "redshift.ap-southeast-2.amazonaws.com", "ca-central-1": "redshift.ca-central-1.amazonaws.com", "cn-north-1": "redshift.cn-north-1.amazonaws.com.cn", "eu-central-1": "redshift.eu-central-1.amazonaws.com", "eu-west-1": "redshift.eu-west-1.amazonaws.com", "eu-west-2": "redshift.eu-west-2.amazonaws.com", "sa-east-1": "redshift.sa-east-1.amazonaws.com", "us-east-1": "redshift.us-east-1.amazonaws.com", "us-east-2": "redshift.us-east-2.amazonaws.com", "us-gov-west-1": "redshift.us-gov-west-1.amazonaws.com", "us-west-1": "redshift.us-west-1.amazonaws.com", "us-west-2": "redshift.us-west-2.amazonaws.com" }, "route53": { "ap-northeast-1": "route53.amazonaws.com", "ap-northeast-2": "route53.amazonaws.com", "ap-south-1": "route53.amazonaws.com", "ap-southeast-1": "route53.amazonaws.com", "ap-southeast-2": "route53.amazonaws.com", "ca-central-1": "route53.amazonaws.com", "eu-central-1": "route53.amazonaws.com", "eu-west-1": "route53.amazonaws.com", "eu-west-2": "route53.amazonaws.com", "sa-east-1": "route53.amazonaws.com", "us-east-1": "route53.amazonaws.com", "us-east-2": "route53.amazonaws.com", "us-west-1": "route53.amazonaws.com", "us-west-2": "route53.amazonaws.com" }, "route53domains": { "us-east-1": "route53domains.us-east-1.amazonaws.com" }, "s3": { "ap-northeast-1": "s3-ap-northeast-1.amazonaws.com", "ap-northeast-2": "s3.ap-northeast-2.amazonaws.com", "ap-south-1": "s3.ap-south-1.amazonaws.com", "ap-southeast-1": "s3-ap-southeast-1.amazonaws.com", "ap-southeast-2": "s3-ap-southeast-2.amazonaws.com", "ca-central-1": "s3.ca-central-1.amazonaws.com", "cn-north-1": "s3.cn-north-1.amazonaws.com.cn", "eu-central-1": "s3.eu-central-1.amazonaws.com", "eu-west-1": "s3-eu-west-1.amazonaws.com", "eu-west-2": "s3.eu-west-2.amazonaws.com", "sa-east-1": "s3-sa-east-1.amazonaws.com", "us-east-1": "s3.amazonaws.com", "us-east-2": "s3.us-east-2.amazonaws.com", "us-gov-west-1": "s3-us-gov-west-1.amazonaws.com", "us-west-1": "s3-us-west-1.amazonaws.com", "us-west-2": "s3-us-west-2.amazonaws.com" }, "sdb": { "ap-northeast-1": "sdb.ap-northeast-1.amazonaws.com", "ap-southeast-1": "sdb.ap-southeast-1.amazonaws.com", "ap-southeast-2": "sdb.ap-southeast-2.amazonaws.com", "eu-west-1": "sdb.eu-west-1.amazonaws.com", "sa-east-1": "sdb.sa-east-1.amazonaws.com", "us-east-1": "sdb.amazonaws.com", "us-west-1": "sdb.us-west-1.amazonaws.com", "us-west-2": "sdb.us-west-2.amazonaws.com" }, "ses": { "eu-west-1": "email.eu-west-1.amazonaws.com", "us-east-1": "email.us-east-1.amazonaws.com", "us-west-2": "email.us-west-2.amazonaws.com" }, "sns": { "ap-northeast-1": "sns.ap-northeast-1.amazonaws.com", "ap-northeast-2": "sns.ap-northeast-2.amazonaws.com", "ap-south-1": "sns.ap-south-1.amazonaws.com", "ap-southeast-1": "sns.ap-southeast-1.amazonaws.com", "ap-southeast-2": "sns.ap-southeast-2.amazonaws.com", "ca-central-1": "sns.ca-central-1.amazonaws.com", "cn-north-1": "sns.cn-north-1.amazonaws.com.cn", "eu-central-1": "sns.eu-central-1.amazonaws.com", "eu-west-1": "sns.eu-west-1.amazonaws.com", "eu-west-2": "sns.eu-west-2.amazonaws.com", "sa-east-1": "sns.sa-east-1.amazonaws.com", "us-east-1": "sns.us-east-1.amazonaws.com", "us-east-2": "sns.us-east-2.amazonaws.com", "us-gov-west-1": "sns.us-gov-west-1.amazonaws.com", "us-west-1": "sns.us-west-1.amazonaws.com", "us-west-2": "sns.us-west-2.amazonaws.com" }, "sqs": { "ap-northeast-1": "ap-northeast-1.queue.amazonaws.com", "ap-northeast-2": "ap-northeast-2.queue.amazonaws.com", "ap-south-1": "ap-south-1.queue.amazonaws.com", "ap-southeast-1": "ap-southeast-1.queue.amazonaws.com", "ap-southeast-2": "ap-southeast-2.queue.amazonaws.com", "ca-central-1": "ca-central-1.queue.amazonaws.com", "cn-north-1": "cn-north-1.queue.amazonaws.com.cn", "eu-central-1": "eu-central-1.queue.amazonaws.com", "eu-west-1": "eu-west-1.queue.amazonaws.com", "eu-west-2": "eu-west-2.queue.amazonaws.com", "sa-east-1": "sa-east-1.queue.amazonaws.com", "us-east-1": "queue.amazonaws.com", "us-east-2": "us-east-2.queue.amazonaws.com", "us-gov-west-1": "us-gov-west-1.queue.amazonaws.com", "us-west-1": "us-west-1.queue.amazonaws.com", "us-west-2": "us-west-2.queue.amazonaws.com" }, "storagegateway": { "ap-northeast-1": "storagegateway.ap-northeast-1.amazonaws.com", "ap-northeast-2": "storagegateway.ap-northeast-2.amazonaws.com", "ap-southeast-1": "storagegateway.ap-southeast-1.amazonaws.com", "ap-southeast-2": "storagegateway.ap-southeast-2.amazonaws.com", "ca-central-1": "storagegateway.ca-central-1.amazonaws.com", "cn-north-1": "storagegateway.cn-north-1.amazonaws.com.cn", "eu-central-1": "storagegateway.eu-central-1.amazonaws.com", "eu-west-1": "storagegateway.eu-west-1.amazonaws.com", "eu-west-2": "storagegateway.eu-west-2.amazonaws.com", "sa-east-1": "storagegateway.sa-east-1.amazonaws.com", "us-east-1": "storagegateway.us-east-1.amazonaws.com", "us-east-2": "storagegateway.us-east-2.amazonaws.com", "us-west-1": "storagegateway.us-west-1.amazonaws.com", "us-west-2": "storagegateway.us-west-2.amazonaws.com" }, "sts": { "ap-northeast-1": "sts.amazonaws.com", "ap-northeast-2": "sts.ap-northeast-2.amazonaws.com", "ap-south-1": "sts.amazonaws.com", "ap-southeast-1": "sts.amazonaws.com", "ap-southeast-2": "sts.amazonaws.com", "ca-central-1": "sts.amazonaws.com", "cn-north-1": "sts.cn-north-1.amazonaws.com.cn", "eu-central-1": "sts.amazonaws.com", "eu-west-1": "sts.amazonaws.com", "eu-west-2": "sts.amazonaws.com", "sa-east-1": "sts.amazonaws.com", "us-east-1": "sts.amazonaws.com", "us-east-2": "sts.amazonaws.com", "us-gov-west-1": "sts.us-gov-west-1.amazonaws.com", "us-west-1": "sts.amazonaws.com", "us-west-2": "sts.amazonaws.com" }, "support": { "us-east-1": "support.us-east-1.amazonaws.com" }, "swf": { "ap-northeast-1": "swf.ap-northeast-1.amazonaws.com", "ap-northeast-2": "swf.ap-northeast-2.amazonaws.com", "ap-south-1": "swf.ap-south-1.amazonaws.com", "ap-southeast-1": "swf.ap-southeast-1.amazonaws.com", "ap-southeast-2": "swf.ap-southeast-2.amazonaws.com", "ca-central-1": "swf.ca-central-1.amazonaws.com", "cn-north-1": "swf.cn-north-1.amazonaws.com.cn", "eu-central-1": "swf.eu-central-1.amazonaws.com", "eu-west-1": "swf.eu-west-1.amazonaws.com", "eu-west-2": "swf.eu-west-2.amazonaws.com", "sa-east-1": "swf.sa-east-1.amazonaws.com", "us-east-1": "swf.us-east-1.amazonaws.com", "us-east-2": "swf.us-east-2.amazonaws.com", "us-gov-west-1": "swf.us-gov-west-1.amazonaws.com", "us-west-1": "swf.us-west-1.amazonaws.com", "us-west-2": "swf.us-west-2.amazonaws.com" } } vendored/boto/tests/unit/utils/test_utils.py 0000644 00000034155 15025013616 0015356 0 ustar 00 # Copyright (c) 2010 Robert Mela # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.compat import mock, unittest import datetime import hashlib import hmac import locale import time import boto.utils from boto.utils import Password from boto.utils import pythonize_name from boto.utils import _build_instance_metadata_url from boto.utils import get_instance_userdata from boto.utils import retry_url from boto.utils import LazyLoadMetadata from boto.compat import json, _thread @unittest.skip("http://bugs.python.org/issue7980") class TestThreadImport(unittest.TestCase): def test_strptime(self): def f(): for m in range(1, 13): for d in range(1,29): boto.utils.parse_ts('2013-01-01T00:00:00Z') for _ in range(10): _thread.start_new_thread(f, ()) time.sleep(3) class TestPassword(unittest.TestCase): """Test basic password functionality""" def clstest(self, cls): """Insure that password.__eq__ hashes test value before compare.""" password = cls('foo') self.assertNotEqual(password, 'foo') password.set('foo') hashed = str(password) self.assertEqual(password, 'foo') self.assertEqual(password.str, hashed) password = cls(hashed) self.assertNotEqual(password.str, 'foo') self.assertEqual(password, 'foo') self.assertEqual(password.str, hashed) def test_aaa_version_1_9_default_behavior(self): self.clstest(Password) def test_custom_hashclass(self): class SHA224Password(Password): hashfunc = hashlib.sha224 password = SHA224Password() password.set('foo') self.assertEqual(hashlib.sha224(b'foo').hexdigest(), str(password)) def test_hmac(self): def hmac_hashfunc(cls, msg): if not isinstance(msg, bytes): msg = msg.encode('utf-8') return hmac.new(b'mysecretkey', msg) class HMACPassword(Password): hashfunc = hmac_hashfunc self.clstest(HMACPassword) password = HMACPassword() password.set('foo') self.assertEqual(str(password), hmac.new(b'mysecretkey', b'foo').hexdigest()) def test_constructor(self): hmac_hashfunc = lambda msg: hmac.new(b'mysecretkey', msg) password = Password(hashfunc=hmac_hashfunc) password.set('foo') self.assertEqual(password.str, hmac.new(b'mysecretkey', b'foo').hexdigest()) class TestPythonizeName(unittest.TestCase): def test_empty_string(self): self.assertEqual(pythonize_name(''), '') def test_all_lower_case(self): self.assertEqual(pythonize_name('lowercase'), 'lowercase') def test_all_upper_case(self): self.assertEqual(pythonize_name('UPPERCASE'), 'uppercase') def test_camel_case(self): self.assertEqual(pythonize_name('OriginallyCamelCased'), 'originally_camel_cased') def test_already_pythonized(self): self.assertEqual(pythonize_name('already_pythonized'), 'already_pythonized') def test_multiple_upper_cased_letters(self): self.assertEqual(pythonize_name('HTTPRequest'), 'http_request') self.assertEqual(pythonize_name('RequestForHTTP'), 'request_for_http') def test_string_with_numbers(self): self.assertEqual(pythonize_name('HTTPStatus200Ok'), 'http_status_200_ok') class TestBuildInstanceMetadataURL(unittest.TestCase): def test_normal(self): # This is the all-defaults case. self.assertEqual(_build_instance_metadata_url( 'http://169.254.169.254', 'latest', 'meta-data/' ), 'http://169.254.169.254/latest/meta-data/' ) def test_custom_path(self): self.assertEqual(_build_instance_metadata_url( 'http://169.254.169.254', 'latest', 'dynamic/' ), 'http://169.254.169.254/latest/dynamic/' ) def test_custom_version(self): self.assertEqual(_build_instance_metadata_url( 'http://169.254.169.254', '1.0', 'meta-data/' ), 'http://169.254.169.254/1.0/meta-data/' ) def test_custom_url(self): self.assertEqual(_build_instance_metadata_url( 'http://10.0.1.5', 'latest', 'meta-data/' ), 'http://10.0.1.5/latest/meta-data/' ) def test_all_custom(self): self.assertEqual(_build_instance_metadata_url( 'http://10.0.1.5', '2013-03-22', 'user-data' ), 'http://10.0.1.5/2013-03-22/user-data' ) class TestRetryURL(unittest.TestCase): def setUp(self): self.urlopen_patch = mock.patch('boto.compat.urllib.request.urlopen') self.opener_patch = mock.patch('boto.compat.urllib.request.build_opener') self.urlopen = self.urlopen_patch.start() self.opener = self.opener_patch.start() def tearDown(self): self.urlopen_patch.stop() self.opener_patch.stop() def set_normal_response(self, response): fake_response = mock.Mock() fake_response.read.return_value = response self.urlopen.return_value = fake_response def set_no_proxy_allowed_response(self, response): fake_response = mock.Mock() fake_response.read.return_value = response self.opener.return_value.open.return_value = fake_response def test_retry_url_uses_proxy(self): self.set_normal_response('normal response') self.set_no_proxy_allowed_response('no proxy response') response = retry_url('http://10.10.10.10/foo', num_retries=1) self.assertEqual(response, 'no proxy response') def test_retry_url_using_bytes_and_string_response(self): test_value = 'normal response' fake_response = mock.Mock() # test using unicode fake_response.read.return_value = test_value self.opener.return_value.open.return_value = fake_response response = retry_url('http://10.10.10.10/foo', num_retries=1) self.assertEqual(response, test_value) # test using bytes fake_response.read.return_value = test_value.encode('utf-8') self.opener.return_value.open.return_value = fake_response response = retry_url('http://10.10.10.10/foo', num_retries=1) self.assertEqual(response, test_value) class TestLazyLoadMetadata(unittest.TestCase): def setUp(self): self.retry_url_patch = mock.patch('boto.utils.retry_url') boto.utils.retry_url = self.retry_url_patch.start() def tearDown(self): self.retry_url_patch.stop() def set_normal_response(self, data): # here "data" should be a list of return values in some order fake_response = mock.Mock() fake_response.side_effect = data boto.utils.retry_url = fake_response def test_meta_data_with_invalid_json_format_happened_once(self): # here "key_data" will be stored in the "self._leaves" # when the class "LazyLoadMetadata" initialized key_data = "test" invalid_data = '{"invalid_json_format" : true,}' valid_data = '{ "%s" : {"valid_json_format": true}}' % key_data url = "/".join(["http://169.254.169.254", key_data]) num_retries = 2 self.set_normal_response([key_data, invalid_data, valid_data]) response = LazyLoadMetadata(url, num_retries) self.assertEqual(list(response.values())[0], json.loads(valid_data)) def test_meta_data_with_invalid_json_format_happened_twice(self): key_data = "test" invalid_data = '{"invalid_json_format" : true,}' valid_data = '{ "%s" : {"valid_json_format": true}}' % key_data url = "/".join(["http://169.254.169.254", key_data]) num_retries = 2 self.set_normal_response([key_data, invalid_data, invalid_data]) response = LazyLoadMetadata(url, num_retries) with self.assertRaises(ValueError): response.values()[0] def test_user_data(self): self.set_normal_response(['foo']) userdata = get_instance_userdata() self.assertEqual('foo', userdata) boto.utils.retry_url.assert_called_with( 'http://169.254.169.254/latest/user-data', retry_on_404=False, num_retries=5, timeout=None) def test_user_data_timeout(self): self.set_normal_response(['foo']) userdata = get_instance_userdata(timeout=1, num_retries=2) self.assertEqual('foo', userdata) boto.utils.retry_url.assert_called_with( 'http://169.254.169.254/latest/user-data', retry_on_404=False, num_retries=2, timeout=1) class TestStringToDatetimeParsing(unittest.TestCase): """ Test string to datetime parsing """ def setUp(self): self._saved = locale.setlocale(locale.LC_ALL) try: locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8') except locale.Error: self.skipTest('Unsupported locale setting') def tearDown(self): locale.setlocale(locale.LC_ALL, self._saved) def test_nonus_locale(self): test_string = 'Thu, 15 May 2014 09:06:03 GMT' # Default strptime shoudl fail with self.assertRaises(ValueError): datetime.datetime.strptime(test_string, boto.utils.RFC1123) # Our parser should succeed result = boto.utils.parse_ts(test_string) self.assertEqual(2014, result.year) self.assertEqual(5, result.month) self.assertEqual(15, result.day) self.assertEqual(9, result.hour) self.assertEqual(6, result.minute) class TestHostIsIPV6(unittest.TestCase): def test_is_ipv6_no_brackets(self): hostname = 'bf1d:cb48:4513:d1f1:efdd:b290:9ff9:64be' result = boto.utils.host_is_ipv6(hostname) self.assertTrue(result) def test_is_ipv6_with_brackets(self): hostname = '[bf1d:cb48:4513:d1f1:efdd:b290:9ff9:64be]' result = boto.utils.host_is_ipv6(hostname) self.assertTrue(result) def test_is_ipv6_with_brackets_and_port(self): hostname = '[bf1d:cb48:4513:d1f1:efdd:b290:9ff9:64be]:8080' result = boto.utils.host_is_ipv6(hostname) self.assertTrue(result) def test_is_ipv6_no_brackets_abbreviated(self): hostname = 'bf1d:cb48:4513::' result = boto.utils.host_is_ipv6(hostname) self.assertTrue(result) def test_is_ipv6_with_brackets_abbreviated(self): hostname = '[bf1d:cb48:4513::' result = boto.utils.host_is_ipv6(hostname) self.assertTrue(result) def test_is_ipv6_with_brackets_and_port_abbreviated(self): hostname = '[bf1d:cb48:4513::]:8080' result = boto.utils.host_is_ipv6(hostname) self.assertTrue(result) def test_empty_string(self): result = boto.utils.host_is_ipv6('') self.assertFalse(result) def test_not_of_string_type(self): hostnames = [None, 0, False, [], {}] for h in hostnames: result = boto.utils.host_is_ipv6(h) self.assertFalse(result) def test_ipv4_no_port(self): result = boto.utils.host_is_ipv6('192.168.1.1') self.assertFalse(result) def test_ipv4_with_port(self): result = boto.utils.host_is_ipv6('192.168.1.1:8080') self.assertFalse(result) def test_hostnames_are_not_ipv6_with_port(self): result = boto.utils.host_is_ipv6('example.org:8080') self.assertFalse(result) def test_hostnames_are_not_ipv6_without_port(self): result = boto.utils.host_is_ipv6('example.org') self.assertFalse(result) class TestParseHost(unittest.TestCase): def test_parses_ipv6_hosts_no_brackets(self): host = 'bf1d:cb48:4513:d1f1:efdd:b290:9ff9:64be' result = boto.utils.parse_host(host) self.assertEqual(result, host) def test_parses_ipv6_hosts_with_brackets_stripping_them(self): host = '[bf1d:cb48:4513:d1f1:efdd:b290:9ff9:64be]' result = boto.utils.parse_host(host) self.assertEqual(result, 'bf1d:cb48:4513:d1f1:efdd:b290:9ff9:64be') def test_parses_ipv6_hosts_with_brackets_and_port(self): host = '[bf1d:cb48:4513:d1f1:efdd:b290:9ff9:64be]:8080' result = boto.utils.parse_host(host) self.assertEqual(result, 'bf1d:cb48:4513:d1f1:efdd:b290:9ff9:64be') def test_parses_ipv4_hosts(self): host = '10.0.1.1' result = boto.utils.parse_host(host) self.assertEqual(result, host) def test_parses_ipv4_hosts_with_port(self): host = '192.168.168.200:8080' result = boto.utils.parse_host(host) self.assertEqual(result, '192.168.168.200') def test_parses_hostnames_with_port(self): host = 'example.org:8080' result = boto.utils.parse_host(host) self.assertEqual(result, 'example.org') def test_parses_hostnames_without_port(self): host = 'example.org' result = boto.utils.parse_host(host) self.assertEqual(result, host) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/utils/__init__.py 0000644 00000000000 15025013616 0014674 0 ustar 00 vendored/boto/tests/unit/test_regioninfo.py 0000644 00000020045 15025013616 0015206 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import os import mock import boto from boto.pyami.config import Config from boto.regioninfo import RegionInfo, load_endpoint_json, merge_endpoints from boto.regioninfo import load_regions, get_regions, connect from tests.unit import unittest class TestRegionInfo(object): def __init__(self, connection=None, name=None, endpoint=None, connection_cls=None): self.connection = connection self.name = name self.endpoint = endpoint self.connection_cls = connection_cls def connect(self, **kwargs): return self.connection_cls(region=self) class FakeConn(object): def __init__(self, region, **kwargs): self.region = region self.kwargs = kwargs class TestEndpointLoading(unittest.TestCase): def setUp(self): super(TestEndpointLoading, self).setUp() def test_load_endpoint_json(self): endpoints = load_endpoint_json(boto.ENDPOINTS_PATH) self.assertTrue('partitions' in endpoints) def test_merge_endpoints(self): defaults = { 'ec2': { 'us-east-1': 'ec2.us-east-1.amazonaws.com', 'us-west-1': 'ec2.us-west-1.amazonaws.com', } } additions = { # Top-level addition. 's3': { 'us-east-1': 's3.amazonaws.com' }, 'ec2': { # Overwrite. This doesn't exist, just test data. 'us-east-1': 'ec2.auto-resolve.amazonaws.com', # Deep addition. 'us-west-2': 'ec2.us-west-2.amazonaws.com', } } endpoints = merge_endpoints(defaults, additions) self.assertEqual(endpoints, { 'ec2': { 'us-east-1': 'ec2.auto-resolve.amazonaws.com', 'us-west-1': 'ec2.us-west-1.amazonaws.com', 'us-west-2': 'ec2.us-west-2.amazonaws.com', }, 's3': { 'us-east-1': 's3.amazonaws.com' } }) def test_load_regions(self): # Just the defaults. endpoints = load_regions() self.assertTrue('us-east-1' in endpoints['ec2']) self.assertFalse('test-1' in endpoints['ec2']) # With ENV overrides. os.environ['BOTO_ENDPOINTS'] = os.path.join( os.path.dirname(__file__), 'test_endpoints.json' ) self.addCleanup(os.environ.pop, 'BOTO_ENDPOINTS') endpoints = load_regions() self.assertTrue('us-east-1' in endpoints['ec2']) self.assertTrue('test-1' in endpoints['ec2']) self.assertEqual(endpoints['ec2']['test-1'], 'ec2.test-1.amazonaws.com') def test_get_regions(self): # With defaults. ec2_regions = get_regions('ec2') self.assertTrue(len(ec2_regions) >= 10) west_2 = None for region_info in ec2_regions: if region_info.name == 'us-west-2': west_2 = region_info break self.assertNotEqual(west_2, None, "Couldn't find the us-west-2 region!") self.assertTrue(isinstance(west_2, RegionInfo)) self.assertEqual(west_2.name, 'us-west-2') self.assertEqual(west_2.endpoint, 'ec2.us-west-2.amazonaws.com') self.assertEqual(west_2.connection_cls, None) def test_get_regions_overrides(self): ec2_regions = get_regions( 'ec2', region_cls=TestRegionInfo, connection_cls=FakeConn ) self.assertTrue(len(ec2_regions) >= 10) west_2 = None for region_info in ec2_regions: if region_info.name == 'us-west-2': west_2 = region_info break self.assertNotEqual(west_2, None, "Couldn't find the us-west-2 region!") self.assertFalse(isinstance(west_2, RegionInfo)) self.assertTrue(isinstance(west_2, TestRegionInfo)) self.assertEqual(west_2.name, 'us-west-2') self.assertEqual(west_2.endpoint, 'ec2.us-west-2.amazonaws.com') self.assertEqual(west_2.connection_cls, FakeConn) class TestConnectToRegion(unittest.TestCase): def test_connect(self): connection = connect( 'ec2', 'us-west-2', connection_cls=FakeConn) self.assertEqual(connection.region.name, 'us-west-2') expected_endpoint = 'ec2.us-west-2.amazonaws.com' self.assertEqual(connection.region.endpoint, expected_endpoint) def test_does_not_use_heuristics_by_default(self): connection = connect( 'ec2', 'us-southeast-43', connection_cls=FakeConn) self.assertIsNone(connection) def test_uses_region_override(self): connection = connect( 'ec2', 'us-west-2', connection_cls=FakeConn, region_cls=TestRegionInfo ) self.assertIsInstance(connection.region, TestRegionInfo) self.assertEqual(connection.region.name, 'us-west-2') expected_endpoint = 'ec2.us-west-2.amazonaws.com' self.assertEqual(connection.region.endpoint, expected_endpoint) def test_use_heuristics_via_env_var(self): # With ENV overrides. os.environ['BOTO_USE_ENDPOINT_HEURISTICS'] = 'True' self.addCleanup(os.environ.pop, 'BOTO_USE_ENDPOINT_HEURISTICS') connection = connect( 'ec2', 'us-southeast-43', connection_cls=FakeConn, region_cls=TestRegionInfo) self.assertIsNotNone(connection) self.assertEqual(connection.region.name, 'us-southeast-43') expected_endpoint = 'ec2.us-southeast-43.amazonaws.com' self.assertEqual(connection.region.endpoint, expected_endpoint) def test_use_heuristics_via_config(self): config = mock.Mock(spec=Config) def _getbool(section, name, default=False): if section == 'Boto' and name == 'use_endpoint_heuristics': return True return default config.getbool = _getbool config.get.return_value = None with mock.patch('boto.config', config): connection = connect( 'ec2', 'us-southeast-43', connection_cls=FakeConn, region_cls=TestRegionInfo) self.assertIsNotNone(connection) self.assertEqual(connection.region.name, 'us-southeast-43') expected_endpoint = 'ec2.us-southeast-43.amazonaws.com' self.assertEqual(connection.region.endpoint, expected_endpoint) def test_connect_with_hueristics_without_explicit_regioninfo(self): os.environ['BOTO_USE_ENDPOINT_HEURISTICS'] = 'True' self.addCleanup(os.environ.pop, 'BOTO_USE_ENDPOINT_HEURISTICS') connection = connect( 'ec2', 'us-southeast-43', connection_cls=FakeConn) self.assertIsNotNone(connection) self.assertIsInstance(connection.region, RegionInfo) self.assertEqual(connection.region.name, 'us-southeast-43') expected_endpoint = 'ec2.us-southeast-43.amazonaws.com' self.assertEqual(connection.region.endpoint, expected_endpoint) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/__init__.py 0000644 00000010031 15025013616 0013541 0 ustar 00 from boto.compat import http_client from tests.compat import mock, unittest class AWSMockServiceTestCase(unittest.TestCase): """Base class for mocking aws services.""" # This param is used by the unittest module to display a full # diff when assert*Equal methods produce an error message. maxDiff = None connection_class = None def setUp(self): self.https_connection = mock.Mock(spec=http_client.HTTPSConnection) self.https_connection.debuglevel = 0 self.https_connection_factory = ( mock.Mock(return_value=self.https_connection), ()) self.service_connection = self.create_service_connection( https_connection_factory=self.https_connection_factory, aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') self.initialize_service_connection() def initialize_service_connection(self): self.actual_request = None self.original_mexe = self.service_connection._mexe self.service_connection._mexe = self._mexe_spy self.proxy = None self.use_proxy = False def create_service_connection(self, **kwargs): if self.connection_class is None: raise ValueError("The connection_class class attribute must be " "set to a non-None value.") return self.connection_class(**kwargs) def _mexe_spy(self, request, *args, **kwargs): self.actual_request = request return self.original_mexe(request, *args, **kwargs) def create_response(self, status_code, reason='', header=[], body=None): if body is None: body = self.default_body() response = mock.Mock(spec=http_client.HTTPResponse) response.status = status_code response.read.return_value = body response.reason = reason response.getheaders.return_value = header response.msg = dict(header) def overwrite_header(arg, default=None): header_dict = dict(header) if arg in header_dict: return header_dict[arg] else: return default response.getheader.side_effect = overwrite_header return response def assert_request_parameters(self, params, ignore_params_values=None): """Verify the actual parameters sent to the service API.""" request_params = self.actual_request.params.copy() if ignore_params_values is not None: for param in ignore_params_values: try: del request_params[param] except KeyError: pass self.assertDictEqual(request_params, params) def set_http_response(self, status_code, reason='', header=[], body=None): http_response = self.create_response(status_code, reason, header, body) self.https_connection.getresponse.return_value = http_response def default_body(self): return '' class MockServiceWithConfigTestCase(AWSMockServiceTestCase): def setUp(self): super(MockServiceWithConfigTestCase, self).setUp() self.environ = {} self.config = {} self.config_patch = mock.patch('boto.provider.config.get', self.get_config) self.has_config_patch = mock.patch('boto.provider.config.has_option', self.has_config) self.environ_patch = mock.patch('os.environ', self.environ) self.config_patch.start() self.has_config_patch.start() self.environ_patch.start() def tearDown(self): self.config_patch.stop() self.has_config_patch.stop() self.environ_patch.stop() def has_config(self, section_name, key): try: self.config[section_name][key] return True except KeyError: return False def get_config(self, section_name, key, default=None): try: return self.config[section_name][key] except KeyError: return None vendored/boto/tests/unit/dynamodb2/test_layer1.py 0000644 00000004343 15025013616 0016126 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests for Layer1 of DynamoDB v2 """ from tests.unit import unittest from boto.dynamodb2.layer1 import DynamoDBConnection from boto.regioninfo import RegionInfo class DynamoDBv2Layer1UnitTest(unittest.TestCase): dynamodb = True def test_init_region(self): dynamodb = DynamoDBConnection( aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key') self.assertEqual(dynamodb.region.name, 'us-east-1') dynamodb = DynamoDBConnection( region=RegionInfo(name='us-west-2', endpoint='dynamodb.us-west-2.amazonaws.com'), aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key', ) self.assertEqual(dynamodb.region.name, 'us-west-2') def test_init_host_override(self): dynamodb = DynamoDBConnection( aws_access_key_id='aws_access_key_id', aws_secret_access_key='aws_secret_access_key', host='localhost', port=8000) self.assertEqual(dynamodb.host, 'localhost') self.assertEqual(dynamodb.port, 8000) vendored/boto/tests/unit/dynamodb2/test_table.py 0000644 00000312763 15025013616 0016030 0 ustar 00 from tests.compat import mock, unittest from boto.dynamodb2 import exceptions from boto.dynamodb2.fields import (HashKey, RangeKey, AllIndex, KeysOnlyIndex, IncludeIndex, GlobalAllIndex, GlobalKeysOnlyIndex, GlobalIncludeIndex) from boto.dynamodb2.items import Item from boto.dynamodb2.layer1 import DynamoDBConnection from boto.dynamodb2.results import ResultSet, BatchGetResultSet from boto.dynamodb2.table import Table from boto.dynamodb2.types import (STRING, NUMBER, BINARY, FILTER_OPERATORS, QUERY_OPERATORS) from boto.exception import JSONResponseError from boto.compat import six, long_type FakeDynamoDBConnection = mock.create_autospec(DynamoDBConnection) class SchemaFieldsTestCase(unittest.TestCase): def test_hash_key(self): hash_key = HashKey('hello') self.assertEqual(hash_key.name, 'hello') self.assertEqual(hash_key.data_type, STRING) self.assertEqual(hash_key.attr_type, 'HASH') self.assertEqual(hash_key.definition(), { 'AttributeName': 'hello', 'AttributeType': 'S' }) self.assertEqual(hash_key.schema(), { 'AttributeName': 'hello', 'KeyType': 'HASH' }) def test_range_key(self): range_key = RangeKey('hello') self.assertEqual(range_key.name, 'hello') self.assertEqual(range_key.data_type, STRING) self.assertEqual(range_key.attr_type, 'RANGE') self.assertEqual(range_key.definition(), { 'AttributeName': 'hello', 'AttributeType': 'S' }) self.assertEqual(range_key.schema(), { 'AttributeName': 'hello', 'KeyType': 'RANGE' }) def test_alternate_type(self): alt_key = HashKey('alt', data_type=NUMBER) self.assertEqual(alt_key.name, 'alt') self.assertEqual(alt_key.data_type, NUMBER) self.assertEqual(alt_key.attr_type, 'HASH') self.assertEqual(alt_key.definition(), { 'AttributeName': 'alt', 'AttributeType': 'N' }) self.assertEqual(alt_key.schema(), { 'AttributeName': 'alt', 'KeyType': 'HASH' }) class IndexFieldTestCase(unittest.TestCase): def test_all_index(self): all_index = AllIndex('AllKeys', parts=[ HashKey('username'), RangeKey('date_joined') ]) self.assertEqual(all_index.name, 'AllKeys') self.assertEqual([part.attr_type for part in all_index.parts], [ 'HASH', 'RANGE' ]) self.assertEqual(all_index.projection_type, 'ALL') self.assertEqual(all_index.definition(), [ {'AttributeName': 'username', 'AttributeType': 'S'}, {'AttributeName': 'date_joined', 'AttributeType': 'S'} ]) self.assertEqual(all_index.schema(), { 'IndexName': 'AllKeys', 'KeySchema': [ { 'AttributeName': 'username', 'KeyType': 'HASH' }, { 'AttributeName': 'date_joined', 'KeyType': 'RANGE' } ], 'Projection': { 'ProjectionType': 'ALL' } }) def test_keys_only_index(self): keys_only = KeysOnlyIndex('KeysOnly', parts=[ HashKey('username'), RangeKey('date_joined') ]) self.assertEqual(keys_only.name, 'KeysOnly') self.assertEqual([part.attr_type for part in keys_only.parts], [ 'HASH', 'RANGE' ]) self.assertEqual(keys_only.projection_type, 'KEYS_ONLY') self.assertEqual(keys_only.definition(), [ {'AttributeName': 'username', 'AttributeType': 'S'}, {'AttributeName': 'date_joined', 'AttributeType': 'S'} ]) self.assertEqual(keys_only.schema(), { 'IndexName': 'KeysOnly', 'KeySchema': [ { 'AttributeName': 'username', 'KeyType': 'HASH' }, { 'AttributeName': 'date_joined', 'KeyType': 'RANGE' } ], 'Projection': { 'ProjectionType': 'KEYS_ONLY' } }) def test_include_index(self): include_index = IncludeIndex('IncludeKeys', parts=[ HashKey('username'), RangeKey('date_joined') ], includes=[ 'gender', 'friend_count' ]) self.assertEqual(include_index.name, 'IncludeKeys') self.assertEqual([part.attr_type for part in include_index.parts], [ 'HASH', 'RANGE' ]) self.assertEqual(include_index.projection_type, 'INCLUDE') self.assertEqual(include_index.definition(), [ {'AttributeName': 'username', 'AttributeType': 'S'}, {'AttributeName': 'date_joined', 'AttributeType': 'S'} ]) self.assertEqual(include_index.schema(), { 'IndexName': 'IncludeKeys', 'KeySchema': [ { 'AttributeName': 'username', 'KeyType': 'HASH' }, { 'AttributeName': 'date_joined', 'KeyType': 'RANGE' } ], 'Projection': { 'ProjectionType': 'INCLUDE', 'NonKeyAttributes': [ 'gender', 'friend_count', ] } }) def test_global_all_index(self): all_index = GlobalAllIndex('AllKeys', parts=[ HashKey('username'), RangeKey('date_joined') ], throughput={ 'read': 6, 'write': 2, }) self.assertEqual(all_index.name, 'AllKeys') self.assertEqual([part.attr_type for part in all_index.parts], [ 'HASH', 'RANGE' ]) self.assertEqual(all_index.projection_type, 'ALL') self.assertEqual(all_index.definition(), [ {'AttributeName': 'username', 'AttributeType': 'S'}, {'AttributeName': 'date_joined', 'AttributeType': 'S'} ]) self.assertEqual(all_index.schema(), { 'IndexName': 'AllKeys', 'KeySchema': [ { 'AttributeName': 'username', 'KeyType': 'HASH' }, { 'AttributeName': 'date_joined', 'KeyType': 'RANGE' } ], 'Projection': { 'ProjectionType': 'ALL' }, 'ProvisionedThroughput': { 'ReadCapacityUnits': 6, 'WriteCapacityUnits': 2 } }) def test_global_keys_only_index(self): keys_only = GlobalKeysOnlyIndex('KeysOnly', parts=[ HashKey('username'), RangeKey('date_joined') ], throughput={ 'read': 3, 'write': 4, }) self.assertEqual(keys_only.name, 'KeysOnly') self.assertEqual([part.attr_type for part in keys_only.parts], [ 'HASH', 'RANGE' ]) self.assertEqual(keys_only.projection_type, 'KEYS_ONLY') self.assertEqual(keys_only.definition(), [ {'AttributeName': 'username', 'AttributeType': 'S'}, {'AttributeName': 'date_joined', 'AttributeType': 'S'} ]) self.assertEqual(keys_only.schema(), { 'IndexName': 'KeysOnly', 'KeySchema': [ { 'AttributeName': 'username', 'KeyType': 'HASH' }, { 'AttributeName': 'date_joined', 'KeyType': 'RANGE' } ], 'Projection': { 'ProjectionType': 'KEYS_ONLY' }, 'ProvisionedThroughput': { 'ReadCapacityUnits': 3, 'WriteCapacityUnits': 4 } }) def test_global_include_index(self): # Lean on the default throughput include_index = GlobalIncludeIndex('IncludeKeys', parts=[ HashKey('username'), RangeKey('date_joined') ], includes=[ 'gender', 'friend_count' ]) self.assertEqual(include_index.name, 'IncludeKeys') self.assertEqual([part.attr_type for part in include_index.parts], [ 'HASH', 'RANGE' ]) self.assertEqual(include_index.projection_type, 'INCLUDE') self.assertEqual(include_index.definition(), [ {'AttributeName': 'username', 'AttributeType': 'S'}, {'AttributeName': 'date_joined', 'AttributeType': 'S'} ]) self.assertEqual(include_index.schema(), { 'IndexName': 'IncludeKeys', 'KeySchema': [ { 'AttributeName': 'username', 'KeyType': 'HASH' }, { 'AttributeName': 'date_joined', 'KeyType': 'RANGE' } ], 'Projection': { 'ProjectionType': 'INCLUDE', 'NonKeyAttributes': [ 'gender', 'friend_count', ] }, 'ProvisionedThroughput': { 'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5 } }) def test_global_include_index_throughput(self): include_index = GlobalIncludeIndex('IncludeKeys', parts=[ HashKey('username'), RangeKey('date_joined') ], includes=[ 'gender', 'friend_count' ], throughput={ 'read': 10, 'write': 8 }) self.assertEqual(include_index.schema(), { 'IndexName': 'IncludeKeys', 'KeySchema': [ { 'AttributeName': 'username', 'KeyType': 'HASH' }, { 'AttributeName': 'date_joined', 'KeyType': 'RANGE' } ], 'Projection': { 'ProjectionType': 'INCLUDE', 'NonKeyAttributes': [ 'gender', 'friend_count', ] }, 'ProvisionedThroughput': { 'ReadCapacityUnits': 10, 'WriteCapacityUnits': 8 } }) class ItemTestCase(unittest.TestCase): if six.PY2: assertCountEqual = unittest.TestCase.assertItemsEqual def setUp(self): super(ItemTestCase, self).setUp() self.table = Table('whatever', connection=FakeDynamoDBConnection()) self.johndoe = self.create_item({ 'username': 'johndoe', 'first_name': 'John', 'date_joined': 12345, }) def create_item(self, data): return Item(self.table, data=data) def test_initialization(self): empty_item = Item(self.table) self.assertEqual(empty_item.table, self.table) self.assertEqual(empty_item._data, {}) full_item = Item(self.table, data={ 'username': 'johndoe', 'date_joined': 12345, }) self.assertEqual(full_item.table, self.table) self.assertEqual(full_item._data, { 'username': 'johndoe', 'date_joined': 12345, }) # The next couple methods make use of ``sorted(...)`` so we get consistent # ordering everywhere & no erroneous failures. def test_keys(self): self.assertCountEqual(self.johndoe.keys(), [ 'date_joined', 'first_name', 'username', ]) def test_values(self): self.assertCountEqual(self.johndoe.values(), [12345, 'John', 'johndoe']) def test_contains(self): self.assertIn('username', self.johndoe) self.assertIn('first_name', self.johndoe) self.assertIn('date_joined', self.johndoe) self.assertNotIn('whatever', self.johndoe) def test_iter(self): self.assertCountEqual(self.johndoe, ['johndoe', 'John', 12345]) def test_get(self): self.assertEqual(self.johndoe.get('username'), 'johndoe') self.assertEqual(self.johndoe.get('first_name'), 'John') self.assertEqual(self.johndoe.get('date_joined'), 12345) # Test a missing key. No default yields ``None``. self.assertEqual(self.johndoe.get('last_name'), None) # This time with a default. self.assertEqual(self.johndoe.get('last_name', True), True) def test_items(self): self.assertCountEqual( self.johndoe.items(), [ ('date_joined', 12345), ('first_name', 'John'), ('username', 'johndoe'), ]) def test_attribute_access(self): self.assertEqual(self.johndoe['username'], 'johndoe') self.assertEqual(self.johndoe['first_name'], 'John') self.assertEqual(self.johndoe['date_joined'], 12345) # Test a missing key. self.assertEqual(self.johndoe['last_name'], None) # Set a key. self.johndoe['last_name'] = 'Doe' # Test accessing the new key. self.assertEqual(self.johndoe['last_name'], 'Doe') # Delete a key. del self.johndoe['last_name'] # Test the now-missing-again key. self.assertEqual(self.johndoe['last_name'], None) def test_needs_save(self): self.johndoe.mark_clean() self.assertFalse(self.johndoe.needs_save()) self.johndoe['last_name'] = 'Doe' self.assertTrue(self.johndoe.needs_save()) def test_needs_save_set_changed(self): # First, ensure we're clean. self.johndoe.mark_clean() self.assertFalse(self.johndoe.needs_save()) # Add a friends collection. self.johndoe['friends'] = set(['jane', 'alice']) self.assertTrue(self.johndoe.needs_save()) # Now mark it clean, then change the collection. # This does NOT call ``__setitem__``, so the item used to be # incorrectly appearing to be clean, when it had in fact been changed. self.johndoe.mark_clean() self.assertFalse(self.johndoe.needs_save()) self.johndoe['friends'].add('bob') self.assertTrue(self.johndoe.needs_save()) def test_mark_clean(self): self.johndoe['last_name'] = 'Doe' self.assertTrue(self.johndoe.needs_save()) self.johndoe.mark_clean() self.assertFalse(self.johndoe.needs_save()) def test_load(self): empty_item = Item(self.table) empty_item.load({ 'Item': { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056668'}, 'friend_count': {'N': '3'}, 'friends': {'SS': ['alice', 'bob', 'jane']}, } }) self.assertEqual(empty_item['username'], 'johndoe') self.assertEqual(empty_item['date_joined'], 1366056668) self.assertEqual(sorted(empty_item['friends']), sorted([ 'alice', 'bob', 'jane' ])) def test_get_keys(self): # Setup the data. self.table.schema = [ HashKey('username'), RangeKey('date_joined'), ] self.assertEqual(self.johndoe.get_keys(), { 'username': 'johndoe', 'date_joined': 12345, }) def test_get_raw_keys(self): # Setup the data. self.table.schema = [ HashKey('username'), RangeKey('date_joined'), ] self.assertEqual(self.johndoe.get_raw_keys(), { 'username': {'S': 'johndoe'}, 'date_joined': {'N': '12345'}, }) def test_build_expects(self): # Pristine. self.assertEqual(self.johndoe.build_expects(), { 'first_name': { 'Exists': False, }, 'username': { 'Exists': False, }, 'date_joined': { 'Exists': False, }, }) # Without modifications. self.johndoe.mark_clean() self.assertEqual(self.johndoe.build_expects(), { 'first_name': { 'Exists': True, 'Value': { 'S': 'John', }, }, 'username': { 'Exists': True, 'Value': { 'S': 'johndoe', }, }, 'date_joined': { 'Exists': True, 'Value': { 'N': '12345', }, }, }) # Change some data. self.johndoe['first_name'] = 'Johann' # Add some data. self.johndoe['last_name'] = 'Doe' # Delete some data. del self.johndoe['date_joined'] # All fields (default). self.assertEqual(self.johndoe.build_expects(), { 'first_name': { 'Exists': True, 'Value': { 'S': 'John', }, }, 'last_name': { 'Exists': False, }, 'username': { 'Exists': True, 'Value': { 'S': 'johndoe', }, }, 'date_joined': { 'Exists': True, 'Value': { 'N': '12345', }, }, }) # Only a subset of the fields. self.assertEqual(self.johndoe.build_expects(fields=[ 'first_name', 'last_name', 'date_joined', ]), { 'first_name': { 'Exists': True, 'Value': { 'S': 'John', }, }, 'last_name': { 'Exists': False, }, 'date_joined': { 'Exists': True, 'Value': { 'N': '12345', }, }, }) def test_prepare_full(self): self.assertEqual(self.johndoe.prepare_full(), { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'date_joined': {'N': '12345'} }) self.johndoe['friends'] = set(['jane', 'alice']) data = self.johndoe.prepare_full() self.assertEqual(data['username'], {'S': 'johndoe'}) self.assertEqual(data['first_name'], {'S': 'John'}) self.assertEqual(data['date_joined'], {'N': '12345'}) self.assertCountEqual(data['friends']['SS'], ['jane', 'alice']) def test_prepare_full_empty_set(self): self.johndoe['friends'] = set() self.assertEqual(self.johndoe.prepare_full(), { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'date_joined': {'N': '12345'} }) def test_prepare_partial(self): self.johndoe.mark_clean() # Change some data. self.johndoe['first_name'] = 'Johann' # Add some data. self.johndoe['last_name'] = 'Doe' # Delete some data. del self.johndoe['date_joined'] final_data, fields = self.johndoe.prepare_partial() self.assertEqual(final_data, { 'date_joined': { 'Action': 'DELETE', }, 'first_name': { 'Action': 'PUT', 'Value': {'S': 'Johann'}, }, 'last_name': { 'Action': 'PUT', 'Value': {'S': 'Doe'}, }, }) self.assertEqual(fields, set([ 'first_name', 'last_name', 'date_joined' ])) def test_prepare_partial_empty_set(self): self.johndoe.mark_clean() # Change some data. self.johndoe['first_name'] = 'Johann' # Add some data. self.johndoe['last_name'] = 'Doe' # Delete some data. del self.johndoe['date_joined'] # Put an empty set on the ``Item``. self.johndoe['friends'] = set() final_data, fields = self.johndoe.prepare_partial() self.assertEqual(final_data, { 'date_joined': { 'Action': 'DELETE', }, 'first_name': { 'Action': 'PUT', 'Value': {'S': 'Johann'}, }, 'last_name': { 'Action': 'PUT', 'Value': {'S': 'Doe'}, }, }) self.assertEqual(fields, set([ 'first_name', 'last_name', 'date_joined' ])) def test_save_no_changes(self): # Unchanged, no save. with mock.patch.object(self.table, '_put_item', return_value=True) \ as mock_put_item: # Pretend we loaded it via ``get_item``... self.johndoe.mark_clean() self.assertFalse(self.johndoe.save()) self.assertFalse(mock_put_item.called) def test_save_with_changes(self): # With changed data. with mock.patch.object(self.table, '_put_item', return_value=True) \ as mock_put_item: self.johndoe.mark_clean() self.johndoe['first_name'] = 'J' self.johndoe['new_attr'] = 'never_seen_before' self.assertTrue(self.johndoe.save()) self.assertFalse(self.johndoe.needs_save()) self.assertTrue(mock_put_item.called) mock_put_item.assert_called_once_with({ 'username': {'S': 'johndoe'}, 'first_name': {'S': 'J'}, 'new_attr': {'S': 'never_seen_before'}, 'date_joined': {'N': '12345'} }, expects={ 'username': { 'Value': { 'S': 'johndoe', }, 'Exists': True, }, 'first_name': { 'Value': { 'S': 'John', }, 'Exists': True, }, 'new_attr': { 'Exists': False, }, 'date_joined': { 'Value': { 'N': '12345', }, 'Exists': True, }, }) def test_save_with_changes_overwrite(self): # With changed data. with mock.patch.object(self.table, '_put_item', return_value=True) \ as mock_put_item: self.johndoe['first_name'] = 'J' self.johndoe['new_attr'] = 'never_seen_before' # OVERWRITE ALL THE THINGS self.assertTrue(self.johndoe.save(overwrite=True)) self.assertFalse(self.johndoe.needs_save()) self.assertTrue(mock_put_item.called) mock_put_item.assert_called_once_with({ 'username': {'S': 'johndoe'}, 'first_name': {'S': 'J'}, 'new_attr': {'S': 'never_seen_before'}, 'date_joined': {'N': '12345'} }, expects=None) def test_partial_no_changes(self): # Unchanged, no save. with mock.patch.object(self.table, '_update_item', return_value=True) \ as mock_update_item: # Pretend we loaded it via ``get_item``... self.johndoe.mark_clean() self.assertFalse(self.johndoe.partial_save()) self.assertFalse(mock_update_item.called) def test_partial_with_changes(self): # Setup the data. self.table.schema = [ HashKey('username'), ] # With changed data. with mock.patch.object(self.table, '_update_item', return_value=True) \ as mock_update_item: # Pretend we loaded it via ``get_item``... self.johndoe.mark_clean() # Now... MODIFY!!! self.johndoe['first_name'] = 'J' self.johndoe['last_name'] = 'Doe' del self.johndoe['date_joined'] self.assertTrue(self.johndoe.partial_save()) self.assertFalse(self.johndoe.needs_save()) self.assertTrue(mock_update_item.called) mock_update_item.assert_called_once_with({ 'username': 'johndoe', }, { 'first_name': { 'Action': 'PUT', 'Value': {'S': 'J'}, }, 'last_name': { 'Action': 'PUT', 'Value': {'S': 'Doe'}, }, 'date_joined': { 'Action': 'DELETE', } }, expects={ 'first_name': { 'Value': { 'S': 'John', }, 'Exists': True }, 'last_name': { 'Exists': False }, 'date_joined': { 'Value': { 'N': '12345', }, 'Exists': True }, }) def test_delete(self): # Setup the data. self.table.schema = [ HashKey('username'), RangeKey('date_joined'), ] with mock.patch.object(self.table, 'delete_item', return_value=True) \ as mock_delete_item: self.johndoe.delete() self.assertTrue(mock_delete_item.called) mock_delete_item.assert_called_once_with( username='johndoe', date_joined=12345 ) def test_nonzero(self): self.assertTrue(self.johndoe) self.assertFalse(self.create_item({})) class ItemFromItemTestCase(ItemTestCase): def setUp(self): super(ItemFromItemTestCase, self).setUp() self.johndoe = self.create_item(self.johndoe) def fake_results(name, greeting='hello', exclusive_start_key=None, limit=None): if exclusive_start_key is None: exclusive_start_key = -1 if limit == 0: raise Exception("Web Service Returns '400 Bad Request'") end_cap = 13 results = [] start_key = exclusive_start_key + 1 for i in range(start_key, start_key + 5): if i < end_cap: results.append("%s %s #%s" % (greeting, name, i)) # Don't return more than limit results if limit < len(results): results = results[:limit] retval = { 'results': results, } if exclusive_start_key + 5 < end_cap: retval['last_key'] = exclusive_start_key + 5 return retval class ResultSetTestCase(unittest.TestCase): def setUp(self): super(ResultSetTestCase, self).setUp() self.results = ResultSet() self.result_function = mock.MagicMock(side_effect=fake_results) self.results.to_call(self.result_function, 'john', greeting='Hello', limit=20) def test_first_key(self): self.assertEqual(self.results.first_key, 'exclusive_start_key') def test_max_page_size_fetch_more(self): self.results = ResultSet(max_page_size=10) self.results.to_call(self.result_function, 'john', greeting='Hello') self.results.fetch_more() self.result_function.assert_called_with('john', greeting='Hello', limit=10) self.result_function.reset_mock() def test_max_page_size_and_smaller_limit_fetch_more(self): self.results = ResultSet(max_page_size=10) self.results.to_call(self.result_function, 'john', greeting='Hello', limit=5) self.results.fetch_more() self.result_function.assert_called_with('john', greeting='Hello', limit=5) self.result_function.reset_mock() def test_max_page_size_and_bigger_limit_fetch_more(self): self.results = ResultSet(max_page_size=10) self.results.to_call(self.result_function, 'john', greeting='Hello', limit=15) self.results.fetch_more() self.result_function.assert_called_with('john', greeting='Hello', limit=10) self.result_function.reset_mock() def test_fetch_more(self): # First "page". self.results.fetch_more() self.assertEqual(self.results._results, [ 'Hello john #0', 'Hello john #1', 'Hello john #2', 'Hello john #3', 'Hello john #4', ]) self.result_function.assert_called_with('john', greeting='Hello', limit=20) self.result_function.reset_mock() # Fake in a last key. self.results._last_key_seen = 4 # Second "page". self.results.fetch_more() self.assertEqual(self.results._results, [ 'Hello john #5', 'Hello john #6', 'Hello john #7', 'Hello john #8', 'Hello john #9', ]) self.result_function.assert_called_with('john', greeting='Hello', limit=20, exclusive_start_key=4) self.result_function.reset_mock() # Fake in a last key. self.results._last_key_seen = 9 # Last "page". self.results.fetch_more() self.assertEqual(self.results._results, [ 'Hello john #10', 'Hello john #11', 'Hello john #12', ]) # Fake in a key outside the range. self.results._last_key_seen = 15 # Empty "page". Nothing new gets added self.results.fetch_more() self.assertEqual(self.results._results, []) # Make sure we won't check for results in the future. self.assertFalse(self.results._results_left) def test_iteration(self): # First page. self.assertEqual(next(self.results), 'Hello john #0') self.assertEqual(next(self.results), 'Hello john #1') self.assertEqual(next(self.results), 'Hello john #2') self.assertEqual(next(self.results), 'Hello john #3') self.assertEqual(next(self.results), 'Hello john #4') self.assertEqual(self.results._limit, 15) # Second page. self.assertEqual(next(self.results), 'Hello john #5') self.assertEqual(next(self.results), 'Hello john #6') self.assertEqual(next(self.results), 'Hello john #7') self.assertEqual(next(self.results), 'Hello john #8') self.assertEqual(next(self.results), 'Hello john #9') self.assertEqual(self.results._limit, 10) # Third page. self.assertEqual(next(self.results), 'Hello john #10') self.assertEqual(next(self.results), 'Hello john #11') self.assertEqual(next(self.results), 'Hello john #12') self.assertRaises(StopIteration, self.results.next) self.assertEqual(self.results._limit, 7) def test_limit_smaller_than_first_page(self): results = ResultSet() results.to_call(fake_results, 'john', greeting='Hello', limit=2) self.assertEqual(next(results), 'Hello john #0') self.assertEqual(next(results), 'Hello john #1') self.assertRaises(StopIteration, results.next) def test_limit_equals_page(self): results = ResultSet() results.to_call(fake_results, 'john', greeting='Hello', limit=5) # First page self.assertEqual(next(results), 'Hello john #0') self.assertEqual(next(results), 'Hello john #1') self.assertEqual(next(results), 'Hello john #2') self.assertEqual(next(results), 'Hello john #3') self.assertEqual(next(results), 'Hello john #4') self.assertRaises(StopIteration, results.next) def test_limit_greater_than_page(self): results = ResultSet() results.to_call(fake_results, 'john', greeting='Hello', limit=6) # First page self.assertEqual(next(results), 'Hello john #0') self.assertEqual(next(results), 'Hello john #1') self.assertEqual(next(results), 'Hello john #2') self.assertEqual(next(results), 'Hello john #3') self.assertEqual(next(results), 'Hello john #4') # Second page self.assertEqual(next(results), 'Hello john #5') self.assertRaises(StopIteration, results.next) def test_iteration_noresults(self): def none(limit=10): return { 'results': [], } results = ResultSet() results.to_call(none, limit=20) self.assertRaises(StopIteration, results.next) def test_iteration_sporadic_pages(self): # Some pages have no/incomplete results but have a ``LastEvaluatedKey`` # (for instance, scans with filters), so we need to accommodate that. def sporadic(): # A dict, because Python closures have read-only access to the # reference itself. count = {'value': -1} def _wrapper(limit=10, exclusive_start_key=None): count['value'] = count['value'] + 1 if count['value'] == 0: # Full page. return { 'results': [ 'Result #0', 'Result #1', 'Result #2', 'Result #3', ], 'last_key': 'page-1' } elif count['value'] == 1: # Empty page but continue. return { 'results': [], 'last_key': 'page-2' } elif count['value'] == 2: # Final page. return { 'results': [ 'Result #4', 'Result #5', 'Result #6', ], } return _wrapper results = ResultSet() results.to_call(sporadic(), limit=20) # First page self.assertEqual(next(results), 'Result #0') self.assertEqual(next(results), 'Result #1') self.assertEqual(next(results), 'Result #2') self.assertEqual(next(results), 'Result #3') # Second page (misses!) # Moves on to the third page self.assertEqual(next(results), 'Result #4') self.assertEqual(next(results), 'Result #5') self.assertEqual(next(results), 'Result #6') self.assertRaises(StopIteration, results.next) def test_list(self): self.assertEqual(list(self.results), [ 'Hello john #0', 'Hello john #1', 'Hello john #2', 'Hello john #3', 'Hello john #4', 'Hello john #5', 'Hello john #6', 'Hello john #7', 'Hello john #8', 'Hello john #9', 'Hello john #10', 'Hello john #11', 'Hello john #12' ]) def fake_batch_results(keys): results = [] simulate_unprocessed = True if len(keys) and keys[0] == 'johndoe': simulate_unprocessed = False for key in keys: if simulate_unprocessed and key == 'johndoe': continue results.append("hello %s" % key) retval = { 'results': results, 'last_key': None, } if simulate_unprocessed: retval['unprocessed_keys'] = ['johndoe'] return retval class BatchGetResultSetTestCase(unittest.TestCase): def setUp(self): super(BatchGetResultSetTestCase, self).setUp() self.results = BatchGetResultSet(keys=[ 'alice', 'bob', 'jane', 'johndoe', ]) self.results.to_call(fake_batch_results) def test_fetch_more(self): # First "page". self.results.fetch_more() self.assertEqual(self.results._results, [ 'hello alice', 'hello bob', 'hello jane', ]) self.assertEqual(self.results._keys_left, ['johndoe']) # Second "page". self.results.fetch_more() self.assertEqual(self.results._results, [ 'hello johndoe', ]) # Empty "page". Nothing new gets added self.results.fetch_more() self.assertEqual(self.results._results, []) # Make sure we won't check for results in the future. self.assertFalse(self.results._results_left) def test_fetch_more_empty(self): self.results.to_call(lambda keys: {'results': [], 'last_key': None}) self.results.fetch_more() self.assertEqual(self.results._results, []) self.assertRaises(StopIteration, self.results.next) def test_iteration(self): # First page. self.assertEqual(next(self.results), 'hello alice') self.assertEqual(next(self.results), 'hello bob') self.assertEqual(next(self.results), 'hello jane') self.assertEqual(next(self.results), 'hello johndoe') self.assertRaises(StopIteration, self.results.next) class TableTestCase(unittest.TestCase): def setUp(self): super(TableTestCase, self).setUp() self.users = Table('users', connection=FakeDynamoDBConnection()) self.default_connection = DynamoDBConnection( aws_access_key_id='access_key', aws_secret_access_key='secret_key' ) def test__introspect_schema(self): raw_schema_1 = [ { "AttributeName": "username", "KeyType": "HASH" }, { "AttributeName": "date_joined", "KeyType": "RANGE" } ] raw_attributes_1 = [ { 'AttributeName': 'username', 'AttributeType': 'S' }, { 'AttributeName': 'date_joined', 'AttributeType': 'S' }, ] schema_1 = self.users._introspect_schema(raw_schema_1, raw_attributes_1) self.assertEqual(len(schema_1), 2) self.assertTrue(isinstance(schema_1[0], HashKey)) self.assertEqual(schema_1[0].name, 'username') self.assertTrue(isinstance(schema_1[1], RangeKey)) self.assertEqual(schema_1[1].name, 'date_joined') raw_schema_2 = [ { "AttributeName": "username", "KeyType": "BTREE" }, ] raw_attributes_2 = [ { 'AttributeName': 'username', 'AttributeType': 'S' }, ] self.assertRaises( exceptions.UnknownSchemaFieldError, self.users._introspect_schema, raw_schema_2, raw_attributes_2 ) # Test a complex schema & ensure the types come back correctly. raw_schema_3 = [ { "AttributeName": "user_id", "KeyType": "HASH" }, { "AttributeName": "junk", "KeyType": "RANGE" } ] raw_attributes_3 = [ { 'AttributeName': 'user_id', 'AttributeType': 'N' }, { 'AttributeName': 'junk', 'AttributeType': 'B' }, ] schema_3 = self.users._introspect_schema(raw_schema_3, raw_attributes_3) self.assertEqual(len(schema_3), 2) self.assertTrue(isinstance(schema_3[0], HashKey)) self.assertEqual(schema_3[0].name, 'user_id') self.assertEqual(schema_3[0].data_type, NUMBER) self.assertTrue(isinstance(schema_3[1], RangeKey)) self.assertEqual(schema_3[1].name, 'junk') self.assertEqual(schema_3[1].data_type, BINARY) def test__introspect_indexes(self): raw_indexes_1 = [ { "IndexName": "MostRecentlyJoinedIndex", "KeySchema": [ { "AttributeName": "username", "KeyType": "HASH" }, { "AttributeName": "date_joined", "KeyType": "RANGE" } ], "Projection": { "ProjectionType": "KEYS_ONLY" } }, { "IndexName": "EverybodyIndex", "KeySchema": [ { "AttributeName": "username", "KeyType": "HASH" }, ], "Projection": { "ProjectionType": "ALL" } }, { "IndexName": "GenderIndex", "KeySchema": [ { "AttributeName": "username", "KeyType": "HASH" }, { "AttributeName": "date_joined", "KeyType": "RANGE" } ], "Projection": { "ProjectionType": "INCLUDE", "NonKeyAttributes": [ 'gender', ] } } ] indexes_1 = self.users._introspect_indexes(raw_indexes_1) self.assertEqual(len(indexes_1), 3) self.assertTrue(isinstance(indexes_1[0], KeysOnlyIndex)) self.assertEqual(indexes_1[0].name, 'MostRecentlyJoinedIndex') self.assertEqual(len(indexes_1[0].parts), 2) self.assertTrue(isinstance(indexes_1[1], AllIndex)) self.assertEqual(indexes_1[1].name, 'EverybodyIndex') self.assertEqual(len(indexes_1[1].parts), 1) self.assertTrue(isinstance(indexes_1[2], IncludeIndex)) self.assertEqual(indexes_1[2].name, 'GenderIndex') self.assertEqual(len(indexes_1[2].parts), 2) self.assertEqual(indexes_1[2].includes_fields, ['gender']) raw_indexes_2 = [ { "IndexName": "MostRecentlyJoinedIndex", "KeySchema": [ { "AttributeName": "username", "KeyType": "HASH" }, { "AttributeName": "date_joined", "KeyType": "RANGE" } ], "Projection": { "ProjectionType": "SOMETHING_CRAZY" } }, ] self.assertRaises( exceptions.UnknownIndexFieldError, self.users._introspect_indexes, raw_indexes_2 ) def test_initialization(self): users = Table('users', connection=self.default_connection) self.assertEqual(users.table_name, 'users') self.assertTrue(isinstance(users.connection, DynamoDBConnection)) self.assertEqual(users.throughput['read'], 5) self.assertEqual(users.throughput['write'], 5) self.assertEqual(users.schema, None) self.assertEqual(users.indexes, None) groups = Table('groups', connection=FakeDynamoDBConnection()) self.assertEqual(groups.table_name, 'groups') self.assertTrue(hasattr(groups.connection, 'assert_called_once_with')) def test_create_simple(self): conn = FakeDynamoDBConnection() with mock.patch.object(conn, 'create_table', return_value={}) \ as mock_create_table: retval = Table.create('users', schema=[ HashKey('username'), RangeKey('date_joined', data_type=NUMBER) ], connection=conn) self.assertTrue(retval) self.assertTrue(mock_create_table.called) mock_create_table.assert_called_once_with(attribute_definitions=[ { 'AttributeName': 'username', 'AttributeType': 'S' }, { 'AttributeName': 'date_joined', 'AttributeType': 'N' } ], table_name='users', key_schema=[ { 'KeyType': 'HASH', 'AttributeName': 'username' }, { 'KeyType': 'RANGE', 'AttributeName': 'date_joined' } ], provisioned_throughput={ 'WriteCapacityUnits': 5, 'ReadCapacityUnits': 5 }) def test_create_full(self): conn = FakeDynamoDBConnection() with mock.patch.object(conn, 'create_table', return_value={}) \ as mock_create_table: retval = Table.create('users', schema=[ HashKey('username'), RangeKey('date_joined', data_type=NUMBER) ], throughput={ 'read':20, 'write': 10, }, indexes=[ KeysOnlyIndex('FriendCountIndex', parts=[ RangeKey('friend_count') ]), ], global_indexes=[ GlobalKeysOnlyIndex('FullFriendCountIndex', parts=[ RangeKey('friend_count') ], throughput={ 'read': 10, 'write': 8, }), ], connection=conn) self.assertTrue(retval) self.assertTrue(mock_create_table.called) mock_create_table.assert_called_once_with(attribute_definitions=[ { 'AttributeName': 'username', 'AttributeType': 'S' }, { 'AttributeName': 'date_joined', 'AttributeType': 'N' }, { 'AttributeName': 'friend_count', 'AttributeType': 'S' } ], key_schema=[ { 'KeyType': 'HASH', 'AttributeName': 'username' }, { 'KeyType': 'RANGE', 'AttributeName': 'date_joined' } ], table_name='users', provisioned_throughput={ 'WriteCapacityUnits': 10, 'ReadCapacityUnits': 20 }, global_secondary_indexes=[ { 'KeySchema': [ { 'KeyType': 'RANGE', 'AttributeName': 'friend_count' } ], 'IndexName': 'FullFriendCountIndex', 'Projection': { 'ProjectionType': 'KEYS_ONLY' }, 'ProvisionedThroughput': { 'WriteCapacityUnits': 8, 'ReadCapacityUnits': 10 } } ], local_secondary_indexes=[ { 'KeySchema': [ { 'KeyType': 'RANGE', 'AttributeName': 'friend_count' } ], 'IndexName': 'FriendCountIndex', 'Projection': { 'ProjectionType': 'KEYS_ONLY' } } ]) def test_describe(self): expected = { "Table": { "AttributeDefinitions": [ { "AttributeName": "username", "AttributeType": "S" } ], "ItemCount": 5, "KeySchema": [ { "AttributeName": "username", "KeyType": "HASH" } ], "LocalSecondaryIndexes": [ { "IndexName": "UsernameIndex", "KeySchema": [ { "AttributeName": "username", "KeyType": "HASH" } ], "Projection": { "ProjectionType": "KEYS_ONLY" } } ], "ProvisionedThroughput": { "ReadCapacityUnits": 20, "WriteCapacityUnits": 6 }, "TableName": "Thread", "TableStatus": "ACTIVE" } } with mock.patch.object( self.users.connection, 'describe_table', return_value=expected) as mock_describe: self.assertEqual(self.users.throughput['read'], 5) self.assertEqual(self.users.throughput['write'], 5) self.assertEqual(self.users.schema, None) self.assertEqual(self.users.indexes, None) self.users.describe() self.assertEqual(self.users.throughput['read'], 20) self.assertEqual(self.users.throughput['write'], 6) self.assertEqual(len(self.users.schema), 1) self.assertEqual(isinstance(self.users.schema[0], HashKey), 1) self.assertEqual(len(self.users.indexes), 1) mock_describe.assert_called_once_with('users') def test_update(self): with mock.patch.object( self.users.connection, 'update_table', return_value={}) as mock_update: self.assertEqual(self.users.throughput['read'], 5) self.assertEqual(self.users.throughput['write'], 5) self.users.update(throughput={ 'read': 7, 'write': 2, }) self.assertEqual(self.users.throughput['read'], 7) self.assertEqual(self.users.throughput['write'], 2) mock_update.assert_called_once_with( 'users', global_secondary_index_updates=None, provisioned_throughput={ 'WriteCapacityUnits': 2, 'ReadCapacityUnits': 7 } ) with mock.patch.object( self.users.connection, 'update_table', return_value={}) as mock_update: self.assertEqual(self.users.throughput['read'], 7) self.assertEqual(self.users.throughput['write'], 2) self.users.update(throughput={ 'read': 9, 'write': 5, }, global_indexes={ 'WhateverIndex': { 'read': 6, 'write': 1 }, 'AnotherIndex': { 'read': 1, 'write': 2 } }) self.assertEqual(self.users.throughput['read'], 9) self.assertEqual(self.users.throughput['write'], 5) args, kwargs = mock_update.call_args self.assertEqual(args, ('users',)) self.assertEqual(kwargs['provisioned_throughput'], { 'WriteCapacityUnits': 5, 'ReadCapacityUnits': 9, }) update = kwargs['global_secondary_index_updates'][:] update.sort(key=lambda x: x['Update']['IndexName']) self.assertDictEqual( update[0], { 'Update': { 'IndexName': 'AnotherIndex', 'ProvisionedThroughput': { 'WriteCapacityUnits': 2, 'ReadCapacityUnits': 1 } } }) self.assertDictEqual( update[1], { 'Update': { 'IndexName': 'WhateverIndex', 'ProvisionedThroughput': { 'WriteCapacityUnits': 1, 'ReadCapacityUnits': 6 } } }) def test_create_global_secondary_index(self): with mock.patch.object( self.users.connection, 'update_table', return_value={}) as mock_update: self.users.create_global_secondary_index( global_index=GlobalAllIndex( 'JustCreatedIndex', parts=[ HashKey('requiredHashKey') ], throughput={ 'read': 2, 'write': 2 } ) ) mock_update.assert_called_once_with( 'users', global_secondary_index_updates=[ { 'Create': { 'IndexName': 'JustCreatedIndex', 'KeySchema': [ { 'KeyType': 'HASH', 'AttributeName': 'requiredHashKey' } ], 'Projection': { 'ProjectionType': 'ALL' }, 'ProvisionedThroughput': { 'WriteCapacityUnits': 2, 'ReadCapacityUnits': 2 } } } ], attribute_definitions=[ { 'AttributeName': 'requiredHashKey', 'AttributeType': 'S' } ] ) def test_delete_global_secondary_index(self): with mock.patch.object( self.users.connection, 'update_table', return_value={}) as mock_update: self.users.delete_global_secondary_index('RandomGSIIndex') mock_update.assert_called_once_with( 'users', global_secondary_index_updates=[ { 'Delete': { 'IndexName': 'RandomGSIIndex', } } ] ) def test_update_global_secondary_index(self): # Updating a single global secondary index with mock.patch.object( self.users.connection, 'update_table', return_value={}) as mock_update: self.users.update_global_secondary_index(global_indexes={ 'A_IndexToBeUpdated': { 'read': 5, 'write': 5 } }) mock_update.assert_called_once_with( 'users', global_secondary_index_updates=[ { 'Update': { 'IndexName': 'A_IndexToBeUpdated', "ProvisionedThroughput": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, } } ] ) # Updating multiple global secondary indexes with mock.patch.object( self.users.connection, 'update_table', return_value={}) as mock_update: self.users.update_global_secondary_index(global_indexes={ 'A_IndexToBeUpdated': { 'read': 5, 'write': 5 }, 'B_IndexToBeUpdated': { 'read': 9, 'write': 9 } }) args, kwargs = mock_update.call_args self.assertEqual(args, ('users',)) update = kwargs['global_secondary_index_updates'][:] update.sort(key=lambda x: x['Update']['IndexName']) self.assertDictEqual( update[0], { 'Update': { 'IndexName': 'A_IndexToBeUpdated', 'ProvisionedThroughput': { 'WriteCapacityUnits': 5, 'ReadCapacityUnits': 5 } } }) self.assertDictEqual( update[1], { 'Update': { 'IndexName': 'B_IndexToBeUpdated', 'ProvisionedThroughput': { 'WriteCapacityUnits': 9, 'ReadCapacityUnits': 9 } } }) def test_delete(self): with mock.patch.object( self.users.connection, 'delete_table', return_value={}) as mock_delete: self.assertTrue(self.users.delete()) mock_delete.assert_called_once_with('users') def test_get_item(self): expected = { 'Item': { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056668'}, 'friend_count': {'N': '3'}, 'friends': {'SS': ['alice', 'bob', 'jane']}, } } with mock.patch.object( self.users.connection, 'get_item', return_value=expected) as mock_get_item: item = self.users.get_item(username='johndoe') self.assertEqual(item['username'], 'johndoe') self.assertEqual(item['first_name'], 'John') mock_get_item.assert_called_once_with('users', { 'username': {'S': 'johndoe'} }, consistent_read=False, attributes_to_get=None) with mock.patch.object( self.users.connection, 'get_item', return_value=expected) as mock_get_item: item = self.users.get_item(username='johndoe', attributes=[ 'username', 'first_name', ]) mock_get_item.assert_called_once_with('users', { 'username': {'S': 'johndoe'} }, consistent_read=False, attributes_to_get=['username', 'first_name']) def test_has_item(self): expected = { 'Item': { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056668'}, 'friend_count': {'N': '3'}, 'friends': {'SS': ['alice', 'bob', 'jane']}, } } with mock.patch.object( self.users.connection, 'get_item', return_value=expected) as mock_get_item: found = self.users.has_item(username='johndoe') self.assertTrue(found) with mock.patch.object( self.users.connection, 'get_item') as mock_get_item: mock_get_item.side_effect = JSONResponseError("Nope.", None, None) found = self.users.has_item(username='mrsmith') self.assertFalse(found) def test_lookup_hash(self): """Tests the "lookup" function with just a hash key""" expected = { 'Item': { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056668'}, 'friend_count': {'N': '3'}, 'friends': {'SS': ['alice', 'bob', 'jane']}, } } # Set the Schema self.users.schema = [ HashKey('username'), RangeKey('date_joined', data_type=NUMBER), ] with mock.patch.object( self.users, 'get_item', return_value=expected) as mock_get_item: self.users.lookup('johndoe') mock_get_item.assert_called_once_with( username= 'johndoe') def test_lookup_hash_and_range(self): """Test the "lookup" function with a hash and range key""" expected = { 'Item': { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056668'}, 'friend_count': {'N': '3'}, 'friends': {'SS': ['alice', 'bob', 'jane']}, } } # Set the Schema self.users.schema = [ HashKey('username'), RangeKey('date_joined', data_type=NUMBER), ] with mock.patch.object( self.users, 'get_item', return_value=expected) as mock_get_item: self.users.lookup('johndoe', 1366056668) mock_get_item.assert_called_once_with( username= 'johndoe', date_joined= 1366056668) def test_put_item(self): with mock.patch.object( self.users.connection, 'put_item', return_value={}) as mock_put_item: self.users.put_item(data={ 'username': 'johndoe', 'last_name': 'Doe', 'date_joined': 12345, }) mock_put_item.assert_called_once_with('users', { 'username': {'S': 'johndoe'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '12345'} }, expected={ 'username': { 'Exists': False, }, 'last_name': { 'Exists': False, }, 'date_joined': { 'Exists': False, } }) def test_private_put_item(self): with mock.patch.object( self.users.connection, 'put_item', return_value={}) as mock_put_item: self.users._put_item({'some': 'data'}) mock_put_item.assert_called_once_with('users', {'some': 'data'}) def test_private_update_item(self): with mock.patch.object( self.users.connection, 'update_item', return_value={}) as mock_update_item: self.users._update_item({ 'username': 'johndoe' }, { 'some': 'data', }) mock_update_item.assert_called_once_with('users', { 'username': {'S': 'johndoe'}, }, { 'some': 'data', }) def test_delete_item(self): with mock.patch.object( self.users.connection, 'delete_item', return_value={}) as mock_delete_item: self.assertTrue(self.users.delete_item(username='johndoe', date_joined=23456)) mock_delete_item.assert_called_once_with('users', { 'username': { 'S': 'johndoe' }, 'date_joined': { 'N': '23456' } }, expected=None, conditional_operator=None) def test_delete_item_conditionally(self): with mock.patch.object( self.users.connection, 'delete_item', return_value={}) as mock_delete_item: self.assertTrue(self.users.delete_item(expected={'balance__eq': 0}, username='johndoe', date_joined=23456)) mock_delete_item.assert_called_once_with('users', { 'username': { 'S': 'johndoe' }, 'date_joined': { 'N': '23456' } }, expected={ 'balance': { 'ComparisonOperator': 'EQ', 'AttributeValueList': [{'N': '0'}] }, }, conditional_operator=None) def side_effect(*args, **kwargs): raise exceptions.ConditionalCheckFailedException(400, '', {}) with mock.patch.object( self.users.connection, 'delete_item', side_effect=side_effect) as mock_delete_item: self.assertFalse(self.users.delete_item(expected={'balance__eq': 0}, username='johndoe', date_joined=23456)) def test_get_key_fields_no_schema_populated(self): expected = { "Table": { "AttributeDefinitions": [ { "AttributeName": "username", "AttributeType": "S" }, { "AttributeName": "date_joined", "AttributeType": "N" } ], "ItemCount": 5, "KeySchema": [ { "AttributeName": "username", "KeyType": "HASH" }, { "AttributeName": "date_joined", "KeyType": "RANGE" } ], "LocalSecondaryIndexes": [ { "IndexName": "UsernameIndex", "KeySchema": [ { "AttributeName": "username", "KeyType": "HASH" } ], "Projection": { "ProjectionType": "KEYS_ONLY" } } ], "ProvisionedThroughput": { "ReadCapacityUnits": 20, "WriteCapacityUnits": 6 }, "TableName": "Thread", "TableStatus": "ACTIVE" } } with mock.patch.object( self.users.connection, 'describe_table', return_value=expected) as mock_describe: self.assertEqual(self.users.schema, None) key_fields = self.users.get_key_fields() self.assertEqual(key_fields, ['username', 'date_joined']) self.assertEqual(len(self.users.schema), 2) mock_describe.assert_called_once_with('users') def test_batch_write_no_writes(self): with mock.patch.object( self.users.connection, 'batch_write_item', return_value={}) as mock_batch: with self.users.batch_write() as batch: pass self.assertFalse(mock_batch.called) def test_batch_write(self): with mock.patch.object( self.users.connection, 'batch_write_item', return_value={}) as mock_batch: with self.users.batch_write() as batch: batch.put_item(data={ 'username': 'jane', 'date_joined': 12342547 }) batch.delete_item(username='johndoe') batch.put_item(data={ 'username': 'alice', 'date_joined': 12342888 }) mock_batch.assert_called_once_with({ 'users': [ { 'PutRequest': { 'Item': { 'username': {'S': 'jane'}, 'date_joined': {'N': '12342547'} } } }, { 'PutRequest': { 'Item': { 'username': {'S': 'alice'}, 'date_joined': {'N': '12342888'} } } }, { 'DeleteRequest': { 'Key': { 'username': {'S': 'johndoe'}, } } }, ] }) def test_batch_write_dont_swallow_exceptions(self): with mock.patch.object( self.users.connection, 'batch_write_item', return_value={}) as mock_batch: try: with self.users.batch_write() as batch: raise Exception('OH NOES') except Exception as e: self.assertEqual(str(e), 'OH NOES') self.assertFalse(mock_batch.called) def test_batch_write_flushing(self): with mock.patch.object( self.users.connection, 'batch_write_item', return_value={}) as mock_batch: with self.users.batch_write() as batch: batch.put_item(data={ 'username': 'jane', 'date_joined': 12342547 }) # This would only be enough for one batch. batch.delete_item(username='johndoe1') batch.delete_item(username='johndoe2') batch.delete_item(username='johndoe3') batch.delete_item(username='johndoe4') batch.delete_item(username='johndoe5') batch.delete_item(username='johndoe6') batch.delete_item(username='johndoe7') batch.delete_item(username='johndoe8') batch.delete_item(username='johndoe9') batch.delete_item(username='johndoe10') batch.delete_item(username='johndoe11') batch.delete_item(username='johndoe12') batch.delete_item(username='johndoe13') batch.delete_item(username='johndoe14') batch.delete_item(username='johndoe15') batch.delete_item(username='johndoe16') batch.delete_item(username='johndoe17') batch.delete_item(username='johndoe18') batch.delete_item(username='johndoe19') batch.delete_item(username='johndoe20') batch.delete_item(username='johndoe21') batch.delete_item(username='johndoe22') batch.delete_item(username='johndoe23') # We're only at 24 items. No flushing yet. self.assertEqual(mock_batch.call_count, 0) # This pushes it over the edge. A flush happens then we start # queuing objects again. batch.delete_item(username='johndoe24') self.assertEqual(mock_batch.call_count, 1) # Since we add another, there's enough for a second call to # flush. batch.delete_item(username='johndoe25') self.assertEqual(mock_batch.call_count, 2) def test_batch_write_unprocessed_items(self): unprocessed = { 'UnprocessedItems': { 'users': [ { 'PutRequest': { 'username': { 'S': 'jane', }, 'date_joined': { 'N': 12342547 } }, }, ], }, } # Test enqueuing the unprocessed bits. with mock.patch.object( self.users.connection, 'batch_write_item', return_value=unprocessed) as mock_batch: with self.users.batch_write() as batch: self.assertEqual(len(batch._unprocessed), 0) # Trash the ``resend_unprocessed`` method so that we don't # infinite loop forever here. batch.resend_unprocessed = lambda: True batch.put_item(data={ 'username': 'jane', 'date_joined': 12342547 }) batch.delete_item(username='johndoe') batch.put_item(data={ 'username': 'alice', 'date_joined': 12342888 }) self.assertEqual(len(batch._unprocessed), 1) # Now test resending those unprocessed items. with mock.patch.object( self.users.connection, 'batch_write_item', return_value={}) as mock_batch: with self.users.batch_write() as batch: self.assertEqual(len(batch._unprocessed), 0) # Toss in faked unprocessed items, as though a previous batch # had failed. batch._unprocessed = [ { 'PutRequest': { 'username': { 'S': 'jane', }, 'date_joined': { 'N': 12342547 } }, }, ] batch.put_item(data={ 'username': 'jane', 'date_joined': 12342547 }) batch.delete_item(username='johndoe') batch.put_item(data={ 'username': 'alice', 'date_joined': 12342888 }) # Flush, to make sure everything has been processed. # Unprocessed items should still be hanging around. batch.flush() self.assertEqual(len(batch._unprocessed), 1) # Post-exit, this should be emptied. self.assertEqual(len(batch._unprocessed), 0) def test__build_filters(self): filters = self.users._build_filters({ 'username__eq': 'johndoe', 'date_joined__gte': 1234567, 'age__in': [30, 31, 32, 33], 'last_name__between': ['danzig', 'only'], 'first_name__null': False, 'gender__null': True, }, using=FILTER_OPERATORS) self.assertEqual(filters, { 'username': { 'AttributeValueList': [ { 'S': 'johndoe', }, ], 'ComparisonOperator': 'EQ', }, 'date_joined': { 'AttributeValueList': [ { 'N': '1234567', }, ], 'ComparisonOperator': 'GE', }, 'age': { 'AttributeValueList': [ {'N': '30'}, {'N': '31'}, {'N': '32'}, {'N': '33'}, ], 'ComparisonOperator': 'IN', }, 'last_name': { 'AttributeValueList': [{'S': 'danzig'}, {'S': 'only'}], 'ComparisonOperator': 'BETWEEN', }, 'first_name': { 'ComparisonOperator': 'NOT_NULL' }, 'gender': { 'ComparisonOperator': 'NULL' }, }) self.assertRaises(exceptions.UnknownFilterTypeError, self.users._build_filters, { 'darling__die': True, } ) q_filters = self.users._build_filters({ 'username__eq': 'johndoe', 'date_joined__gte': 1234567, 'last_name__between': ['danzig', 'only'], 'gender__beginswith': 'm', }, using=QUERY_OPERATORS) self.assertEqual(q_filters, { 'username': { 'AttributeValueList': [ { 'S': 'johndoe', }, ], 'ComparisonOperator': 'EQ', }, 'date_joined': { 'AttributeValueList': [ { 'N': '1234567', }, ], 'ComparisonOperator': 'GE', }, 'last_name': { 'AttributeValueList': [{'S': 'danzig'}, {'S': 'only'}], 'ComparisonOperator': 'BETWEEN', }, 'gender': { 'AttributeValueList': [{'S': 'm'}], 'ComparisonOperator': 'BEGINS_WITH', }, }) self.assertRaises(exceptions.UnknownFilterTypeError, self.users._build_filters, { 'darling__die': True, }, using=QUERY_OPERATORS ) self.assertRaises(exceptions.UnknownFilterTypeError, self.users._build_filters, { 'first_name__null': True, }, using=QUERY_OPERATORS ) def test_private_query(self): expected = { "ConsumedCapacity": { "CapacityUnits": 0.5, "TableName": "users" }, "Count": 4, "Items": [ { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056668'}, 'friend_count': {'N': '3'}, 'friends': {'SS': ['alice', 'bob', 'jane']}, }, { 'username': {'S': 'jane'}, 'first_name': {'S': 'Jane'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366057777'}, 'friend_count': {'N': '2'}, 'friends': {'SS': ['alice', 'johndoe']}, }, { 'username': {'S': 'alice'}, 'first_name': {'S': 'Alice'}, 'last_name': {'S': 'Expert'}, 'date_joined': {'N': '1366056680'}, 'friend_count': {'N': '1'}, 'friends': {'SS': ['jane']}, }, { 'username': {'S': 'bob'}, 'first_name': {'S': 'Bob'}, 'last_name': {'S': 'Smith'}, 'date_joined': {'N': '1366056888'}, 'friend_count': {'N': '1'}, 'friends': {'SS': ['johndoe']}, }, ], "ScannedCount": 4 } with mock.patch.object( self.users.connection, 'query', return_value=expected) as mock_query: results = self.users._query( limit=4, reverse=True, username__between=['aaa', 'mmm'] ) usernames = [res['username'] for res in results['results']] self.assertEqual(usernames, ['johndoe', 'jane', 'alice', 'bob']) self.assertEqual(len(results['results']), 4) self.assertEqual(results['last_key'], None) mock_query.assert_called_once_with('users', consistent_read=False, scan_index_forward=False, index_name=None, attributes_to_get=None, limit=4, key_conditions={ 'username': { 'AttributeValueList': [{'S': 'aaa'}, {'S': 'mmm'}], 'ComparisonOperator': 'BETWEEN', } }, select=None, query_filter=None, conditional_operator=None ) # Now alter the expected. expected['LastEvaluatedKey'] = { 'username': { 'S': 'johndoe', }, } with mock.patch.object( self.users.connection, 'query', return_value=expected) as mock_query_2: results = self.users._query( limit=4, reverse=True, username__between=['aaa', 'mmm'], exclusive_start_key={ 'username': 'adam', }, consistent=True, query_filter=None, conditional_operator='AND' ) usernames = [res['username'] for res in results['results']] self.assertEqual(usernames, ['johndoe', 'jane', 'alice', 'bob']) self.assertEqual(len(results['results']), 4) self.assertEqual(results['last_key'], {'username': 'johndoe'}) mock_query_2.assert_called_once_with('users', key_conditions={ 'username': { 'AttributeValueList': [{'S': 'aaa'}, {'S': 'mmm'}], 'ComparisonOperator': 'BETWEEN', } }, index_name=None, attributes_to_get=None, scan_index_forward=False, limit=4, exclusive_start_key={ 'username': { 'S': 'adam', }, }, consistent_read=True, select=None, query_filter=None, conditional_operator='AND' ) def test_private_scan(self): expected = { "ConsumedCapacity": { "CapacityUnits": 0.5, "TableName": "users" }, "Count": 4, "Items": [ { 'username': {'S': 'alice'}, 'first_name': {'S': 'Alice'}, 'last_name': {'S': 'Expert'}, 'date_joined': {'N': '1366056680'}, 'friend_count': {'N': '1'}, 'friends': {'SS': ['jane']}, }, { 'username': {'S': 'bob'}, 'first_name': {'S': 'Bob'}, 'last_name': {'S': 'Smith'}, 'date_joined': {'N': '1366056888'}, 'friend_count': {'N': '1'}, 'friends': {'SS': ['johndoe']}, }, { 'username': {'S': 'jane'}, 'first_name': {'S': 'Jane'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366057777'}, 'friend_count': {'N': '2'}, 'friends': {'SS': ['alice', 'johndoe']}, }, ], "ScannedCount": 4 } with mock.patch.object( self.users.connection, 'scan', return_value=expected) as mock_scan: results = self.users._scan( limit=2, friend_count__lte=2 ) usernames = [res['username'] for res in results['results']] self.assertEqual(usernames, ['alice', 'bob', 'jane']) self.assertEqual(len(results['results']), 3) self.assertEqual(results['last_key'], None) mock_scan.assert_called_once_with('users', scan_filter={ 'friend_count': { 'AttributeValueList': [{'N': '2'}], 'ComparisonOperator': 'LE', } }, limit=2, segment=None, attributes_to_get=None, total_segments=None, conditional_operator=None ) # Now alter the expected. expected['LastEvaluatedKey'] = { 'username': { 'S': 'jane', }, } with mock.patch.object( self.users.connection, 'scan', return_value=expected) as mock_scan_2: results = self.users._scan( limit=3, friend_count__lte=2, exclusive_start_key={ 'username': 'adam', }, segment=None, total_segments=None ) usernames = [res['username'] for res in results['results']] self.assertEqual(usernames, ['alice', 'bob', 'jane']) self.assertEqual(len(results['results']), 3) self.assertEqual(results['last_key'], {'username': 'jane'}) mock_scan_2.assert_called_once_with('users', scan_filter={ 'friend_count': { 'AttributeValueList': [{'N': '2'}], 'ComparisonOperator': 'LE', } }, limit=3, exclusive_start_key={ 'username': { 'S': 'adam', }, }, segment=None, attributes_to_get=None, total_segments=None, conditional_operator=None ) def test_query(self): items_1 = { 'results': [ Item(self.users, data={ 'username': 'johndoe', 'first_name': 'John', 'last_name': 'Doe', }), Item(self.users, data={ 'username': 'jane', 'first_name': 'Jane', 'last_name': 'Doe', }), ], 'last_key': 'jane', } results = self.users.query_2(last_name__eq='Doe') self.assertTrue(isinstance(results, ResultSet)) self.assertEqual(len(results._results), 0) self.assertEqual(results.the_callable, self.users._query) with mock.patch.object( results, 'the_callable', return_value=items_1) as mock_query: res_1 = next(results) # Now it should be populated. self.assertEqual(len(results._results), 2) self.assertEqual(res_1['username'], 'johndoe') res_2 = next(results) self.assertEqual(res_2['username'], 'jane') self.assertEqual(mock_query.call_count, 1) items_2 = { 'results': [ Item(self.users, data={ 'username': 'foodoe', 'first_name': 'Foo', 'last_name': 'Doe', }), ], } with mock.patch.object( results, 'the_callable', return_value=items_2) as mock_query_2: res_3 = next(results) # New results should have been found. self.assertEqual(len(results._results), 1) self.assertEqual(res_3['username'], 'foodoe') self.assertRaises(StopIteration, results.next) self.assertEqual(mock_query_2.call_count, 1) def test_query_with_specific_attributes(self): items_1 = { 'results': [ Item(self.users, data={ 'username': 'johndoe', }), Item(self.users, data={ 'username': 'jane', }), ], 'last_key': 'jane', } results = self.users.query_2(last_name__eq='Doe', attributes=['username']) self.assertTrue(isinstance(results, ResultSet)) self.assertEqual(len(results._results), 0) self.assertEqual(results.the_callable, self.users._query) with mock.patch.object( results, 'the_callable', return_value=items_1) as mock_query: res_1 = next(results) # Now it should be populated. self.assertEqual(len(results._results), 2) self.assertEqual(res_1['username'], 'johndoe') self.assertEqual(list(res_1.keys()), ['username']) res_2 = next(results) self.assertEqual(res_2['username'], 'jane') self.assertEqual(mock_query.call_count, 1) def test_scan(self): items_1 = { 'results': [ Item(self.users, data={ 'username': 'johndoe', 'first_name': 'John', 'last_name': 'Doe', }), Item(self.users, data={ 'username': 'jane', 'first_name': 'Jane', 'last_name': 'Doe', }), ], 'last_key': 'jane', } results = self.users.scan(last_name__eq='Doe') self.assertTrue(isinstance(results, ResultSet)) self.assertEqual(len(results._results), 0) self.assertEqual(results.the_callable, self.users._scan) with mock.patch.object( results, 'the_callable', return_value=items_1) as mock_scan: res_1 = next(results) # Now it should be populated. self.assertEqual(len(results._results), 2) self.assertEqual(res_1['username'], 'johndoe') res_2 = next(results) self.assertEqual(res_2['username'], 'jane') self.assertEqual(mock_scan.call_count, 1) items_2 = { 'results': [ Item(self.users, data={ 'username': 'zoeydoe', 'first_name': 'Zoey', 'last_name': 'Doe', }), ], } with mock.patch.object( results, 'the_callable', return_value=items_2) as mock_scan_2: res_3 = next(results) # New results should have been found. self.assertEqual(len(results._results), 1) self.assertEqual(res_3['username'], 'zoeydoe') self.assertRaises(StopIteration, results.next) self.assertEqual(mock_scan_2.call_count, 1) def test_scan_with_specific_attributes(self): items_1 = { 'results': [ Item(self.users, data={ 'username': 'johndoe', }), Item(self.users, data={ 'username': 'jane', }), ], 'last_key': 'jane', } results = self.users.scan(attributes=['username']) self.assertTrue(isinstance(results, ResultSet)) self.assertEqual(len(results._results), 0) self.assertEqual(results.the_callable, self.users._scan) with mock.patch.object( results, 'the_callable', return_value=items_1) as mock_query: res_1 = next(results) # Now it should be populated. self.assertEqual(len(results._results), 2) self.assertEqual(res_1['username'], 'johndoe') self.assertEqual(list(res_1.keys()), ['username']) res_2 = next(results) self.assertEqual(res_2['username'], 'jane') self.assertEqual(mock_query.call_count, 1) def test_count(self): expected = { "Table": { "AttributeDefinitions": [ { "AttributeName": "username", "AttributeType": "S" } ], "ItemCount": 5, "KeySchema": [ { "AttributeName": "username", "KeyType": "HASH" } ], "LocalSecondaryIndexes": [ { "IndexName": "UsernameIndex", "KeySchema": [ { "AttributeName": "username", "KeyType": "HASH" } ], "Projection": { "ProjectionType": "KEYS_ONLY" } } ], "ProvisionedThroughput": { "ReadCapacityUnits": 20, "WriteCapacityUnits": 6 }, "TableName": "Thread", "TableStatus": "ACTIVE" } } with mock.patch.object( self.users, 'describe', return_value=expected) as mock_count: self.assertEqual(self.users.count(), 5) def test_query_count_simple(self): expected_0 = { 'Count': 0.0, } expected_1 = { 'Count': 10.0, } with mock.patch.object( self.users.connection, 'query', return_value=expected_0) as mock_query: results = self.users.query_count(username__eq='notmyname') self.assertTrue(isinstance(results, int)) self.assertEqual(results, 0) self.assertEqual(mock_query.call_count, 1) self.assertIn('scan_index_forward', mock_query.call_args[1]) self.assertEqual(True, mock_query.call_args[1]['scan_index_forward']) self.assertIn('limit', mock_query.call_args[1]) self.assertEqual(None, mock_query.call_args[1]['limit']) with mock.patch.object( self.users.connection, 'query', return_value=expected_1) as mock_query: results = self.users.query_count(username__gt='somename', consistent=True, scan_index_forward=False, limit=10) self.assertTrue(isinstance(results, int)) self.assertEqual(results, 10) self.assertEqual(mock_query.call_count, 1) self.assertIn('scan_index_forward', mock_query.call_args[1]) self.assertEqual(False, mock_query.call_args[1]['scan_index_forward']) self.assertIn('limit', mock_query.call_args[1]) self.assertEqual(10, mock_query.call_args[1]['limit']) def test_query_count_paginated(self): def return_side_effect(*args, **kwargs): if kwargs.get('exclusive_start_key'): return {'Count': 10, 'LastEvaluatedKey': None} else: return { 'Count': 20, 'LastEvaluatedKey': { 'username': { 'S': 'johndoe' }, 'date_joined': { 'N': '4118642633' } } } with mock.patch.object( self.users.connection, 'query', side_effect=return_side_effect ) as mock_query: count = self.users.query_count(username__eq='johndoe') self.assertTrue(isinstance(count, int)) self.assertEqual(30, count) self.assertEqual(mock_query.call_count, 2) def test_private_batch_get(self): expected = { "ConsumedCapacity": { "CapacityUnits": 0.5, "TableName": "users" }, 'Responses': { 'users': [ { 'username': {'S': 'alice'}, 'first_name': {'S': 'Alice'}, 'last_name': {'S': 'Expert'}, 'date_joined': {'N': '1366056680'}, 'friend_count': {'N': '1'}, 'friends': {'SS': ['jane']}, }, { 'username': {'S': 'bob'}, 'first_name': {'S': 'Bob'}, 'last_name': {'S': 'Smith'}, 'date_joined': {'N': '1366056888'}, 'friend_count': {'N': '1'}, 'friends': {'SS': ['johndoe']}, }, { 'username': {'S': 'jane'}, 'first_name': {'S': 'Jane'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366057777'}, 'friend_count': {'N': '2'}, 'friends': {'SS': ['alice', 'johndoe']}, }, ], }, "UnprocessedKeys": { }, } with mock.patch.object( self.users.connection, 'batch_get_item', return_value=expected) as mock_batch_get: results = self.users._batch_get(keys=[ {'username': 'alice', 'friend_count': 1}, {'username': 'bob', 'friend_count': 1}, {'username': 'jane'}, ]) usernames = [res['username'] for res in results['results']] self.assertEqual(usernames, ['alice', 'bob', 'jane']) self.assertEqual(len(results['results']), 3) self.assertEqual(results['last_key'], None) self.assertEqual(results['unprocessed_keys'], []) mock_batch_get.assert_called_once_with(request_items={ 'users': { 'Keys': [ { 'username': {'S': 'alice'}, 'friend_count': {'N': '1'} }, { 'username': {'S': 'bob'}, 'friend_count': {'N': '1'} }, { 'username': {'S': 'jane'}, } ] } }) # Now alter the expected. del expected['Responses']['users'][2] expected['UnprocessedKeys'] = { 'users': { 'Keys': [ {'username': {'S': 'jane',}}, ], }, } with mock.patch.object( self.users.connection, 'batch_get_item', return_value=expected) as mock_batch_get_2: results = self.users._batch_get(keys=[ {'username': 'alice', 'friend_count': 1}, {'username': 'bob', 'friend_count': 1}, {'username': 'jane'}, ]) usernames = [res['username'] for res in results['results']] self.assertEqual(usernames, ['alice', 'bob']) self.assertEqual(len(results['results']), 2) self.assertEqual(results['last_key'], None) self.assertEqual(results['unprocessed_keys'], [ {'username': 'jane'} ]) mock_batch_get_2.assert_called_once_with(request_items={ 'users': { 'Keys': [ { 'username': {'S': 'alice'}, 'friend_count': {'N': '1'} }, { 'username': {'S': 'bob'}, 'friend_count': {'N': '1'} }, { 'username': {'S': 'jane'}, } ] } }) def test_private_batch_get_attributes(self): # test if AttributesToGet parameter is passed to DynamoDB API expected = { "ConsumedCapacity": { "CapacityUnits": 0.5, "TableName": "users" }, 'Responses': { 'users': [ { 'username': {'S': 'alice'}, 'first_name': {'S': 'Alice'}, }, { 'username': {'S': 'bob'}, 'first_name': {'S': 'Bob'}, }, ], }, "UnprocessedKeys": {}, } with mock.patch.object( self.users.connection, 'batch_get_item', return_value=expected) as mock_batch_get_attr: results = self.users._batch_get(keys=[ {'username': 'alice'}, {'username': 'bob'}, ], attributes=['username', 'first_name']) usernames = [res['username'] for res in results['results']] first_names = [res['first_name'] for res in results['results']] self.assertEqual(usernames, ['alice', 'bob']) self.assertEqual(first_names, ['Alice', 'Bob']) self.assertEqual(len(results['results']), 2) self.assertEqual(results['last_key'], None) self.assertEqual(results['unprocessed_keys'], []) mock_batch_get_attr.assert_called_once_with(request_items={ 'users': { 'Keys': [ { 'username': {'S': 'alice'} }, { 'username': {'S': 'bob'} }, ], 'AttributesToGet': ['username', 'first_name'], }, }) def test_batch_get(self): items_1 = { 'results': [ Item(self.users, data={ 'username': 'johndoe', 'first_name': 'John', 'last_name': 'Doe', }), Item(self.users, data={ 'username': 'jane', 'first_name': 'Jane', 'last_name': 'Doe', }), ], 'last_key': None, 'unprocessed_keys': [ 'zoeydoe', ] } results = self.users.batch_get(keys=[ {'username': 'johndoe'}, {'username': 'jane'}, {'username': 'zoeydoe'}, ]) self.assertTrue(isinstance(results, BatchGetResultSet)) self.assertEqual(len(results._results), 0) self.assertEqual(results.the_callable, self.users._batch_get) with mock.patch.object( results, 'the_callable', return_value=items_1) as mock_batch_get: res_1 = next(results) # Now it should be populated. self.assertEqual(len(results._results), 2) self.assertEqual(res_1['username'], 'johndoe') res_2 = next(results) self.assertEqual(res_2['username'], 'jane') self.assertEqual(mock_batch_get.call_count, 1) self.assertEqual(results._keys_left, ['zoeydoe']) items_2 = { 'results': [ Item(self.users, data={ 'username': 'zoeydoe', 'first_name': 'Zoey', 'last_name': 'Doe', }), ], } with mock.patch.object( results, 'the_callable', return_value=items_2) as mock_batch_get_2: res_3 = next(results) # New results should have been found. self.assertEqual(len(results._results), 1) self.assertEqual(res_3['username'], 'zoeydoe') self.assertRaises(StopIteration, results.next) self.assertEqual(mock_batch_get_2.call_count, 1) self.assertEqual(results._keys_left, []) vendored/boto/tests/unit/dynamodb2/__init__.py 0000644 00000000000 15025013616 0015413 0 ustar 00 vendored/boto/tests/unit/directconnect/test_layer1.py 0000644 00000004232 15025013616 0017070 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.directconnect.layer1 import DirectConnectConnection from tests.unit import AWSMockServiceTestCase class TestDescribeTrails(AWSMockServiceTestCase): connection_class = DirectConnectConnection def default_body(self): return b''' { "connections": [ { "bandwidth": "string", "connectionId": "string", "connectionName": "string", "connectionState": "string", "location": "string", "ownerAccount": "string", "partnerName": "string", "region": "string", "vlan": 1 } ] }''' def test_describe(self): self.set_http_response(status_code=200) api_response = self.service_connection.describe_connections() self.assertEqual(1, len(api_response['connections'])) self.assertEqual('string', api_response['connections'][0]['region']) self.assert_request_parameters({}) target = self.actual_request.headers['X-Amz-Target'] self.assertTrue('DescribeConnections' in target) vendored/boto/tests/unit/directconnect/__init__.py 0000644 00000000000 15025013616 0016360 0 ustar 00 vendored/boto/tests/unit/logs/test_layer1.py 0000644 00000001217 15025013616 0015210 0 ustar 00 #!/usr/bin/env python from boto.logs.layer1 import CloudWatchLogsConnection from tests.unit import AWSMockServiceTestCase class TestDescribeLogs(AWSMockServiceTestCase): connection_class = CloudWatchLogsConnection def default_body(self): return b'{"logGroups": []}' def test_describe(self): self.set_http_response(status_code=200) api_response = self.service_connection.describe_log_groups() self.assertEqual(0, len(api_response['logGroups'])) self.assert_request_parameters({}) target = self.actual_request.headers['X-Amz-Target'] self.assertTrue('DescribeLogGroups' in target) vendored/boto/tests/unit/logs/__init__.py 0000644 00000000000 15025013616 0014500 0 ustar 00 vendored/boto/tests/unit/emr/test_emr_responses.py 0000644 00000041557 15025013616 0016531 0 ustar 00 # Copyright (c) 2010 Jeremy Thurgood <firxen+boto@gmail.com> # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # NOTE: These tests only cover the very simple cases I needed to test # for the InstanceGroup fix. import xml.sax from boto import handler from boto.emr import emrobject from boto.resultset import ResultSet from tests.compat import unittest JOB_FLOW_EXAMPLE = b""" <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-01-15"> <DescribeJobFlowsResult> <JobFlows> <member> <ExecutionStatusDetail> <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime> <StartDateTime>2009-01-28T21:49:16Z</StartDateTime> <State>STARTING</State> </ExecutionStatusDetail> <BootstrapActions> <member> <BootstrapActionConfig> <ScriptBootstrapAction> <Args/> <Path>s3://elasticmapreduce/libs/hue/install-hue</Path> </ScriptBootstrapAction> <Name>Install Hue</Name> </BootstrapActionConfig> </member> </BootstrapActions> <VisibleToAllUsers>true</VisibleToAllUsers> <SupportedProducts> <member>Hue</member> </SupportedProducts> <Name>MyJobFlowName</Name> <LogUri>mybucket/subdir/</LogUri> <Steps> <member> <ExecutionStatusDetail> <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime> <State>PENDING</State> </ExecutionStatusDetail> <StepConfig> <HadoopJarStep> <Jar>MyJarFile</Jar> <MainClass>MyMailClass</MainClass> <Args> <member>arg1</member> <member>arg2</member> </Args> <Properties/> </HadoopJarStep> <Name>MyStepName</Name> <ActionOnFailure>CONTINUE</ActionOnFailure> </StepConfig> </member> </Steps> <JobFlowId>j-3UN6WX5RRO2AG</JobFlowId> <Instances> <Placement> <AvailabilityZone>us-east-1a</AvailabilityZone> </Placement> <SlaveInstanceType>m1.small</SlaveInstanceType> <MasterInstanceType>m1.small</MasterInstanceType> <Ec2KeyName>myec2keyname</Ec2KeyName> <InstanceCount>4</InstanceCount> <KeepJobFlowAliveWhenNoSteps>true</KeepJobFlowAliveWhenNoSteps> </Instances> </member> </JobFlows> </DescribeJobFlowsResult> <ResponseMetadata> <RequestId>9cea3229-ed85-11dd-9877-6fad448a8419</RequestId> </ResponseMetadata> </DescribeJobFlowsResponse> """ JOB_FLOW_COMPLETED = b""" <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <DescribeJobFlowsResult> <JobFlows> <member> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <LastStateChangeReason>Steps completed</LastStateChangeReason> <StartDateTime>2010-10-21T01:03:59Z</StartDateTime> <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:44:18Z</EndDateTime> </ExecutionStatusDetail> <BootstrapActions/> <Name>RealJobFlowName</Name> <LogUri>s3n://example.emrtest.scripts/jobflow_logs/</LogUri> <Steps> <member> <StepConfig> <HadoopJarStep> <Jar>s3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar</Jar> <Args> <member>s3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch</member> </Args> <Properties/> </HadoopJarStep> <Name>Setup Hadoop Debugging</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:03:59Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:04:22Z</EndDateTime> </ExecutionStatusDetail> </member> <member> <StepConfig> <HadoopJarStep> <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar> <Args> <member>-mapper</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialMapper.py</member> <member>-reducer</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialReducer.py</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/20/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/19/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/18/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/17/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/16/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/15/*</member> <member>-input</member> <member>s3://example.emrtest.data/raw/2010/10/14/*</member> <member>-output</member> <member>s3://example.emrtest.crunched/</member> </Args> <Properties/> </HadoopJarStep> <Name>testjob_Initial</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:04:22Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:36:18Z</EndDateTime> </ExecutionStatusDetail> </member> <member> <StepConfig> <HadoopJarStep> <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar> <Args> <member>-mapper</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Mapper.py</member> <member>-reducer</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Reducer.py</member> <member>-input</member> <member>s3://example.emrtest.crunched/*</member> <member>-output</member> <member>s3://example.emrtest.step1/</member> </Args> <Properties/> </HadoopJarStep> <Name>testjob_step1</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:36:18Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:37:51Z</EndDateTime> </ExecutionStatusDetail> </member> <member> <StepConfig> <HadoopJarStep> <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar> <Args> <member>-mapper</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Mapper.py</member> <member>-reducer</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Reducer.py</member> <member>-input</member> <member>s3://example.emrtest.crunched/*</member> <member>-output</member> <member>s3://example.emrtest.step2/</member> </Args> <Properties/> </HadoopJarStep> <Name>testjob_step2</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:37:51Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:39:32Z</EndDateTime> </ExecutionStatusDetail> </member> <member> <StepConfig> <HadoopJarStep> <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar> <Args> <member>-mapper</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Mapper.py</member> <member>-reducer</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Reducer.py</member> <member>-input</member> <member>s3://example.emrtest.step1/*</member> <member>-output</member> <member>s3://example.emrtest.step3/</member> </Args> <Properties/> </HadoopJarStep> <Name>testjob_step3</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:39:32Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:41:22Z</EndDateTime> </ExecutionStatusDetail> </member> <member> <StepConfig> <HadoopJarStep> <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar> <Args> <member>-mapper</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Mapper.py</member> <member>-reducer</member> <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Reducer.py</member> <member>-input</member> <member>s3://example.emrtest.step1/*</member> <member>-output</member> <member>s3://example.emrtest.step4/</member> </Args> <Properties/> </HadoopJarStep> <Name>testjob_step4</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> <ExecutionStatusDetail> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <StartDateTime>2010-10-21T01:41:22Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2010-10-21T01:43:03Z</EndDateTime> </ExecutionStatusDetail> </member> </Steps> <JobFlowId>j-3H3Q13JPFLU22</JobFlowId> <Instances> <SlaveInstanceType>m1.large</SlaveInstanceType> <MasterInstanceId>i-64c21609</MasterInstanceId> <Placement> <AvailabilityZone>us-east-1b</AvailabilityZone> </Placement> <InstanceGroups> <member> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <InstanceRunningCount>0</InstanceRunningCount> <StartDateTime>2010-10-21T01:02:09Z</StartDateTime> <ReadyDateTime>2010-10-21T01:03:03Z</ReadyDateTime> <State>ENDED</State> <EndDateTime>2010-10-21T01:44:18Z</EndDateTime> <InstanceRequestCount>1</InstanceRequestCount> <InstanceType>m1.large</InstanceType> <Market>ON_DEMAND</Market> <LastStateChangeReason>Job flow terminated</LastStateChangeReason> <InstanceRole>MASTER</InstanceRole> <InstanceGroupId>ig-EVMHOZJ2SCO8</InstanceGroupId> <Name>master</Name> </member> <member> <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime> <InstanceRunningCount>0</InstanceRunningCount> <StartDateTime>2010-10-21T01:03:59Z</StartDateTime> <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime> <State>ENDED</State> <EndDateTime>2010-10-21T01:44:18Z</EndDateTime> <InstanceRequestCount>9</InstanceRequestCount> <InstanceType>m1.large</InstanceType> <Market>ON_DEMAND</Market> <LastStateChangeReason>Job flow terminated</LastStateChangeReason> <InstanceRole>CORE</InstanceRole> <InstanceGroupId>ig-YZHDYVITVHKB</InstanceGroupId> <Name>slave</Name> </member> </InstanceGroups> <NormalizedInstanceHours>40</NormalizedInstanceHours> <HadoopVersion>0.20</HadoopVersion> <MasterInstanceType>m1.large</MasterInstanceType> <MasterPublicDnsName>ec2-184-72-153-139.compute-1.amazonaws.com</MasterPublicDnsName> <Ec2KeyName>myubersecurekey</Ec2KeyName> <InstanceCount>10</InstanceCount> <KeepJobFlowAliveWhenNoSteps>false</KeepJobFlowAliveWhenNoSteps> </Instances> </member> </JobFlows> </DescribeJobFlowsResult> <ResponseMetadata> <RequestId>c31e701d-dcb4-11df-b5d9-337fc7fe4773</RequestId> </ResponseMetadata> </DescribeJobFlowsResponse> """ class TestEMRResponses(unittest.TestCase): def _parse_xml(self, body, markers): rs = ResultSet(markers) h = handler.XmlHandler(rs, None) xml.sax.parseString(body, h) return rs def _assert_fields(self, response, **fields): for field, expected in fields.items(): actual = getattr(response, field) self.assertEqual(expected, actual, "Field %s: %r != %r" % (field, expected, actual)) def test_JobFlows_example(self): [jobflow] = self._parse_xml(JOB_FLOW_EXAMPLE, [('member', emrobject.JobFlow)]) self._assert_fields(jobflow, creationdatetime='2009-01-28T21:49:16Z', startdatetime='2009-01-28T21:49:16Z', state='STARTING', instancecount='4', jobflowid='j-3UN6WX5RRO2AG', loguri='mybucket/subdir/', name='MyJobFlowName', availabilityzone='us-east-1a', slaveinstancetype='m1.small', masterinstancetype='m1.small', ec2keyname='myec2keyname', keepjobflowalivewhennosteps='true') def test_JobFlows_completed(self): [jobflow] = self._parse_xml(JOB_FLOW_COMPLETED, [('member', emrobject.JobFlow)]) self._assert_fields(jobflow, creationdatetime='2010-10-21T01:00:25Z', startdatetime='2010-10-21T01:03:59Z', enddatetime='2010-10-21T01:44:18Z', state='COMPLETED', instancecount='10', jobflowid='j-3H3Q13JPFLU22', loguri='s3n://example.emrtest.scripts/jobflow_logs/', name='RealJobFlowName', availabilityzone='us-east-1b', slaveinstancetype='m1.large', masterinstancetype='m1.large', ec2keyname='myubersecurekey', keepjobflowalivewhennosteps='false') self.assertEqual(6, len(jobflow.steps)) self.assertEqual(2, len(jobflow.instancegroups)) vendored/boto/tests/unit/emr/test_instance_group_args.py 0000644 00000004004 15025013616 0017663 0 ustar 00 #!/usr/bin/env python # Author: Charlie Schluting <charlie@schluting.com> # # Test to ensure initalization of InstanceGroup object emits appropriate errors # if bidprice is not specified, but allows float, int, Decimal. from decimal import Decimal from tests.compat import unittest from boto.emr.instance_group import InstanceGroup class TestInstanceGroupArgs(unittest.TestCase): def test_bidprice_missing_spot(self): """ Test InstanceGroup init raises ValueError when market==spot and bidprice is not specified. """ with self.assertRaisesRegex(ValueError, 'bidprice must be specified'): InstanceGroup(1, 'MASTER', 'm1.small', 'SPOT', 'master') def test_bidprice_missing_ondemand(self): """ Test InstanceGroup init accepts a missing bidprice arg, when market is ON_DEMAND. """ instance_group = InstanceGroup(1, 'MASTER', 'm1.small', 'ON_DEMAND', 'master') def test_bidprice_Decimal(self): """ Test InstanceGroup init works with bidprice type = Decimal. """ instance_group = InstanceGroup(1, 'MASTER', 'm1.small', 'SPOT', 'master', bidprice=Decimal(1.10)) self.assertEqual('1.10', instance_group.bidprice[:4]) def test_bidprice_float(self): """ Test InstanceGroup init works with bidprice type = float. """ instance_group = InstanceGroup(1, 'MASTER', 'm1.small', 'SPOT', 'master', bidprice=1.1) self.assertEqual('1.1', instance_group.bidprice) def test_bidprice_string(self): """ Test InstanceGroup init works with bidprice type = string. """ instance_group = InstanceGroup(1, 'MASTER', 'm1.small', 'SPOT', 'master', bidprice='1.1') self.assertEqual('1.1', instance_group.bidprice) if __name__ == "__main__": unittest.main() vendored/boto/tests/unit/emr/test_connection.py 0000644 00000114303 15025013616 0015772 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import boto.utils from datetime import datetime from time import time from tests.unit import AWSMockServiceTestCase from boto.compat import six from boto.emr.connection import EmrConnection from boto.emr.emrobject import BootstrapAction, BootstrapActionList, \ ClusterStateChangeReason, ClusterStatus, ClusterSummaryList, \ ClusterSummary, ClusterTimeline, InstanceInfo, \ InstanceList, InstanceGroupInfo, \ InstanceGroup, InstanceGroupList, JobFlow, \ JobFlowStepList, Step, StepSummaryList, \ Cluster, RunJobFlowResponse # These tests are just checking the basic structure of # the Elastic MapReduce code, by picking a few calls # and verifying we get the expected results with mocked # responses. The integration tests actually verify the # API calls interact with the service correctly. class TestListClusters(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b""" <ListClustersResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <ListClustersResult> <Clusters> <member> <Id>j-aaaaaaaaaaaa</Id> <Status> <StateChangeReason> <Message>Terminated by user request</Message> <Code>USER_REQUEST</Code> </StateChangeReason> <State>TERMINATED</State> <Timeline> <CreationDateTime>2014-01-24T01:21:21Z</CreationDateTime> <ReadyDateTime>2014-01-24T01:25:26Z</ReadyDateTime> <EndDateTime>2014-01-24T02:19:46Z</EndDateTime> </Timeline> </Status> <Name>analytics test</Name> <NormalizedInstanceHours>10</NormalizedInstanceHours> </member> <member> <Id>j-aaaaaaaaaaaab</Id> <Status> <StateChangeReason> <Message>Terminated by user request</Message> <Code>USER_REQUEST</Code> </StateChangeReason> <State>TERMINATED</State> <Timeline> <CreationDateTime>2014-01-21T02:53:08Z</CreationDateTime> <ReadyDateTime>2014-01-21T02:56:40Z</ReadyDateTime> <EndDateTime>2014-01-21T03:40:22Z</EndDateTime> </Timeline> </Status> <Name>test job</Name> <NormalizedInstanceHours>20</NormalizedInstanceHours> </member> </Clusters> </ListClustersResult> <ResponseMetadata> <RequestId>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</RequestId> </ResponseMetadata> </ListClustersResponse> """ def test_list_clusters(self): self.set_http_response(status_code=200) response = self.service_connection.list_clusters() self.assert_request_parameters({ 'Action': 'ListClusters', 'Version': '2009-03-31', }) self.assertTrue(isinstance(response, ClusterSummaryList)) self.assertEqual(len(response.clusters), 2) self.assertTrue(isinstance(response.clusters[0], ClusterSummary)) self.assertEqual(response.clusters[0].name, 'analytics test') self.assertEqual(response.clusters[0].normalizedinstancehours, '10') self.assertTrue(isinstance(response.clusters[0].status, ClusterStatus)) self.assertEqual(response.clusters[0].status.state, 'TERMINATED') self.assertTrue( isinstance(response.clusters[0].status.timeline, ClusterTimeline)) self.assertEqual( response.clusters[0].status.timeline.creationdatetime, '2014-01-24T01:21:21Z') self.assertEqual( response.clusters[0].status.timeline.readydatetime, '2014-01-24T01:25:26Z') self.assertEqual( response.clusters[0].status.timeline.enddatetime, '2014-01-24T02:19:46Z') self.assertTrue(isinstance( response.clusters[0].status.statechangereason, ClusterStateChangeReason)) self.assertEqual( response.clusters[0].status.statechangereason.code, 'USER_REQUEST') self.assertEqual(response.clusters[ 0].status.statechangereason.message, 'Terminated by user request') def test_list_clusters_created_before(self): self.set_http_response(status_code=200) date = datetime.now() response = self.service_connection.list_clusters(created_before=date) self.assert_request_parameters({ 'Action': 'ListClusters', 'CreatedBefore': date.strftime(boto.utils.ISO8601), 'Version': '2009-03-31' }) def test_list_clusters_created_after(self): self.set_http_response(status_code=200) date = datetime.now() response = self.service_connection.list_clusters(created_after=date) self.assert_request_parameters({ 'Action': 'ListClusters', 'CreatedAfter': date.strftime(boto.utils.ISO8601), 'Version': '2009-03-31' }) def test_list_clusters_states(self): self.set_http_response(status_code=200) response = self.service_connection.list_clusters(cluster_states=[ 'RUNNING', 'WAITING' ]) self.assert_request_parameters({ 'Action': 'ListClusters', 'ClusterStates.member.1': 'RUNNING', 'ClusterStates.member.2': 'WAITING', 'Version': '2009-03-31' }) class TestListInstanceGroups(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b""" <ListInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <ListInstanceGroupsResult> <InstanceGroups> <member> <Id>ig-aaaaaaaaaaaaa</Id> <InstanceType>m1.large</InstanceType> <Market>ON_DEMAND</Market> <Status> <StateChangeReason> <Message>Job flow terminated</Message> <Code>CLUSTER_TERMINATED</Code> </StateChangeReason> <State>TERMINATED</State> <Timeline> <CreationDateTime>2014-01-24T01:21:21Z</CreationDateTime> <ReadyDateTime>2014-01-24T01:25:08Z</ReadyDateTime> <EndDateTime>2014-01-24T02:19:46Z</EndDateTime> </Timeline> </Status> <Name>Master instance group</Name> <RequestedInstanceCount>1</RequestedInstanceCount> <RunningInstanceCount>0</RunningInstanceCount> <InstanceGroupType>MASTER</InstanceGroupType> </member> <member> <Id>ig-aaaaaaaaaaab</Id> <InstanceType>m1.large</InstanceType> <Market>ON_DEMAND</Market> <Status> <StateChangeReason> <Message>Job flow terminated</Message> <Code>CLUSTER_TERMINATED</Code> </StateChangeReason> <State>TERMINATED</State> <Timeline> <CreationDateTime>2014-01-24T01:21:21Z</CreationDateTime> <ReadyDateTime>2014-01-24T01:25:26Z</ReadyDateTime> <EndDateTime>2014-01-24T02:19:46Z</EndDateTime> </Timeline> </Status> <Name>Core instance group</Name> <RequestedInstanceCount>2</RequestedInstanceCount> <RunningInstanceCount>0</RunningInstanceCount> <InstanceGroupType>CORE</InstanceGroupType> </member> </InstanceGroups> </ListInstanceGroupsResult> <ResponseMetadata> <RequestId>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</RequestId> </ResponseMetadata> </ListInstanceGroupsResponse> """ def test_list_instance_groups(self): self.set_http_response(200) with self.assertRaises(TypeError): self.service_connection.list_instance_groups() response = self.service_connection.list_instance_groups( cluster_id='j-123') self.assert_request_parameters({ 'Action': 'ListInstanceGroups', 'ClusterId': 'j-123', 'Version': '2009-03-31' }) self.assertTrue(isinstance(response, InstanceGroupList)) self.assertEqual(len(response.instancegroups), 2) self.assertTrue( isinstance(response.instancegroups[0], InstanceGroupInfo)) self.assertEqual(response.instancegroups[0].id, 'ig-aaaaaaaaaaaaa') self.assertEqual( response.instancegroups[0].instancegrouptype, "MASTER") self.assertEqual(response.instancegroups[0].instancetype, "m1.large") self.assertEqual(response.instancegroups[0].market, "ON_DEMAND") self.assertEqual( response.instancegroups[0].name, "Master instance group") self.assertEqual( response.instancegroups[0].requestedinstancecount, '1') self.assertEqual(response.instancegroups[0].runninginstancecount, '0') self.assertTrue( isinstance(response.instancegroups[0].status, ClusterStatus)) self.assertEqual(response.instancegroups[0].status.state, 'TERMINATED') # status.statechangereason is not parsed into an object #self.assertEqual(response.instancegroups[0].status.statechangereason.code, 'CLUSTER_TERMINATED') class TestListInstances(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b""" <ListInstancesResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <ListInstancesResult> <Instances> <member> <Id>ci-123456789abc</Id> <Status> <StateChangeReason> <Message>Cluster was terminated.</Message> <Code>CLUSTER_TERMINATED</Code> </StateChangeReason> <State>TERMINATED</State> <Timeline> <CreationDateTime>2014-01-24T01:21:26Z</CreationDateTime> <ReadyDateTime>2014-01-24T01:25:25Z</ReadyDateTime> <EndDateTime>2014-01-24T02:19:46Z</EndDateTime> </Timeline> </Status> <PrivateDnsName>ip-10-0-0-60.us-west-1.compute.internal</PrivateDnsName> <PublicIpAddress>54.0.0.1</PublicIpAddress> <PublicDnsName>ec2-54-0-0-1.us-west-1.compute.amazonaws.com</PublicDnsName> <Ec2InstanceId>i-aaaaaaaa</Ec2InstanceId> <PrivateIpAddress>10.0.0.60</PrivateIpAddress> </member> <member> <Id>ci-123456789abd</Id> <Status> <StateChangeReason> <Message>Cluster was terminated.</Message> <Code>CLUSTER_TERMINATED</Code> </StateChangeReason> <State>TERMINATED</State> <Timeline> <CreationDateTime>2014-01-24T01:21:26Z</CreationDateTime> <ReadyDateTime>2014-01-24T01:25:25Z</ReadyDateTime> <EndDateTime>2014-01-24T02:19:46Z</EndDateTime> </Timeline> </Status> <PrivateDnsName>ip-10-0-0-61.us-west-1.compute.internal</PrivateDnsName> <PublicIpAddress>54.0.0.2</PublicIpAddress> <PublicDnsName>ec2-54-0-0-2.us-west-1.compute.amazonaws.com</PublicDnsName> <Ec2InstanceId>i-aaaaaaab</Ec2InstanceId> <PrivateIpAddress>10.0.0.61</PrivateIpAddress> </member> <member> <Id>ci-123456789abe3</Id> <Status> <StateChangeReason> <Message>Cluster was terminated.</Message> <Code>CLUSTER_TERMINATED</Code> </StateChangeReason> <State>TERMINATED</State> <Timeline> <CreationDateTime>2014-01-24T01:21:33Z</CreationDateTime> <ReadyDateTime>2014-01-24T01:25:08Z</ReadyDateTime> <EndDateTime>2014-01-24T02:19:46Z</EndDateTime> </Timeline> </Status> <PrivateDnsName>ip-10-0-0-62.us-west-1.compute.internal</PrivateDnsName> <PublicIpAddress>54.0.0.3</PublicIpAddress> <PublicDnsName>ec2-54-0-0-3.us-west-1.compute.amazonaws.com</PublicDnsName> <Ec2InstanceId>i-aaaaaaac</Ec2InstanceId> <PrivateIpAddress>10.0.0.62</PrivateIpAddress> </member> </Instances> </ListInstancesResult> <ResponseMetadata> <RequestId>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</RequestId> </ResponseMetadata> </ListInstancesResponse> """ def test_list_instances(self): self.set_http_response(200) with self.assertRaises(TypeError): self.service_connection.list_instances() response = self.service_connection.list_instances(cluster_id='j-123') self.assertTrue(isinstance(response, InstanceList)) self.assertEqual(len(response.instances), 3) self.assertTrue(isinstance(response.instances[0], InstanceInfo)) self.assertEqual(response.instances[0].ec2instanceid, 'i-aaaaaaaa') self.assertEqual(response.instances[0].id, 'ci-123456789abc') self.assertEqual( response.instances[0].privatednsname, 'ip-10-0-0-60.us-west-1.compute.internal') self.assertEqual(response.instances[0].privateipaddress, '10.0.0.60') self.assertEqual(response.instances[ 0].publicdnsname, 'ec2-54-0-0-1.us-west-1.compute.amazonaws.com') self.assertEqual(response.instances[0].publicipaddress, '54.0.0.1') self.assert_request_parameters({ 'Action': 'ListInstances', 'ClusterId': 'j-123', 'Version': '2009-03-31' }) def test_list_instances_with_group_id(self): self.set_http_response(200) response = self.service_connection.list_instances( cluster_id='j-123', instance_group_id='abc') self.assert_request_parameters({ 'Action': 'ListInstances', 'ClusterId': 'j-123', 'InstanceGroupId': 'abc', 'Version': '2009-03-31' }) def test_list_instances_with_types(self): self.set_http_response(200) response = self.service_connection.list_instances( cluster_id='j-123', instance_group_types=[ 'MASTER', 'TASK' ]) self.assert_request_parameters({ 'Action': 'ListInstances', 'ClusterId': 'j-123', 'InstanceGroupTypes.member.1': 'MASTER', 'InstanceGroupTypes.member.2': 'TASK', 'Version': '2009-03-31' }) class TestListSteps(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b"""<ListStepsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <ListStepsResult> <Steps> <member> <Id>abc123</Id> <Status> <StateChangeReason/> <Timeline> <CreationDateTime>2014-07-01T00:00:00.000Z</CreationDateTime> </Timeline> <State>PENDING</State> </Status> <Name>Step 1</Name> <Config> <Jar>/home/hadoop/lib/emr-s3distcp-1.0.jar</Jar> <Args> <member>--src</member> <member>hdfs:///data/test/</member> <member>--dest</member> <member>s3n://test/data</member> </Args> <Properties/> </Config> <ActionOnFailure>CONTINUE</ActionOnFailure> </member> <member> <Id>def456</Id> <Status> <StateChangeReason/> <Timeline> <CreationDateTime>2014-07-01T00:00:00.000Z</CreationDateTime> </Timeline> <State>COMPLETED</State> </Status> <Name>Step 2</Name> <Config> <MainClass>my.main.SomeClass</MainClass> <Jar>s3n://test/jars/foo.jar</Jar> </Config> <ActionOnFailure>CONTINUE</ActionOnFailure> </member> <member> <Id>ghi789</Id> <Status> <StateChangeReason/> <Timeline> <CreationDateTime>2014-07-01T00:00:00.000Z</CreationDateTime> </Timeline> <State>FAILED</State> </Status> <Name>Step 3</Name> <Config> <Jar>s3n://test/jars/bar.jar</Jar> <Args> <member>-arg</member> <member>value</member> </Args> <Properties/> </Config> <ActionOnFailure>TERMINATE_CLUSTER</ActionOnFailure> </member> </Steps> </ListStepsResult> <ResponseMetadata> <RequestId>eff31ee5-0342-11e4-b3c7-9de5a93f6fcb</RequestId> </ResponseMetadata> </ListStepsResponse> """ def test_list_steps(self): self.set_http_response(200) with self.assertRaises(TypeError): self.service_connection.list_steps() response = self.service_connection.list_steps(cluster_id='j-123') self.assert_request_parameters({ 'Action': 'ListSteps', 'ClusterId': 'j-123', 'Version': '2009-03-31' }) self.assertTrue(isinstance(response, StepSummaryList)) self.assertEqual(response.steps[0].name, 'Step 1') valid_states = [ 'PENDING', 'RUNNING', 'COMPLETED', 'CANCELLED', 'FAILED', 'INTERRUPTED' ] # Check for step states for step in response.steps: self.assertIn(step.status.state, valid_states) # Check for step config step = response.steps[0] self.assertEqual(step.config.jar, '/home/hadoop/lib/emr-s3distcp-1.0.jar') self.assertEqual(len(step.config.args), 4) self.assertEqual(step.config.args[0].value, '--src') self.assertEqual(step.config.args[1].value, 'hdfs:///data/test/') step = response.steps[1] self.assertEqual(step.config.mainclass, 'my.main.SomeClass') def test_list_steps_with_states(self): self.set_http_response(200) response = self.service_connection.list_steps( cluster_id='j-123', step_states=[ 'COMPLETED', 'FAILED' ]) self.assert_request_parameters({ 'Action': 'ListSteps', 'ClusterId': 'j-123', 'StepStates.member.1': 'COMPLETED', 'StepStates.member.2': 'FAILED', 'Version': '2009-03-31' }) self.assertTrue(isinstance(response, StepSummaryList)) self.assertEqual(response.steps[0].name, 'Step 1') class TestListBootstrapActions(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b"""<ListBootstrapActionsOutput></ListBootstrapActionsOutput>""" def test_list_bootstrap_actions(self): self.set_http_response(200) with self.assertRaises(TypeError): self.service_connection.list_bootstrap_actions() response = self.service_connection.list_bootstrap_actions( cluster_id='j-123') self.assert_request_parameters({ 'Action': 'ListBootstrapActions', 'ClusterId': 'j-123', 'Version': '2009-03-31' }) class TestDescribeCluster(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b""" <DescribeClusterResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <DescribeClusterResult> <Cluster> <Id>j-aaaaaaaaa</Id> <Tags/> <Ec2InstanceAttributes> <Ec2AvailabilityZone>us-west-1c</Ec2AvailabilityZone> <Ec2KeyName>my_secret_key</Ec2KeyName> </Ec2InstanceAttributes> <RunningAmiVersion>2.4.2</RunningAmiVersion> <VisibleToAllUsers>true</VisibleToAllUsers> <Status> <StateChangeReason> <Message>Terminated by user request</Message> <Code>USER_REQUEST</Code> </StateChangeReason> <State>TERMINATED</State> <Timeline> <CreationDateTime>2014-01-24T01:21:21Z</CreationDateTime> <ReadyDateTime>2014-01-24T01:25:26Z</ReadyDateTime> <EndDateTime>2014-01-24T02:19:46Z</EndDateTime> </Timeline> </Status> <AutoTerminate>false</AutoTerminate> <Name>test analytics</Name> <RequestedAmiVersion>2.4.2</RequestedAmiVersion> <Applications> <member> <Name>hadoop</Name> <Version>1.0.3</Version> </member> </Applications> <TerminationProtected>false</TerminationProtected> <MasterPublicDnsName>ec2-184-0-0-1.us-west-1.compute.amazonaws.com</MasterPublicDnsName> <NormalizedInstanceHours>10</NormalizedInstanceHours> <ServiceRole>my-service-role</ServiceRole> </Cluster> </DescribeClusterResult> <ResponseMetadata> <RequestId>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</RequestId> </ResponseMetadata> </DescribeClusterResponse> """ def test_describe_cluster(self): self.set_http_response(200) with self.assertRaises(TypeError): self.service_connection.describe_cluster() response = self.service_connection.describe_cluster(cluster_id='j-123') self.assertTrue(isinstance(response, Cluster)) self.assertEqual(response.id, 'j-aaaaaaaaa') self.assertEqual(response.runningamiversion, '2.4.2') self.assertEqual(response.visibletoallusers, 'true') self.assertEqual(response.autoterminate, 'false') self.assertEqual(response.name, 'test analytics') self.assertEqual(response.requestedamiversion, '2.4.2') self.assertEqual(response.terminationprotected, 'false') self.assertEqual( response.ec2instanceattributes.ec2availabilityzone, "us-west-1c") self.assertEqual( response.ec2instanceattributes.ec2keyname, 'my_secret_key') self.assertEqual(response.status.state, 'TERMINATED') self.assertEqual(response.applications[0].name, 'hadoop') self.assertEqual(response.applications[0].version, '1.0.3') self.assertEqual( response.masterpublicdnsname, 'ec2-184-0-0-1.us-west-1.compute.amazonaws.com') self.assertEqual(response.normalizedinstancehours, '10') self.assertEqual(response.servicerole, 'my-service-role') self.assert_request_parameters({ 'Action': 'DescribeCluster', 'ClusterId': 'j-123', 'Version': '2009-03-31' }) class TestDescribeStep(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b"""<DescribeStepOutput></DescribeStepOutput>""" def test_describe_step(self): self.set_http_response(200) with self.assertRaises(TypeError): self.service_connection.describe_step() with self.assertRaises(TypeError): self.service_connection.describe_step(cluster_id='j-123') with self.assertRaises(TypeError): self.service_connection.describe_step(step_id='abc') response = self.service_connection.describe_step( cluster_id='j-123', step_id='abc') self.assert_request_parameters({ 'Action': 'DescribeStep', 'ClusterId': 'j-123', 'StepId': 'abc', 'Version': '2009-03-31' }) class TestAddJobFlowSteps(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b""" <AddJobFlowStepsOutput> <StepIds> <member>Foo</member> <member>Bar</member> </StepIds> </AddJobFlowStepsOutput> """ def test_add_jobflow_steps(self): self.set_http_response(200) response = self.service_connection.add_jobflow_steps( jobflow_id='j-123', steps=[]) # Make sure the correct object is returned, as this was # previously set to incorrectly return an empty instance # of RunJobFlowResponse. self.assertTrue(isinstance(response, JobFlowStepList)) self.assertEqual(response.stepids[0].value, 'Foo') self.assertEqual(response.stepids[1].value, 'Bar') class TestBuildTagList(AWSMockServiceTestCase): connection_class = EmrConnection def test_key_without_value_encoding(self): input_dict = { 'KeyWithNoValue': '', 'AnotherKeyWithNoValue': None } res = self.service_connection._build_tag_list(input_dict) # Keys are outputted in ascending key order. expected = { 'Tags.member.1.Key': 'AnotherKeyWithNoValue', 'Tags.member.2.Key': 'KeyWithNoValue' } self.assertEqual(expected, res) def test_key_full_key_value_encoding(self): input_dict = { 'FirstKey': 'One', 'SecondKey': 'Two' } res = self.service_connection._build_tag_list(input_dict) # Keys are outputted in ascending key order. expected = { 'Tags.member.1.Key': 'FirstKey', 'Tags.member.1.Value': 'One', 'Tags.member.2.Key': 'SecondKey', 'Tags.member.2.Value': 'Two' } self.assertEqual(expected, res) class TestAddTag(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b"""<AddTagsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <AddTagsResult/> <ResponseMetadata> <RequestId>88888888-8888-8888-8888-888888888888</RequestId> </ResponseMetadata> </AddTagsResponse> """ def test_add_mix_of_tags_with_without_values(self): input_tags = { 'FirstKey': 'One', 'SecondKey': 'Two', 'ZzzNoValue': '' } self.set_http_response(200) with self.assertRaises(TypeError): self.service_connection.add_tags() with self.assertRaises(TypeError): self.service_connection.add_tags('j-123') with self.assertRaises(AssertionError): self.service_connection.add_tags('j-123', []) response = self.service_connection.add_tags('j-123', input_tags) self.assertTrue(response) self.assert_request_parameters({ 'Action': 'AddTags', 'ResourceId': 'j-123', 'Tags.member.1.Key': 'FirstKey', 'Tags.member.1.Value': 'One', 'Tags.member.2.Key': 'SecondKey', 'Tags.member.2.Value': 'Two', 'Tags.member.3.Key': 'ZzzNoValue', 'Version': '2009-03-31' }) class TestRemoveTag(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b"""<RemoveTagsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <RemoveTagsResult/> <ResponseMetadata> <RequestId>88888888-8888-8888-8888-888888888888</RequestId> </ResponseMetadata> </RemoveTagsResponse> """ def test_remove_tags(self): input_tags = { 'FirstKey': 'One', 'SecondKey': 'Two', 'ZzzNoValue': '' } self.set_http_response(200) with self.assertRaises(TypeError): self.service_connection.add_tags() with self.assertRaises(TypeError): self.service_connection.add_tags('j-123') with self.assertRaises(AssertionError): self.service_connection.add_tags('j-123', []) response = self.service_connection.remove_tags( 'j-123', ['FirstKey', 'SecondKey']) self.assertTrue(response) self.assert_request_parameters({ 'Action': 'RemoveTags', 'ResourceId': 'j-123', 'TagKeys.member.1': 'FirstKey', 'TagKeys.member.2': 'SecondKey', 'Version': '2009-03-31' }) class DescribeJobFlowsTestBase(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b""" <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <DescribeJobFlowsResult> <JobFlows> <member> <AmiVersion>2.4.2</AmiVersion> <ExecutionStatusDetail> <CreationDateTime>2014-01-24T01:21:21Z</CreationDateTime> <LastStateChangeReason>Terminated by user request</LastStateChangeReason> <StartDateTime>2014-01-24T01:25:26Z</StartDateTime> <ReadyDateTime>2014-01-24T01:25:26Z</ReadyDateTime> <State>TERMINATED</State> <EndDateTime>2014-01-24T02:19:46Z</EndDateTime> </ExecutionStatusDetail> <BootstrapActions/> <VisibleToAllUsers>true</VisibleToAllUsers> <SupportedProducts/> <Name>test analytics</Name> <JobFlowId>j-aaaaaa</JobFlowId> <Steps> <member> <ExecutionStatusDetail> <CreationDateTime>2014-01-24T01:21:21Z</CreationDateTime> <StartDateTime>2014-01-24T01:25:26Z</StartDateTime> <State>COMPLETED</State> <EndDateTime>2014-01-24T01:26:08Z</EndDateTime> </ExecutionStatusDetail> <StepConfig> <HadoopJarStep> <Args> <member>s3://us-west-1.elasticmapreduce/libs/hive/hive-script</member> <member>--base-path</member> <member>s3://us-west-1.elasticmapreduce/libs/hive/</member> <member>--install-hive</member> <member>--hive-versions</member> <member>0.11.0.1</member> </Args> <Jar>s3://us-west-1.elasticmapreduce/libs/script-runner/script-runner.jar</Jar> <Properties/> </HadoopJarStep> <Name>Setup hive</Name> <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure> </StepConfig> </member> </Steps> <Instances> <Placement> <AvailabilityZone>us-west-1c</AvailabilityZone> </Placement> <MasterInstanceType>m1.large</MasterInstanceType> <Ec2KeyName>my_key</Ec2KeyName> <KeepJobFlowAliveWhenNoSteps>true</KeepJobFlowAliveWhenNoSteps> <InstanceGroups> <member> <CreationDateTime>2014-01-24T01:21:21Z</CreationDateTime> <InstanceRunningCount>0</InstanceRunningCount> <StartDateTime>2014-01-24T01:23:56Z</StartDateTime> <ReadyDateTime>2014-01-24T01:25:08Z</ReadyDateTime> <State>ENDED</State> <EndDateTime>2014-01-24T02:19:46Z</EndDateTime> <InstanceRequestCount>1</InstanceRequestCount> <InstanceType>m1.large</InstanceType> <LastStateChangeReason>Job flow terminated</LastStateChangeReason> <Market>ON_DEMAND</Market> <InstanceGroupId>ig-aaaaaa</InstanceGroupId> <InstanceRole>MASTER</InstanceRole> <Name>Master instance group</Name> </member> <member> <CreationDateTime>2014-01-24T01:21:21Z</CreationDateTime> <InstanceRunningCount>0</InstanceRunningCount> <StartDateTime>2014-01-24T01:25:26Z</StartDateTime> <ReadyDateTime>2014-01-24T01:25:26Z</ReadyDateTime> <State>ENDED</State> <EndDateTime>2014-01-24T02:19:46Z</EndDateTime> <InstanceRequestCount>2</InstanceRequestCount> <InstanceType>m1.large</InstanceType> <LastStateChangeReason>Job flow terminated</LastStateChangeReason> <Market>ON_DEMAND</Market> <InstanceGroupId>ig-aaaaab</InstanceGroupId> <InstanceRole>CORE</InstanceRole> <Name>Core instance group</Name> </member> </InstanceGroups> <SlaveInstanceType>m1.large</SlaveInstanceType> <MasterInstanceId>i-aaaaaa</MasterInstanceId> <HadoopVersion>1.0.3</HadoopVersion> <NormalizedInstanceHours>12</NormalizedInstanceHours> <MasterPublicDnsName>ec2-184-0-0-1.us-west-1.compute.amazonaws.com</MasterPublicDnsName> <InstanceCount>3</InstanceCount> <TerminationProtected>false</TerminationProtected> </Instances> </member> </JobFlows> </DescribeJobFlowsResult> <ResponseMetadata> <RequestId>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</RequestId> </ResponseMetadata> </DescribeJobFlowsResponse> """ class TestDescribeJobFlows(DescribeJobFlowsTestBase): def test_describe_jobflows_response(self): self.set_http_response(200) response = self.service_connection.describe_jobflows() self.assertTrue(isinstance(response, list)) jf = response[0] self.assertTrue(isinstance(jf, JobFlow)) self.assertEqual(jf.amiversion, '2.4.2') self.assertEqual(jf.visibletoallusers, 'true') self.assertEqual(jf.name, 'test analytics') self.assertEqual(jf.jobflowid, 'j-aaaaaa') self.assertEqual(jf.ec2keyname, 'my_key') self.assertEqual(jf.masterinstancetype, 'm1.large') self.assertEqual(jf.availabilityzone, 'us-west-1c') self.assertEqual(jf.keepjobflowalivewhennosteps, 'true') self.assertEqual(jf.slaveinstancetype, 'm1.large') self.assertEqual(jf.masterinstanceid, 'i-aaaaaa') self.assertEqual(jf.hadoopversion, '1.0.3') self.assertEqual(jf.normalizedinstancehours, '12') self.assertEqual( jf.masterpublicdnsname, 'ec2-184-0-0-1.us-west-1.compute.amazonaws.com') self.assertEqual(jf.instancecount, '3') self.assertEqual(jf.terminationprotected, 'false') self.assertTrue(isinstance(jf.steps, list)) step = jf.steps[0] self.assertTrue(isinstance(step, Step)) self.assertEqual( step.jar, 's3://us-west-1.elasticmapreduce/libs/script-runner/script-runner.jar') self.assertEqual(step.name, 'Setup hive') self.assertEqual(step.actiononfailure, 'TERMINATE_JOB_FLOW') self.assertTrue(isinstance(jf.instancegroups, list)) ig = jf.instancegroups[0] self.assertTrue(isinstance(ig, InstanceGroup)) self.assertEqual(ig.creationdatetime, '2014-01-24T01:21:21Z') self.assertEqual(ig.state, 'ENDED') self.assertEqual(ig.instancerequestcount, '1') self.assertEqual(ig.instancetype, 'm1.large') self.assertEqual(ig.laststatechangereason, 'Job flow terminated') self.assertEqual(ig.market, 'ON_DEMAND') self.assertEqual(ig.instancegroupid, 'ig-aaaaaa') self.assertEqual(ig.instancerole, 'MASTER') self.assertEqual(ig.name, 'Master instance group') def test_describe_jobflows_no_args(self): self.set_http_response(200) self.service_connection.describe_jobflows() self.assert_request_parameters({ 'Action': 'DescribeJobFlows', }, ignore_params_values=['Version']) def test_describe_jobflows_filtered(self): self.set_http_response(200) now = datetime.now() a_bit_before = datetime.fromtimestamp(time() - 1000) self.service_connection.describe_jobflows(states=['WAITING', 'RUNNING'], jobflow_ids=[ 'j-aaaaaa', 'j-aaaaab'], created_after=a_bit_before, created_before=now) self.assert_request_parameters({ 'Action': 'DescribeJobFlows', 'JobFlowIds.member.1': 'j-aaaaaa', 'JobFlowIds.member.2': 'j-aaaaab', 'JobFlowStates.member.1': 'WAITING', 'JobFlowStates.member.2': 'RUNNING', 'CreatedAfter': a_bit_before.strftime(boto.utils.ISO8601), 'CreatedBefore': now.strftime(boto.utils.ISO8601), }, ignore_params_values=['Version']) class TestDescribeJobFlow(DescribeJobFlowsTestBase): def test_describe_jobflow(self): self.set_http_response(200) response = self.service_connection.describe_jobflow('j-aaaaaa') self.assertTrue(isinstance(response, JobFlow)) self.assert_request_parameters({ 'Action': 'DescribeJobFlows', 'JobFlowIds.member.1': 'j-aaaaaa', }, ignore_params_values=['Version']) class TestRunJobFlow(AWSMockServiceTestCase): connection_class = EmrConnection def default_body(self): return b""" <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31"> <RunJobFlowResult> <JobFlowId>j-ZKIY4CKQRX72</JobFlowId> </RunJobFlowResult> <ResponseMetadata> <RequestId>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</RequestId> </ResponseMetadata> </RunJobFlowResponse> """ def test_run_jobflow_service_role(self): self.set_http_response(200) response = self.service_connection.run_jobflow( 'EmrCluster', service_role='EMR_DefaultRole') self.assertTrue(response) self.assert_request_parameters({ 'Action': 'RunJobFlow', 'Version': '2009-03-31', 'ServiceRole': 'EMR_DefaultRole', 'Name': 'EmrCluster'}, ignore_params_values=['ActionOnFailure', 'Instances.InstanceCount', 'Instances.KeepJobFlowAliveWhenNoSteps', 'Instances.MasterInstanceType', 'Instances.SlaveInstanceType']) def test_run_jobflow_enable_debugging(self): self.region = 'ap-northeast-2' self.set_http_response(200) self.service_connection.run_jobflow( 'EmrCluster', enable_debugging=True) actual_params = set(self.actual_request.params.copy().items()) expected_params = set([ ('Steps.member.1.HadoopJarStep.Jar', 's3://ap-northeast-2.elasticmapreduce/libs/script-runner/script-runner.jar'), ('Steps.member.1.HadoopJarStep.Args.member.1', 's3://ap-northeast-2.elasticmapreduce/libs/state-pusher/0.1/fetch'), ]) self.assertTrue(expected_params <= actual_params) vendored/boto/tests/unit/rds/test_connection.py 0000644 00000104325 15025013616 0016002 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.ec2.securitygroup import SecurityGroup from boto.rds import RDSConnection from boto.rds.vpcsecuritygroupmembership import VPCSecurityGroupMembership from boto.rds.parametergroup import ParameterGroup from boto.rds.logfile import LogFile, LogFileObject import xml.sax.saxutils as saxutils class TestRDSConnection(AWSMockServiceTestCase): connection_class = RDSConnection def setUp(self): super(TestRDSConnection, self).setUp() def default_body(self): return """ <DescribeDBInstancesResponse> <DescribeDBInstancesResult> <DBInstances> <DBInstance> <Iops>2000</Iops> <BackupRetentionPeriod>1</BackupRetentionPeriod> <MultiAZ>false</MultiAZ> <DBInstanceStatus>backing-up</DBInstanceStatus> <DBInstanceIdentifier>mydbinstance2</DBInstanceIdentifier> <PreferredBackupWindow>10:30-11:00</PreferredBackupWindow> <PreferredMaintenanceWindow>wed:06:30-wed:07:00</PreferredMaintenanceWindow> <OptionGroupMembership> <OptionGroupName>default:mysql-5-5</OptionGroupName> <Status>in-sync</Status> </OptionGroupMembership> <AvailabilityZone>us-west-2b</AvailabilityZone> <ReadReplicaDBInstanceIdentifiers/> <Engine>mysql</Engine> <PendingModifiedValues/> <LicenseModel>general-public-license</LicenseModel> <DBParameterGroups> <DBParameterGroup> <ParameterApplyStatus>in-sync</ParameterApplyStatus> <DBParameterGroupName>default.mysql5.5</DBParameterGroupName> </DBParameterGroup> </DBParameterGroups> <Endpoint> <Port>3306</Port> <Address>mydbinstance2.c0hjqouvn9mf.us-west-2.rds.amazonaws.com</Address> </Endpoint> <EngineVersion>5.5.27</EngineVersion> <DBSecurityGroups> <DBSecurityGroup> <Status>active</Status> <DBSecurityGroupName>default</DBSecurityGroupName> </DBSecurityGroup> </DBSecurityGroups> <VpcSecurityGroups> <VpcSecurityGroupMembership> <VpcSecurityGroupId>sg-1</VpcSecurityGroupId> <Status>active</Status> </VpcSecurityGroupMembership> </VpcSecurityGroups> <DBName>mydb2</DBName> <AutoMinorVersionUpgrade>true</AutoMinorVersionUpgrade> <InstanceCreateTime>2012-10-03T22:01:51.047Z</InstanceCreateTime> <AllocatedStorage>200</AllocatedStorage> <DBInstanceClass>db.m1.large</DBInstanceClass> <MasterUsername>awsuser</MasterUsername> <StatusInfos> <DBInstanceStatusInfo> <Message></Message> <Normal>true</Normal> <Status>replicating</Status> <StatusType>read replication</StatusType> </DBInstanceStatusInfo> </StatusInfos> <DBSubnetGroup> <VpcId>990524496922</VpcId> <SubnetGroupStatus>Complete</SubnetGroupStatus> <DBSubnetGroupDescription>My modified DBSubnetGroup</DBSubnetGroupDescription> <DBSubnetGroupName>mydbsubnetgroup</DBSubnetGroupName> <Subnets> <Subnet> <SubnetStatus>Active</SubnetStatus> <SubnetIdentifier>subnet-7c5b4115</SubnetIdentifier> <SubnetAvailabilityZone> <Name>us-east-1c</Name> </SubnetAvailabilityZone> </Subnet> <Subnet> <SubnetStatus>Active</SubnetStatus> <SubnetIdentifier>subnet-7b5b4112</SubnetIdentifier> <SubnetAvailabilityZone> <Name>us-east-1b</Name> </SubnetAvailabilityZone> </Subnet> <Subnet> <SubnetStatus>Active</SubnetStatus> <SubnetIdentifier>subnet-3ea6bd57</SubnetIdentifier> <SubnetAvailabilityZone> <Name>us-east-1d</Name> </SubnetAvailabilityZone> </Subnet> </Subnets> </DBSubnetGroup> </DBInstance> </DBInstances> </DescribeDBInstancesResult> </DescribeDBInstancesResponse> """ def test_get_all_db_instances(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_dbinstances('instance_id') self.assertEqual(len(response), 1) self.assert_request_parameters({ 'Action': 'DescribeDBInstances', 'DBInstanceIdentifier': 'instance_id', }, ignore_params_values=['Version']) db = response[0] self.assertEqual(db.id, 'mydbinstance2') self.assertEqual(db.create_time, '2012-10-03T22:01:51.047Z') self.assertEqual(db.engine, 'mysql') self.assertEqual(db.status, 'backing-up') self.assertEqual(db.allocated_storage, 200) self.assertEqual( db.endpoint, (u'mydbinstance2.c0hjqouvn9mf.us-west-2.rds.amazonaws.com', 3306)) self.assertEqual(db.instance_class, 'db.m1.large') self.assertEqual(db.master_username, 'awsuser') self.assertEqual(db.availability_zone, 'us-west-2b') self.assertEqual(db.backup_retention_period, 1) self.assertEqual(db.preferred_backup_window, '10:30-11:00') self.assertEqual(db.preferred_maintenance_window, 'wed:06:30-wed:07:00') self.assertEqual(db.latest_restorable_time, None) self.assertEqual(db.multi_az, False) self.assertEqual(db.iops, 2000) self.assertEqual(db.pending_modified_values, {}) self.assertEqual(db.parameter_group.name, 'default.mysql5.5') self.assertEqual(db.parameter_group.description, None) self.assertEqual(db.parameter_group.engine, None) self.assertEqual(db.security_group.owner_id, None) self.assertEqual(db.security_group.name, 'default') self.assertEqual(db.security_group.description, None) self.assertEqual(db.security_group.ec2_groups, []) self.assertEqual(db.security_group.ip_ranges, []) self.assertEqual(len(db.status_infos), 1) self.assertEqual(db.status_infos[0].message, '') self.assertEqual(db.status_infos[0].normal, True) self.assertEqual(db.status_infos[0].status, 'replicating') self.assertEqual(db.status_infos[0].status_type, 'read replication') self.assertEqual(db.vpc_security_groups[0].status, 'active') self.assertEqual(db.vpc_security_groups[0].vpc_group, 'sg-1') self.assertEqual(db.license_model, 'general-public-license') self.assertEqual(db.engine_version, '5.5.27') self.assertEqual(db.auto_minor_version_upgrade, True) self.assertEqual(db.subnet_group.name, 'mydbsubnetgroup') class TestRDSCCreateDBInstance(AWSMockServiceTestCase): connection_class = RDSConnection def setUp(self): super(TestRDSCCreateDBInstance, self).setUp() def default_body(self): return """ <CreateDBInstanceResponse xmlns="http://rds.amazonaws.com/doc/2013-05-15/"> <CreateDBInstanceResult> <DBInstance> <ReadReplicaDBInstanceIdentifiers/> <Engine>mysql</Engine> <PendingModifiedValues> <MasterUserPassword>****</MasterUserPassword> </PendingModifiedValues> <BackupRetentionPeriod>0</BackupRetentionPeriod> <MultiAZ>false</MultiAZ> <LicenseModel>general-public-license</LicenseModel> <DBSubnetGroup> <VpcId>990524496922</VpcId> <SubnetGroupStatus>Complete</SubnetGroupStatus> <DBSubnetGroupDescription>description</DBSubnetGroupDescription> <DBSubnetGroupName>subnet_grp1</DBSubnetGroupName> <Subnets> <Subnet> <SubnetStatus>Active</SubnetStatus> <SubnetIdentifier>subnet-7c5b4115</SubnetIdentifier> <SubnetAvailabilityZone> <Name>us-east-1c</Name> </SubnetAvailabilityZone> </Subnet> <Subnet> <SubnetStatus>Active</SubnetStatus> <SubnetIdentifier>subnet-7b5b4112</SubnetIdentifier> <SubnetAvailabilityZone> <Name>us-east-1b</Name> </SubnetAvailabilityZone> </Subnet> <Subnet> <SubnetStatus>Active</SubnetStatus> <SubnetIdentifier>subnet-3ea6bd57</SubnetIdentifier> <SubnetAvailabilityZone> <Name>us-east-1d</Name> </SubnetAvailabilityZone> </Subnet> </Subnets> </DBSubnetGroup> <DBInstanceStatus>creating</DBInstanceStatus> <EngineVersion>5.1.50</EngineVersion> <DBInstanceIdentifier>simcoprod01</DBInstanceIdentifier> <DBParameterGroups> <DBParameterGroup> <ParameterApplyStatus>in-sync</ParameterApplyStatus> <DBParameterGroupName>default.mysql5.1</DBParameterGroupName> </DBParameterGroup> </DBParameterGroups> <DBSecurityGroups> <DBSecurityGroup> <Status>active</Status> <DBSecurityGroupName>default</DBSecurityGroupName> </DBSecurityGroup> </DBSecurityGroups> <PreferredBackupWindow>00:00-00:30</PreferredBackupWindow> <AutoMinorVersionUpgrade>true</AutoMinorVersionUpgrade> <PreferredMaintenanceWindow>sat:07:30-sat:08:00</PreferredMaintenanceWindow> <AllocatedStorage>10</AllocatedStorage> <DBInstanceClass>db.m1.large</DBInstanceClass> <MasterUsername>master</MasterUsername> </DBInstance> </CreateDBInstanceResult> <ResponseMetadata> <RequestId>2e5d4270-8501-11e0-bd9b-a7b1ece36d51</RequestId> </ResponseMetadata> </CreateDBInstanceResponse> """ def test_create_db_instance_param_group_name(self): self.set_http_response(status_code=200) db = self.service_connection.create_dbinstance( 'SimCoProd01', 10, 'db.m1.large', 'master', 'Password01', param_group='default.mysql5.1', db_subnet_group_name='dbSubnetgroup01', backup_retention_period=0) self.assert_request_parameters({ 'Action': 'CreateDBInstance', 'AllocatedStorage': 10, 'AutoMinorVersionUpgrade': 'true', 'BackupRetentionPeriod': 0, 'DBInstanceClass': 'db.m1.large', 'DBInstanceIdentifier': 'SimCoProd01', 'DBParameterGroupName': 'default.mysql5.1', 'DBSubnetGroupName': 'dbSubnetgroup01', 'Engine': 'MySQL5.1', 'MasterUsername': 'master', 'MasterUserPassword': 'Password01', 'Port': 3306 }, ignore_params_values=['Version']) self.assertEqual(db.id, 'simcoprod01') self.assertEqual(db.engine, 'mysql') self.assertEqual(db.status, 'creating') self.assertEqual(db.allocated_storage, 10) self.assertEqual(db.instance_class, 'db.m1.large') self.assertEqual(db.master_username, 'master') self.assertEqual(db.multi_az, False) self.assertEqual(db.pending_modified_values, {'MasterUserPassword': '****'}) self.assertEqual(db.parameter_group.name, 'default.mysql5.1') self.assertEqual(db.parameter_group.description, None) self.assertEqual(db.parameter_group.engine, None) self.assertEqual(db.backup_retention_period, 0) def test_create_db_instance_param_group_instance(self): self.set_http_response(status_code=200) param_group = ParameterGroup() param_group.name = 'default.mysql5.1' db = self.service_connection.create_dbinstance( 'SimCoProd01', 10, 'db.m1.large', 'master', 'Password01', param_group=param_group, db_subnet_group_name='dbSubnetgroup01') self.assert_request_parameters({ 'Action': 'CreateDBInstance', 'AllocatedStorage': 10, 'AutoMinorVersionUpgrade': 'true', 'DBInstanceClass': 'db.m1.large', 'DBInstanceIdentifier': 'SimCoProd01', 'DBParameterGroupName': 'default.mysql5.1', 'DBSubnetGroupName': 'dbSubnetgroup01', 'Engine': 'MySQL5.1', 'MasterUsername': 'master', 'MasterUserPassword': 'Password01', 'Port': 3306, }, ignore_params_values=['Version']) self.assertEqual(db.id, 'simcoprod01') self.assertEqual(db.engine, 'mysql') self.assertEqual(db.status, 'creating') self.assertEqual(db.allocated_storage, 10) self.assertEqual(db.instance_class, 'db.m1.large') self.assertEqual(db.master_username, 'master') self.assertEqual(db.multi_az, False) self.assertEqual(db.pending_modified_values, {'MasterUserPassword': '****'}) self.assertEqual(db.parameter_group.name, 'default.mysql5.1') self.assertEqual(db.parameter_group.description, None) self.assertEqual(db.parameter_group.engine, None) class TestRDSConnectionRestoreDBInstanceFromPointInTime(AWSMockServiceTestCase): connection_class = RDSConnection def setUp(self): super(TestRDSConnectionRestoreDBInstanceFromPointInTime, self).setUp() def default_body(self): return """ <RestoreDBInstanceToPointInTimeResponse xmlns="http://rds.amazonaws.com/doc/2013-05-15/"> <RestoreDBInstanceToPointInTimeResult> <DBInstance> <ReadReplicaDBInstanceIdentifiers/> <Engine>mysql</Engine> <PendingModifiedValues/> <BackupRetentionPeriod>1</BackupRetentionPeriod> <MultiAZ>false</MultiAZ> <LicenseModel>general-public-license</LicenseModel> <DBInstanceStatus>creating</DBInstanceStatus> <EngineVersion>5.1.50</EngineVersion> <DBInstanceIdentifier>restored-db</DBInstanceIdentifier> <DBParameterGroups> <DBParameterGroup> <ParameterApplyStatus>in-sync</ParameterApplyStatus> <DBParameterGroupName>default.mysql5.1</DBParameterGroupName> </DBParameterGroup> </DBParameterGroups> <DBSecurityGroups> <DBSecurityGroup> <Status>active</Status> <DBSecurityGroupName>default</DBSecurityGroupName> </DBSecurityGroup> </DBSecurityGroups> <PreferredBackupWindow>00:00-00:30</PreferredBackupWindow> <AutoMinorVersionUpgrade>true</AutoMinorVersionUpgrade> <PreferredMaintenanceWindow>sat:07:30-sat:08:00</PreferredMaintenanceWindow> <AllocatedStorage>10</AllocatedStorage> <DBInstanceClass>db.m1.large</DBInstanceClass> <MasterUsername>master</MasterUsername> </DBInstance> </RestoreDBInstanceToPointInTimeResult> <ResponseMetadata> <RequestId>1ef546bc-850b-11e0-90aa-eb648410240d</RequestId> </ResponseMetadata> </RestoreDBInstanceToPointInTimeResponse> """ def test_restore_dbinstance_from_point_in_time(self): self.set_http_response(status_code=200) db = self.service_connection.restore_dbinstance_from_point_in_time( 'simcoprod01', 'restored-db', True) self.assert_request_parameters({ 'Action': 'RestoreDBInstanceToPointInTime', 'SourceDBInstanceIdentifier': 'simcoprod01', 'TargetDBInstanceIdentifier': 'restored-db', 'UseLatestRestorableTime': 'true', }, ignore_params_values=['Version']) self.assertEqual(db.id, 'restored-db') self.assertEqual(db.engine, 'mysql') self.assertEqual(db.status, 'creating') self.assertEqual(db.allocated_storage, 10) self.assertEqual(db.instance_class, 'db.m1.large') self.assertEqual(db.master_username, 'master') self.assertEqual(db.multi_az, False) self.assertEqual(db.parameter_group.name, 'default.mysql5.1') self.assertEqual(db.parameter_group.description, None) self.assertEqual(db.parameter_group.engine, None) def test_restore_dbinstance_from_point_in_time__db_subnet_group_name(self): self.set_http_response(status_code=200) db = self.service_connection.restore_dbinstance_from_point_in_time( 'simcoprod01', 'restored-db', True, db_subnet_group_name='dbsubnetgroup') self.assert_request_parameters({ 'Action': 'RestoreDBInstanceToPointInTime', 'SourceDBInstanceIdentifier': 'simcoprod01', 'TargetDBInstanceIdentifier': 'restored-db', 'UseLatestRestorableTime': 'true', 'DBSubnetGroupName': 'dbsubnetgroup', }, ignore_params_values=['Version']) def test_create_db_instance_vpc_sg_str(self): self.set_http_response(status_code=200) vpc_security_groups = [ VPCSecurityGroupMembership(self.service_connection, 'active', 'sg-1'), VPCSecurityGroupMembership(self.service_connection, None, 'sg-2')] db = self.service_connection.create_dbinstance( 'SimCoProd01', 10, 'db.m1.large', 'master', 'Password01', param_group='default.mysql5.1', db_subnet_group_name='dbSubnetgroup01', vpc_security_groups=vpc_security_groups) self.assert_request_parameters({ 'Action': 'CreateDBInstance', 'AllocatedStorage': 10, 'AutoMinorVersionUpgrade': 'true', 'DBInstanceClass': 'db.m1.large', 'DBInstanceIdentifier': 'SimCoProd01', 'DBParameterGroupName': 'default.mysql5.1', 'DBSubnetGroupName': 'dbSubnetgroup01', 'Engine': 'MySQL5.1', 'MasterUsername': 'master', 'MasterUserPassword': 'Password01', 'Port': 3306, 'VpcSecurityGroupIds.member.1': 'sg-1', 'VpcSecurityGroupIds.member.2': 'sg-2' }, ignore_params_values=['Version']) def test_create_db_instance_vpc_sg_obj(self): self.set_http_response(status_code=200) sg1 = SecurityGroup(name='sg-1') sg2 = SecurityGroup(name='sg-2') vpc_security_groups = [ VPCSecurityGroupMembership(self.service_connection, 'active', sg1.name), VPCSecurityGroupMembership(self.service_connection, None, sg2.name)] db = self.service_connection.create_dbinstance( 'SimCoProd01', 10, 'db.m1.large', 'master', 'Password01', param_group='default.mysql5.1', db_subnet_group_name='dbSubnetgroup01', vpc_security_groups=vpc_security_groups) self.assert_request_parameters({ 'Action': 'CreateDBInstance', 'AllocatedStorage': 10, 'AutoMinorVersionUpgrade': 'true', 'DBInstanceClass': 'db.m1.large', 'DBInstanceIdentifier': 'SimCoProd01', 'DBParameterGroupName': 'default.mysql5.1', 'DBSubnetGroupName': 'dbSubnetgroup01', 'Engine': 'MySQL5.1', 'MasterUsername': 'master', 'MasterUserPassword': 'Password01', 'Port': 3306, 'VpcSecurityGroupIds.member.1': 'sg-1', 'VpcSecurityGroupIds.member.2': 'sg-2' }, ignore_params_values=['Version']) class TestRDSOptionGroups(AWSMockServiceTestCase): connection_class = RDSConnection def setUp(self): super(TestRDSOptionGroups, self).setUp() def default_body(self): return """ <DescribeOptionGroupsResponse xmlns="http://rds.amazonaws.com/doc/2013-05-15/"> <DescribeOptionGroupsResult> <OptionGroupsList> <OptionGroup> <MajorEngineVersion>11.2</MajorEngineVersion> <OptionGroupName>myoptiongroup</OptionGroupName> <EngineName>oracle-se1</EngineName> <OptionGroupDescription>Test option group</OptionGroupDescription> <Options/> </OptionGroup> <OptionGroup> <MajorEngineVersion>11.2</MajorEngineVersion> <OptionGroupName>default:oracle-se1-11-2</OptionGroupName> <EngineName>oracle-se1</EngineName> <OptionGroupDescription>Default Option Group.</OptionGroupDescription> <Options/> </OptionGroup> </OptionGroupsList> </DescribeOptionGroupsResult> <ResponseMetadata> <RequestId>e4b234d9-84d5-11e1-87a6-71059839a52b</RequestId> </ResponseMetadata> </DescribeOptionGroupsResponse> """ def test_describe_option_groups(self): self.set_http_response(status_code=200) response = self.service_connection.describe_option_groups() self.assertEqual(len(response), 2) options = response[0] self.assertEqual(options.name, 'myoptiongroup') self.assertEqual(options.description, 'Test option group') self.assertEqual(options.engine_name, 'oracle-se1') self.assertEqual(options.major_engine_version, '11.2') options = response[1] self.assertEqual(options.name, 'default:oracle-se1-11-2') self.assertEqual(options.description, 'Default Option Group.') self.assertEqual(options.engine_name, 'oracle-se1') self.assertEqual(options.major_engine_version, '11.2') class TestRDSLogFile(AWSMockServiceTestCase): connection_class = RDSConnection def setUp(self): super(TestRDSLogFile, self).setUp() def default_body(self): return """ <DescribeDBLogFilesResponse xmlns="http://rds.amazonaws.com/doc/2013-02-12/"> <DescribeDBLogFilesResult> <DescribeDBLogFiles> <DescribeDBLogFilesDetails> <LastWritten>1364403600000</LastWritten> <LogFileName>error/mysql-error-running.log</LogFileName> <Size>0</Size> </DescribeDBLogFilesDetails> <DescribeDBLogFilesDetails> <LastWritten>1364338800000</LastWritten> <LogFileName>error/mysql-error-running.log.0</LogFileName> <Size>0</Size> </DescribeDBLogFilesDetails> <DescribeDBLogFilesDetails> <LastWritten>1364342400000</LastWritten> <LogFileName>error/mysql-error-running.log.1</LogFileName> <Size>0</Size> </DescribeDBLogFilesDetails> <DescribeDBLogFilesDetails> <LastWritten>1364346000000</LastWritten> <LogFileName>error/mysql-error-running.log.2</LogFileName> <Size>0</Size> </DescribeDBLogFilesDetails> <DescribeDBLogFilesDetails> <LastWritten>1364349600000</LastWritten> <LogFileName>error/mysql-error-running.log.3</LogFileName> <Size>0</Size> </DescribeDBLogFilesDetails> <DescribeDBLogFilesDetails> <LastWritten>1364405700000</LastWritten> <LogFileName>error/mysql-error.log</LogFileName> <Size>0</Size> </DescribeDBLogFilesDetails> </DescribeDBLogFiles> </DescribeDBLogFilesResult> <ResponseMetadata> <RequestId>d70fb3b3-9704-11e2-a0db-871552e0ef19</RequestId> </ResponseMetadata> </DescribeDBLogFilesResponse> """ def test_get_all_logs_simple(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_logs('db1') self.assert_request_parameters({ 'Action': 'DescribeDBLogFiles', 'DBInstanceIdentifier': 'db1', }, ignore_params_values=['Version']) self.assertEqual(len(response), 6) self.assertTrue(isinstance(response[0], LogFile)) self.assertEqual(response[0].log_filename, 'error/mysql-error-running.log') self.assertEqual(response[0].last_written, '1364403600000') self.assertEqual(response[0].size, '0') def test_get_all_logs_filtered(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_logs('db_instance_1', max_records=100, marker='error/mysql-error.log', file_size=2000000, filename_contains='error', file_last_written=12345678) self.assert_request_parameters({ 'Action': 'DescribeDBLogFiles', 'DBInstanceIdentifier': 'db_instance_1', 'MaxRecords': 100, 'Marker': 'error/mysql-error.log', 'FileSize': 2000000, 'FilenameContains': 'error', 'FileLastWritten': 12345678, }, ignore_params_values=['Version']) self.assertEqual(len(response), 6) self.assertTrue(isinstance(response[0], LogFile)) self.assertEqual(response[0].log_filename, 'error/mysql-error-running.log') self.assertEqual(response[0].last_written, '1364403600000') self.assertEqual(response[0].size, '0') class TestRDSLogFileDownload(AWSMockServiceTestCase): connection_class = RDSConnection logfile_sample = """ ??2014-01-26 23:59:00.01 spid54 Microsoft SQL Server 2012 - 11.0.2100.60 (X64) Feb 10 2012 19:39:15 Copyright (c) Microsoft Corporation Web Edition (64-bit) on Windows NT 6.1 <X64> (Build 7601: Service Pack 1) (Hypervisor) 2014-01-26 23:59:00.01 spid54 (c) Microsoft Corporation. 2014-01-26 23:59:00.01 spid54 All rights reserved. 2014-01-26 23:59:00.01 spid54 Server process ID is 2976. 2014-01-26 23:59:00.01 spid54 System Manufacturer: 'Xen', System Model: 'HVM domU'. 2014-01-26 23:59:00.01 spid54 Authentication mode is MIXED. 2014-01-26 23:59:00.01 spid54 Logging SQL Server messages in file 'D:\\RDSDBDATA\\Log\\ERROR'. 2014-01-26 23:59:00.01 spid54 The service account is 'WORKGROUP\\AMAZONA-NUQUUMV$'. This is an informational message; no user action is required. 2014-01-26 23:59:00.01 spid54 The error log has been reinitialized. See the previous log for older entries. 2014-01-27 00:00:56.42 spid25s This instance of SQL Server has been using a process ID of 2976 since 10/21/2013 2:16:50 AM (local) 10/21/2013 2:16:50 AM (UTC). This is an informational message only; no user action is required. 2014-01-27 09:35:15.43 spid71 I/O is frozen on database model. No user action is required. However, if I/O is not resumed promptly, you could cancel the backup. 2014-01-27 09:35:15.44 spid72 I/O is frozen on database msdb. No user action is required. However, if I/O is not resumed promptly, you could cancel the backup. 2014-01-27 09:35:15.44 spid74 I/O is frozen on database rdsadmin. No user action is required. However, if I/O is not resumed promptly, you could cancel the backup. 2014-01-27 09:35:15.44 spid73 I/O is frozen on database master. No user action is required. However, if I/O is not resumed promptly, you could cancel the backup. 2014-01-27 09:35:25.57 spid73 I/O was resumed on database master. No user action is required. 2014-01-27 09:35:25.57 spid74 I/O was resumed on database rdsadmin. No user action is required. 2014-01-27 09:35:25.57 spid71 I/O was resumed on database model. No user action is required. 2014-01-27 09:35:25.57 spid72 I/O was resumed on database msdb. No user action is required. """ def setUp(self): super(TestRDSLogFileDownload, self).setUp() def default_body(self): return """ <DownloadDBLogFilePortionResponse xmlns="http://rds.amazonaws.com/doc/2013-09-09/"> <DownloadDBLogFilePortionResult> <Marker>0:4485</Marker> <LogFileData>%s</LogFileData> <AdditionalDataPending>false</AdditionalDataPending> </DownloadDBLogFilePortionResult> <ResponseMetadata> <RequestId>27143615-87ae-11e3-acc9-fb64b157268e</RequestId> </ResponseMetadata> </DownloadDBLogFilePortionResponse> """ % self.logfile_sample def test_single_download(self): self.set_http_response(status_code=200) response = self.service_connection.get_log_file('db1', 'foo.log') self.assertTrue(isinstance(response, LogFileObject)) self.assertEqual(response.marker, '0:4485') self.assertEqual(response.dbinstance_id, 'db1') self.assertEqual(response.log_filename, 'foo.log') self.assertEqual(response.data, saxutils.unescape(self.logfile_sample)) self.assert_request_parameters({ 'Action': 'DownloadDBLogFilePortion', 'DBInstanceIdentifier': 'db1', 'LogFileName': 'foo.log', }, ignore_params_values=['Version']) def test_multi_args(self): self.set_http_response(status_code=200) response = self.service_connection.get_log_file('db1', 'foo.log', marker='0:4485', number_of_lines=10) self.assertTrue(isinstance(response, LogFileObject)) self.assert_request_parameters({ 'Action': 'DownloadDBLogFilePortion', 'DBInstanceIdentifier': 'db1', 'Marker': '0:4485', 'NumberOfLines': 10, 'LogFileName': 'foo.log', }, ignore_params_values=['Version']) class TestRDSOptionGroupOptions(AWSMockServiceTestCase): connection_class = RDSConnection def setUp(self): super(TestRDSOptionGroupOptions, self).setUp() def default_body(self): return """ <DescribeOptionGroupOptionsResponse xmlns="http://rds.amazonaws.com/doc/2013-05-15/"> <DescribeOptionGroupOptionsResult> <OptionGroupOptions> <OptionGroupOption> <MajorEngineVersion>11.2</MajorEngineVersion> <PortRequired>true</PortRequired> <OptionsDependedOn/> <Description>Oracle Enterprise Manager</Description> <DefaultPort>1158</DefaultPort> <Name>OEM</Name> <EngineName>oracle-se1</EngineName> <MinimumRequiredMinorEngineVersion>0.2.v3</MinimumRequiredMinorEngineVersion> <Persistent>false</Persistent> <Permanent>false</Permanent> </OptionGroupOption> </OptionGroupOptions> </DescribeOptionGroupOptionsResult> <ResponseMetadata> <RequestId>d9c8f6a1-84c7-11e1-a264-0b23c28bc344</RequestId> </ResponseMetadata> </DescribeOptionGroupOptionsResponse> """ def test_describe_option_group_options(self): self.set_http_response(status_code=200) response = self.service_connection.describe_option_group_options() self.assertEqual(len(response), 1) options = response[0] self.assertEqual(options.name, 'OEM') self.assertEqual(options.description, 'Oracle Enterprise Manager') self.assertEqual(options.engine_name, 'oracle-se1') self.assertEqual(options.major_engine_version, '11.2') self.assertEqual(options.min_minor_engine_version, '0.2.v3') self.assertEqual(options.port_required, True) self.assertEqual(options.default_port, 1158) self.assertEqual(options.permanent, False) self.assertEqual(options.persistent, False) self.assertEqual(options.depends_on, []) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/rds/test_snapshot.py 0000644 00000034155 15025013616 0015505 0 ustar 00 from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.rds import RDSConnection from boto.rds.dbsnapshot import DBSnapshot from boto.rds import DBInstance class TestDescribeDBSnapshots(AWSMockServiceTestCase): connection_class = RDSConnection def default_body(self): return """ <DescribeDBSnapshotsResponse xmlns="http://rds.amazonaws.com/doc/2013-05-15/"> <DescribeDBSnapshotsResult> <DBSnapshots> <DBSnapshot> <Port>3306</Port> <SnapshotCreateTime>2011-05-23T06:29:03.483Z</SnapshotCreateTime> <Engine>mysql</Engine> <Status>available</Status> <AvailabilityZone>us-east-1a</AvailabilityZone> <LicenseModel>general-public-license</LicenseModel> <InstanceCreateTime>2011-05-23T06:06:43.110Z</InstanceCreateTime> <AllocatedStorage>10</AllocatedStorage> <DBInstanceIdentifier>simcoprod01</DBInstanceIdentifier> <EngineVersion>5.1.50</EngineVersion> <DBSnapshotIdentifier>mydbsnapshot</DBSnapshotIdentifier> <SnapshotType>manual</SnapshotType> <MasterUsername>master</MasterUsername> <OptionGroupName>myoptiongroupname</OptionGroupName> <Iops>1000</Iops> <PercentProgress>100</PercentProgress> <SourceRegion>eu-west-1</SourceRegion> <VpcId>myvpc</VpcId> </DBSnapshot> <DBSnapshot> <Port>3306</Port> <SnapshotCreateTime>2011-03-11T07:20:24.082Z</SnapshotCreateTime> <Engine>mysql</Engine> <Status>available</Status> <AvailabilityZone>us-east-1a</AvailabilityZone> <LicenseModel>general-public-license</LicenseModel> <InstanceCreateTime>2010-08-04T23:27:36.420Z</InstanceCreateTime> <AllocatedStorage>50</AllocatedStorage> <DBInstanceIdentifier>mydbinstance</DBInstanceIdentifier> <EngineVersion>5.1.49</EngineVersion> <DBSnapshotIdentifier>mysnapshot1</DBSnapshotIdentifier> <SnapshotType>manual</SnapshotType> <MasterUsername>sa</MasterUsername> <OptionGroupName>myoptiongroupname</OptionGroupName> <Iops>1000</Iops> </DBSnapshot> <DBSnapshot> <Port>3306</Port> <SnapshotCreateTime>2012-04-02T00:01:24.082Z</SnapshotCreateTime> <Engine>mysql</Engine> <Status>available</Status> <AvailabilityZone>us-east-1d</AvailabilityZone> <LicenseModel>general-public-license</LicenseModel> <InstanceCreateTime>2010-07-16T00:06:59.107Z</InstanceCreateTime> <AllocatedStorage>60</AllocatedStorage> <DBInstanceIdentifier>simcoprod01</DBInstanceIdentifier> <EngineVersion>5.1.47</EngineVersion> <DBSnapshotIdentifier>rds:simcoprod01-2012-04-02-00-01</DBSnapshotIdentifier> <SnapshotType>automated</SnapshotType> <MasterUsername>master</MasterUsername> <OptionGroupName>myoptiongroupname</OptionGroupName> <Iops>1000</Iops> </DBSnapshot> </DBSnapshots> </DescribeDBSnapshotsResult> <ResponseMetadata> <RequestId>c4191173-8506-11e0-90aa-eb648410240d</RequestId> </ResponseMetadata> </DescribeDBSnapshotsResponse> """ def test_describe_dbinstances_by_instance(self): self.set_http_response(status_code=200) response = self.service_connection.get_all_dbsnapshots(instance_id='simcoprod01') self.assert_request_parameters({ 'Action': 'DescribeDBSnapshots', 'DBInstanceIdentifier': 'simcoprod01' }, ignore_params_values=['Version']) self.assertEqual(len(response), 3) self.assertIsInstance(response[0], DBSnapshot) self.assertEqual(response[0].id, 'mydbsnapshot') self.assertEqual(response[0].status, 'available') self.assertEqual(response[0].instance_id, 'simcoprod01') self.assertEqual(response[0].engine_version, '5.1.50') self.assertEqual(response[0].license_model, 'general-public-license') self.assertEqual(response[0].iops, 1000) self.assertEqual(response[0].option_group_name, 'myoptiongroupname') self.assertEqual(response[0].percent_progress, 100) self.assertEqual(response[0].snapshot_type, 'manual') self.assertEqual(response[0].source_region, 'eu-west-1') self.assertEqual(response[0].vpc_id, 'myvpc') class TestCreateDBSnapshot(AWSMockServiceTestCase): connection_class = RDSConnection def default_body(self): return """ <CreateDBSnapshotResponse xmlns="http://rds.amazonaws.com/doc/2013-05-15/"> <CreateDBSnapshotResult> <DBSnapshot> <Port>3306</Port> <Engine>mysql</Engine> <Status>creating</Status> <AvailabilityZone>us-east-1a</AvailabilityZone> <LicenseModel>general-public-license</LicenseModel> <InstanceCreateTime>2011-05-23T06:06:43.110Z</InstanceCreateTime> <AllocatedStorage>10</AllocatedStorage> <DBInstanceIdentifier>simcoprod01</DBInstanceIdentifier> <EngineVersion>5.1.50</EngineVersion> <DBSnapshotIdentifier>mydbsnapshot</DBSnapshotIdentifier> <SnapshotType>manual</SnapshotType> <MasterUsername>master</MasterUsername> </DBSnapshot> </CreateDBSnapshotResult> <ResponseMetadata> <RequestId>c4181d1d-8505-11e0-90aa-eb648410240d</RequestId> </ResponseMetadata> </CreateDBSnapshotResponse> """ def test_create_dbinstance(self): self.set_http_response(status_code=200) response = self.service_connection.create_dbsnapshot('mydbsnapshot', 'simcoprod01') self.assert_request_parameters({ 'Action': 'CreateDBSnapshot', 'DBSnapshotIdentifier': 'mydbsnapshot', 'DBInstanceIdentifier': 'simcoprod01' }, ignore_params_values=['Version']) self.assertIsInstance(response, DBSnapshot) self.assertEqual(response.id, 'mydbsnapshot') self.assertEqual(response.instance_id, 'simcoprod01') self.assertEqual(response.status, 'creating') class TestCopyDBSnapshot(AWSMockServiceTestCase): connection_class = RDSConnection def default_body(self): return """ <CopyDBSnapshotResponse xmlns="http://rds.amazonaws.com/doc/2013-05-15/"> <CopyDBSnapshotResult> <DBSnapshot> <Port>3306</Port> <Engine>mysql</Engine> <Status>available</Status> <AvailabilityZone>us-east-1a</AvailabilityZone> <LicenseModel>general-public-license</LicenseModel> <InstanceCreateTime>2011-05-23T06:06:43.110Z</InstanceCreateTime> <AllocatedStorage>10</AllocatedStorage> <DBInstanceIdentifier>simcoprod01</DBInstanceIdentifier> <EngineVersion>5.1.50</EngineVersion> <DBSnapshotIdentifier>mycopieddbsnapshot</DBSnapshotIdentifier> <SnapshotType>manual</SnapshotType> <MasterUsername>master</MasterUsername> </DBSnapshot> </CopyDBSnapshotResult> <ResponseMetadata> <RequestId>c4181d1d-8505-11e0-90aa-eb648410240d</RequestId> </ResponseMetadata> </CopyDBSnapshotResponse> """ def test_copy_dbinstance(self): self.set_http_response(status_code=200) response = self.service_connection.copy_dbsnapshot('myautomaticdbsnapshot', 'mycopieddbsnapshot') self.assert_request_parameters({ 'Action': 'CopyDBSnapshot', 'SourceDBSnapshotIdentifier': 'myautomaticdbsnapshot', 'TargetDBSnapshotIdentifier': 'mycopieddbsnapshot' }, ignore_params_values=['Version']) self.assertIsInstance(response, DBSnapshot) self.assertEqual(response.id, 'mycopieddbsnapshot') self.assertEqual(response.status, 'available') class TestDeleteDBSnapshot(AWSMockServiceTestCase): connection_class = RDSConnection def default_body(self): return """ <DeleteDBSnapshotResponse xmlns="http://rds.amazonaws.com/doc/2013-05-15/"> <DeleteDBSnapshotResult> <DBSnapshot> <Port>3306</Port> <SnapshotCreateTime>2011-03-11T07:20:24.082Z</SnapshotCreateTime> <Engine>mysql</Engine> <Status>deleted</Status> <AvailabilityZone>us-east-1d</AvailabilityZone> <LicenseModel>general-public-license</LicenseModel> <InstanceCreateTime>2010-07-16T00:06:59.107Z</InstanceCreateTime> <AllocatedStorage>60</AllocatedStorage> <DBInstanceIdentifier>simcoprod01</DBInstanceIdentifier> <EngineVersion>5.1.47</EngineVersion> <DBSnapshotIdentifier>mysnapshot2</DBSnapshotIdentifier> <SnapshotType>manual</SnapshotType> <MasterUsername>master</MasterUsername> </DBSnapshot> </DeleteDBSnapshotResult> <ResponseMetadata> <RequestId>627a43a1-8507-11e0-bd9b-a7b1ece36d51</RequestId> </ResponseMetadata> </DeleteDBSnapshotResponse> """ def test_delete_dbinstance(self): self.set_http_response(status_code=200) response = self.service_connection.delete_dbsnapshot('mysnapshot2') self.assert_request_parameters({ 'Action': 'DeleteDBSnapshot', 'DBSnapshotIdentifier': 'mysnapshot2' }, ignore_params_values=['Version']) self.assertIsInstance(response, DBSnapshot) self.assertEqual(response.id, 'mysnapshot2') self.assertEqual(response.status, 'deleted') class TestRestoreDBInstanceFromDBSnapshot(AWSMockServiceTestCase): connection_class = RDSConnection def default_body(self): return """ <RestoreDBInstanceFromDBSnapshotResponse xmlns="http://rds.amazonaws.com/doc/2013-05-15/"> <RestoreDBInstanceFromDBSnapshotResult> <DBInstance> <ReadReplicaDBInstanceIdentifiers/> <Engine>mysql</Engine> <PendingModifiedValues/> <BackupRetentionPeriod>1</BackupRetentionPeriod> <MultiAZ>false</MultiAZ> <LicenseModel>general-public-license</LicenseModel> <DBInstanceStatus>creating</DBInstanceStatus> <EngineVersion>5.1.50</EngineVersion> <DBInstanceIdentifier>myrestoreddbinstance</DBInstanceIdentifier> <DBParameterGroups> <DBParameterGroup> <ParameterApplyStatus>in-sync</ParameterApplyStatus> <DBParameterGroupName>default.mysql5.1</DBParameterGroupName> </DBParameterGroup> </DBParameterGroups> <DBSecurityGroups> <DBSecurityGroup> <Status>active</Status> <DBSecurityGroupName>default</DBSecurityGroupName> </DBSecurityGroup> </DBSecurityGroups> <PreferredBackupWindow>00:00-00:30</PreferredBackupWindow> <AutoMinorVersionUpgrade>true</AutoMinorVersionUpgrade> <PreferredMaintenanceWindow>sat:07:30-sat:08:00</PreferredMaintenanceWindow> <AllocatedStorage>10</AllocatedStorage> <DBInstanceClass>db.m1.large</DBInstanceClass> <MasterUsername>master</MasterUsername> </DBInstance> </RestoreDBInstanceFromDBSnapshotResult> <ResponseMetadata> <RequestId>7ca622e8-8508-11e0-bd9b-a7b1ece36d51</RequestId> </ResponseMetadata> </RestoreDBInstanceFromDBSnapshotResponse> """ def test_restore_dbinstance_from_dbsnapshot(self): self.set_http_response(status_code=200) response = self.service_connection.restore_dbinstance_from_dbsnapshot('mydbsnapshot', 'myrestoreddbinstance', 'db.m1.large', '3306', 'us-east-1a', 'false', 'true') self.assert_request_parameters({ 'Action': 'RestoreDBInstanceFromDBSnapshot', 'DBSnapshotIdentifier': 'mydbsnapshot', 'DBInstanceIdentifier': 'myrestoreddbinstance', 'DBInstanceClass': 'db.m1.large', 'Port': '3306', 'AvailabilityZone': 'us-east-1a', 'MultiAZ': 'false', 'AutoMinorVersionUpgrade': 'true' }, ignore_params_values=['Version']) self.assertIsInstance(response, DBInstance) self.assertEqual(response.id, 'myrestoreddbinstance') self.assertEqual(response.status, 'creating') self.assertEqual(response.instance_class, 'db.m1.large') self.assertEqual(response.multi_az, False) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/rds/__init__.py 0000644 00000000000 15025013616 0014324 0 ustar 00 vendored/boto/tests/unit/glacier/test_layer2.py 0000644 00000034073 15025013616 0015661 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2012 Thomas Parslow http://almostobsolete.net/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from mock import call, Mock, patch, sentinel import codecs from boto.glacier.layer1 import Layer1 from boto.glacier.layer2 import Layer2 import boto.glacier.vault from boto.glacier.vault import Vault from boto.glacier.vault import Job from datetime import datetime, tzinfo, timedelta # Some fixture data from the Glacier docs FIXTURE_VAULT = { "CreationDate": "2012-02-20T17:01:45.198Z", "LastInventoryDate": "2012-03-20T17:03:43.221Z", "NumberOfArchives": 192, "SizeInBytes": 78088912, "VaultARN": "arn:aws:glacier:us-east-1:012345678901:vaults/examplevault", "VaultName": "examplevault" } FIXTURE_VAULTS = { 'RequestId': 'vuXO7SHTw-luynJ0Zu31AYjR3TcCn7X25r7ykpuulxY2lv8', 'VaultList': [{'SizeInBytes': 0, 'LastInventoryDate': None, 'VaultARN': 'arn:aws:glacier:us-east-1:686406519478:vaults/vault0', 'VaultName': 'vault0', 'NumberOfArchives': 0, 'CreationDate': '2013-05-17T02:38:39.049Z'}, {'SizeInBytes': 0, 'LastInventoryDate': None, 'VaultARN': 'arn:aws:glacier:us-east-1:686406519478:vaults/vault3', 'VaultName': 'vault3', 'NumberOfArchives': 0, 'CreationDate': '2013-05-17T02:31:18.659Z'}]} FIXTURE_PAGINATED_VAULTS = { 'Marker': 'arn:aws:glacier:us-east-1:686406519478:vaults/vault2', 'RequestId': 'vuXO7SHTw-luynJ0Zu31AYjR3TcCn7X25r7ykpuulxY2lv8', 'VaultList': [{'SizeInBytes': 0, 'LastInventoryDate': None, 'VaultARN': 'arn:aws:glacier:us-east-1:686406519478:vaults/vault0', 'VaultName': 'vault0', 'NumberOfArchives': 0, 'CreationDate': '2013-05-17T02:38:39.049Z'}, {'SizeInBytes': 0, 'LastInventoryDate': None, 'VaultARN': 'arn:aws:glacier:us-east-1:686406519478:vaults/vault1', 'VaultName': 'vault1', 'NumberOfArchives': 0, 'CreationDate': '2013-05-17T02:31:18.659Z'}]} FIXTURE_PAGINATED_VAULTS_CONT = { 'Marker': None, 'RequestId': 'vuXO7SHTw-luynJ0Zu31AYjR3TcCn7X25r7ykpuulxY2lv8', 'VaultList': [{'SizeInBytes': 0, 'LastInventoryDate': None, 'VaultARN': 'arn:aws:glacier:us-east-1:686406519478:vaults/vault2', 'VaultName': 'vault2', 'NumberOfArchives': 0, 'CreationDate': '2013-05-17T02:38:39.049Z'}, {'SizeInBytes': 0, 'LastInventoryDate': None, 'VaultARN': 'arn:aws:glacier:us-east-1:686406519478:vaults/vault3', 'VaultName': 'vault3', 'NumberOfArchives': 0, 'CreationDate': '2013-05-17T02:31:18.659Z'}]} FIXTURE_ARCHIVE_JOB = { "Action": "ArchiveRetrieval", "ArchiveId": ("NkbByEejwEggmBz2fTHgJrg0XBoDfjP4q6iu87-TjhqG6eGoOY9Z8i1_AUyUs" "uhPAdTqLHy8pTl5nfCFJmDl2yEZONi5L26Omw12vcs01MNGntHEQL8MBfGlqr" "EXAMPLEArchiveId"), "ArchiveSizeInBytes": 16777216, "Completed": False, "CreationDate": "2012-05-15T17:21:39.339Z", "CompletionDate": "2012-05-15T17:21:43.561Z", "InventorySizeInBytes": None, "JobDescription": "My ArchiveRetrieval Job", "JobId": ("HkF9p6o7yjhFx-K3CGl6fuSm6VzW9T7esGQfco8nUXVYwS0jlb5gq1JZ55yHgt5v" "P54ZShjoQzQVVh7vEXAMPLEjobID"), "SHA256TreeHash": ("beb0fe31a1c7ca8c6c04d574ea906e3f97b31fdca7571defb5b44dc" "a89b5af60"), "SNSTopic": "arn:aws:sns:us-east-1:012345678901:mytopic", "StatusCode": "InProgress", "StatusMessage": "Operation in progress.", "VaultARN": "arn:aws:glacier:us-east-1:012345678901:vaults/examplevault" } EXAMPLE_PART_LIST_RESULT_PAGE_1 = { "ArchiveDescription": "archive description 1", "CreationDate": "2012-03-20T17:03:43.221Z", "Marker": "MfgsKHVjbQ6EldVl72bn3_n5h2TaGZQUO-Qb3B9j3TITf7WajQ", "MultipartUploadId": "OW2fM5iVylEpFEMM9_HpKowRapC3vn5sSL39_396UW9zLFUWVrnRHaPjUJddQ5OxSHVXjYtrN47NBZ-khxOjyEXAMPLE", "PartSizeInBytes": 4194304, "Parts": [{ "RangeInBytes": "4194304-8388607", "SHA256TreeHash": "01d34dabf7be316472c93b1ef80721f5d4" }], "VaultARN": "arn:aws:glacier:us-east-1:012345678901:vaults/demo1-vault" } # The documentation doesn't say whether the non-Parts fields are defined in # future pages, so assume they are not. EXAMPLE_PART_LIST_RESULT_PAGE_2 = { "ArchiveDescription": None, "CreationDate": None, "Marker": None, "MultipartUploadId": None, "PartSizeInBytes": None, "Parts": [{ "RangeInBytes": "0-4194303", "SHA256TreeHash": "01d34dabf7be316472c93b1ef80721f5d4" }], "VaultARN": None } EXAMPLE_PART_LIST_COMPLETE = { "ArchiveDescription": "archive description 1", "CreationDate": "2012-03-20T17:03:43.221Z", "Marker": None, "MultipartUploadId": "OW2fM5iVylEpFEMM9_HpKowRapC3vn5sSL39_396UW9zLFUWVrnRHaPjUJddQ5OxSHVXjYtrN47NBZ-khxOjyEXAMPLE", "PartSizeInBytes": 4194304, "Parts": [{ "RangeInBytes": "4194304-8388607", "SHA256TreeHash": "01d34dabf7be316472c93b1ef80721f5d4" }, { "RangeInBytes": "0-4194303", "SHA256TreeHash": "01d34dabf7be316472c93b1ef80721f5d4" }], "VaultARN": "arn:aws:glacier:us-east-1:012345678901:vaults/demo1-vault" } class GlacierLayer2Base(unittest.TestCase): def setUp(self): self.mock_layer1 = Mock(spec=Layer1) class TestGlacierLayer2Connection(GlacierLayer2Base): def setUp(self): GlacierLayer2Base.setUp(self) self.layer2 = Layer2(layer1=self.mock_layer1) def test_create_vault(self): self.mock_layer1.describe_vault.return_value = FIXTURE_VAULT self.layer2.create_vault("My Vault") self.mock_layer1.create_vault.assert_called_with("My Vault") def test_get_vault(self): self.mock_layer1.describe_vault.return_value = FIXTURE_VAULT vault = self.layer2.get_vault("examplevault") self.assertEqual(vault.layer1, self.mock_layer1) self.assertEqual(vault.name, "examplevault") self.assertEqual(vault.size, 78088912) self.assertEqual(vault.number_of_archives, 192) def test_list_vaults(self): self.mock_layer1.list_vaults.return_value = FIXTURE_VAULTS vaults = self.layer2.list_vaults() self.assertEqual(vaults[0].name, "vault0") self.assertEqual(len(vaults), 2) def test_list_vaults_paginated(self): resps = [FIXTURE_PAGINATED_VAULTS, FIXTURE_PAGINATED_VAULTS_CONT] def return_paginated_vaults_resp(marker=None, limit=None): return resps.pop(0) self.mock_layer1.list_vaults = Mock(side_effect=return_paginated_vaults_resp) vaults = self.layer2.list_vaults() self.assertEqual(vaults[0].name, "vault0") self.assertEqual(vaults[3].name, "vault3") self.assertEqual(len(vaults), 4) class TestVault(GlacierLayer2Base): def setUp(self): GlacierLayer2Base.setUp(self) self.vault = Vault(self.mock_layer1, FIXTURE_VAULT) # TODO: Tests for the other methods of uploading def test_create_archive_writer(self): self.mock_layer1.initiate_multipart_upload.return_value = { "UploadId": "UPLOADID"} writer = self.vault.create_archive_writer(description="stuff") self.mock_layer1.initiate_multipart_upload.assert_called_with( "examplevault", self.vault.DefaultPartSize, "stuff") self.assertEqual(writer.vault, self.vault) self.assertEqual(writer.upload_id, "UPLOADID") def test_delete_vault(self): self.vault.delete_archive("archive") self.mock_layer1.delete_archive.assert_called_with("examplevault", "archive") def test_initiate_job(self): class UTC(tzinfo): """UTC""" def utcoffset(self, dt): return timedelta(0) def tzname(self, dt): return "Z" def dst(self, dt): return timedelta(0) self.mock_layer1.initiate_job.return_value = {'JobId': 'job-id'} self.vault.retrieve_inventory(start_date=datetime(2014, 0o1, 0o1, tzinfo=UTC()), end_date=datetime(2014, 0o1, 0o2, tzinfo=UTC()), limit=100) self.mock_layer1.initiate_job.assert_called_with( 'examplevault', { 'Type': 'inventory-retrieval', 'InventoryRetrievalParameters': { 'StartDate': '2014-01-01T00:00:00Z', 'EndDate': '2014-01-02T00:00:00Z', 'Limit': 100 } }) def test_get_job(self): self.mock_layer1.describe_job.return_value = FIXTURE_ARCHIVE_JOB job = self.vault.get_job( "NkbByEejwEggmBz2fTHgJrg0XBoDfjP4q6iu87-TjhqG6eGoOY9Z8i1_AUyUsuhPA" "dTqLHy8pTl5nfCFJmDl2yEZONi5L26Omw12vcs01MNGntHEQL8MBfGlqrEXAMPLEA" "rchiveId") self.assertEqual(job.action, "ArchiveRetrieval") def test_list_jobs(self): self.mock_layer1.list_jobs.return_value = { "JobList": [FIXTURE_ARCHIVE_JOB]} jobs = self.vault.list_jobs(False, "InProgress") self.mock_layer1.list_jobs.assert_called_with("examplevault", False, "InProgress") self.assertEqual(jobs[0].archive_id, "NkbByEejwEggmBz2fTHgJrg0XBoDfjP4q6iu87-TjhqG6eGoOY9Z" "8i1_AUyUsuhPAdTqLHy8pTl5nfCFJmDl2yEZONi5L26Omw12vcs0" "1MNGntHEQL8MBfGlqrEXAMPLEArchiveId") def test_list_all_parts_one_page(self): self.mock_layer1.list_parts.return_value = ( dict(EXAMPLE_PART_LIST_COMPLETE)) # take a copy parts_result = self.vault.list_all_parts(sentinel.upload_id) expected = [call('examplevault', sentinel.upload_id)] self.assertEqual(expected, self.mock_layer1.list_parts.call_args_list) self.assertEqual(EXAMPLE_PART_LIST_COMPLETE, parts_result) def test_list_all_parts_two_pages(self): self.mock_layer1.list_parts.side_effect = [ # take copies dict(EXAMPLE_PART_LIST_RESULT_PAGE_1), dict(EXAMPLE_PART_LIST_RESULT_PAGE_2) ] parts_result = self.vault.list_all_parts(sentinel.upload_id) expected = [call('examplevault', sentinel.upload_id), call('examplevault', sentinel.upload_id, marker=EXAMPLE_PART_LIST_RESULT_PAGE_1['Marker'])] self.assertEqual(expected, self.mock_layer1.list_parts.call_args_list) self.assertEqual(EXAMPLE_PART_LIST_COMPLETE, parts_result) @patch('boto.glacier.vault.resume_file_upload') def test_resume_archive_from_file(self, mock_resume_file_upload): part_size = 4 mock_list_parts = Mock() mock_list_parts.return_value = { 'PartSizeInBytes': part_size, 'Parts': [{ 'RangeInBytes': '0-3', 'SHA256TreeHash': '12', }, { 'RangeInBytes': '4-6', 'SHA256TreeHash': '34', }], } self.vault.list_all_parts = mock_list_parts self.vault.resume_archive_from_file( sentinel.upload_id, file_obj=sentinel.file_obj) mock_resume_file_upload.assert_called_once_with( self.vault, sentinel.upload_id, part_size, sentinel.file_obj, {0: codecs.decode('12', 'hex_codec'), 1: codecs.decode('34', 'hex_codec')}) class TestJob(GlacierLayer2Base): def setUp(self): GlacierLayer2Base.setUp(self) self.vault = Vault(self.mock_layer1, FIXTURE_VAULT) self.job = Job(self.vault, FIXTURE_ARCHIVE_JOB) def test_get_job_output(self): self.mock_layer1.get_job_output.return_value = "TEST_OUTPUT" self.job.get_output((0, 100)) self.mock_layer1.get_job_output.assert_called_with( "examplevault", "HkF9p6o7yjhFx-K3CGl6fuSm6VzW9T7esGQfco8nUXVYwS0jlb5gq1JZ55yHgt5vP" "54ZShjoQzQVVh7vEXAMPLEjobID", (0, 100)) class TestRangeStringParsing(unittest.TestCase): def test_simple_range(self): self.assertEqual( Vault._range_string_to_part_index('0-3', 4), 0) def test_range_one_too_big(self): # Off-by-one bug in Amazon's Glacier implementation # See: https://forums.aws.amazon.com/thread.jspa?threadID=106866&tstart=0 # Workaround is to assume that if a (start, end] range appears to be # returned then that is what it is. self.assertEqual( Vault._range_string_to_part_index('0-4', 4), 0) def test_range_too_big(self): self.assertRaises( AssertionError, Vault._range_string_to_part_index, '0-5', 4) def test_range_start_mismatch(self): self.assertRaises( AssertionError, Vault._range_string_to_part_index, '1-3', 4) def test_range_end_mismatch(self): # End mismatch is OK, since the last part might be short self.assertEqual( Vault._range_string_to_part_index('0-2', 4), 0) vendored/boto/tests/unit/glacier/test_job.py 0000644 00000007177 15025013616 0015242 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.compat import StringIO from tests.compat import mock, unittest from boto.glacier.job import Job from boto.glacier.layer1 import Layer1 from boto.glacier.response import GlacierResponse from boto.glacier.exceptions import TreeHashDoesNotMatchError class TestJob(unittest.TestCase): def setUp(self): self.api = mock.Mock(spec=Layer1) self.vault = mock.Mock() self.vault.layer1 = self.api self.job = Job(self.vault) def test_get_job_validate_checksum_success(self): response = GlacierResponse(mock.Mock(), None) response['TreeHash'] = 'tree_hash' self.api.get_job_output.return_value = response with mock.patch('boto.glacier.job.tree_hash_from_str') as t: t.return_value = 'tree_hash' self.job.get_output(byte_range=(1, 1024), validate_checksum=True) def test_get_job_validation_fails(self): response = GlacierResponse(mock.Mock(), None) response['TreeHash'] = 'tree_hash' self.api.get_job_output.return_value = response with mock.patch('boto.glacier.job.tree_hash_from_str') as t: t.return_value = 'BAD_TREE_HASH_VALUE' with self.assertRaises(TreeHashDoesNotMatchError): # With validate_checksum set to True, this call fails. self.job.get_output(byte_range=(1, 1024), validate_checksum=True) # With validate_checksum set to False, this call succeeds. self.job.get_output(byte_range=(1, 1024), validate_checksum=False) def test_download_to_fileobj(self): http_response = mock.Mock(read=mock.Mock(return_value='xyz')) response = GlacierResponse(http_response, None) response['TreeHash'] = 'tree_hash' self.api.get_job_output.return_value = response fileobj = StringIO() self.job.archive_size = 3 with mock.patch('boto.glacier.job.tree_hash_from_str') as t: t.return_value = 'tree_hash' self.job.download_to_fileobj(fileobj) fileobj.seek(0) self.assertEqual(http_response.read.return_value, fileobj.read()) def test_calc_num_chunks(self): self.job.archive_size = 0 self.assertEqual(self.job._calc_num_chunks(self.job.DefaultPartSize), 0) self.job.archive_size = 1 self.assertEqual(self.job._calc_num_chunks(self.job.DefaultPartSize), 1) self.job.archive_size = self.job.DefaultPartSize + 1 self.assertEqual(self.job._calc_num_chunks(self.job.DefaultPartSize), 2) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/glacier/test_response.py 0000644 00000003167 15025013616 0016321 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import AWSMockServiceTestCase from boto.glacier.layer1 import Layer1 from boto.glacier.response import GlacierResponse class TestResponse(AWSMockServiceTestCase): connection_class = Layer1 def test_204_body_isnt_passed_to_json(self): response = self.create_response(status_code=204,header=[('Content-Type','application/json')]) result = GlacierResponse(response,response.getheaders()) self.assertEqual(result.status, response.status) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/glacier/test_writer.py 0000644 00000020257 15025013616 0015776 0 ustar 00 # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from hashlib import sha256 import itertools from boto.compat import StringIO from tests.unit import unittest from mock import ( call, Mock, sentinel, ) from nose.tools import assert_equal from boto.glacier.layer1 import Layer1 from boto.glacier.vault import Vault from boto.glacier.writer import Writer, resume_file_upload from boto.glacier.utils import bytes_to_hex, chunk_hashes, tree_hash def create_mock_vault(): vault = Mock(spec=Vault) vault.layer1 = Mock(spec=Layer1) vault.layer1.complete_multipart_upload.return_value = dict( ArchiveId=sentinel.archive_id) vault.name = sentinel.vault_name return vault def partify(data, part_size): for i in itertools.count(0): start = i * part_size part = data[start:start + part_size] if part: yield part else: return def calculate_mock_vault_calls(data, part_size, chunk_size): upload_part_calls = [] data_tree_hashes = [] for i, data_part in enumerate(partify(data, part_size)): start = i * part_size end = start + len(data_part) data_part_tree_hash_blob = tree_hash( chunk_hashes(data_part, chunk_size)) data_part_tree_hash = bytes_to_hex(data_part_tree_hash_blob) data_part_linear_hash = sha256(data_part).hexdigest() upload_part_calls.append( call( sentinel.vault_name, sentinel.upload_id, data_part_linear_hash, data_part_tree_hash, (start, end - 1), data_part)) data_tree_hashes.append(data_part_tree_hash_blob) return upload_part_calls, data_tree_hashes def check_mock_vault_calls(vault, upload_part_calls, data_tree_hashes, data_len): vault.layer1.upload_part.assert_has_calls( upload_part_calls, any_order=True) assert_equal( len(upload_part_calls), vault.layer1.upload_part.call_count) data_tree_hash = bytes_to_hex(tree_hash(data_tree_hashes)) vault.layer1.complete_multipart_upload.assert_called_once_with( sentinel.vault_name, sentinel.upload_id, data_tree_hash, data_len) class TestWriter(unittest.TestCase): def setUp(self): super(TestWriter, self).setUp() self.vault = create_mock_vault() self.chunk_size = 2 # power of 2 self.part_size = 4 # power of 2 upload_id = sentinel.upload_id self.writer = Writer( self.vault, upload_id, self.part_size, self.chunk_size) def check_write(self, write_list): for write_data in write_list: self.writer.write(write_data) self.writer.close() data = b''.join(write_list) upload_part_calls, data_tree_hashes = calculate_mock_vault_calls( data, self.part_size, self.chunk_size) check_mock_vault_calls( self.vault, upload_part_calls, data_tree_hashes, len(data)) def test_single_byte_write(self): self.check_write([b'1']) def test_one_part_write(self): self.check_write([b'1234']) def test_split_write_1(self): self.check_write([b'1', b'234']) def test_split_write_2(self): self.check_write([b'12', b'34']) def test_split_write_3(self): self.check_write([b'123', b'4']) def test_one_part_plus_one_write(self): self.check_write([b'12345']) def test_returns_archive_id(self): self.writer.write(b'1') self.writer.close() self.assertEqual(sentinel.archive_id, self.writer.get_archive_id()) def test_current_tree_hash(self): self.writer.write(b'1234') self.writer.write(b'567') hash_1 = self.writer.current_tree_hash self.assertEqual(hash_1, b'\x0e\xb0\x11Z\x1d\x1f\n\x10|\xf76\xa6\xf5' + b'\x83\xd1\xd5"bU\x0c\x95\xa8<\xf5\x81\xef\x0e\x0f\x95\n\xb7k' ) # This hash will be different, since the content has changed. self.writer.write(b'22i3uy') hash_2 = self.writer.current_tree_hash self.assertEqual(hash_2, b'\x7f\xf4\x97\x82U]\x81R\x05#^\xe8\x1c\xd19' + b'\xe8\x1f\x9e\xe0\x1aO\xaad\xe5\x06"\xa5\xc0\xa8AdL' ) self.writer.close() # Check the final tree hash, post-close. final_hash = self.writer.current_tree_hash self.assertEqual(final_hash, b';\x1a\xb8!=\xf0\x14#\x83\x11\xd5\x0b\x0f' + b'\xc7D\xe4\x8e\xd1W\x99z\x14\x06\xb9D\xd0\xf0*\x93\xa2\x8e\xf9' ) # Then assert we don't get a different one on a subsequent call. self.assertEqual(final_hash, self.writer.current_tree_hash) def test_current_uploaded_size(self): self.writer.write(b'1234') self.writer.write(b'567') size_1 = self.writer.current_uploaded_size self.assertEqual(size_1, 4) # This hash will be different, since the content has changed. self.writer.write(b'22i3uy') size_2 = self.writer.current_uploaded_size self.assertEqual(size_2, 12) self.writer.close() # Get the final size, post-close. final_size = self.writer.current_uploaded_size self.assertEqual(final_size, 13) # Then assert we don't get a different one on a subsequent call. self.assertEqual(final_size, self.writer.current_uploaded_size) def test_upload_id(self): self.assertEqual(sentinel.upload_id, self.writer.upload_id) class TestResume(unittest.TestCase): def setUp(self): super(TestResume, self).setUp() self.vault = create_mock_vault() self.chunk_size = 2 # power of 2 self.part_size = 4 # power of 2 def check_no_resume(self, data, resume_set=set()): fobj = StringIO(data.decode('utf-8')) part_hash_map = {} for part_index in resume_set: start = self.part_size * part_index end = start + self.part_size part_data = data[start:end] part_hash_map[part_index] = tree_hash( chunk_hashes(part_data, self.chunk_size)) resume_file_upload( self.vault, sentinel.upload_id, self.part_size, fobj, part_hash_map, self.chunk_size) upload_part_calls, data_tree_hashes = calculate_mock_vault_calls( data, self.part_size, self.chunk_size) resume_upload_part_calls = [ call for part_index, call in enumerate(upload_part_calls) if part_index not in resume_set] check_mock_vault_calls( self.vault, resume_upload_part_calls, data_tree_hashes, len(data)) def test_one_part_no_resume(self): self.check_no_resume(b'1234') def test_two_parts_no_resume(self): self.check_no_resume(b'12345678') def test_one_part_resume(self): self.check_no_resume(b'1234', resume_set=set([0])) def test_two_parts_one_resume(self): self.check_no_resume(b'12345678', resume_set=set([1])) def test_returns_archive_id(self): archive_id = resume_file_upload( self.vault, sentinel.upload_id, self.part_size, StringIO('1'), {}, self.chunk_size) self.assertEqual(sentinel.archive_id, archive_id) vendored/boto/tests/unit/glacier/test_utils.py 0000644 00000015450 15025013616 0015621 0 ustar 00 # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import logging import os import tempfile import time from hashlib import sha256 from tests.unit import unittest from boto.compat import BytesIO, six, StringIO from boto.glacier.utils import minimum_part_size, chunk_hashes, tree_hash, \ bytes_to_hex, compute_hashes_from_fileobj class TestPartSizeCalculations(unittest.TestCase): def test_small_values_still_use_default_part_size(self): self.assertEqual(minimum_part_size(1), 4 * 1024 * 1024) def test_under_the_maximum_value(self): # If we're under the maximum, we can use 4MB part sizes. self.assertEqual(minimum_part_size(8 * 1024 * 1024), 4 * 1024 * 1024) def test_gigabyte_size(self): # If we're over the maximum default part size, we go up to the next # power of two until we find a part size that keeps us under 10,000 # parts. self.assertEqual(minimum_part_size(8 * 1024 * 1024 * 10000), 8 * 1024 * 1024) def test_terabyte_size(self): # For a 4 TB file we need at least a 512 MB part size. self.assertEqual(minimum_part_size(4 * 1024 * 1024 * 1024 * 1024), 512 * 1024 * 1024) def test_file_size_too_large(self): with self.assertRaises(ValueError): minimum_part_size((40000 * 1024 * 1024 * 1024) + 1) def test_default_part_size_can_be_specified(self): default_part_size = 2 * 1024 * 1024 self.assertEqual(minimum_part_size(8 * 1024 * 1024, default_part_size), default_part_size) class TestChunking(unittest.TestCase): def test_chunk_hashes_exact(self): chunks = chunk_hashes(b'a' * (2 * 1024 * 1024)) self.assertEqual(len(chunks), 2) self.assertEqual(chunks[0], sha256(b'a' * 1024 * 1024).digest()) def test_chunks_with_leftovers(self): bytestring = b'a' * (2 * 1024 * 1024 + 20) chunks = chunk_hashes(bytestring) self.assertEqual(len(chunks), 3) self.assertEqual(chunks[0], sha256(b'a' * 1024 * 1024).digest()) self.assertEqual(chunks[1], sha256(b'a' * 1024 * 1024).digest()) self.assertEqual(chunks[2], sha256(b'a' * 20).digest()) def test_less_than_one_chunk(self): chunks = chunk_hashes(b'aaaa') self.assertEqual(len(chunks), 1) self.assertEqual(chunks[0], sha256(b'aaaa').digest()) class TestTreeHash(unittest.TestCase): # For these tests, a set of reference tree hashes were computed. # This will at least catch any regressions to the tree hash # calculations. def calculate_tree_hash(self, bytestring): start = time.time() calculated = bytes_to_hex(tree_hash(chunk_hashes(bytestring))) end = time.time() logging.debug("Tree hash calc time for length %s: %s", len(bytestring), end - start) return calculated def test_tree_hash_calculations(self): one_meg_bytestring = b'a' * (1 * 1024 * 1024) two_meg_bytestring = b'a' * (2 * 1024 * 1024) four_meg_bytestring = b'a' * (4 * 1024 * 1024) bigger_bytestring = four_meg_bytestring + b'a' * 20 self.assertEqual( self.calculate_tree_hash(one_meg_bytestring), b'9bc1b2a288b26af7257a36277ae3816a7d4f16e89c1e7e77d0a5c48bad62b360') self.assertEqual( self.calculate_tree_hash(two_meg_bytestring), b'560c2c9333c719cb00cfdffee3ba293db17f58743cdd1f7e4055373ae6300afa') self.assertEqual( self.calculate_tree_hash(four_meg_bytestring), b'9491cb2ed1d4e7cd53215f4017c23ec4ad21d7050a1e6bb636c4f67e8cddb844') self.assertEqual( self.calculate_tree_hash(bigger_bytestring), b'12f3cbd6101b981cde074039f6f728071da8879d6f632de8afc7cdf00661b08f') def test_empty_tree_hash(self): self.assertEqual( self.calculate_tree_hash(''), b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') class TestFileHash(unittest.TestCase): def _gen_data(self): # Generate some pseudo-random bytes of data. We include the # hard-coded blob as an example that fails to decode via UTF-8. return os.urandom(5000) + b'\xc2\x00' def test_compute_hash_tempfile(self): # Compute a hash from a file object. On Python 2 this uses a non- # binary mode. On Python 3, however, binary mode is required for # binary files. If not used, you will get UTF-8 code errors. if six.PY2: mode = "w+" else: mode = "wb+" with tempfile.TemporaryFile(mode=mode) as f: f.write(self._gen_data()) f.seek(0) compute_hashes_from_fileobj(f, chunk_size=512) @unittest.skipUnless(six.PY3, 'Python 3 requires reading binary!') def test_compute_hash_tempfile_py3(self): # Note the missing 'b' in the mode! with tempfile.TemporaryFile(mode='w+') as f: with self.assertRaises(ValueError): compute_hashes_from_fileobj(f, chunk_size=512) # What about file-like objects without a mode? If it has an # encoding we use it, otherwise attempt UTF-8 encoding to # bytes for hashing. f = StringIO('test data' * 500) compute_hashes_from_fileobj(f, chunk_size=512) @unittest.skipUnless(six.PY2, 'Python 3 requires reading binary!') def test_compute_hash_stringio(self): # Python 2 binary data in StringIO example f = StringIO(self._gen_data()) compute_hashes_from_fileobj(f, chunk_size=512) def test_compute_hash_bytesio(self): # Compute a hash from a file-like BytesIO object. f = BytesIO(self._gen_data()) compute_hashes_from_fileobj(f, chunk_size=512) vendored/boto/tests/unit/glacier/test_layer1.py 0000644 00000012777 15025013616 0015667 0 ustar 00 import json import copy import tempfile from tests.unit import unittest from tests.unit import AWSMockServiceTestCase from boto.glacier.layer1 import Layer1 from boto.compat import six class GlacierLayer1ConnectionBase(AWSMockServiceTestCase): connection_class = Layer1 def setUp(self): super(GlacierLayer1ConnectionBase, self).setUp() self.json_header = [('Content-Type', 'application/json')] self.vault_name = u'examplevault' self.vault_arn = 'arn:aws:glacier:us-east-1:012345678901:vaults/' + \ self.vault_name self.vault_info = {u'CreationDate': u'2012-03-16T22:22:47.214Z', u'LastInventoryDate': u'2012-03-21T22:06:51.218Z', u'NumberOfArchives': 2, u'SizeInBytes': 12334, u'VaultARN': self.vault_arn, u'VaultName': self.vault_name} class GlacierVaultsOperations(GlacierLayer1ConnectionBase): def test_create_vault_parameters(self): self.set_http_response(status_code=201) self.service_connection.create_vault(self.vault_name) def test_list_vaults(self): content = {u'Marker': None, u'RequestId': None, u'VaultList': [self.vault_info]} self.set_http_response(status_code=200, header=self.json_header, body=json.dumps(content).encode('utf-8')) api_response = self.service_connection.list_vaults() self.assertDictEqual(content, api_response) def test_describe_vaults(self): content = copy.copy(self.vault_info) content[u'RequestId'] = None self.set_http_response(status_code=200, header=self.json_header, body=json.dumps(content).encode('utf-8')) api_response = self.service_connection.describe_vault(self.vault_name) self.assertDictEqual(content, api_response) def test_delete_vault(self): self.set_http_response(status_code=204) self.service_connection.delete_vault(self.vault_name) class GlacierJobOperations(GlacierLayer1ConnectionBase): def setUp(self): super(GlacierJobOperations, self).setUp() self.job_content = 'abc' * 1024 def test_initiate_archive_job(self): content = {u'Type': u'archive-retrieval', u'ArchiveId': u'AAABZpJrTyioDC_HsOmHae8EZp_uBSJr6cnGOLKp_XJCl-Q', u'Description': u'Test Archive', u'SNSTopic': u'Topic', u'JobId': None, u'Location': None, u'RequestId': None} self.set_http_response(status_code=202, header=self.json_header, body=json.dumps(content).encode('utf-8')) api_response = self.service_connection.initiate_job(self.vault_name, self.job_content) self.assertDictEqual(content, api_response) def test_get_archive_output(self): header = [('Content-Type', 'application/octet-stream')] self.set_http_response(status_code=200, header=header, body=self.job_content) response = self.service_connection.get_job_output(self.vault_name, 'example-job-id') self.assertEqual(self.job_content, response.read()) class TestGlacierUploadPart(GlacierLayer1ConnectionBase): def test_upload_part_content_range_header(self): fake_data = b'\xe2' self.set_http_response(status_code=204) self.service_connection.upload_part( u'unicode_vault_name', 'upload_id', 'linear_hash', 'tree_hash', (1,2), fake_data) self.assertEqual( self.actual_request.headers['Content-Range'], 'bytes 1-2/*') def test_upload_part_with_unicode_name(self): fake_data = b'\xe2' self.set_http_response(status_code=204) self.service_connection.upload_part( u'unicode_vault_name', 'upload_id', 'linear_hash', 'tree_hash', (1,2), fake_data) self.assertEqual( self.actual_request.path, '/-/vaults/unicode_vault_name/multipart-uploads/upload_id') # If the path is unicode in python2, it triggers the following bug # noted in this PR: https://github.com/boto/boto/pull/2697 # httplib notices that if the path is unicode, it will try to encode # body which may be impossible if there is the data is binary. self.assertIsInstance(self.actual_request.body, six.binary_type) self.assertEqual(self.actual_request.body, fake_data) class GlacierUploadArchiveResets(GlacierLayer1ConnectionBase): def test_upload_archive(self): fake_data = tempfile.NamedTemporaryFile() fake_data.write(b'foobarbaz') # First seek to a non zero offset. fake_data.seek(2) self.set_http_response(status_code=201) # Simulate reading the request body when we send the request. self.service_connection.connection.request.side_effect = \ lambda *args: fake_data.read() self.service_connection.upload_archive('vault_name', fake_data, 'linear_hash', 'tree_hash') # Verify that we seek back to the original offset after making # a request. This ensures that if we need to resend the request we're # back at the correct location within the file. self.assertEqual(fake_data.tell(), 2) vendored/boto/tests/unit/glacier/test_concurrent.py 0000644 00000016610 15025013616 0016642 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import tempfile from boto.compat import Queue from tests.compat import mock, unittest from tests.unit import AWSMockServiceTestCase from boto.glacier.concurrent import ConcurrentUploader, ConcurrentDownloader from boto.glacier.concurrent import UploadWorkerThread from boto.glacier.concurrent import _END_SENTINEL class FakeThreadedConcurrentUploader(ConcurrentUploader): def _start_upload_threads(self, results_queue, upload_id, worker_queue, filename): self.results_queue = results_queue self.worker_queue = worker_queue self.upload_id = upload_id def _wait_for_upload_threads(self, hash_chunks, result_queue, total_parts): for i in range(total_parts): hash_chunks[i] = b'foo' class FakeThreadedConcurrentDownloader(ConcurrentDownloader): def _start_download_threads(self, results_queue, worker_queue): self.results_queue = results_queue self.worker_queue = worker_queue def _wait_for_download_threads(self, filename, result_queue, total_parts): pass class TestConcurrentUploader(unittest.TestCase): def setUp(self): super(TestConcurrentUploader, self).setUp() self.stat_patch = mock.patch('os.stat') self.addCleanup(self.stat_patch.stop) self.stat_mock = self.stat_patch.start() # Give a default value for tests that don't care # what the file size is. self.stat_mock.return_value.st_size = 1024 * 1024 * 8 def test_calculate_required_part_size(self): self.stat_mock.return_value.st_size = 1024 * 1024 * 8 uploader = ConcurrentUploader(mock.Mock(), 'vault_name') total_parts, part_size = uploader._calculate_required_part_size( 1024 * 1024 * 8) self.assertEqual(total_parts, 2) self.assertEqual(part_size, 4 * 1024 * 1024) def test_calculate_required_part_size_too_small(self): too_small = 1 * 1024 * 1024 self.stat_mock.return_value.st_size = 1024 * 1024 * 1024 uploader = ConcurrentUploader(mock.Mock(), 'vault_name', part_size=too_small) total_parts, part_size = uploader._calculate_required_part_size( 1024 * 1024 * 1024) self.assertEqual(total_parts, 256) # Part size if 4MB not the passed in 1MB. self.assertEqual(part_size, 4 * 1024 * 1024) def test_work_queue_is_correctly_populated(self): uploader = FakeThreadedConcurrentUploader(mock.MagicMock(), 'vault_name') uploader.upload('foofile') q = uploader.worker_queue items = [q.get() for i in range(q.qsize())] self.assertEqual(items[0], (0, 4 * 1024 * 1024)) self.assertEqual(items[1], (1, 4 * 1024 * 1024)) # 2 for the parts, 10 for the end sentinels (10 threads). self.assertEqual(len(items), 12) def test_correct_low_level_api_calls(self): api_mock = mock.MagicMock() upload_id = '0898d645-ea45-4548-9a67-578f507ead49' initiate_upload_mock = mock.Mock( return_value={'UploadId': upload_id}) # initiate_multipart_upload must return a body containing an `UploadId` api_mock.attach_mock(initiate_upload_mock, 'initiate_multipart_upload') uploader = FakeThreadedConcurrentUploader(api_mock, 'vault_name') uploader.upload('foofile') # The threads call the upload_part, so we're just verifying the # initiate/complete multipart API calls. initiate_upload_mock.assert_called_with( 'vault_name', 4 * 1024 * 1024, None) api_mock.complete_multipart_upload.assert_called_with( 'vault_name', upload_id, mock.ANY, 8 * 1024 * 1024) def test_downloader_work_queue_is_correctly_populated(self): job = mock.MagicMock() job.archive_size = 8 * 1024 * 1024 downloader = FakeThreadedConcurrentDownloader(job) downloader.download('foofile') q = downloader.worker_queue items = [q.get() for i in range(q.qsize())] self.assertEqual(items[0], (0, 4 * 1024 * 1024)) self.assertEqual(items[1], (1, 4 * 1024 * 1024)) # 2 for the parts, 10 for the end sentinels (10 threads). self.assertEqual(len(items), 12) class TestUploaderThread(unittest.TestCase): def setUp(self): self.fileobj = tempfile.NamedTemporaryFile() self.filename = self.fileobj.name def test_fileobj_closed_when_thread_shuts_down(self): thread = UploadWorkerThread(mock.Mock(), 'vault_name', self.filename, 'upload_id', Queue(), Queue()) fileobj = thread._fileobj self.assertFalse(fileobj.closed) # By settings should_continue to False, it should immediately # exit, and we can still verify cleanup behavior. thread.should_continue = False thread.run() self.assertTrue(fileobj.closed) def test_upload_errors_have_exception_messages(self): api = mock.Mock() job_queue = Queue() result_queue = Queue() upload_thread = UploadWorkerThread( api, 'vault_name', self.filename, 'upload_id', job_queue, result_queue, num_retries=1, time_between_retries=0) api.upload_part.side_effect = Exception("exception message") job_queue.put((0, 1024)) job_queue.put(_END_SENTINEL) upload_thread.run() result = result_queue.get(timeout=1) self.assertIn("exception message", str(result)) def test_num_retries_is_obeyed(self): # total attempts is 1 + num_retries so if I have num_retries of 2, # I'll attempt the upload once, and if that fails I'll retry up to # 2 more times for a total of 3 attempts. api = mock.Mock() job_queue = Queue() result_queue = Queue() upload_thread = UploadWorkerThread( api, 'vault_name', self.filename, 'upload_id', job_queue, result_queue, num_retries=2, time_between_retries=0) api.upload_part.side_effect = Exception() job_queue.put((0, 1024)) job_queue.put(_END_SENTINEL) upload_thread.run() self.assertEqual(api.upload_part.call_count, 3) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/glacier/test_vault.py 0000644 00000016470 15025013616 0015617 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.compat import StringIO from tests.compat import mock, unittest ANY = mock.ANY from boto.glacier import vault from boto.glacier.job import Job from boto.glacier.response import GlacierResponse class TestVault(unittest.TestCase): def setUp(self): self.size_patch = mock.patch('os.path.getsize') self.getsize = self.size_patch.start() self.api = mock.Mock() self.vault = vault.Vault(self.api, None) self.vault.name = 'myvault' self.mock_open = mock.mock_open() stringio = StringIO('content') self.mock_open.return_value.read = stringio.read def tearDown(self): self.size_patch.stop() @mock.patch('boto.glacier.vault.compute_hashes_from_fileobj', return_value=[b'abc', b'123']) def test_upload_archive_small_file(self, compute_hashes): self.getsize.return_value = 1 self.api.upload_archive.return_value = {'ArchiveId': 'archive_id'} with mock.patch('boto.glacier.vault.open', self.mock_open, create=True): archive_id = self.vault.upload_archive( 'filename', 'my description') self.assertEqual(archive_id, 'archive_id') self.api.upload_archive.assert_called_with( 'myvault', self.mock_open.return_value, mock.ANY, mock.ANY, 'my description') def test_small_part_size_is_obeyed(self): self.vault.DefaultPartSize = 2 * 1024 * 1024 self.vault.create_archive_writer = mock.Mock() self.getsize.return_value = 1 with mock.patch('boto.glacier.vault.open', self.mock_open, create=True): self.vault.create_archive_from_file('myfile') # The write should be created with the default part size of the # instance (2 MB). self.vault.create_archive_writer.assert_called_with( description=mock.ANY, part_size=self.vault.DefaultPartSize) def test_large_part_size_is_obeyed(self): self.vault.DefaultPartSize = 8 * 1024 * 1024 self.vault.create_archive_writer = mock.Mock() self.getsize.return_value = 1 with mock.patch('boto.glacier.vault.open', self.mock_open, create=True): self.vault.create_archive_from_file('myfile') # The write should be created with the default part size of the # instance (8 MB). self.vault.create_archive_writer.assert_called_with( description=mock.ANY, part_size=self.vault.DefaultPartSize) def test_part_size_needs_to_be_adjusted(self): # If we have a large file (400 GB) self.getsize.return_value = 400 * 1024 * 1024 * 1024 self.vault.create_archive_writer = mock.Mock() # When we try to upload the file. with mock.patch('boto.glacier.vault.open', self.mock_open, create=True): self.vault.create_archive_from_file('myfile') # We should automatically bump up the part size used to # 64 MB. expected_part_size = 64 * 1024 * 1024 self.vault.create_archive_writer.assert_called_with( description=mock.ANY, part_size=expected_part_size) def test_retrieve_inventory(self): class FakeResponse(object): status = 202 def getheader(self, key, default=None): if key == 'x-amz-job-id': return 'HkF9p6' elif key == 'Content-Type': return 'application/json' return 'something' def read(self, amt=None): return b"""{ "Action": "ArchiveRetrieval", "ArchiveId": "NkbByEejwEggmBz2fTHgJrg0XBoDfjP4q6iu87-EXAMPLEArchiveId", "ArchiveSizeInBytes": 16777216, "ArchiveSHA256TreeHash": "beb0fe31a1c7ca8c6c04d574ea906e3f97", "Completed": false, "CreationDate": "2012-05-15T17:21:39.339Z", "CompletionDate": "2012-05-15T17:21:43.561Z", "InventorySizeInBytes": null, "JobDescription": "My ArchiveRetrieval Job", "JobId": "HkF9p6", "RetrievalByteRange": "0-16777215", "SHA256TreeHash": "beb0fe31a1c7ca8c6c04d574ea906e3f97b31fd", "SNSTopic": "arn:aws:sns:us-east-1:012345678901:mytopic", "StatusCode": "InProgress", "StatusMessage": "Operation in progress.", "VaultARN": "arn:aws:glacier:us-east-1:012345678901:vaults/examplevault" }""" raw_resp = FakeResponse() init_resp = GlacierResponse(raw_resp, [('x-amz-job-id', 'JobId')]) raw_resp_2 = FakeResponse() desc_resp = GlacierResponse(raw_resp_2, []) with mock.patch.object(self.vault.layer1, 'initiate_job', return_value=init_resp): with mock.patch.object(self.vault.layer1, 'describe_job', return_value=desc_resp): # The old/back-compat variant of the call. self.assertEqual(self.vault.retrieve_inventory(), 'HkF9p6') # The variant the returns a full ``Job`` object. job = self.vault.retrieve_inventory_job() self.assertTrue(isinstance(job, Job)) self.assertEqual(job.id, 'HkF9p6') class TestConcurrentUploads(unittest.TestCase): def test_concurrent_upload_file(self): v = vault.Vault(None, None) with mock.patch('boto.glacier.vault.ConcurrentUploader') as c: c.return_value.upload.return_value = 'archive_id' archive_id = v.concurrent_create_archive_from_file( 'filename', 'my description') c.return_value.upload.assert_called_with('filename', 'my description') self.assertEqual(archive_id, 'archive_id') def test_concurrent_upload_forwards_kwargs(self): v = vault.Vault(None, None) with mock.patch('boto.glacier.vault.ConcurrentUploader') as c: c.return_value.upload.return_value = 'archive_id' archive_id = v.concurrent_create_archive_from_file( 'filename', 'my description', num_threads=10, part_size=1024 * 1024 * 1024 * 8) c.assert_called_with(None, None, num_threads=10, part_size=1024 * 1024 * 1024 * 8) if __name__ == '__main__': unittest.main() vendored/boto/tests/unit/glacier/__init__.py 0000644 00000000000 15025013616 0015142 0 ustar 00 vendored/boto/tests/unit/mturk/test_locale_qualification_notin.py 0000644 00000013521 15025013616 0021570 0 ustar 00 # -*- coding: utf-8 -*- from tests.unit import AWSMockServiceTestCase from boto.mturk.connection import MTurkConnection from boto.mturk.question import ExternalQuestion from boto.mturk.qualification import \ Qualifications, LocaleRequirement MOCK_SERVER_RESPONSE = b""" <MockServerResponse> <Request> <IsValid>True</IsValid> </Request> </MockServerResponse>""" class TestMTurkPostingWithQualificationsNotin(AWSMockServiceTestCase): connection_class = MTurkConnection def setUp(self): super(TestMTurkPostingWithQualificationsNotin, self).setUp() def test_locale_qualification_notin(self): self.set_http_response( status_code=200, body=MOCK_SERVER_RESPONSE) q = ExternalQuestion( external_url="http://samplesite", frame_height=800) keywords = ['boto', 'test', 'doctest'] title = "Boto External Question Test" annotation = 'An annotation from boto external question test' qualifications = Qualifications() test_requirement = LocaleRequirement( comparator='NotIn', locale=[('US', 'WA'), 'CA']) qualifications.add(test_requirement) create_hit_rs = self.service_connection.create_hit( question=q, lifetime=60*65, max_assignments=2, title=title, keywords=keywords, reward=0.05, duration=60*6, approval_delay=60*60, annotation=annotation, qualifications=qualifications) self.assert_request_parameters({ 'QualificationRequirement.1.Comparator': 'NotIn', 'QualificationRequirement.1.LocaleValue.1.Country': 'US', 'QualificationRequirement.1.LocaleValue.1.Subdivision': 'WA', 'QualificationRequirement.1.LocaleValue.2.Country': 'CA', 'QualificationRequirement.1.QualificationTypeId': '00000000000000000071'}, ignore_params_values=['AWSAccessKeyId', 'SignatureVersion', 'Timestamp', 'Title', 'Question', 'AssignmentDurationInSeconds', 'RequesterAnnotation', 'Version', 'LifetimeInSeconds', 'AutoApprovalDelayInSeconds', 'Reward.1.Amount', 'Description', 'MaxAssignments', 'Reward.1.CurrencyCode', 'Keywords', 'Operation']) self.assertEqual(create_hit_rs.status, True) def test_locale_qualification_in_notin(self): self.set_http_response( status_code=200, body=MOCK_SERVER_RESPONSE) q = ExternalQuestion( external_url="http://samplesite", frame_height=800) keywords = ['boto', 'test', 'doctest'] title = "Boto External Question Test" annotation = 'An annotation from boto external question test' qualifications = Qualifications() test_requirement1 = LocaleRequirement( comparator='In', locale=[('US', 'CA')]) test_requirement2 = LocaleRequirement( comparator='NotIn', locale=[('US', 'WA'), 'CA']) qualifications.add(test_requirement1) qualifications.add(test_requirement2) create_hit_rs = self.service_connection.create_hit( question=q, lifetime=60*65, max_assignments=2, title=title, keywords=keywords, reward=0.05, duration=60*6, approval_delay=60*60, annotation=annotation, qualifications=qualifications) self.assert_request_parameters({ 'QualificationRequirement.1.Comparator': 'In', 'QualificationRequirement.1.LocaleValue.1.Country': 'US', 'QualificationRequirement.1.LocaleValue.1.Subdivision': 'CA', 'QualificationRequirement.1.QualificationTypeId': '00000000000000000071', 'QualificationRequirement.2.Comparator': 'NotIn', 'QualificationRequirement.2.LocaleValue.1.Country': 'US', 'QualificationRequirement.2.LocaleValue.1.Subdivision': 'WA', 'QualificationRequirement.2.LocaleValue.2.Country': 'CA', 'QualificationRequirement.2.QualificationTypeId': '00000000000000000071'}, ignore_params_values=['AWSAccessKeyId', 'SignatureVersion', 'Timestamp', 'Title', 'Question', 'AssignmentDurationInSeconds', 'RequesterAnnotation', 'Version', 'LifetimeInSeconds', 'AutoApprovalDelayInSeconds', 'Reward.1.Amount', 'Description', 'MaxAssignments', 'Reward.1.CurrencyCode', 'Keywords', 'Operation']) self.assertEqual(create_hit_rs.status, True) vendored/boto/tests/unit/mturk/test_connection.py 0000644 00000001532 15025013616 0016350 0 ustar 00 # -*- coding: utf-8 -*- from tests.unit import AWSMockServiceTestCase from boto.mturk.connection import MTurkConnection GET_FILE_UPLOAD_URL = b""" <GetFileUploadURLResult> <Request> <IsValid>True</IsValid> </Request> <FileUploadURL>http://s3.amazonaws.com/myawsbucket/puppy.jpg</FileUploadURL> </GetFileUploadURLResult>""" class TestMTurkConnection(AWSMockServiceTestCase): connection_class = MTurkConnection def setUp(self): super(TestMTurkConnection, self).setUp() def test_get_file_upload_url_success(self): self.set_http_response(status_code=200, body=GET_FILE_UPLOAD_URL) rset = self.service_connection.get_file_upload_url('aid', 'qid') self.assertEqual(len(rset), 1) self.assertEqual(rset[0].FileUploadURL, 'http://s3.amazonaws.com/myawsbucket/puppy.jpg') vendored/boto/tests/unit/mturk/test_locale_qualification_in.py 0000644 00000013500 15025013616 0021044 0 ustar 00 # -*- coding: utf-8 -*- from tests.unit import AWSMockServiceTestCase from boto.mturk.connection import MTurkConnection from boto.mturk.question import ExternalQuestion from boto.mturk.qualification import \ Qualifications, LocaleRequirement MOCK_SERVER_RESPONSE = b""" <MockServerResponse> <Request> <IsValid>True</IsValid> </Request> </MockServerResponse>""" class TestMTurkPostingWithQualificationsIn(AWSMockServiceTestCase): connection_class = MTurkConnection def setUp(self): super(TestMTurkPostingWithQualificationsIn, self).setUp() def test_locale_qualification_in(self): self.set_http_response( status_code=200, body=MOCK_SERVER_RESPONSE) q = ExternalQuestion( external_url="http://samplesite", frame_height=800) keywords = ['boto', 'test', 'doctest'] title = "Boto External Question Test" annotation = 'An annotation from boto external question test' qualifications = Qualifications() test_requirement = LocaleRequirement( comparator='In', locale=[('US', 'WA'), 'CA']) qualifications.add(test_requirement) create_hit_rs = self.service_connection.create_hit( question=q, lifetime=60*65, max_assignments=2, title=title, keywords=keywords, reward=0.05, duration=60*6, approval_delay=60*60, annotation=annotation, qualifications=qualifications) self.assert_request_parameters({ 'QualificationRequirement.1.Comparator': 'In', 'QualificationRequirement.1.LocaleValue.1.Country': 'US', 'QualificationRequirement.1.LocaleValue.1.Subdivision': 'WA', 'QualificationRequirement.1.LocaleValue.2.Country': 'CA', 'QualificationRequirement.1.QualificationTypeId': '00000000000000000071'}, ignore_params_values=['AWSAccessKeyId', 'SignatureVersion', 'Timestamp', 'Title', 'Question', 'AssignmentDurationInSeconds', 'RequesterAnnotation', 'Version', 'LifetimeInSeconds', 'AutoApprovalDelayInSeconds', 'Reward.1.Amount', 'Description', 'MaxAssignments', 'Reward.1.CurrencyCode', 'Keywords', 'Operation']) self.assertEqual(create_hit_rs.status, True) def test_locale_qualification_notin_in(self): self.set_http_response( status_code=200, body=MOCK_SERVER_RESPONSE) q = ExternalQuestion( external_url="http://samplesite", frame_height=800) keywords = ['boto', 'test', 'doctest'] title = "Boto External Question Test" annotation = 'An annotation from boto external question test' qualifications = Qualifications() test_requirement1 = LocaleRequirement( comparator='NotIn', locale=[('US', 'WA'), 'CA']) test_requirement2 = LocaleRequirement( comparator='In', locale=[('US', 'CA')]) qualifications.add(test_requirement1) qualifications.add(test_requirement2) create_hit_rs = self.service_connection.create_hit( question=q, lifetime=60*65, max_assignments=2, title=title, keywords=keywords, reward=0.05, duration=60*6, approval_delay=60*60, annotation=annotation, qualifications=qualifications) self.assert_request_parameters({ 'QualificationRequirement.1.Comparator': 'NotIn', 'QualificationRequirement.1.LocaleValue.1.Country': 'US', 'QualificationRequirement.1.LocaleValue.1.Subdivision': 'WA', 'QualificationRequirement.1.LocaleValue.2.Country': 'CA', 'QualificationRequirement.1.QualificationTypeId': '00000000000000000071', 'QualificationRequirement.2.Comparator': 'In', 'QualificationRequirement.2.LocaleValue.1.Country': 'US', 'QualificationRequirement.2.LocaleValue.1.Subdivision': 'CA', 'QualificationRequirement.2.QualificationTypeId': '00000000000000000071'}, ignore_params_values=['AWSAccessKeyId', 'SignatureVersion', 'Timestamp', 'Title', 'Question', 'AssignmentDurationInSeconds', 'RequesterAnnotation', 'Version', 'LifetimeInSeconds', 'AutoApprovalDelayInSeconds', 'Reward.1.Amount', 'Description', 'MaxAssignments', 'Reward.1.CurrencyCode', 'Keywords', 'Operation']) self.assertEqual(create_hit_rs.status, True) vendored/boto/tests/unit/mturk/test_qualification_doesnotexist.py 0000644 00000005562 15025013616 0021660 0 ustar 00 # -*- coding: utf-8 -*- from tests.unit import AWSMockServiceTestCase from boto.mturk.connection import MTurkConnection from boto.mturk.question import ExternalQuestion from boto.mturk.qualification import \ Qualifications, Requirement QUAL_NO_ONE_ELSE_HAS_ID = "333333333333333333333333333333" MOCK_SERVER_RESPONSE = b""" <MockServerResponse> <Request> <IsValid>True</IsValid> </Request> </MockServerResponse>""" class TestMTurkPostingWithQualificationsDoesnotexist(AWSMockServiceTestCase): connection_class = MTurkConnection def setUp(self): super(TestMTurkPostingWithQualificationsDoesnotexist, self).setUp() def test_qualification_doesnotexist(self): self.set_http_response( status_code=200, body=MOCK_SERVER_RESPONSE) q = ExternalQuestion( external_url="http://samplesite", frame_height=800) keywords = ['boto', 'test', 'doctest'] title = "Boto External Question Test" annotation = 'An annotation from boto external question test' qualifications = Qualifications() test_requirement = Requirement( qualification_type_id=QUAL_NO_ONE_ELSE_HAS_ID, comparator='DoesNotExist') qualifications.add(test_requirement) create_hit_rs = self.service_connection.create_hit( question=q, lifetime=60*65, max_assignments=2, title=title, keywords=keywords, reward=0.05, duration=60*6, approval_delay=60*60, annotation=annotation, qualifications=qualifications) self.assert_request_parameters({ 'QualificationRequirement.1.Comparator': 'DoesNotExist', 'QualificationRequirement.1.QualificationTypeId': '333333333333333333333333333333'}, ignore_params_values=['AWSAccessKeyId', 'SignatureVersion', 'Timestamp', 'Title', 'Question', 'AssignmentDurationInSeconds', 'RequesterAnnotation', 'Version', 'LifetimeInSeconds', 'AutoApprovalDelayInSeconds', 'Reward.1.Amount', 'Description', 'MaxAssignments', 'Reward.1.CurrencyCode', 'Keywords', 'Operation']) self.assertEqual(create_hit_rs.status, True) vendored/boto/tests/unit/mturk/test_qualification_qualtypewithscore_in.py 0000644 00000006166 15025013616 0023413 0 ustar 00 # -*- coding: utf-8 -*- from tests.unit import AWSMockServiceTestCase from boto.mturk.connection import MTurkConnection from boto.mturk.question import ExternalQuestion from boto.mturk.qualification import \ Qualifications, Requirement QUAL_WITH_SCORE_ID = "333333333333333333333333333333" MOCK_SERVER_RESPONSE = b""" <MockServerResponse> <Request> <IsValid>True</IsValid> </Request> </MockServerResponse>""" class TestMTurkPostingWithQualQualtypewithscoreIn(AWSMockServiceTestCase): connection_class = MTurkConnection def setUp(self): super(TestMTurkPostingWithQualQualtypewithscoreIn, self).setUp() def test_qualification_qualtypewithscore_in(self): self.set_http_response( status_code=200, body=MOCK_SERVER_RESPONSE) q = ExternalQuestion( external_url="http://samplesite", frame_height=800) keywords = ['boto', 'test', 'doctest'] title = "Boto External Question Test" annotation = 'An annotation from boto external question test' qualifications = Qualifications() test_requirement = Requirement( qualification_type_id=QUAL_WITH_SCORE_ID, comparator='In', integer_value=[100, 90, 80]) qualifications.add(test_requirement) create_hit_rs = self.service_connection.create_hit( question=q, lifetime=60*65, max_assignments=2, title=title, keywords=keywords, reward=0.05, duration=60*6, approval_delay=60*60, annotation=annotation, qualifications=qualifications) self.assert_request_parameters({ 'QualificationRequirement.1.Comparator': 'In', 'QualificationRequirement.1.QualificationTypeId': '333333333333333333333333333333', 'QualificationRequirement.1.IntegerValue.1': 100, 'QualificationRequirement.1.IntegerValue.2': 90, 'QualificationRequirement.1.IntegerValue.3': 80}, ignore_params_values=['AWSAccessKeyId', 'SignatureVersion', 'Timestamp', 'Title', 'Question', 'AssignmentDurationInSeconds', 'RequesterAnnotation', 'Version', 'LifetimeInSeconds', 'AutoApprovalDelayInSeconds', 'Reward.1.Amount', 'Description', 'MaxAssignments', 'Reward.1.CurrencyCode', 'Keywords', 'Operation']) self.assertEqual(create_hit_rs.status, True) vendored/boto/tests/unit/mturk/test_qualification_exists.py 0000644 00000005524 15025013616 0020445 0 ustar 00 # -*- coding: utf-8 -*- from tests.unit import AWSMockServiceTestCase from boto.mturk.connection import MTurkConnection from boto.mturk.question import ExternalQuestion from boto.mturk.qualification import \ Qualifications, Requirement QUAL_NO_ONE_ELSE_HAS_ID = "333333333333333333333333333333" MOCK_SERVER_RESPONSE = b""" <MockServerResponse> <Request> <IsValid>True</IsValid> </Request> </MockServerResponse>""" class TestMTurkPostingWithQualificationsExists(AWSMockServiceTestCase): connection_class = MTurkConnection def setUp(self): super(TestMTurkPostingWithQualificationsExists, self).setUp() def test_qualification_exists(self): self.set_http_response( status_code=200, body=MOCK_SERVER_RESPONSE) q = ExternalQuestion( external_url="http://samplesite", frame_height=800) keywords = ['boto', 'test', 'doctest'] title = "Boto External Question Test" annotation = 'An annotation from boto external question test' qualifications = Qualifications() test_requirement = Requirement( qualification_type_id=QUAL_NO_ONE_ELSE_HAS_ID, comparator='Exists') qualifications.add(test_requirement) create_hit_rs = self.service_connection.create_hit( question=q, lifetime=60*65, max_assignments=2, title=title, keywords=keywords, reward=0.05, duration=60*6, approval_delay=60*60, annotation=annotation, qualifications=qualifications) self.assert_request_parameters({ 'QualificationRequirement.1.Comparator': 'Exists', 'QualificationRequirement.1.QualificationTypeId': '333333333333333333333333333333'}, ignore_params_values=['AWSAccessKeyId', 'SignatureVersion', 'Timestamp', 'Title', 'Question', 'AssignmentDurationInSeconds', 'RequesterAnnotation', 'Version', 'LifetimeInSeconds', 'AutoApprovalDelayInSeconds', 'Reward.1.Amount', 'Description', 'MaxAssignments', 'Reward.1.CurrencyCode', 'Keywords', 'Operation']) self.assertEqual(create_hit_rs.status, True) vendored/boto/tests/unit/mturk/__init__.py 0000644 00000000000 15025013616 0014676 0 ustar 00 vendored/boto/tests/devpay/test_s3.py 0000644 00000016364 15025013616 0013716 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the S3Connection """ import time import os import urllib from boto.s3.connection import S3Connection from boto.exception import S3PermissionsError # this test requires a devpay product and user token to run: AMAZON_USER_TOKEN = '{UserToken}...your token here...' DEVPAY_HEADERS = { 'x-amz-security-token': AMAZON_USER_TOKEN } def test(): print('--- running S3Connection tests (DevPay) ---') c = S3Connection() # create a new, empty bucket bucket_name = 'test-%d' % int(time.time()) bucket = c.create_bucket(bucket_name, headers=DEVPAY_HEADERS) # now try a get_bucket call and see if it's really there bucket = c.get_bucket(bucket_name, headers=DEVPAY_HEADERS) # test logging logging_bucket = c.create_bucket(bucket_name + '-log', headers=DEVPAY_HEADERS) logging_bucket.set_as_logging_target(headers=DEVPAY_HEADERS) bucket.enable_logging(target_bucket=logging_bucket, target_prefix=bucket.name, headers=DEVPAY_HEADERS) bucket.disable_logging(headers=DEVPAY_HEADERS) c.delete_bucket(logging_bucket, headers=DEVPAY_HEADERS) # create a new key and store it's content from a string k = bucket.new_key() k.name = 'foobar' s1 = 'This is a test of file upload and download' s2 = 'This is a second string to test file upload and download' k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) fp = open('foobar', 'wb') # now get the contents from s3 to a local file k.get_contents_to_file(fp, headers=DEVPAY_HEADERS) fp.close() fp = open('foobar') # check to make sure content read from s3 is identical to original assert s1 == fp.read(), 'corrupted file' fp.close() # test generated URLs url = k.generate_url(3600, headers=DEVPAY_HEADERS) file = urllib.urlopen(url) assert s1 == file.read(), 'invalid URL %s' % url url = k.generate_url(3600, force_http=True, headers=DEVPAY_HEADERS) file = urllib.urlopen(url) assert s1 == file.read(), 'invalid URL %s' % url bucket.delete_key(k, headers=DEVPAY_HEADERS) # test a few variations on get_all_keys - first load some data # for the first one, let's override the content type phony_mimetype = 'application/x-boto-test' headers = {'Content-Type': phony_mimetype} headers.update(DEVPAY_HEADERS) k.name = 'foo/bar' k.set_contents_from_string(s1, headers) k.name = 'foo/bas' k.set_contents_from_filename('foobar', headers=DEVPAY_HEADERS) k.name = 'foo/bat' k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) k.name = 'fie/bar' k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) k.name = 'fie/bas' k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) k.name = 'fie/bat' k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) # try resetting the contents to another value md5 = k.md5 k.set_contents_from_string(s2, headers=DEVPAY_HEADERS) assert k.md5 != md5 os.unlink('foobar') all = bucket.get_all_keys(headers=DEVPAY_HEADERS) assert len(all) == 6 rs = bucket.get_all_keys(prefix='foo', headers=DEVPAY_HEADERS) assert len(rs) == 3 rs = bucket.get_all_keys(prefix='', delimiter='/', headers=DEVPAY_HEADERS) assert len(rs) == 2 rs = bucket.get_all_keys(maxkeys=5, headers=DEVPAY_HEADERS) assert len(rs) == 5 # test the lookup method k = bucket.lookup('foo/bar', headers=DEVPAY_HEADERS) assert isinstance(k, bucket.key_class) assert k.content_type == phony_mimetype k = bucket.lookup('notthere', headers=DEVPAY_HEADERS) assert k == None # try some metadata stuff k = bucket.new_key() k.name = 'has_metadata' mdkey1 = 'meta1' mdval1 = 'This is the first metadata value' k.set_metadata(mdkey1, mdval1) mdkey2 = 'meta2' mdval2 = 'This is the second metadata value' k.set_metadata(mdkey2, mdval2) k.set_contents_from_string(s1, headers=DEVPAY_HEADERS) k = bucket.lookup('has_metadata', headers=DEVPAY_HEADERS) assert k.get_metadata(mdkey1) == mdval1 assert k.get_metadata(mdkey2) == mdval2 k = bucket.new_key() k.name = 'has_metadata' k.get_contents_as_string(headers=DEVPAY_HEADERS) assert k.get_metadata(mdkey1) == mdval1 assert k.get_metadata(mdkey2) == mdval2 bucket.delete_key(k, headers=DEVPAY_HEADERS) # test list and iterator rs1 = bucket.list(headers=DEVPAY_HEADERS) num_iter = 0 for r in rs1: num_iter = num_iter + 1 rs = bucket.get_all_keys(headers=DEVPAY_HEADERS) num_keys = len(rs) assert num_iter == num_keys # try a key with a funny character k = bucket.new_key() k.name = 'testnewline\n' k.set_contents_from_string('This is a test', headers=DEVPAY_HEADERS) rs = bucket.get_all_keys(headers=DEVPAY_HEADERS) assert len(rs) == num_keys + 1 bucket.delete_key(k, headers=DEVPAY_HEADERS) rs = bucket.get_all_keys(headers=DEVPAY_HEADERS) assert len(rs) == num_keys # try some acl stuff bucket.set_acl('public-read', headers=DEVPAY_HEADERS) policy = bucket.get_acl(headers=DEVPAY_HEADERS) assert len(policy.acl.grants) == 2 bucket.set_acl('private', headers=DEVPAY_HEADERS) policy = bucket.get_acl(headers=DEVPAY_HEADERS) assert len(policy.acl.grants) == 1 k = bucket.lookup('foo/bar', headers=DEVPAY_HEADERS) k.set_acl('public-read', headers=DEVPAY_HEADERS) policy = k.get_acl(headers=DEVPAY_HEADERS) assert len(policy.acl.grants) == 2 k.set_acl('private', headers=DEVPAY_HEADERS) policy = k.get_acl(headers=DEVPAY_HEADERS) assert len(policy.acl.grants) == 1 # try the convenience methods for grants # this doesn't work with devpay #bucket.add_user_grant('FULL_CONTROL', # 'c1e724fbfa0979a4448393c59a8c055011f739b6d102fb37a65f26414653cd67', # headers=DEVPAY_HEADERS) try: bucket.add_email_grant('foobar', 'foo@bar.com', headers=DEVPAY_HEADERS) except S3PermissionsError: pass # now delete all keys in bucket for k in all: bucket.delete_key(k, headers=DEVPAY_HEADERS) # now delete bucket c.delete_bucket(bucket, headers=DEVPAY_HEADERS) print('--- tests completed ---') if __name__ == '__main__': test() vendored/boto/tests/devpay/__init__.py 0000644 00000000000 15025013616 0014045 0 ustar 00 vendored/boto/tests/fps/test_verify_signature.py 0000644 00000001667 15025013616 0016256 0 ustar 00 from boto.fps.connection import FPSConnection def test(): conn = FPSConnection() # example response from the docs params = 'expiry=08%2F2015&signature=ynDukZ9%2FG77uSJVb5YM0cadwHVwYKPMKOO3PNvgADbv6VtymgBxeOWEhED6KGHsGSvSJnMWDN%2FZl639AkRe9Ry%2F7zmn9CmiM%2FZkp1XtshERGTqi2YL10GwQpaH17MQqOX3u1cW4LlyFoLy4celUFBPq1WM2ZJnaNZRJIEY%2FvpeVnCVK8VIPdY3HMxPAkNi5zeF2BbqH%2BL2vAWef6vfHkNcJPlOuOl6jP4E%2B58F24ni%2B9ek%2FQH18O4kw%2FUJ7ZfKwjCCI13%2BcFybpofcKqddq8CuUJj5Ii7Pdw1fje7ktzHeeNhF0r9siWcYmd4JaxTP3NmLJdHFRq2T%2FgsF3vK9m3gw%3D%3D&signatureVersion=2&signatureMethod=RSA-SHA1&certificateUrl=https%3A%2F%2Ffps.sandbox.amazonaws.com%2Fcerts%2F090909%2FPKICert.pem&tokenID=A5BB3HUNAZFJ5CRXIPH72LIODZUNAUZIVP7UB74QNFQDSQ9MN4HPIKISQZWPLJXF&status=SC&callerReference=callerReferenceMultiUse1' endpoint = 'http://vamsik.desktop.amazon.com:8080/ipn.jsp' conn.verify_signature(endpoint, params) if __name__ == '__main__': test() vendored/boto/tests/fps/__init__.py 0000644 00000000000 15025013616 0013345 0 ustar 00 vendored/boto/tests/fps/test.py 0000644 00000007320 15025013616 0012601 0 ustar 00 #!/usr/bin/env python from tests.unit import unittest import sys import os import os.path simple = True advanced = False if __name__ == "__main__": devpath = os.path.relpath(os.path.join('..', '..'), start=os.path.dirname(__file__)) sys.path = [devpath] + sys.path print '>>> advanced FPS tests; using local boto sources' advanced = True from boto.fps.connection import FPSConnection from boto.fps.response import ComplexAmount class FPSTestCase(unittest.TestCase): def setUp(self): self.fps = FPSConnection(host='fps.sandbox.amazonaws.com') if advanced: self.activity = self.fps.get_account_activity(\ StartDate='2012-01-01') result = self.activity.GetAccountActivityResult self.transactions = result.Transaction @unittest.skipUnless(simple, "skipping simple test") def test_get_account_balance(self): response = self.fps.get_account_balance() self.assertTrue(hasattr(response, 'GetAccountBalanceResult')) self.assertTrue(hasattr(response.GetAccountBalanceResult, 'AccountBalance')) accountbalance = response.GetAccountBalanceResult.AccountBalance self.assertTrue(hasattr(accountbalance, 'TotalBalance')) self.assertIsInstance(accountbalance.TotalBalance, ComplexAmount) self.assertTrue(hasattr(accountbalance, 'AvailableBalances')) availablebalances = accountbalance.AvailableBalances self.assertTrue(hasattr(availablebalances, 'RefundBalance')) @unittest.skipUnless(simple, "skipping simple test") def test_complex_amount(self): response = self.fps.get_account_balance() accountbalance = response.GetAccountBalanceResult.AccountBalance asfloat = float(accountbalance.TotalBalance.Value) self.assertIn('.', str(asfloat)) @unittest.skipUnless(simple, "skipping simple test") def test_required_arguments(self): with self.assertRaises(KeyError): self.fps.write_off_debt(AdjustmentAmount=123.45) @unittest.skipUnless(simple, "skipping simple test") def test_cbui_url(self): inputs = { 'transactionAmount': 123.45, 'pipelineName': 'SingleUse', 'returnURL': 'https://localhost/', 'paymentReason': 'a reason for payment', 'callerReference': 'foo', } result = self.fps.cbui_url(**inputs) print "cbui_url() yields {0}".format(result) @unittest.skipUnless(simple, "skipping simple test") def test_get_account_activity(self): response = self.fps.get_account_activity(StartDate='2012-01-01') self.assertTrue(hasattr(response, 'GetAccountActivityResult')) result = response.GetAccountActivityResult self.assertTrue(hasattr(result, 'BatchSize')) try: int(result.BatchSize) except: self.assertTrue(False) @unittest.skipUnless(advanced, "skipping advanced test") def test_get_transaction(self): assert len(self.transactions) transactionid = self.transactions[0].TransactionId result = self.fps.get_transaction(TransactionId=transactionid) self.assertTrue(hasattr(result.GetTransactionResult, 'Transaction')) @unittest.skip('cosmetic') def test_bad_request(self): try: self.fps.write_off_debt(CreditInstrumentId='foo', AdjustmentAmount=123.45) except Exception as e: print e @unittest.skip('cosmetic') def test_repr(self): print self.fps.get_account_balance() if __name__ == "__main__": unittest.main() vendored/boto/tests/db/test_query.py 0000644 00000011553 15025013616 0013626 0 ustar 00 # Copyright (c) 2010 Chris Moyer http://coredumped.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.sdb.db.property import ListProperty, StringProperty, ReferenceProperty, IntegerProperty from boto.sdb.db.model import Model import time class SimpleModel(Model): """Simple Test Model""" name = StringProperty() strs = ListProperty(str) num = IntegerProperty() class SubModel(SimpleModel): """Simple Subclassed Model""" ref = ReferenceProperty(SimpleModel, collection_name="reverse_ref") class TestQuerying(object): """Test different querying capabilities""" def setup_class(cls): """Setup this class""" cls.objs = [] o = SimpleModel() o.name = "Simple Object" o.strs = ["B", "A", "C", "Foo"] o.num = 1 o.put() cls.objs.append(o) o2 = SimpleModel() o2.name = "Referenced Object" o2.num = 2 o2.put() cls.objs.append(o2) o3 = SubModel() o3.name = "Sub Object" o3.num = 3 o3.ref = o2 o3.put() cls.objs.append(o3) time.sleep(3) def teardown_class(cls): """Remove our objects""" for o in cls.objs: try: o.delete() except: pass def test_find(self): """Test using the "Find" method""" assert(SimpleModel.find(name="Simple Object").next().id == self.objs[0].id) assert(SimpleModel.find(name="Referenced Object").next().id == self.objs[1].id) assert(SimpleModel.find(name="Sub Object").next().id == self.objs[2].id) def test_like_filter(self): """Test a "like" filter""" query = SimpleModel.all() query.filter("name like", "% Object") assert(query.count() == 3) query = SimpleModel.all() query.filter("name not like", "% Object") assert(query.count() == 0) def test_equals_filter(self): """Test an "=" and "!=" filter""" query = SimpleModel.all() query.filter("name =", "Simple Object") assert(query.count() == 1) query = SimpleModel.all() query.filter("name !=", "Simple Object") assert(query.count() == 2) def test_or_filter(self): """Test a filter function as an "or" """ query = SimpleModel.all() query.filter("name =", ["Simple Object", "Sub Object"]) assert(query.count() == 2) def test_and_filter(self): """Test Multiple filters which are an "and" """ query = SimpleModel.all() query.filter("name like", "% Object") query.filter("name like", "Simple %") assert(query.count() == 1) def test_none_filter(self): """Test filtering for a value that's not set""" query = SimpleModel.all() query.filter("ref =", None) assert(query.count() == 2) def test_greater_filter(self): """Test filtering Using >, >=""" query = SimpleModel.all() query.filter("num >", 1) assert(query.count() == 2) query = SimpleModel.all() query.filter("num >=", 1) assert(query.count() == 3) def test_less_filter(self): """Test filtering Using <, <=""" query = SimpleModel.all() query.filter("num <", 3) assert(query.count() == 2) query = SimpleModel.all() query.filter("num <=", 3) assert(query.count() == 3) def test_query_on_list(self): """Test querying on a list""" assert(SimpleModel.find(strs="A").next().id == self.objs[0].id) assert(SimpleModel.find(strs="B").next().id == self.objs[0].id) assert(SimpleModel.find(strs="C").next().id == self.objs[0].id) def test_like(self): """Test with a "like" expression""" query = SimpleModel.all() query.filter("strs like", "%oo%") print query.get_query() assert(query.count() == 1) vendored/boto/tests/db/test_sequence.py 0000644 00000007741 15025013616 0014275 0 ustar 00 # Copyright (c) 2010 Chris Moyer http://coredumped.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. class TestDBHandler(object): """Test the DBHandler""" def setup_class(cls): """Setup this class""" cls.sequences = [] def teardown_class(cls): """Remove our sequences""" for s in cls.sequences: try: s.delete() except: pass def test_sequence_generator_no_rollover(self): """Test the sequence generator without rollover""" from boto.sdb.db.sequence import SequenceGenerator gen = SequenceGenerator("ABC") assert(gen("") == "A") assert(gen("A") == "B") assert(gen("B") == "C") assert(gen("C") == "AA") assert(gen("AC") == "BA") def test_sequence_generator_with_rollover(self): """Test the sequence generator with rollover""" from boto.sdb.db.sequence import SequenceGenerator gen = SequenceGenerator("ABC", rollover=True) assert(gen("") == "A") assert(gen("A") == "B") assert(gen("B") == "C") assert(gen("C") == "A") def test_sequence_simple_int(self): """Test a simple counter sequence""" from boto.sdb.db.sequence import Sequence s = Sequence() self.sequences.append(s) assert(s.val == 0) assert(s.next() == 1) assert(s.next() == 2) s2 = Sequence(s.id) assert(s2.val == 2) assert(s.next() == 3) assert(s.val == 3) assert(s2.val == 3) def test_sequence_simple_string(self): from boto.sdb.db.sequence import Sequence, increment_string s = Sequence(fnc=increment_string) self.sequences.append(s) assert(s.val == "A") assert(s.next() == "B") def test_fib(self): """Test the fibonacci sequence generator""" from boto.sdb.db.sequence import fib # Just check the first few numbers in the sequence lv = 0 for v in [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144]: assert(fib(v, lv) == lv+v) lv = fib(v, lv) def test_sequence_fib(self): """Test the fibonacci sequence""" from boto.sdb.db.sequence import Sequence, fib s = Sequence(fnc=fib) s2 = Sequence(s.id) self.sequences.append(s) assert(s.val == 1) # Just check the first few numbers in the sequence for v in [1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144]: assert(s.next() == v) assert(s.val == v) assert(s2.val == v) # it shouldn't matter which reference we use since it's garunteed to be consistent def test_sequence_string(self): """Test the String incrementation sequence""" from boto.sdb.db.sequence import Sequence, increment_string s = Sequence(fnc=increment_string) self.sequences.append(s) assert(s.val == "A") assert(s.next() == "B") s.val = "Z" assert(s.val == "Z") assert(s.next() == "AA") vendored/boto/tests/db/test_password.py 0000644 00000010507 15025013616 0014321 0 ustar 00 # Copyright (c) 2010 Robert Mela # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import unittest import logging import time log= logging.getLogger('password_property_test') log.setLevel(logging.DEBUG) class PasswordPropertyTest(unittest.TestCase): """Test the PasswordProperty""" def tearDown(self): cls=self.test_model() for obj in cls.all(): obj.delete() def hmac_hashfunc(self): import hmac def hashfunc(msg): return hmac.new('mysecret', msg) return hashfunc def test_model(self,hashfunc=None): from boto.utils import Password from boto.sdb.db.model import Model from boto.sdb.db.property import PasswordProperty import hashlib class MyModel(Model): password=PasswordProperty(hashfunc=hashfunc) return MyModel def test_custom_password_class(self): from boto.utils import Password from boto.sdb.db.model import Model from boto.sdb.db.property import PasswordProperty import hmac, hashlib myhashfunc = hashlib.md5 ## Define a new Password class class MyPassword(Password): hashfunc = myhashfunc #hashlib.md5 #lambda cls,msg: hmac.new('mysecret',msg) ## Define a custom password property using the new Password class class MyPasswordProperty(PasswordProperty): data_type=MyPassword type_name=MyPassword.__name__ ## Define a model using the new password property class MyModel(Model): password=MyPasswordProperty()#hashfunc=hashlib.md5) obj = MyModel() obj.password = 'bar' expected = myhashfunc('bar').hexdigest() #hmac.new('mysecret','bar').hexdigest() log.debug("\npassword=%s\nexpected=%s" % (obj.password, expected)) self.assertTrue(obj.password == 'bar' ) obj.save() id= obj.id time.sleep(5) obj = MyModel.get_by_id(id) self.assertEqual(obj.password, 'bar') self.assertEqual(str(obj.password), expected) #hmac.new('mysecret','bar').hexdigest()) def test_aaa_default_password_property(self): cls = self.test_model() obj = cls(id='passwordtest') obj.password = 'foo' self.assertEqual('foo', obj.password) obj.save() time.sleep(5) obj = cls.get_by_id('passwordtest') self.assertEqual('foo', obj.password) def test_password_constructor_hashfunc(self): import hmac myhashfunc=lambda msg: hmac.new('mysecret', msg) cls = self.test_model(hashfunc=myhashfunc) obj = cls() obj.password='hello' expected = myhashfunc('hello').hexdigest() self.assertEqual(obj.password, 'hello') self.assertEqual(str(obj.password), expected) obj.save() id = obj.id time.sleep(5) obj = cls.get_by_id(id) log.debug("\npassword=%s" % obj.password) self.assertTrue(obj.password == 'hello') if __name__ == '__main__': import sys, os curdir = os.path.dirname( os.path.abspath(__file__) ) srcroot = curdir + "/../.." sys.path = [ srcroot ] + sys.path logging.basicConfig() log.setLevel(logging.INFO) suite = unittest.TestLoader().loadTestsFromTestCase(PasswordPropertyTest) unittest.TextTestRunner(verbosity=2).run(suite) import boto vendored/boto/tests/db/test_lists.py 0000644 00000006622 15025013616 0013620 0 ustar 00 # Copyright (c) 2010 Chris Moyer http://coredumped.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.sdb.db.property import ListProperty from boto.sdb.db.model import Model import time class SimpleListModel(Model): """Test the List Property""" nums = ListProperty(int) strs = ListProperty(str) class TestLists(object): """Test the List property""" def setup_class(cls): """Setup this class""" cls.objs = [] def teardown_class(cls): """Remove our objects""" for o in cls.objs: try: o.delete() except: pass def test_list_order(self): """Testing the order of lists""" t = SimpleListModel() t.nums = [5, 4, 1, 3, 2] t.strs = ["B", "C", "A", "D", "Foo"] t.put() self.objs.append(t) time.sleep(3) t = SimpleListModel.get_by_id(t.id) assert(t.nums == [5, 4, 1, 3, 2]) assert(t.strs == ["B", "C", "A", "D", "Foo"]) def test_old_compat(self): """Testing to make sure the old method of encoding lists will still return results""" t = SimpleListModel() t.put() self.objs.append(t) time.sleep(3) item = t._get_raw_item() item['strs'] = ["A", "B", "C"] item.save() time.sleep(3) t = SimpleListModel.get_by_id(t.id) i1 = sorted(item['strs']) i2 = t.strs i2.sort() assert(i1 == i2) def test_query_equals(self): """We noticed a slight problem with querying, since the query uses the same encoder, it was asserting that the value was at the same position in the list, not just "in" the list""" t = SimpleListModel() t.strs = ["Bizzle", "Bar"] t.put() self.objs.append(t) time.sleep(3) assert(SimpleListModel.find(strs="Bizzle").count() == 1) assert(SimpleListModel.find(strs="Bar").count() == 1) assert(SimpleListModel.find(strs=["Bar", "Bizzle"]).count() == 1) def test_query_not_equals(self): """Test a not equal filter""" t = SimpleListModel() t.strs = ["Fizzle"] t.put() self.objs.append(t) time.sleep(3) print SimpleListModel.all().filter("strs !=", "Fizzle").get_query() for tt in SimpleListModel.all().filter("strs !=", "Fizzle"): print tt.strs assert("Fizzle" not in tt.strs) vendored/boto/tests/__init__.py 0000644 00000002117 15025013616 0012570 0 ustar 00 # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/cognito/identity/test_cognito_identity.py 0000644 00000004761 15025013616 0023270 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.cognito.identity.exceptions import ResourceNotFoundException from tests.integration.cognito import CognitoTest class TestCognitoIdentity(CognitoTest): """ Test Cognitoy identity pools operations since individual Cognito identities require an AWS account ID. """ def test_cognito_identity(self): # Ensure the identity pool is in the list of pools. response = self.cognito_identity.list_identity_pools(max_results=5) expected_identity = {'IdentityPoolId': self.identity_pool_id, 'IdentityPoolName': self.identity_pool_name} self.assertIn(expected_identity, response['IdentityPools']) # Ensure the pool's attributes are as expected. response = self.cognito_identity.describe_identity_pool( identity_pool_id=self.identity_pool_id ) self.assertEqual(response['IdentityPoolName'], self.identity_pool_name) self.assertEqual(response['IdentityPoolId'], self.identity_pool_id) self.assertFalse(response['AllowUnauthenticatedIdentities']) def test_resource_not_found_exception(self): with self.assertRaises(ResourceNotFoundException): # Note the region is us-east-0 which is an invalid region name. self.cognito_identity.describe_identity_pool( identity_pool_id='us-east-0:c09e640-b014-4822-86b9-ec77c40d8d6f' ) vendored/boto/tests/integration/cognito/identity/__init__.py 0000644 00000002143 15025013616 0020405 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/integration/cognito/__init__.py 0000644 00000003363 15025013616 0016561 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import boto from tests.compat import unittest class CognitoTest(unittest.TestCase): def setUp(self): self.cognito_identity = boto.connect_cognito_identity() self.cognito_sync = boto.connect_cognito_sync() self.identity_pool_name = 'myIdentityPool' response = self.cognito_identity.create_identity_pool( identity_pool_name=self.identity_pool_name, allow_unauthenticated_identities=False ) self.identity_pool_id = response['IdentityPoolId'] def tearDown(self): self.cognito_identity.delete_identity_pool( identity_pool_id=self.identity_pool_id ) vendored/boto/tests/integration/cognito/sync/__init__.py 0000644 00000002143 15025013616 0017530 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/integration/cognito/sync/test_cognito_sync.py 0000644 00000004226 15025013616 0021532 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.cognito.sync.exceptions import ResourceNotFoundException from tests.integration.cognito import CognitoTest class TestCognitoSync(CognitoTest): """ Even more so for Cognito Sync, Cognito identites are required. However, AWS account IDs are required to aqcuire a Cognito identity so only Cognito pool identity related operations are tested. """ def test_cognito_sync(self): response = self.cognito_sync.describe_identity_pool_usage( identity_pool_id=self.identity_pool_id ) identity_pool_usage = response['IdentityPoolUsage'] self.assertEqual(identity_pool_usage['SyncSessionsCount'], None) self.assertEqual(identity_pool_usage['DataStorage'], 0) def test_resource_not_found_exception(self): with self.assertRaises(ResourceNotFoundException): # Note the region is us-east-0 which is an invalid region name. self.cognito_sync.describe_identity_pool_usage( identity_pool_id='us-east-0:c09e640-b014-4822-86b9-ec77c40d8d6f' ) vendored/boto/tests/integration/route53/test_cert_verification.py 0000644 00000003150 15025013616 0021416 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ from tests.compat import unittest from nose.plugins.attrib import attr from tests.integration import ServiceCertVerificationTest import boto.route53 @attr(route53=True) class Route53CertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): route53 = True regions = boto.route53.regions() def sample_service_call(self, conn): conn.get_all_hosted_zones() vendored/boto/tests/integration/route53/test_zone.py 0000644 00000017037 15025013616 0016703 0 ustar 00 # Copyright (c) 2011 Blue Pines Technologies LLC, Brad Carleton # www.bluepines.org # Copyright (c) 2012 42 Lines Inc., Jim Browne # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import time from tests.compat import unittest from nose.plugins.attrib import attr from boto.route53.connection import Route53Connection from boto.exception import TooManyRecordsException from boto.vpc import VPCConnection @attr(route53=True) class TestRoute53Zone(unittest.TestCase): @classmethod def setUpClass(self): route53 = Route53Connection() self.base_domain = 'boto-test-%s.com' % str(int(time.time())) zone = route53.get_zone(self.base_domain) if zone is not None: zone.delete() self.zone = route53.create_zone(self.base_domain) def test_nameservers(self): self.zone.get_nameservers() def test_a(self): self.zone.add_a(self.base_domain, '102.11.23.1', 80) record = self.zone.get_a(self.base_domain) self.assertEqual(record.name, u'%s.' % self.base_domain) self.assertEqual(record.resource_records, [u'102.11.23.1']) self.assertEqual(record.ttl, u'80') self.zone.update_a(self.base_domain, '186.143.32.2', '800') record = self.zone.get_a(self.base_domain) self.assertEqual(record.name, u'%s.' % self.base_domain) self.assertEqual(record.resource_records, [u'186.143.32.2']) self.assertEqual(record.ttl, u'800') def test_cname(self): self.zone.add_cname( 'www.%s' % self.base_domain, 'webserver.%s' % self.base_domain, 200 ) record = self.zone.get_cname('www.%s' % self.base_domain) self.assertEqual(record.name, u'www.%s.' % self.base_domain) self.assertEqual(record.resource_records, [ u'webserver.%s.' % self.base_domain ]) self.assertEqual(record.ttl, u'200') self.zone.update_cname( 'www.%s' % self.base_domain, 'web.%s' % self.base_domain, 45 ) record = self.zone.get_cname('www.%s' % self.base_domain) self.assertEqual(record.name, u'www.%s.' % self.base_domain) self.assertEqual(record.resource_records, [ u'web.%s.' % self.base_domain ]) self.assertEqual(record.ttl, u'45') def test_mx(self): self.zone.add_mx( self.base_domain, [ '10 mx1.%s' % self.base_domain, '20 mx2.%s' % self.base_domain, ], 1000 ) record = self.zone.get_mx(self.base_domain) self.assertEqual(set(record.resource_records), set([u'10 mx1.%s.' % self.base_domain, u'20 mx2.%s.' % self.base_domain])) self.assertEqual(record.ttl, u'1000') self.zone.update_mx( self.base_domain, [ '10 mail1.%s' % self.base_domain, '20 mail2.%s' % self.base_domain, ], 50 ) record = self.zone.get_mx(self.base_domain) self.assertEqual(set(record.resource_records), set([u'10 mail1.%s.' % self.base_domain, '20 mail2.%s.' % self.base_domain])) self.assertEqual(record.ttl, u'50') def test_get_records(self): self.zone.get_records() def test_get_nameservers(self): self.zone.get_nameservers() def test_get_zones(self): route53 = Route53Connection() route53.get_zones() def test_identifiers_wrrs(self): self.zone.add_a('wrr.%s' % self.base_domain, '1.2.3.4', identifier=('foo', '20')) self.zone.add_a('wrr.%s' % self.base_domain, '5.6.7.8', identifier=('bar', '10')) wrrs = self.zone.find_records( 'wrr.%s' % self.base_domain, 'A', all=True ) self.assertEqual(len(wrrs), 2) self.zone.delete_a('wrr.%s' % self.base_domain, all=True) def test_identifiers_lbrs(self): self.zone.add_a('lbr.%s' % self.base_domain, '4.3.2.1', identifier=('baz', 'us-east-1')) self.zone.add_a('lbr.%s' % self.base_domain, '8.7.6.5', identifier=('bam', 'us-west-1')) lbrs = self.zone.find_records( 'lbr.%s' % self.base_domain, 'A', all=True ) self.assertEqual(len(lbrs), 2) self.zone.delete_a('lbr.%s' % self.base_domain, identifier=('bam', 'us-west-1')) self.zone.delete_a('lbr.%s' % self.base_domain, identifier=('baz', 'us-east-1')) def test_toomany_exception(self): self.zone.add_a('exception.%s' % self.base_domain, '4.3.2.1', identifier=('baz', 'us-east-1')) self.zone.add_a('exception.%s' % self.base_domain, '8.7.6.5', identifier=('bam', 'us-west-1')) self.assertRaises(TooManyRecordsException, lambda: self.zone.get_a('exception.%s' % self.base_domain)) self.zone.delete_a('exception.%s' % self.base_domain, all=True) @classmethod def tearDownClass(self): self.zone.delete_a(self.base_domain) self.zone.delete_cname('www.%s' % self.base_domain) self.zone.delete_mx(self.base_domain) self.zone.delete() @attr(route53=True) class TestRoute53PrivateZone(unittest.TestCase): @classmethod def setUpClass(self): time_str = str(int(time.time())) self.route53 = Route53Connection() self.base_domain = 'boto-private-zone-test-%s.com' % time_str self.vpc = VPCConnection() self.test_vpc = self.vpc.create_vpc(cidr_block='10.11.0.0/16') # tag the vpc to make it easily identifiable if things go spang self.test_vpc.add_tag("Name", self.base_domain) self.zone = self.route53.get_zone(self.base_domain) if self.zone is not None: self.zone.delete() def test_create_private_zone(self): self.zone = self.route53.create_hosted_zone(self.base_domain, private_zone=True, vpc_id=self.test_vpc.id, vpc_region='us-east-1') @classmethod def tearDownClass(self): if self.zone is not None: self.zone.delete() self.test_vpc.delete() if __name__ == '__main__': unittest.main(verbosity=3) vendored/boto/tests/integration/route53/test_resourcerecordsets.py 0000644 00000006335 15025013616 0021654 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.compat import unittest from tests.integration.route53 import Route53TestCase from boto.route53.record import ResourceRecordSets class TestRoute53ResourceRecordSets(Route53TestCase): def test_add_change(self): rrs = ResourceRecordSets(self.conn, self.zone.id) created = rrs.add_change("CREATE", "vpn.%s." % self.base_domain, "A") created.add_value('192.168.0.25') rrs.commit() rrs = ResourceRecordSets(self.conn, self.zone.id) deleted = rrs.add_change('DELETE', "vpn.%s." % self.base_domain, "A") deleted.add_value('192.168.0.25') rrs.commit() def test_record_count(self): rrs = ResourceRecordSets(self.conn, self.zone.id) hosts = 101 for hostid in range(hosts): rec = "test" + str(hostid) + ".%s" % self.base_domain created = rrs.add_change("CREATE", rec, "A") ip = '192.168.0.' + str(hostid) created.add_value(ip) # Max 100 changes per commit if (hostid + 1) % 100 == 0: rrs.commit() rrs = ResourceRecordSets(self.conn, self.zone.id) rrs.commit() all_records = self.conn.get_all_rrsets(self.zone.id) # First time around was always fine i = 0 for rset in all_records: i += 1 # Second time was a failure i = 0 for rset in all_records: i += 1 # Cleanup indivual records rrs = ResourceRecordSets(self.conn, self.zone.id) for hostid in range(hosts): rec = "test" + str(hostid) + ".%s" % self.base_domain deleted = rrs.add_change("DELETE", rec, "A") ip = '192.168.0.' + str(hostid) deleted.add_value(ip) # Max 100 changes per commit if (hostid + 1) % 100 == 0: rrs.commit() rrs = ResourceRecordSets(self.conn, self.zone.id) rrs.commit() # 2nd count should match the number of hosts plus NS/SOA records records = hosts + 2 self.assertEqual(i, records) if __name__ == '__main__': unittest.main() vendored/boto/tests/integration/route53/domains/test_route53domains.py 0000644 00000003537 15025013616 0022243 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import boto from boto.route53.domains.exceptions import InvalidInput from tests.compat import unittest class TestRoute53Domains(unittest.TestCase): def setUp(self): self.route53domains = boto.connect_route53domains() def test_check_domain_availability(self): response = self.route53domains.check_domain_availability( domain_name='amazon.com', idn_lang_code='eng' ) self.assertEqual(response, {'Availability': 'UNAVAILABLE'}) def test_handle_invalid_input_error(self): with self.assertRaises(InvalidInput): # Note the invalid character in the domain name. self.route53domains.check_domain_availability( domain_name='!amazon.com', ) vendored/boto/tests/integration/route53/domains/__init__.py 0000644 00000002142 15025013616 0020051 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/integration/route53/test_alias_resourcerecordsets.py 0000644 00000011414 15025013616 0023017 0 ustar 00 # Copyright (c) 2014 Netflix, Inc. Stefan Praszalowicz # Copyright (c) 2014 42Lines, Inc. Jim Browne # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import time from tests.compat import unittest from boto.route53.connection import Route53Connection from boto.route53.record import ResourceRecordSets from boto.route53.exception import DNSServerError class TestRoute53AliasResourceRecordSets(unittest.TestCase): route53 = True def setUp(self): super(TestRoute53AliasResourceRecordSets, self).setUp() self.conn = Route53Connection() self.base_domain = 'boto-test-%s.com' % str(int(time.time())) self.zone = self.conn.create_zone(self.base_domain) # a standard record to use as the target for our alias self.zone.add_a('target.%s' % self.base_domain, '102.11.23.1') def tearDown(self): self.zone.delete_a('target.%s' % self.base_domain) self.zone.delete() super(TestRoute53AliasResourceRecordSets, self).tearDown() def test_incomplete_add_alias_failure(self): base_record = dict(name="alias.%s." % self.base_domain, type="A", alias_dns_name="target.%s" % self.base_domain, alias_hosted_zone_id=self.zone.id, identifier="boto:TestRoute53AliasResourceRecordSets") rrs = ResourceRecordSets(self.conn, self.zone.id) rrs.add_change(action="UPSERT", **base_record) try: self.assertRaises(DNSServerError, rrs.commit) except: # if the call somehow goes through, delete our unexpected new record before failing test rrs = ResourceRecordSets(self.conn, self.zone.id) rrs.add_change(action="DELETE", **base_record) rrs.commit() raise def test_add_alias(self): base_record = dict(name="alias.%s." % self.base_domain, type="A", alias_evaluate_target_health=False, alias_dns_name="target.%s" % self.base_domain, alias_hosted_zone_id=self.zone.id, identifier="boto:TestRoute53AliasResourceRecordSets") rrs = ResourceRecordSets(self.conn, self.zone.id) rrs.add_change(action="UPSERT", **base_record) rrs.commit() rrs = ResourceRecordSets(self.conn, self.zone.id) rrs.add_change(action="DELETE", **base_record) rrs.commit() def test_set_alias(self): base_record = dict(name="alias.%s." % self.base_domain, type="A", identifier="boto:TestRoute53AliasResourceRecordSets") rrs = ResourceRecordSets(self.conn, self.zone.id) new = rrs.add_change(action="UPSERT", **base_record) new.set_alias(self.zone.id, "target.%s" % self.base_domain, False) rrs.commit() rrs = ResourceRecordSets(self.conn, self.zone.id) delete = rrs.add_change(action="DELETE", **base_record) delete.set_alias(self.zone.id, "target.%s" % self.base_domain, False) rrs.commit() def test_set_alias_backwards_compatability(self): base_record = dict(name="alias.%s." % self.base_domain, type="A", identifier="boto:TestRoute53AliasResourceRecordSets") rrs = ResourceRecordSets(self.conn, self.zone.id) new = rrs.add_change(action="UPSERT", **base_record) new.set_alias(self.zone.id, "target.%s" % self.base_domain) rrs.commit() rrs = ResourceRecordSets(self.conn, self.zone.id) delete = rrs.add_change(action="DELETE", **base_record) delete.set_alias(self.zone.id, "target.%s" % self.base_domain) rrs.commit() if __name__ == '__main__': unittest.main() vendored/boto/tests/integration/route53/test_health_check.py 0000644 00000024641 15025013616 0020331 0 ustar 00 # Copyright (c) 2014 Tellybug, Matt Millar # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.integration.route53 import Route53TestCase from boto.compat import six from boto.route53.healthcheck import HealthCheck from boto.route53.record import ResourceRecordSets class TestRoute53HealthCheck(Route53TestCase): def test_create_health_check(self): hc = HealthCheck(ip_addr="54.217.7.118", port=80, hc_type="HTTP", resource_path="/testing") result = self.conn.create_health_check(hc) self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'Type'], 'HTTP') self.assertEqual(result[u'CreateHealthCheckResponse'][ u'HealthCheck'][u'HealthCheckConfig'][u'IPAddress'], '54.217.7.118') self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'Port'], '80') self.assertEqual(result[u'CreateHealthCheckResponse'][ u'HealthCheck'][u'HealthCheckConfig'][u'ResourcePath'], '/testing') self.conn.delete_health_check(result['CreateHealthCheckResponse']['HealthCheck']['Id']) def test_create_https_health_check(self): hc = HealthCheck(ip_addr="54.217.7.118", port=443, hc_type="HTTPS", resource_path="/testing") result = self.conn.create_health_check(hc) self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'Type'], 'HTTPS') self.assertEqual(result[u'CreateHealthCheckResponse'][ u'HealthCheck'][u'HealthCheckConfig'][u'IPAddress'], '54.217.7.118') self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'Port'], '443') self.assertEqual(result[u'CreateHealthCheckResponse'][ u'HealthCheck'][u'HealthCheckConfig'][u'ResourcePath'], '/testing') self.assertFalse('FullyQualifiedDomainName' in result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig']) self.conn.delete_health_check(result['CreateHealthCheckResponse']['HealthCheck']['Id']) def test_create_https_health_check_fqdn(self): hc = HealthCheck(ip_addr=None, port=443, hc_type="HTTPS", resource_path="/", fqdn="google.com") result = self.conn.create_health_check(hc) self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'Type'], 'HTTPS') self.assertEqual(result[u'CreateHealthCheckResponse'][ u'HealthCheck'][u'HealthCheckConfig'][u'FullyQualifiedDomainName'], 'google.com') self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'Port'], '443') self.assertEqual(result[u'CreateHealthCheckResponse'][ u'HealthCheck'][u'HealthCheckConfig'][u'ResourcePath'], '/') self.assertFalse('IPAddress' in result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig']) self.conn.delete_health_check(result['CreateHealthCheckResponse']['HealthCheck']['Id']) def test_create_and_list_health_check(self): hc = HealthCheck(ip_addr="54.217.7.118", port=80, hc_type="HTTP", resource_path="/testing") result1 = self.conn.create_health_check(hc) hc = HealthCheck(ip_addr="54.217.7.119", port=80, hc_type="HTTP", resource_path="/testing") result2 = self.conn.create_health_check(hc) result = self.conn.get_list_health_checks() self.assertTrue(len(result['ListHealthChecksResponse']['HealthChecks']) > 1) self.conn.delete_health_check(result1['CreateHealthCheckResponse']['HealthCheck']['Id']) self.conn.delete_health_check(result2['CreateHealthCheckResponse']['HealthCheck']['Id']) def test_delete_health_check(self): hc = HealthCheck(ip_addr="54.217.7.118", port=80, hc_type="HTTP", resource_path="/testing") result = self.conn.create_health_check(hc) hc_id = result['CreateHealthCheckResponse']['HealthCheck']['Id'] result = self.conn.get_list_health_checks() found = False for hc in result['ListHealthChecksResponse']['HealthChecks']: if hc['Id'] == hc_id: found = True break self.assertTrue(found) result = self.conn.delete_health_check(hc_id) result = self.conn.get_list_health_checks() for hc in result['ListHealthChecksResponse']['HealthChecks']: self.assertFalse(hc['Id'] == hc_id) def test_create_health_check_string_match(self): hc = HealthCheck(ip_addr="54.217.7.118", port=80, hc_type="HTTP_STR_MATCH", resource_path="/testing", string_match="test") result = self.conn.create_health_check(hc) self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'Type'], 'HTTP_STR_MATCH') self.assertEqual(result[u'CreateHealthCheckResponse'][ u'HealthCheck'][u'HealthCheckConfig'][u'IPAddress'], '54.217.7.118') self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'Port'], '80') self.assertEqual(result[u'CreateHealthCheckResponse'][ u'HealthCheck'][u'HealthCheckConfig'][u'ResourcePath'], '/testing') self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'SearchString'], 'test') self.conn.delete_health_check(result['CreateHealthCheckResponse']['HealthCheck']['Id']) def test_create_health_check_https_string_match(self): hc = HealthCheck(ip_addr="54.217.7.118", port=80, hc_type="HTTPS_STR_MATCH", resource_path="/testing", string_match="test") result = self.conn.create_health_check(hc) self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'Type'], 'HTTPS_STR_MATCH') self.assertEqual(result[u'CreateHealthCheckResponse'][ u'HealthCheck'][u'HealthCheckConfig'][u'IPAddress'], '54.217.7.118') self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'Port'], '80') self.assertEqual(result[u'CreateHealthCheckResponse'][ u'HealthCheck'][u'HealthCheckConfig'][u'ResourcePath'], '/testing') self.assertEqual(result[u'CreateHealthCheckResponse'][u'HealthCheck'][u'HealthCheckConfig'][u'SearchString'], 'test') self.conn.delete_health_check(result['CreateHealthCheckResponse']['HealthCheck']['Id']) def test_create_resource_record_set(self): hc = HealthCheck(ip_addr="54.217.7.118", port=80, hc_type="HTTP", resource_path="/testing") result = self.conn.create_health_check(hc) records = ResourceRecordSets( connection=self.conn, hosted_zone_id=self.zone.id, comment='Create DNS entry for test') change = records.add_change('CREATE', 'unittest.%s.' % self.base_domain, 'A', ttl=30, identifier='test', weight=1, health_check=result['CreateHealthCheckResponse']['HealthCheck']['Id']) change.add_value("54.217.7.118") records.commit() records = ResourceRecordSets(self.conn, self.zone.id) deleted = records.add_change('DELETE', "unittest.%s." % self.base_domain, "A", ttl=30, identifier='test', weight=1, health_check=result['CreateHealthCheckResponse']['HealthCheck']['Id']) deleted.add_value('54.217.7.118') records.commit() def test_create_health_check_invalid_request_interval(self): """Test that health checks cannot be created with an invalid 'request_interval'. """ self.assertRaises(AttributeError, lambda: HealthCheck(**self.health_check_params(request_interval=5))) def test_create_health_check_invalid_failure_threshold(self): """ Test that health checks cannot be created with an invalid 'failure_threshold'. """ self.assertRaises(AttributeError, lambda: HealthCheck(**self.health_check_params(failure_threshold=0))) self.assertRaises(AttributeError, lambda: HealthCheck(**self.health_check_params(failure_threshold=11))) def test_create_health_check_request_interval(self): hc_params = self.health_check_params(request_interval=10) hc = HealthCheck(**hc_params) result = self.conn.create_health_check(hc) hc_config = (result[u'CreateHealthCheckResponse'] [u'HealthCheck'][u'HealthCheckConfig']) self.assertEqual(hc_config[u'RequestInterval'], six.text_type(hc_params['request_interval'])) self.conn.delete_health_check(result['CreateHealthCheckResponse']['HealthCheck']['Id']) def test_create_health_check_failure_threshold(self): hc_params = self.health_check_params(failure_threshold=1) hc = HealthCheck(**hc_params) result = self.conn.create_health_check(hc) hc_config = (result[u'CreateHealthCheckResponse'] [u'HealthCheck'][u'HealthCheckConfig']) self.assertEqual(hc_config[u'FailureThreshold'], six.text_type(hc_params['failure_threshold'])) self.conn.delete_health_check(result['CreateHealthCheckResponse']['HealthCheck']['Id']) def health_check_params(self, **kwargs): params = { 'ip_addr': "54.217.7.118", 'port': 80, 'hc_type': 'HTTP', 'resource_path': '/testing', } params.update(kwargs) return params vendored/boto/tests/integration/route53/__init__.py 0000644 00000003175 15025013616 0016426 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2014 Tellybug, Matt Millar # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import time import unittest from nose.plugins.attrib import attr from boto.route53.connection import Route53Connection @attr(route53=True) class Route53TestCase(unittest.TestCase): def setUp(self): super(Route53TestCase, self).setUp() self.conn = Route53Connection() self.base_domain = 'boto-test-%s.com' % str(int(time.time())) self.zone = self.conn.create_zone(self.base_domain) def tearDown(self): self.zone.delete() super(Route53TestCase, self).tearDown() vendored/boto/tests/integration/kinesis/test_cert_verification.py 0000644 00000002736 15025013616 0021566 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.kinesis class KinesisCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): kinesis = True regions = boto.kinesis.regions() def sample_service_call(self, conn): conn.list_streams() vendored/boto/tests/integration/kinesis/test_kinesis.py 0000644 00000010464 15025013616 0017531 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import time import boto from tests.compat import unittest from boto.kinesis.exceptions import ResourceNotFoundException class TimeoutError(Exception): pass class TestKinesis(unittest.TestCase): def setUp(self): self.kinesis = boto.connect_kinesis() def test_kinesis(self): kinesis = self.kinesis # Create a new stream kinesis.create_stream('test', 1) self.addCleanup(self.kinesis.delete_stream, 'test') # Wait for the stream to be ready tries = 0 while tries < 10: tries += 1 time.sleep(15) response = kinesis.describe_stream('test') if response['StreamDescription']['StreamStatus'] == 'ACTIVE': shard_id = response['StreamDescription']['Shards'][0]['ShardId'] break else: raise TimeoutError('Stream is still not active, aborting...') # Make a tag. kinesis.add_tags_to_stream(stream_name='test', tags={'foo': 'bar'}) # Check that the correct tag is there. response = kinesis.list_tags_for_stream(stream_name='test') self.assertEqual(len(response['Tags']), 1) self.assertEqual(response['Tags'][0], {'Key':'foo', 'Value': 'bar'}) # Remove the tag and ensure it is removed. kinesis.remove_tags_from_stream(stream_name='test', tag_keys=['foo']) response = kinesis.list_tags_for_stream(stream_name='test') self.assertEqual(len(response['Tags']), 0) # Get ready to process some data from the stream response = kinesis.get_shard_iterator('test', shard_id, 'TRIM_HORIZON') shard_iterator = response['ShardIterator'] # Write some data to the stream data = 'Some data ...' record = { 'Data': data, 'PartitionKey': data, } response = kinesis.put_record('test', data, data) response = kinesis.put_records([record, record.copy()], 'test') # Wait for the data to show up tries = 0 num_collected = 0 num_expected_records = 3 collected_records = [] while tries < 100: tries += 1 time.sleep(1) response = kinesis.get_records(shard_iterator) shard_iterator = response['NextShardIterator'] for record in response['Records']: if 'Data' in record: collected_records.append(record['Data']) num_collected += 1 if num_collected >= num_expected_records: self.assertEqual(num_expected_records, num_collected) break else: raise TimeoutError('No records found, aborting...') # Read the data, which should be the same as what we wrote for record in collected_records: self.assertEqual(data, record) def test_describe_non_existent_stream(self): with self.assertRaises(ResourceNotFoundException) as cm: self.kinesis.describe_stream('this-stream-shouldnt-exist') # Assert things about the data we passed along. self.assertEqual(cm.exception.error_code, None) self.assertTrue('not found' in cm.exception.message) vendored/boto/tests/integration/kinesis/__init__.py 0000644 00000000000 15025013616 0016545 0 ustar 00 vendored/boto/tests/integration/dynamodb/test_cert_verification.py 0000644 00000003037 15025013616 0021711 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.dynamodb class DynamoDBCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): dynamodb = True regions = boto.dynamodb.regions() def sample_service_call(self, conn): conn.layer1.list_tables() vendored/boto/tests/integration/dynamodb/test_layer2.py 0000644 00000046546 15025013616 0017424 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests for Layer2 of Amazon DynamoDB """ import time import uuid from decimal import Decimal from tests.unit import unittest from boto.dynamodb.exceptions import DynamoDBKeyNotFoundError from boto.dynamodb.exceptions import DynamoDBConditionalCheckFailedError from boto.dynamodb.layer2 import Layer2 from boto.dynamodb.types import get_dynamodb_type, Binary from boto.dynamodb.condition import BEGINS_WITH, CONTAINS, GT from boto.compat import six, long_type class DynamoDBLayer2Test(unittest.TestCase): dynamodb = True def setUp(self): self.dynamodb = Layer2() self.hash_key_name = 'forum_name' self.hash_key_proto_value = '' self.range_key_name = 'subject' self.range_key_proto_value = '' self.table_name = 'sample_data_%s' % int(time.time()) def create_sample_table(self): schema = self.dynamodb.create_schema( self.hash_key_name, self.hash_key_proto_value, self.range_key_name, self.range_key_proto_value) table = self.create_table(self.table_name, schema, 5, 5) table.refresh(wait_for_active=True) return table def create_table(self, table_name, schema, read_units, write_units): result = self.dynamodb.create_table(table_name, schema, read_units, write_units) self.addCleanup(self.dynamodb.delete_table, result) return result def test_layer2_basic(self): print('--- running Amazon DynamoDB Layer2 tests ---') c = self.dynamodb # First create a schema for the table schema = c.create_schema(self.hash_key_name, self.hash_key_proto_value, self.range_key_name, self.range_key_proto_value) # Create another schema without a range key schema2 = c.create_schema('post_id', '') # Now create a table index = int(time.time()) table_name = 'test-%d' % index read_units = 5 write_units = 5 table = self.create_table(table_name, schema, read_units, write_units) assert table.name == table_name assert table.schema.hash_key_name == self.hash_key_name assert table.schema.hash_key_type == get_dynamodb_type(self.hash_key_proto_value) assert table.schema.range_key_name == self.range_key_name assert table.schema.range_key_type == get_dynamodb_type(self.range_key_proto_value) assert table.read_units == read_units assert table.write_units == write_units assert table.item_count == 0 assert table.size_bytes == 0 # Create the second table table2_name = 'test-%d' % (index + 1) table2 = self.create_table(table2_name, schema2, read_units, write_units) # Wait for table to become active table.refresh(wait_for_active=True) table2.refresh(wait_for_active=True) # List tables and make sure new one is there table_names = c.list_tables() assert table_name in table_names assert table2_name in table_names # Update the tables ProvisionedThroughput new_read_units = 10 new_write_units = 5 table.update_throughput(new_read_units, new_write_units) # Wait for table to be updated table.refresh(wait_for_active=True) assert table.read_units == new_read_units assert table.write_units == new_write_units # Put an item item1_key = 'Amazon DynamoDB' item1_range = 'DynamoDB Thread 1' item1_attrs = { 'Message': 'DynamoDB thread 1 message text', 'LastPostedBy': 'User A', 'Views': 0, 'Replies': 0, 'Answered': 0, 'Public': True, 'Tags': set(['index', 'primarykey', 'table']), 'LastPostDateTime': '12/9/2011 11:36:03 PM'} # Test a few corner cases with new_item # Try supplying a hash_key as an arg and as an item in attrs item1_attrs[self.hash_key_name] = 'foo' foobar_item = table.new_item(item1_key, item1_range, item1_attrs) assert foobar_item.hash_key == item1_key # Try supplying a range_key as an arg and as an item in attrs item1_attrs[self.range_key_name] = 'bar' foobar_item = table.new_item(item1_key, item1_range, item1_attrs) assert foobar_item.range_key == item1_range # Try supplying hash and range key in attrs dict foobar_item = table.new_item(attrs=item1_attrs) assert foobar_item.hash_key == 'foo' assert foobar_item.range_key == 'bar' del item1_attrs[self.hash_key_name] del item1_attrs[self.range_key_name] item1 = table.new_item(item1_key, item1_range, item1_attrs) # make sure the put() succeeds try: item1.put() except c.layer1.ResponseError as e: raise Exception("Item put failed: %s" % e) # Try to get an item that does not exist. self.assertRaises(DynamoDBKeyNotFoundError, table.get_item, 'bogus_key', item1_range) # Now do a consistent read and check results item1_copy = table.get_item(item1_key, item1_range, consistent_read=True) assert item1_copy.hash_key == item1.hash_key assert item1_copy.range_key == item1.range_key for attr_name in item1_attrs: val = item1_copy[attr_name] if isinstance(val, (int, long_type, float, six.string_types)): assert val == item1[attr_name] # Try retrieving only select attributes attributes = ['Message', 'Views'] item1_small = table.get_item(item1_key, item1_range, attributes_to_get=attributes, consistent_read=True) for attr_name in item1_small: # The item will include the attributes we asked for as # well as the hashkey and rangekey, so filter those out. if attr_name not in (item1_small.hash_key_name, item1_small.range_key_name): assert attr_name in attributes self.assertTrue(table.has_item(item1_key, range_key=item1_range, consistent_read=True)) # Try to delete the item with the wrong Expected value expected = {'Views': 1} self.assertRaises(DynamoDBConditionalCheckFailedError, item1.delete, expected_value=expected) # Try to delete a value while expecting a non-existant attribute expected = {'FooBar': True} try: item1.delete(expected_value=expected) except c.layer1.ResponseError: pass # Now update the existing object item1.add_attribute('Replies', 2) removed_attr = 'Public' item1.delete_attribute(removed_attr) removed_tag = item1_attrs['Tags'].copy().pop() item1.delete_attribute('Tags', set([removed_tag])) replies_by_set = set(['Adam', 'Arnie']) item1.put_attribute('RepliesBy', replies_by_set) retvals = item1.save(return_values='ALL_OLD') # Need more tests here for variations on return_values assert 'Attributes' in retvals # Check for correct updates item1_updated = table.get_item(item1_key, item1_range, consistent_read=True) assert item1_updated['Replies'] == item1_attrs['Replies'] + 2 self.assertFalse(removed_attr in item1_updated) self.assertTrue(removed_tag not in item1_updated['Tags']) self.assertTrue('RepliesBy' in item1_updated) self.assertTrue(item1_updated['RepliesBy'] == replies_by_set) # Put a few more items into the table item2_key = 'Amazon DynamoDB' item2_range = 'DynamoDB Thread 2' item2_attrs = { 'Message': 'DynamoDB thread 2 message text', 'LastPostedBy': 'User A', 'Views': 0, 'Replies': 0, 'Answered': 0, 'Tags': set(["index", "primarykey", "table"]), 'LastPost2DateTime': '12/9/2011 11:36:03 PM'} item2 = table.new_item(item2_key, item2_range, item2_attrs) item2.put() item3_key = 'Amazon S3' item3_range = 'S3 Thread 1' item3_attrs = { 'Message': 'S3 Thread 1 message text', 'LastPostedBy': 'User A', 'Views': 0, 'Replies': 0, 'Answered': 0, 'Tags': set(['largeobject', 'multipart upload']), 'LastPostDateTime': '12/9/2011 11:36:03 PM' } item3 = table.new_item(item3_key, item3_range, item3_attrs) item3.put() # Put an item into the second table table2_item1_key = uuid.uuid4().hex table2_item1_attrs = { 'DateTimePosted': '25/1/2011 12:34:56 PM', 'Text': 'I think boto rocks and so does DynamoDB' } table2_item1 = table2.new_item(table2_item1_key, attrs=table2_item1_attrs) table2_item1.put() # Try a few queries items = table.query('Amazon DynamoDB', range_key_condition=BEGINS_WITH('DynamoDB')) n = 0 for item in items: n += 1 assert n == 2 assert items.consumed_units > 0 items = table.query('Amazon DynamoDB', range_key_condition=BEGINS_WITH('DynamoDB'), request_limit=1, max_results=1) n = 0 for item in items: n += 1 assert n == 1 assert items.consumed_units > 0 # Try a few scans items = table.scan() n = 0 for item in items: n += 1 assert n == 3 assert items.consumed_units > 0 items = table.scan(scan_filter={'Replies': GT(0)}) n = 0 for item in items: n += 1 assert n == 1 assert items.consumed_units > 0 # Test some integer and float attributes integer_value = 42 float_value = 345.678 item3['IntAttr'] = integer_value item3['FloatAttr'] = float_value # Test booleans item3['TrueBoolean'] = True item3['FalseBoolean'] = False # Test some set values integer_set = set([1, 2, 3, 4, 5]) float_set = set([1.1, 2.2, 3.3, 4.4, 5.5]) mixed_set = set([1, 2, 3.3, 4, 5.555]) str_set = set(['foo', 'bar', 'fie', 'baz']) item3['IntSetAttr'] = integer_set item3['FloatSetAttr'] = float_set item3['MixedSetAttr'] = mixed_set item3['StrSetAttr'] = str_set item3.put() # Now do a consistent read item4 = table.get_item(item3_key, item3_range, consistent_read=True) assert item4['IntAttr'] == integer_value assert item4['FloatAttr'] == float_value assert bool(item4['TrueBoolean']) is True assert bool(item4['FalseBoolean']) is False # The values will not necessarily be in the same order as when # we wrote them to the DB. for i in item4['IntSetAttr']: assert i in integer_set for i in item4['FloatSetAttr']: assert i in float_set for i in item4['MixedSetAttr']: assert i in mixed_set for i in item4['StrSetAttr']: assert i in str_set # Try a batch get batch_list = c.new_batch_list() batch_list.add_batch(table, [(item2_key, item2_range), (item3_key, item3_range)]) response = batch_list.submit() assert len(response['Responses'][table.name]['Items']) == 2 # Try an empty batch get batch_list = c.new_batch_list() batch_list.add_batch(table, []) response = batch_list.submit() assert response == {} # Try a few batch write operations item4_key = 'Amazon S3' item4_range = 'S3 Thread 2' item4_attrs = { 'Message': 'S3 Thread 2 message text', 'LastPostedBy': 'User A', 'Views': 0, 'Replies': 0, 'Answered': 0, 'Tags': set(['largeobject', 'multipart upload']), 'LastPostDateTime': '12/9/2011 11:36:03 PM' } item5_key = 'Amazon S3' item5_range = 'S3 Thread 3' item5_attrs = { 'Message': 'S3 Thread 3 message text', 'LastPostedBy': 'User A', 'Views': 0, 'Replies': 0, 'Answered': 0, 'Tags': set(['largeobject', 'multipart upload']), 'LastPostDateTime': '12/9/2011 11:36:03 PM' } item4 = table.new_item(item4_key, item4_range, item4_attrs) item5 = table.new_item(item5_key, item5_range, item5_attrs) batch_list = c.new_batch_write_list() batch_list.add_batch(table, puts=[item4, item5]) response = batch_list.submit() # should really check for unprocessed items # Do some generator gymnastics results = table.scan(scan_filter={'Tags': CONTAINS('table')}) assert results.scanned_count == 5 results = table.scan(request_limit=2, max_results=5) assert results.count == 2 for item in results: if results.count == 2: assert results.remaining == 4 results.remaining -= 2 results.next_response() else: assert results.count == 4 assert results.remaining in (0, 1) assert results.count == 4 results = table.scan(request_limit=6, max_results=4) assert len(list(results)) == 4 assert results.count == 4 batch_list = c.new_batch_write_list() batch_list.add_batch(table, deletes=[(item4_key, item4_range), (item5_key, item5_range)]) response = batch_list.submit() # Try queries results = table.query('Amazon DynamoDB', range_key_condition=BEGINS_WITH('DynamoDB')) n = 0 for item in results: n += 1 assert n == 2 # Try to delete the item with the right Expected value expected = {'Views': 0} item1.delete(expected_value=expected) self.assertFalse(table.has_item(item1_key, range_key=item1_range, consistent_read=True)) # Now delete the remaining items ret_vals = item2.delete(return_values='ALL_OLD') # some additional checks here would be useful assert ret_vals['Attributes'][self.hash_key_name] == item2_key assert ret_vals['Attributes'][self.range_key_name] == item2_range item3.delete() table2_item1.delete() print('--- tests completed ---') def test_binary_attrs(self): c = self.dynamodb schema = c.create_schema(self.hash_key_name, self.hash_key_proto_value, self.range_key_name, self.range_key_proto_value) index = int(time.time()) table_name = 'test-%d' % index read_units = 5 write_units = 5 table = self.create_table(table_name, schema, read_units, write_units) table.refresh(wait_for_active=True) item1_key = 'Amazon S3' item1_range = 'S3 Thread 1' item1_attrs = { 'Message': 'S3 Thread 1 message text', 'LastPostedBy': 'User A', 'Views': 0, 'Replies': 0, 'Answered': 0, 'BinaryData': Binary(b'\x01\x02\x03\x04'), 'BinarySequence': set([Binary(b'\x01\x02'), Binary(b'\x03\x04')]), 'Tags': set(['largeobject', 'multipart upload']), 'LastPostDateTime': '12/9/2011 11:36:03 PM' } item1 = table.new_item(item1_key, item1_range, item1_attrs) item1.put() retrieved = table.get_item(item1_key, item1_range, consistent_read=True) self.assertEqual(retrieved['Message'], 'S3 Thread 1 message text') self.assertEqual(retrieved['Views'], 0) self.assertEqual(retrieved['Tags'], set(['largeobject', 'multipart upload'])) self.assertEqual(retrieved['BinaryData'], Binary(b'\x01\x02\x03\x04')) # Also comparable directly to bytes: self.assertEqual(retrieved['BinaryData'], b'\x01\x02\x03\x04') self.assertEqual(retrieved['BinarySequence'], set([Binary(b'\x01\x02'), Binary(b'\x03\x04')])) def test_put_decimal_attrs(self): self.dynamodb.use_decimals() table = self.create_sample_table() item = table.new_item('foo', 'bar') item['decimalvalue'] = Decimal('1.12345678912345') item.put() retrieved = table.get_item('foo', 'bar') self.assertEqual(retrieved['decimalvalue'], Decimal('1.12345678912345')) @unittest.skipIf(six.PY3, "skipping lossy_float_conversion test for Python 3.x") def test_lossy_float_conversion(self): table = self.create_sample_table() item = table.new_item('foo', 'bar') item['floatvalue'] = 1.12345678912345 item.put() retrieved = table.get_item('foo', 'bar')['floatvalue'] # Notice how this is not equal to the original value. self.assertNotEqual(1.12345678912345, retrieved) # Instead, it's truncated: self.assertEqual(1.12345678912, retrieved) def test_large_integers(self): # It's not just floating point numbers, large integers # can trigger rouding issues. self.dynamodb.use_decimals() table = self.create_sample_table() item = table.new_item('foo', 'bar') item['decimalvalue'] = Decimal('129271300103398600') item.put() retrieved = table.get_item('foo', 'bar') self.assertEqual(retrieved['decimalvalue'], Decimal('129271300103398600')) # Also comparable directly to an int. self.assertEqual(retrieved['decimalvalue'], 129271300103398600) def test_put_single_letter_attr(self): # When an attr is added that is a single letter, if it overlaps with # the built-in "types", the decoding used to fall down. Assert that # it's now working correctly. table = self.create_sample_table() item = table.new_item('foo', 'foo1') item.put_attribute('b', 4) stored = item.save(return_values='UPDATED_NEW') self.assertEqual(stored['Attributes'], {'b': 4}) vendored/boto/tests/integration/dynamodb/test_layer1.py 0000644 00000027000 15025013616 0017403 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests for Layer1 of DynamoDB """ import time import base64 from tests.unit import unittest from boto.dynamodb.exceptions import DynamoDBKeyNotFoundError from boto.dynamodb.exceptions import DynamoDBConditionalCheckFailedError from boto.dynamodb.exceptions import DynamoDBValidationError from boto.dynamodb.layer1 import Layer1 class DynamoDBLayer1Test(unittest.TestCase): dynamodb = True def setUp(self): self.dynamodb = Layer1() self.table_name = 'test-%d' % int(time.time()) self.hash_key_name = 'forum_name' self.hash_key_type = 'S' self.range_key_name = 'subject' self.range_key_type = 'S' self.read_units = 5 self.write_units = 5 self.schema = {'HashKeyElement': {'AttributeName': self.hash_key_name, 'AttributeType': self.hash_key_type}, 'RangeKeyElement': {'AttributeName': self.range_key_name, 'AttributeType': self.range_key_type}} self.provisioned_throughput = {'ReadCapacityUnits': self.read_units, 'WriteCapacityUnits': self.write_units} def tearDown(self): pass def create_table(self, table_name, schema, provisioned_throughput): result = self.dynamodb.create_table(table_name, schema, provisioned_throughput) self.addCleanup(self.dynamodb.delete_table, table_name) return result def test_layer1_basic(self): print('--- running DynamoDB Layer1 tests ---') c = self.dynamodb # First create a table table_name = self.table_name hash_key_name = self.hash_key_name hash_key_type = self.hash_key_type range_key_name = self.range_key_name range_key_type = self.range_key_type read_units = self.read_units write_units = self.write_units schema = self.schema provisioned_throughput = self.provisioned_throughput result = self.create_table(table_name, schema, provisioned_throughput) assert result['TableDescription']['TableName'] == table_name result_schema = result['TableDescription']['KeySchema'] assert result_schema['HashKeyElement']['AttributeName'] == hash_key_name assert result_schema['HashKeyElement']['AttributeType'] == hash_key_type assert result_schema['RangeKeyElement']['AttributeName'] == range_key_name assert result_schema['RangeKeyElement']['AttributeType'] == range_key_type result_thruput = result['TableDescription']['ProvisionedThroughput'] assert result_thruput['ReadCapacityUnits'] == read_units assert result_thruput['WriteCapacityUnits'] == write_units # Wait for table to become active result = c.describe_table(table_name) while result['Table']['TableStatus'] != 'ACTIVE': time.sleep(5) result = c.describe_table(table_name) # List tables and make sure new one is there result = c.list_tables() assert table_name in result['TableNames'] # Update the tables ProvisionedThroughput new_read_units = 10 new_write_units = 5 new_provisioned_throughput = {'ReadCapacityUnits': new_read_units, 'WriteCapacityUnits': new_write_units} result = c.update_table(table_name, new_provisioned_throughput) # Wait for table to be updated result = c.describe_table(table_name) while result['Table']['TableStatus'] == 'UPDATING': time.sleep(5) result = c.describe_table(table_name) result_thruput = result['Table']['ProvisionedThroughput'] assert result_thruput['ReadCapacityUnits'] == new_read_units assert result_thruput['WriteCapacityUnits'] == new_write_units # Put an item item1_key = 'Amazon DynamoDB' item1_range = 'DynamoDB Thread 1' item1_data = { hash_key_name: {hash_key_type: item1_key}, range_key_name: {range_key_type: item1_range}, 'Message': {'S': 'DynamoDB thread 1 message text'}, 'LastPostedBy': {'S': 'User A'}, 'Views': {'N': '0'}, 'Replies': {'N': '0'}, 'Answered': {'N': '0'}, 'Tags': {'SS': ["index", "primarykey", "table"]}, 'LastPostDateTime': {'S': '12/9/2011 11:36:03 PM'} } result = c.put_item(table_name, item1_data) # Now do a consistent read and check results key1 = {'HashKeyElement': {hash_key_type: item1_key}, 'RangeKeyElement': {range_key_type: item1_range}} result = c.get_item(table_name, key=key1, consistent_read=True) for name in item1_data: assert name in result['Item'] # Try to get an item that does not exist. invalid_key = {'HashKeyElement': {hash_key_type: 'bogus_key'}, 'RangeKeyElement': {range_key_type: item1_range}} self.assertRaises(DynamoDBKeyNotFoundError, c.get_item, table_name, key=invalid_key) # Try retrieving only select attributes attributes = ['Message', 'Views'] result = c.get_item(table_name, key=key1, consistent_read=True, attributes_to_get=attributes) for name in result['Item']: assert name in attributes # Try to delete the item with the wrong Expected value expected = {'Views': {'Value': {'N': '1'}}} self.assertRaises(DynamoDBConditionalCheckFailedError, c.delete_item, table_name, key=key1, expected=expected) # Now update the existing object attribute_updates = {'Views': {'Value': {'N': '5'}, 'Action': 'PUT'}, 'Tags': {'Value': {'SS': ['foobar']}, 'Action': 'ADD'}} result = c.update_item(table_name, key=key1, attribute_updates=attribute_updates) # Try and update an item, in a fashion which makes it too large. # The new message text is the item size limit minus 32 bytes and # the current object is larger than 32 bytes. item_size_overflow_text = 'Text to be padded'.zfill(64 * 1024 - 32) attribute_updates = {'Message': {'Value': {'S': item_size_overflow_text}, 'Action': 'PUT'}} self.assertRaises(DynamoDBValidationError, c.update_item, table_name, key=key1, attribute_updates=attribute_updates) # Put a few more items into the table item2_key = 'Amazon DynamoDB' item2_range = 'DynamoDB Thread 2' item2_data = { hash_key_name: {hash_key_type: item2_key}, range_key_name: {range_key_type: item2_range}, 'Message': {'S': 'DynamoDB thread 2 message text'}, 'LastPostedBy': {'S': 'User A'}, 'Views': {'N': '0'}, 'Replies': {'N': '0'}, 'Answered': {'N': '0'}, 'Tags': {'SS': ["index", "primarykey", "table"]}, 'LastPostDateTime': {'S': '12/9/2011 11:36:03 PM'} } result = c.put_item(table_name, item2_data) key2 = {'HashKeyElement': {hash_key_type: item2_key}, 'RangeKeyElement': {range_key_type: item2_range}} item3_key = 'Amazon S3' item3_range = 'S3 Thread 1' item3_data = { hash_key_name: {hash_key_type: item3_key}, range_key_name: {range_key_type: item3_range}, 'Message': {'S': 'S3 Thread 1 message text'}, 'LastPostedBy': {'S': 'User A'}, 'Views': {'N': '0'}, 'Replies': {'N': '0'}, 'Answered': {'N': '0'}, 'Tags': {'SS': ['largeobject', 'multipart upload']}, 'LastPostDateTime': {'S': '12/9/2011 11:36:03 PM'} } result = c.put_item(table_name, item3_data) key3 = {'HashKeyElement': {hash_key_type: item3_key}, 'RangeKeyElement': {range_key_type: item3_range}} # Try a few queries result = c.query(table_name, {'S': 'Amazon DynamoDB'}, {'AttributeValueList': [{'S': 'DynamoDB'}], 'ComparisonOperator': 'BEGINS_WITH'}) assert 'Count' in result assert result['Count'] == 2 # Try a few scans result = c.scan(table_name, {'Tags': {'AttributeValueList': [{'S': 'table'}], 'ComparisonOperator': 'CONTAINS'}}) assert 'Count' in result assert result['Count'] == 2 # Now delete the items result = c.delete_item(table_name, key=key1) result = c.delete_item(table_name, key=key2) result = c.delete_item(table_name, key=key3) print('--- tests completed ---') def test_binary_attributes(self): c = self.dynamodb result = self.create_table(self.table_name, self.schema, self.provisioned_throughput) # Wait for table to become active result = c.describe_table(self.table_name) while result['Table']['TableStatus'] != 'ACTIVE': time.sleep(5) result = c.describe_table(self.table_name) # Put an item item1_key = 'Amazon DynamoDB' item1_range = 'DynamoDB Thread 1' item1_data = { self.hash_key_name: {self.hash_key_type: item1_key}, self.range_key_name: {self.range_key_type: item1_range}, 'Message': {'S': 'DynamoDB thread 1 message text'}, 'LastPostedBy': {'S': 'User A'}, 'Views': {'N': '0'}, 'Replies': {'N': '0'}, 'BinaryData': {'B': base64.b64encode(b'\x01\x02\x03\x04').decode('utf-8')}, 'Answered': {'N': '0'}, 'Tags': {'SS': ["index", "primarykey", "table"]}, 'LastPostDateTime': {'S': '12/9/2011 11:36:03 PM'} } result = c.put_item(self.table_name, item1_data) # Now do a consistent read and check results key1 = {'HashKeyElement': {self.hash_key_type: item1_key}, 'RangeKeyElement': {self.range_key_type: item1_range}} result = c.get_item(self.table_name, key=key1, consistent_read=True) self.assertEqual(result['Item']['BinaryData'], {'B': base64.b64encode(b'\x01\x02\x03\x04').decode('utf-8')}) vendored/boto/tests/integration/dynamodb/test_table.py 0000644 00000006741 15025013616 0017306 0 ustar 00 # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import time from tests.unit import unittest from boto.dynamodb.layer2 import Layer2 from boto.dynamodb.table import Table from boto.dynamodb.schema import Schema class TestDynamoDBTable(unittest.TestCase): dynamodb = True def setUp(self): self.dynamodb = Layer2() self.schema = Schema.create(('foo', 'N'), ('bar', 'S')) self.table_name = 'testtable%s' % int(time.time()) def create_table(self, table_name, schema, read_units, write_units): result = self.dynamodb.create_table(table_name, schema, read_units, write_units) self.addCleanup(self.dynamodb.delete_table, result) return result def assertAllEqual(self, *items): first = items[0] for item in items[1:]: self.assertEqual(first, item) def test_table_retrieval_parity(self): created_table = self.dynamodb.create_table( self.table_name, self.schema, 1, 1) created_table.refresh(wait_for_active=True) retrieved_table = self.dynamodb.get_table(self.table_name) constructed_table = self.dynamodb.table_from_schema(self.table_name, self.schema) # All three tables should have the same name # and schema attributes. self.assertAllEqual(created_table.name, retrieved_table.name, constructed_table.name) self.assertAllEqual(created_table.schema, retrieved_table.schema, constructed_table.schema) # However for create_time, status, read/write units, # only the created/retrieved table will have equal # values. self.assertEqual(created_table.create_time, retrieved_table.create_time) self.assertEqual(created_table.status, retrieved_table.status) self.assertEqual(created_table.read_units, retrieved_table.read_units) self.assertEqual(created_table.write_units, retrieved_table.write_units) # The constructed table will have values of None. self.assertIsNone(constructed_table.create_time) self.assertIsNone(constructed_table.status) self.assertIsNone(constructed_table.read_units) self.assertIsNone(constructed_table.write_units) vendored/boto/tests/integration/dynamodb/__init__.py 0000644 00000002113 15025013616 0016704 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/sts/test_cert_verification.py 0000644 00000003012 15025013616 0020716 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.sts class STSCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): sts = True regions = boto.sts.regions() def sample_service_call(self, conn): conn.get_session_token() vendored/boto/tests/integration/sts/test_session_token.py 0000644 00000006515 15025013616 0020115 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests for Session Tokens """ import unittest import os from boto.exception import BotoServerError from boto.sts.connection import STSConnection from boto.sts.credentials import Credentials from boto.s3.connection import S3Connection class SessionTokenTest(unittest.TestCase): sts = True def test_session_token(self): print('--- running Session Token tests ---') c = STSConnection() # Create a session token token = c.get_session_token() # Save session token to a file token.save('token.json') # Now load up a copy of that token token_copy = Credentials.load('token.json') assert token_copy.access_key == token.access_key assert token_copy.secret_key == token.secret_key assert token_copy.session_token == token.session_token assert token_copy.expiration == token.expiration assert token_copy.request_id == token.request_id os.unlink('token.json') assert not token.is_expired() # Try using the session token with S3 s3 = S3Connection(aws_access_key_id=token.access_key, aws_secret_access_key=token.secret_key, security_token=token.session_token) buckets = s3.get_all_buckets() print('--- tests completed ---') def test_assume_role_with_web_identity(self): c = STSConnection(anon=True) arn = 'arn:aws:iam::000240903217:role/FederatedWebIdentityRole' wit = 'b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9' try: creds = c.assume_role_with_web_identity( role_arn=arn, role_session_name='guestuser', web_identity_token=wit, provider_id='www.amazon.com', ) except BotoServerError as err: self.assertEqual(err.status, 403) self.assertTrue('Not authorized' in err.body) def test_decode_authorization_message(self): c = STSConnection() try: creds = c.decode_authorization_message('b94d27b9934') except BotoServerError as err: self.assertEqual(err.status, 400) self.assertIn('InvalidAuthorizationMessageException', err.body) vendored/boto/tests/integration/sts/__init__.py 0000644 00000002113 15025013616 0015720 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/configservice/test_configservice.py 0000644 00000004041 15025013616 0022065 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import boto from boto.configservice.exceptions import NoSuchConfigurationRecorderException from tests.compat import unittest class TestConfigService(unittest.TestCase): def setUp(self): self.configservice = boto.connect_configservice() def test_describe_configuration_recorders(self): response = self.configservice.describe_configuration_recorders() self.assertIn('ConfigurationRecorders', response) def test_handle_no_such_configuration_recorder(self): with self.assertRaises(NoSuchConfigurationRecorderException): self.configservice.describe_configuration_recorders( configuration_recorder_names=['non-existant-recorder']) def test_connect_to_non_us_east_1(self): self.configservice = boto.configservice.connect_to_region('us-west-2') response = self.configservice.describe_configuration_recorders() self.assertIn('ConfigurationRecorders', response) vendored/boto/tests/integration/configservice/__init__.py 0000644 00000002143 15025013616 0017740 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/integration/opsworks/test_layer1.py 0000644 00000004111 15025013616 0017473 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.exception import JSONResponseError from boto.opsworks import connect_to_region, regions, RegionInfo from boto.opsworks.layer1 import OpsWorksConnection from tests.compat import unittest class TestOpsWorksConnection(unittest.TestCase): opsworks = True def setUp(self): self.api = OpsWorksConnection() def test_describe_stacks(self): response = self.api.describe_stacks() self.assertIn('Stacks', response) def test_validation_errors(self): with self.assertRaises(JSONResponseError): self.api.create_stack('testbotostack', 'us-east-1', 'badarn', 'badarn2') class TestOpsWorksHelpers(unittest.TestCase): opsworks = True def test_regions(self): response = regions() self.assertIsInstance(response[0], RegionInfo) def test_connect_to_region(self): connection = connect_to_region('us-east-1') self.assertIsInstance(connection, OpsWorksConnection) vendored/boto/tests/integration/opsworks/__init__.py 0000644 00000000000 15025013616 0016767 0 ustar 00 vendored/boto/tests/integration/cloudhsm/__init__.py 0000644 00000002143 15025013616 0016730 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/integration/cloudhsm/test_cloudhsm.py 0000644 00000003533 15025013616 0020052 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import boto from tests.compat import unittest from boto.cloudhsm.exceptions import InvalidRequestException class TestCloudHSM(unittest.TestCase): def setUp(self): self.cloudhsm = boto.connect_cloudhsm() def test_hapgs(self): label = 'my-hapg' response = self.cloudhsm.create_hapg(label=label) hapg_arn = response['HapgArn'] self.addCleanup(self.cloudhsm.delete_hapg, hapg_arn) response = self.cloudhsm.list_hapgs() self.assertIn(hapg_arn, response['HapgList']) def test_validation_exception(self): invalid_arn = 'arn:aws:cloudhsm:us-east-1:123456789012:hapg-55214b8d' with self.assertRaises(InvalidRequestException): self.cloudhsm.describe_hapg(invalid_arn) vendored/boto/tests/integration/datapipeline/test_cert_verification.py 0000644 00000002764 15025013616 0022561 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.datapipeline class DatapipelineCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): datapipeline = True regions = boto.datapipeline.regions() def sample_service_call(self, conn): conn.list_pipelines() vendored/boto/tests/integration/datapipeline/test_layer1.py 0000644 00000012505 15025013616 0020251 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import time from tests.unit import unittest from boto.datapipeline import layer1 class TestDataPipeline(unittest.TestCase): datapipeline = True def setUp(self): self.connection = layer1.DataPipelineConnection() self.sample_pipeline_objects = [ {'fields': [ {'key': 'workerGroup', 'stringValue': 'MyworkerGroup'}], 'id': 'Default', 'name': 'Default'}, {'fields': [ {'key': 'startDateTime', 'stringValue': '2012-09-25T17:00:00'}, {'key': 'type', 'stringValue': 'Schedule'}, {'key': 'period', 'stringValue': '1 hour'}, {'key': 'endDateTime', 'stringValue': '2012-09-25T18:00:00'}], 'id': 'Schedule', 'name': 'Schedule'}, {'fields': [ {'key': 'type', 'stringValue': 'ShellCommandActivity'}, {'key': 'command', 'stringValue': 'echo hello'}, {'key': 'parent', 'refValue': 'Default'}, {'key': 'schedule', 'refValue': 'Schedule'}], 'id': 'SayHello', 'name': 'SayHello'} ] self.connection.auth_service_name = 'datapipeline' def create_pipeline(self, name, unique_id, description=None): response = self.connection.create_pipeline(name, unique_id, description) pipeline_id = response['pipelineId'] self.addCleanup(self.connection.delete_pipeline, pipeline_id) return pipeline_id def get_pipeline_state(self, pipeline_id): response = self.connection.describe_pipelines([pipeline_id]) for attr in response['pipelineDescriptionList'][0]['fields']: if attr['key'] == '@pipelineState': return attr['stringValue'] def test_can_create_and_delete_a_pipeline(self): response = self.connection.create_pipeline('name', 'unique_id', 'description') self.connection.delete_pipeline(response['pipelineId']) def test_validate_pipeline(self): pipeline_id = self.create_pipeline('name2', 'unique_id2') self.connection.validate_pipeline_definition( self.sample_pipeline_objects, pipeline_id) def test_put_pipeline_definition(self): pipeline_id = self.create_pipeline('name3', 'unique_id3') self.connection.put_pipeline_definition(self.sample_pipeline_objects, pipeline_id) # We should now be able to get the pipeline definition and see # that it matches what we put. response = self.connection.get_pipeline_definition(pipeline_id) objects = response['pipelineObjects'] self.assertEqual(len(objects), 3) self.assertEqual(objects[0]['id'], 'Default') self.assertEqual(objects[0]['name'], 'Default') self.assertEqual(objects[0]['fields'], [{'key': 'workerGroup', 'stringValue': 'MyworkerGroup'}]) def test_activate_pipeline(self): pipeline_id = self.create_pipeline('name4', 'unique_id4') self.connection.put_pipeline_definition(self.sample_pipeline_objects, pipeline_id) self.connection.activate_pipeline(pipeline_id) attempts = 0 state = self.get_pipeline_state(pipeline_id) while state != 'SCHEDULED' and attempts < 10: time.sleep(10) attempts += 1 state = self.get_pipeline_state(pipeline_id) if attempts > 10: self.fail("Pipeline did not become scheduled " "after 10 attempts.") objects = self.connection.describe_objects(['Default'], pipeline_id) field = objects['pipelineObjects'][0]['fields'][0] self.assertDictEqual(field, {'stringValue': 'COMPONENT', 'key': '@sphere'}) def test_list_pipelines(self): pipeline_id = self.create_pipeline('name5', 'unique_id5') pipeline_id_list = [p['id'] for p in self.connection.list_pipelines()['pipelineIdList']] self.assertTrue(pipeline_id in pipeline_id_list) if __name__ == '__main__': unittest.main() vendored/boto/tests/integration/datapipeline/__init__.py 0000644 00000000000 15025013616 0017537 0 ustar 00 vendored/boto/tests/integration/codedeploy/test_codedeploy.py 0000644 00000003534 15025013616 0020675 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import boto from boto.codedeploy.exceptions import ApplicationDoesNotExistException from tests.compat import unittest class TestCodeDeploy(unittest.TestCase): def setUp(self): self.codedeploy = boto.connect_codedeploy() def test_applications(self): application_name = 'my-boto-application' self.codedeploy.create_application(application_name=application_name) self.addCleanup(self.codedeploy.delete_application, application_name) response = self.codedeploy.list_applications() self.assertIn(application_name, response['applications']) def test_exception(self): with self.assertRaises(ApplicationDoesNotExistException): self.codedeploy.get_application('some-non-existant-app') vendored/boto/tests/integration/codedeploy/__init__.py 0000644 00000002143 15025013616 0017241 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/integration/rds2/test_cert_verification.py 0000644 00000003020 15025013616 0020756 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.rds2 class RDSCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): rds = True regions = boto.rds2.regions() def sample_service_call(self, conn): conn.describe_db_instances() vendored/boto/tests/integration/rds2/test_connection.py 0000644 00000007066 15025013616 0017434 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import unittest import time from boto.rds2.layer1 import RDSConnection class TestRDS2Connection(unittest.TestCase): rds = True def setUp(self): self.conn = RDSConnection() self.db_name = "test-db-%s" % str(int(time.time())) def test_connect_rds(self): # Upon release, this did not function correct. Ensure that # args are passed correctly. import boto conn = boto.connect_rds2() def test_integration(self): resp = self.conn.create_db_instance( db_instance_identifier=self.db_name, allocated_storage=5, db_instance_class='db.t1.micro', engine='postgres', master_username='bototestuser', master_user_password='testtestt3st', # Try to limit the impact & test options. multi_az=False, backup_retention_period=0 ) self.addCleanup( self.conn.delete_db_instance, self.db_name, skip_final_snapshot=True ) # Wait for 6 minutes for it to come up. time.sleep(60 * 6) instances = self.conn.describe_db_instances(self.db_name) inst = instances['DescribeDBInstancesResponse']\ ['DescribeDBInstancesResult']['DBInstances'][0] self.assertEqual(inst['DBInstanceStatus'], 'available') self.assertEqual(inst['Engine'], 'postgres') self.assertEqual(inst['AllocatedStorage'], 5) # Try renaming it. resp = self.conn.modify_db_instance( self.db_name, allocated_storage=10, apply_immediately=True ) # Give it a chance to start modifying... time.sleep(60) instances = self.conn.describe_db_instances(self.db_name) inst = instances['DescribeDBInstancesResponse']\ ['DescribeDBInstancesResult']['DBInstances'][0] self.assertEqual(inst['DBInstanceStatus'], 'modifying') self.assertEqual(inst['Engine'], 'postgres') # ...then finish the remainder of 10 minutes for the change. time.sleep(60 * 9) instances = self.conn.describe_db_instances(self.db_name) inst = instances['DescribeDBInstancesResponse']\ ['DescribeDBInstancesResult']['DBInstances'][0] self.assertEqual(inst['DBInstanceStatus'], 'available') self.assertEqual(inst['Engine'], 'postgres') self.assertEqual(inst['AllocatedStorage'], 10) vendored/boto/tests/integration/rds2/__init__.py 0000644 00000002227 15025013616 0015767 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All Rights Reserved # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/s3/__pycache__/mock_storage_service.cpython-39.pyc 0000644 00000051471 15025013616 0024374 0 ustar 00 a sFg�\ � @ s� d Z ddlZddlZddlZddlZddlZddlmZ ddlm Z ddlm Z ddlm Z ddlm Z ddl mZ dd lmZ dZG d d � d e�ZG d d � d e�ZG dd� de�ZG dd� de�ZG dd� de�Ze� ZG dd� de�ZdS )a Provides basic mocks of core storage service classes, for unit testing: ACL, Key, Bucket, Connection, and StorageUri. We implement a subset of the interfaces defined in the real boto classes, but don't handle most of the optional params (which we indicate with the constant "NOT_IMPL"). � N)�md5)� compute_md5)�find_matching_headers)�merge_headers_by_name)� write_to_fd)�Prefix)�sixc @ s0 e Zd Zefdd�Zdd� Zdd� Zdd� Zd S ) �MockAclc C s d S �N� )�self�parentr r �L/opt/gsutil/gslib/vendored/boto/tests/integration/s3/mock_storage_service.py�__init__0 s zMockAcl.__init__c C s d S r r )r �name�attrs� connectionr r r � startElement3 s zMockAcl.startElementc C s d S r r )r r �valuer r r r � endElement6 s zMockAcl.endElementc C s dS )Nz<mock_ACL_XML/>r �r r r r �to_xml9 s zMockAcl.to_xmlN)�__name__� __module__� __qualname__�NOT_IMPLr r r r r r r r r . s r c @ s e Zd Zd%dd�Zdd� Zeeeeefdd�Zeeeeeefdd �Zeeeeeefd d �Zd d � Z deefdd�Z efdd�Z d&dd�Z deeeeeefdd�Z deeeeeeefdd�Zeeeeeeefdd�Zdeeeeeefdd �Zeeefd d �Zed d � �Zd!d"� Zd#d$� ZdS )'�MockKeyNc C sF || _ || _d | _d | _d | _d| _d | _d | _d | _d| _ d| _ d S )NTz Wed, 06 Oct 2010 05:11:54 GMTi ) �bucketr �data�etag�size�closed�content_encoding�content_language� content_type� last_modified� BufferSize)r r r r r r r ? s zMockKey.__init__c C s& | j rd| j j| jf S d| j S d S )Nz<MockKey: %s,%s>z <MockKey: %s>�r r r r r r �__repr__L s zMockKey.__repr__c C s | j S r )r )r �headers�cb�num_cb�torrent� version_idr r r �get_contents_as_stringR s z MockKey.get_contents_as_stringc C s t �| j�}t||� d S r �r � ensure_binaryr r ) r �fpr) r* r+ r, r- �res_download_handlerr r r r �get_contents_to_fileX s z MockKey.get_contents_to_filec C s t �| j�}t||� d S r r/ ) r r1 r) r* r+ r, r- �override_num_retriesr r r r �get_file` s zMockKey.get_filec C sN |sd S t d|�r td|�| _t d|�r4td|�| _t d|�rJtd|�| _d S )NzContent-Encodingz Content-TypezContent-Language)r r r" r$ r# �r r) r r r �_handle_headersf s � �zMockKey._handle_headersc C sF | j r d| _d| _ |rBd|v rBt�d|d �}|rBt|�d��| _d S )Nr F�Rangezbytes=([0-9]+)-$� )r! �read_pos�re�match�int�group)r r) � query_argsr4 r< r r r � open_readt s zMockKey.open_readc C s d| _ d S �NT)r! )r �fastr r r �close~ s z MockKey.closer c C s^ | � � |dkr*| j| jd � }| j| _n$| j| j| j| � }| j|7 _|sZ| �� |S )Nr )r@ r r: r rC )r r r r r r �read� s z MockKey.readc C s, |� � | _| �� t| j�| _| �|� d S r )rD r �set_etag�lenr r7 ) r r1 r) �replacer* r+ �policyr �res_upload_handlerr r r �set_contents_from_file� s z MockKey.set_contents_from_filec C sT d| _ |�| j�} | r2| j | 7 _ |�| j�} q| �� t| j �| _| �|� d S )N� )r rD r& rE rF r r7 ) r r1 r) rG r* r+ rH �reduced_redundancyr? r �chunkr r r �set_contents_from_stream� s z MockKey.set_contents_from_streamc C s, t � |�| _| �� t|�| _| �|� d S r )�copyr rE rF r r7 ) r �sr) rG r* r+ rH r rL r r r �set_contents_from_string� s z MockKey.set_contents_from_stringc C s. t |d�} | �| |||||||� | �� d S )N�rb)�openrJ rC ) r �filenamer) rG r* r+ rH r rI r1 r r r �set_contents_from_filename� s �z"MockKey.set_contents_from_filenamec C s$ | j j�|�}|�|| j j| j|�S r )r r � get_bucket�copy_keyr )r �dst_bucket_name�dst_key�metadatarL � preserve_acl� dst_bucketr r r rO � s �z MockKey.copyc C s d }| j r | j jr | j jj}|S r )r r �provider�r r] r r r r] � s zMockKey.providerc C sD t � }t| jt�s&|�| j�d�� n |�| j� |�� }|| _dS )zl Set etag attribute by generating hex MD5 checksum on current contents of mock key. zutf-8N)r � isinstancer �bytes�update�encode� hexdigestr )r �mZhex_md5r r r rE � s zMockKey.set_etagc C s t |�}|d | _|dd� S )a� :type fp: file :param fp: File pointer to the file to MD5 hash. The file pointer will be reset to the beginning of the file before the method returns. :rtype: tuple :return: A tuple containing the hex digest version of the MD5 hash as the first element and the base64 encoded version of the plain digest as the second element. � r )r r )r r1 �tupr r r r � s zMockKey.compute_md5)NN)r )r r r r r( r r. r3 r5 r7 r@ rC rD rJ rN rQ rU rO �propertyr] rE r r r r r r = sb � � � � � � � � � r c @ s e Zd Zddefdd�Zdd� Zeeeeeeefdd�Zdd � Zd d � Zd d � Z efdd�Z deefdd�Z eeefdd�Z eeefdd�Z dd� Zd)dd�Zeeefdd �Zefd d �Zeefd d �Zddeefd!d"�Zdeefd#d$�Zeeefd%d&�Zeeefd'd(�ZdS )*� MockBucketNc C s6 || _ i | _|t� i| _t� | _i | _|| _d| _d S �NF)r �keysr �acls�def_acl� subresourcesr �logging)r r r � key_classr r r r � s zMockBucket.__init__c C s d| j S )NzMockBucket: %s)r r r r r r( � s zMockBucket.__repr__c C s< | j |d�} | j�|��|�} t�| j�| _t| j�| _| S )N)�key_name)�new_keyr rV �get_keyrO r rF r ) r � new_key_name�src_bucket_name� src_key_namerZ �src_version_id� storage_classr[ � encrypt_keyr) r? rq Zsrc_keyr r r rW � s �� zMockBucket.copy_keyc C s d| _ d S ri �rn r r r r �disable_logging� s zMockBucket.disable_loggingc C s d| _ d S rA ry )r Ztarget_bucket_prefixr r r �enable_logging s zMockBucket.enable_loggingc C s di iS )N�Loggingr r r r r �get_logging_config s z MockBucket.get_logging_configc C s dS ri r r6 r r r �get_versioning_status s z MockBucket.get_versioning_statusrK c C s |r| j | S | j | j S d S r )rk r �r rp r) r- r r r �get_acl s zMockBucket.get_aclc C s | j S r �rl r r r r � get_def_acl s zMockBucket.get_def_aclc C s || j v r| j | S dS d S )Nz<Subresource/>�rm )r � subresourcerp r) r- r r r �get_subresource s zMockBucket.get_subresourcec C s g S r r r r r r �get_tags s zMockBucket.get_tagsc C s$ t | |�}|| j|< t� | j|< |S r )r rj r rk )r rp Zmock_keyr r r rq " s zMockBucket.new_keyc C s$ || j vrtj�dd��| j |= d S )N� � Not Found)rj �boto� exception�StorageResponseError)r rp r) r- � mfa_tokenr r r � delete_key( s zMockBucket.delete_keyc C s t �| j�S r )r � itervaluesrj r6 r r r � get_all_keys. s zMockBucket.get_all_keysc C s || j vrd S | j | S r )rj r r r r rr 1 s zMockBucket.get_keyc C s� |pd}g }t � }t�| j�D ]�}|j�|�r |jt|�d � }|rP|�|�} nd} | dkr�t| |jd t|�| d � d�} nt | |jd�} | j|vr |� | j� |� | � q |S )NrK ���r9 r' ) �setr r� rj r � startswithrF �findr r �add�append) r �prefix� delimiter�markerr) �resultZ key_name_set�kZk_name_past_prefix�posZ key_or_prefixr r r �list7 s$ � zMockBucket.listc C s( |rt |�| j|< nt |�| j| j< d S r )r rk r �r � acl_or_strrp r) r- r r r �set_aclP s zMockBucket.set_aclc C s || _ d S r r� r� r r r � set_def_acl[ s zMockBucket.set_def_aclc C s || j |< d S r r� )r r� r rp r) r- r r r �set_subresourceb s zMockBucket.set_subresource)N)r r r r r r( rW rz r{ r} r~ r� r� r� r� rq r� r� rr r� r� r� r� r r r r rh � sH � � � � � � � �rh c @ s e Zd Zdd� Zdd� ZdS )� MockProviderc C s || _ d S r �r] r^ r r r r i s zMockProvider.__init__c C s | j S r r� r r r r �get_provider_namel s z MockProvider.get_provider_nameN)r r r r r� r r r r r� g s r� c @ sl e Zd Zeeeeeeeeeeeeedefdd�Zeeeefdd�Zefdd�Zeefdd �Zefd d �Zd S ) �MockConnection�s3c C s i | _ t|�| _d S r )�bucketsr� r] )r �aws_access_key_id�aws_secret_access_key� is_secure�port�proxy� proxy_port� proxy_user� proxy_pass�host�debug�https_connection_factory�calling_format�pathr] � bucket_classr r r r r s zMockConnection.__init__c C s4 || j v rtj�ddd��t|| d�}|| j |< |S )Ni� ZBucketAlreadyOwnedByYouze<Message>Your previous request to create the named bucket succeeded and you already own it.</Message>)r r )r� r� r� �StorageCreateErrorrh )r � bucket_namer) �locationrH rw Z mock_bucketr r r � create_bucket~ s � z MockConnection.create_bucketc C s& || j vrtj�ddd��| j |= d S )Nr� � NoSuchBucketz!<Message>no such bucket</Message>�r� r� r� r� )r r r) r r r � delete_bucket� s �z MockConnection.delete_bucketc C s$ || j vrtj�ddd��| j | S )Nr� r� r� r� )r r� �validater) r r r rV � s �zMockConnection.get_bucketc C s t �| j�S r )r r� r� r6 r r r �get_all_buckets� s z MockConnection.get_all_bucketsN) r r r r r r� r� rV r� r r r r r� p s � � r� c @ sx e Zd ZdZddeedddfdd�Zdd� Zdd � Zd d � Zd d � Z dd� Z eefdd�Z eeeefdd�Z efdd�Z efdd�Zdd� Zeeeefdd�Zeeefd d �Zeeefd d �Zeeefd d!�Zd"d#� Zeeefd$d%�Zeeefd&d'�Zeeefd(d)�Zefd*d+�Zeefd,d-�Zd.d.eefd/d0�Zeefd1d2�Zeeefd3d4�Z d5d6� Z d7d8� Z d9d:� Z d;d<� Z d=d>� Z!d?d@� Z"dAdB� Z#dCdD� Z$dEdF� Z%dGdH� Z&eefdIdJ�Z'd.eeefdKdL�Z(eeeefdMdN�Z)eeefdOdP�Z*eeeeeeeefdQdR�Z+eeeeeeefdSdT�Z,deeeeeeeef dUdV�Z-eeeeeeeefdWdX�Z.eeeeeeefdYdZ�Z/eeeefd[d\�Z0dS )]�MockBucketStorageUri�/NFc C s� || _ || _|| _|| _| jr<| jr<d| j | j| jf | _n&| jrVd| j | jf | _n d| j | _|| _|ort|�| _t| j�p�t| j�| _ || _ |r�|r�d|||f | _ d S )Nz %s://%s/%sz%s://%s/z%s://) �schemer� � object_name�suppress_consec_slashes�urir- r= � generation�bool�is_version_specific� is_latest�versionless_uri) r r� r� r� r� r� r- r� r� r r r r � s* � ��z MockBucketStorageUri.__init__c C s | j S )z%Returns string representation of URI.�r� r r r r r( � s z MockBucketStorageUri.__repr__c C s t S r )r r r r r � acl_class� s z MockBucketStorageUri.acl_classc C s t j�d�jS )N�aws)r� r] �Provider� canned_aclsr r r r r� � s z MockBucketStorageUri.canned_aclsc C s | � | j| j|�S r )� __class__r� r� )r �new_namer r r �clone_replace_name� s z'MockBucketStorageUri.clone_replace_namec C s> | j |j�� |jj|j| jt|dd �t|dd �t|dd �d�S )Nr- r� r� )r� r� r� r- r� r� )r� r] r� r r r� �getattr)r �keyr r r �clone_replace_key� s �z&MockBucketStorageUri.clone_replace_keyc C s t S r )�mock_connection)r � access_key_id�secret_access_keyr r r �connect� s z MockBucketStorageUri.connectc C s | � � �| j�S r )r� r� r� )r r) r� rH rw r r r r� � s z"MockBucketStorageUri.create_bucketc C s | � � �| j�S r )r� r� r� r6 r r r r� � s z"MockBucketStorageUri.delete_bucketc C s | � � �|� d S r )rV r~ r6 r r r �get_versioning_config� s z*MockBucketStorageUri.get_versioning_configc C s" t t| �t�o | jd up | jd uS r )� issubclass�typer� r- r� r r r r � has_version� s �z MockBucketStorageUri.has_versionc C s | � � �| j� d S r )rV r� r� )r r� r) r- r� r r r r� � s z MockBucketStorageUri.delete_keyc C s | � � �� d S r )rV rz �r r� r) r- r r r rz � s z$MockBucketStorageUri.disable_loggingc C s | � � �|� d S r )rV r{ )r � target_bucket� target_prefixr� r) r- r r r r{ � s z#MockBucketStorageUri.enable_loggingc C s | � � �� S r )rV r} r� r r r r} � s z'MockBucketStorageUri.get_logging_configc C s | j |j kS r r� )r r� r r r �equals� s zMockBucketStorageUri.equalsc C s | � � �| j�S r )rV r� r� r� r r r r� � s z MockBucketStorageUri.get_aclc C s | � � �| j�S r )rV r� r� r� r r r r� � s z MockBucketStorageUri.get_def_aclc C s | � � �|| j�S r )rV r� r� )r r� r� r) r- r r r r� � s z$MockBucketStorageUri.get_subresourcec C s | � � �� S r )r� r� r6 r r r r� s z$MockBucketStorageUri.get_all_bucketsc C s | � � �| �S r )rV r� �r r� r) r r r r� s z!MockBucketStorageUri.get_all_keysrK c C s | � � j||d�S )N)r� r� )rV r� )r r� r� r) � all_versionsr r r � list_bucket s z MockBucketStorageUri.list_bucketc C s | � � �| j�S r )r� rV r� r� r r r rV s z MockBucketStorageUri.get_bucketc C s | � � �| j�S r )rV rr r� r� r r r rr s z MockBucketStorageUri.get_keyc C s dS ri r r r r r � is_file_uri s z MockBucketStorageUri.is_file_uric C s dS rA r r r r r � is_cloud_uri s z!MockBucketStorageUri.is_cloud_uric C s t | j �S r �r� r� r r r r �names_container s z$MockBucketStorageUri.names_containerc C s t | j�S r r� r r r r �names_singleton s z$MockBucketStorageUri.names_singletonc C s dS ri r r r r r �names_directory s z$MockBucketStorageUri.names_directoryc C s t | j �S r )r� r� r r r r �names_provider s z#MockBucketStorageUri.names_providerc C s | � � S r �r� r r r r � names_bucket# s z!MockBucketStorageUri.names_bucketc C s dS ri r r r r r � names_file& s z MockBucketStorageUri.names_filec C s | � � S r r� r r r r � names_object) s z!MockBucketStorageUri.names_objectc C s dS ri r r r r r � is_stream, s z MockBucketStorageUri.is_streamc C s | � � }|�| j�S r )rV rq r� )r r� r) r r r r rq / s z MockBucketStorageUri.new_keyc C s | � � �||� d S r )rV r� �r r� rp r� r) r- r r r r� 3 s z MockBucketStorageUri.set_aclc C s | � � �|� d S r )rV r� r� r r r r� 7 s z MockBucketStorageUri.set_def_aclc C s | � � �||| j� d S r )rV r� r� )r r� r r� r) r- r r r r� ; s z$MockBucketStorageUri.set_subresourcec C s | � � } | j| j||d�S )N)rs rt ru )rV rW r� ) r rt ru rZ rv rw r[ rx r) r? �src_generationr\ r r r rW ? s �z MockBucketStorageUri.copy_keyc C s | � � } | �|� d S r )rq rQ ) r rP r) rG r* r+ rH r rL r� r r r rQ H s z-MockBucketStorageUri.set_contents_from_stringc C s | � � } | j||d�S )N)r) )rq rJ ) r r1 r) rG r* r+ rH r r �rewindrI r� r r r rJ N s z+MockBucketStorageUri.set_contents_from_filec C s t �|� d S r )rY rN ) r r1 r) rG r* r+ rH rL r? r r r r rN U s z-MockBucketStorageUri.set_contents_from_streamc C s | � � } | �|� d S r �rr r3 ) r r1 r) r* r+ r, r- r2 �response_headersr� r r r r3 [ s z)MockBucketStorageUri.get_contents_to_filec C s | � � }|�|�S r r� )r r1 r) r* r+ r- r� r r r �get_contents_to_streamb s z+MockBucketStorageUri.get_contents_to_stream)1r r r �delimr r r( r� r� r� r� r� r� r� r� r� r� rz r{ r} r� r� r� r� r� r� r� rV rr r� r� r� r� r� r� r� r� r� r� rq r� r� r� rW rQ rJ rN r3 r� r r r r r� � s� � � � � � � � � � � � � � � � � � � �r� )�__doc__rO r� �base64r; r �hashlibr � boto.utilsr r r r �boto.s3.prefixr � boto.compatr �objectr r rh r� r� r� r� r r r r �<module> s, ) * vendored/boto/tests/integration/s3/test_cert_verification.py 0000644 00000002774 15025013616 0020450 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on SQS endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.s3 class S3CertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): s3 = True regions = boto.s3.regions() def sample_service_call(self, conn): conn.get_all_buckets() vendored/boto/tests/integration/s3/test_key.py 0000644 00000051744 15025013616 0015542 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for S3 Key """ from tests.unit import unittest import time import random import boto.s3 from boto.compat import six, StringIO, urllib from boto.s3.connection import S3Connection from boto.s3.key import Key from boto.exception import S3ResponseError class S3KeyTest(unittest.TestCase): s3 = True def setUp(self): self.conn = S3Connection() random.seed() self.bucket_name = 'keytest-%d-%d' % ( time.time(), random.randint(1, 99999999)) self.bucket = self.conn.create_bucket(self.bucket_name) def tearDown(self): for key in self.bucket: key.delete() self.bucket.delete() def test_set_contents_from_file_dataloss(self): # Create an empty stringio and write to it. content = "abcde" sfp = StringIO() sfp.write(content) # Try set_contents_from_file() without rewinding sfp k = self.bucket.new_key("k") try: k.set_contents_from_file(sfp) self.fail("forgot to rewind so should fail.") except AttributeError: pass # call with rewind and check if we wrote 5 bytes k.set_contents_from_file(sfp, rewind=True) self.assertEqual(k.size, 5) # check actual contents by getting it. kn = self.bucket.new_key("k") ks = kn.get_contents_as_string().decode('utf-8') self.assertEqual(ks, content) # finally, try with a 0 length string sfp = StringIO() k = self.bucket.new_key("k") k.set_contents_from_file(sfp) self.assertEqual(k.size, 0) # check actual contents by getting it. kn = self.bucket.new_key("k") ks = kn.get_contents_as_string().decode('utf-8') self.assertEqual(ks, "") def test_set_contents_as_file(self): content="01234567890123456789" sfp = StringIO(content) # fp is set at 0 for just opened (for read) files. # set_contents should write full content to key. k = self.bucket.new_key("k") k.set_contents_from_file(sfp) self.assertEqual(k.size, 20) kn = self.bucket.new_key("k") ks = kn.get_contents_as_string().decode('utf-8') self.assertEqual(ks, content) # set fp to 5 and set contents. this should # set "567890123456789" to the key sfp.seek(5) k = self.bucket.new_key("k") k.set_contents_from_file(sfp) self.assertEqual(k.size, 15) kn = self.bucket.new_key("k") ks = kn.get_contents_as_string().decode('utf-8') self.assertEqual(ks, content[5:]) # set fp to 5 and only set 5 bytes. this should # write the value "56789" to the key. sfp.seek(5) k = self.bucket.new_key("k") k.set_contents_from_file(sfp, size=5) self.assertEqual(k.size, 5) self.assertEqual(sfp.tell(), 10) kn = self.bucket.new_key("k") ks = kn.get_contents_as_string().decode('utf-8') self.assertEqual(ks, content[5:10]) def test_set_contents_with_md5(self): content="01234567890123456789" sfp = StringIO(content) # fp is set at 0 for just opened (for read) files. # set_contents should write full content to key. k = self.bucket.new_key("k") good_md5 = k.compute_md5(sfp) k.set_contents_from_file(sfp, md5=good_md5) kn = self.bucket.new_key("k") ks = kn.get_contents_as_string().decode('utf-8') self.assertEqual(ks, content) # set fp to 5 and only set 5 bytes. this should # write the value "56789" to the key. sfp.seek(5) k = self.bucket.new_key("k") good_md5 = k.compute_md5(sfp, size=5) k.set_contents_from_file(sfp, size=5, md5=good_md5) self.assertEqual(sfp.tell(), 10) kn = self.bucket.new_key("k") ks = kn.get_contents_as_string().decode('utf-8') self.assertEqual(ks, content[5:10]) # let's try a wrong md5 by just altering it. k = self.bucket.new_key("k") sfp.seek(0) hexdig, base64 = k.compute_md5(sfp) bad_md5 = (hexdig, base64[3:]) try: k.set_contents_from_file(sfp, md5=bad_md5) self.fail("should fail with bad md5") except S3ResponseError: pass def test_get_contents_with_md5(self): content="01234567890123456789" sfp = StringIO(content) k = self.bucket.new_key("k") k.set_contents_from_file(sfp) kn = self.bucket.new_key("k") s = kn.get_contents_as_string().decode('utf-8') self.assertEqual(kn.md5, k.md5) self.assertEqual(s, content) def test_file_callback(self): def callback(wrote, total): self.my_cb_cnt += 1 self.assertNotEqual(wrote, self.my_cb_last, "called twice with same value") self.my_cb_last = wrote # Zero bytes written => 1 call self.my_cb_cnt = 0 self.my_cb_last = None k = self.bucket.new_key("k") k.BufferSize = 2 sfp = StringIO("") k.set_contents_from_file(sfp, cb=callback, num_cb=10) self.assertEqual(self.my_cb_cnt, 1) self.assertEqual(self.my_cb_last, 0) sfp.close() # Read back zero bytes => 1 call self.my_cb_cnt = 0 self.my_cb_last = None s = k.get_contents_as_string(cb=callback) self.assertEqual(self.my_cb_cnt, 1) self.assertEqual(self.my_cb_last, 0) content="01234567890123456789" sfp = StringIO(content) # expect 2 calls due start/finish self.my_cb_cnt = 0 self.my_cb_last = None k = self.bucket.new_key("k") k.set_contents_from_file(sfp, cb=callback, num_cb=10) self.assertEqual(self.my_cb_cnt, 2) self.assertEqual(self.my_cb_last, 20) # Read back all bytes => 2 calls self.my_cb_cnt = 0 self.my_cb_last = None s = k.get_contents_as_string(cb=callback).decode('utf-8') self.assertEqual(self.my_cb_cnt, 2) self.assertEqual(self.my_cb_last, 20) self.assertEqual(s, content) # rewind sfp and try upload again. -1 should call # for every read/write so that should make 11 when bs=2 sfp.seek(0) self.my_cb_cnt = 0 self.my_cb_last = None k = self.bucket.new_key("k") k.BufferSize = 2 k.set_contents_from_file(sfp, cb=callback, num_cb=-1) self.assertEqual(self.my_cb_cnt, 11) self.assertEqual(self.my_cb_last, 20) # Read back all bytes => 11 calls self.my_cb_cnt = 0 self.my_cb_last = None s = k.get_contents_as_string(cb=callback, num_cb=-1).decode('utf-8') self.assertEqual(self.my_cb_cnt, 11) self.assertEqual(self.my_cb_last, 20) self.assertEqual(s, content) # no more than 1 times => 2 times # last time always 20 bytes sfp.seek(0) self.my_cb_cnt = 0 self.my_cb_last = None k = self.bucket.new_key("k") k.BufferSize = 2 k.set_contents_from_file(sfp, cb=callback, num_cb=1) self.assertTrue(self.my_cb_cnt <= 2) self.assertEqual(self.my_cb_last, 20) # no more than 1 times => 2 times self.my_cb_cnt = 0 self.my_cb_last = None s = k.get_contents_as_string(cb=callback, num_cb=1).decode('utf-8') self.assertTrue(self.my_cb_cnt <= 2) self.assertEqual(self.my_cb_last, 20) self.assertEqual(s, content) # no more than 2 times # last time always 20 bytes sfp.seek(0) self.my_cb_cnt = 0 self.my_cb_last = None k = self.bucket.new_key("k") k.BufferSize = 2 k.set_contents_from_file(sfp, cb=callback, num_cb=2) self.assertTrue(self.my_cb_cnt <= 2) self.assertEqual(self.my_cb_last, 20) # no more than 2 times self.my_cb_cnt = 0 self.my_cb_last = None s = k.get_contents_as_string(cb=callback, num_cb=2).decode('utf-8') self.assertTrue(self.my_cb_cnt <= 2) self.assertEqual(self.my_cb_last, 20) self.assertEqual(s, content) # no more than 3 times # last time always 20 bytes sfp.seek(0) self.my_cb_cnt = 0 self.my_cb_last = None k = self.bucket.new_key("k") k.BufferSize = 2 k.set_contents_from_file(sfp, cb=callback, num_cb=3) self.assertTrue(self.my_cb_cnt <= 3) self.assertEqual(self.my_cb_last, 20) # no more than 3 times self.my_cb_cnt = 0 self.my_cb_last = None s = k.get_contents_as_string(cb=callback, num_cb=3).decode('utf-8') self.assertTrue(self.my_cb_cnt <= 3) self.assertEqual(self.my_cb_last, 20) self.assertEqual(s, content) # no more than 4 times # last time always 20 bytes sfp.seek(0) self.my_cb_cnt = 0 self.my_cb_last = None k = self.bucket.new_key("k") k.BufferSize = 2 k.set_contents_from_file(sfp, cb=callback, num_cb=4) self.assertTrue(self.my_cb_cnt <= 4) self.assertEqual(self.my_cb_last, 20) # no more than 4 times self.my_cb_cnt = 0 self.my_cb_last = None s = k.get_contents_as_string(cb=callback, num_cb=4).decode('utf-8') self.assertTrue(self.my_cb_cnt <= 4) self.assertEqual(self.my_cb_last, 20) self.assertEqual(s, content) # no more than 6 times # last time always 20 bytes sfp.seek(0) self.my_cb_cnt = 0 self.my_cb_last = None k = self.bucket.new_key("k") k.BufferSize = 2 k.set_contents_from_file(sfp, cb=callback, num_cb=6) self.assertTrue(self.my_cb_cnt <= 6) self.assertEqual(self.my_cb_last, 20) # no more than 6 times self.my_cb_cnt = 0 self.my_cb_last = None s = k.get_contents_as_string(cb=callback, num_cb=6).decode('utf-8') self.assertTrue(self.my_cb_cnt <= 6) self.assertEqual(self.my_cb_last, 20) self.assertEqual(s, content) # no more than 10 times # last time always 20 bytes sfp.seek(0) self.my_cb_cnt = 0 self.my_cb_last = None k = self.bucket.new_key("k") k.BufferSize = 2 k.set_contents_from_file(sfp, cb=callback, num_cb=10) self.assertTrue(self.my_cb_cnt <= 10) self.assertEqual(self.my_cb_last, 20) # no more than 10 times self.my_cb_cnt = 0 self.my_cb_last = None s = k.get_contents_as_string(cb=callback, num_cb=10).decode('utf-8') self.assertTrue(self.my_cb_cnt <= 10) self.assertEqual(self.my_cb_last, 20) self.assertEqual(s, content) # no more than 1000 times # last time always 20 bytes sfp.seek(0) self.my_cb_cnt = 0 self.my_cb_last = None k = self.bucket.new_key("k") k.BufferSize = 2 k.set_contents_from_file(sfp, cb=callback, num_cb=1000) self.assertTrue(self.my_cb_cnt <= 1000) self.assertEqual(self.my_cb_last, 20) # no more than 1000 times self.my_cb_cnt = 0 self.my_cb_last = None s = k.get_contents_as_string(cb=callback, num_cb=1000).decode('utf-8') self.assertTrue(self.my_cb_cnt <= 1000) self.assertEqual(self.my_cb_last, 20) self.assertEqual(s, content) def test_website_redirects(self): self.bucket.configure_website('index.html') key = self.bucket.new_key('redirect-key') self.assertTrue(key.set_redirect('http://www.amazon.com/')) self.assertEqual(key.get_redirect(), 'http://www.amazon.com/') self.assertTrue(key.set_redirect('http://aws.amazon.com/')) self.assertEqual(key.get_redirect(), 'http://aws.amazon.com/') def test_website_redirect_none_configured(self): key = self.bucket.new_key('redirect-key') key.set_contents_from_string('') self.assertEqual(key.get_redirect(), None) def test_website_redirect_with_bad_value(self): self.bucket.configure_website('index.html') key = self.bucket.new_key('redirect-key') with self.assertRaises(key.provider.storage_response_error): # Must start with a / or http key.set_redirect('ftp://ftp.example.org') with self.assertRaises(key.provider.storage_response_error): # Must start with a / or http key.set_redirect('') def test_setting_date(self): key = self.bucket.new_key('test_date') # This should actually set x-amz-meta-date & not fail miserably. key.set_metadata('date', '20130524T155935Z') key.set_contents_from_string('Some text here.') check = self.bucket.get_key('test_date') self.assertEqual(check.get_metadata('date'), u'20130524T155935Z') self.assertTrue('x-amz-meta-date' in check._get_remote_metadata()) def test_header_casing(self): key = self.bucket.new_key('test_header_case') # Using anything but CamelCase on ``Content-Type`` or ``Content-MD5`` # used to cause a signature error (when using ``s3`` for signing). key.set_metadata('Content-type', 'application/json') key.set_metadata('Content-md5', 'XmUKnus7svY1frWsVskxXg==') key.set_contents_from_string('{"abc": 123}') check = self.bucket.get_key('test_header_case') self.assertEqual(check.content_type, 'application/json') def test_header_encoding(self): key = self.bucket.new_key('test_header_encoding') key.set_metadata('Cache-control', u'public, max-age=500') key.set_metadata('Test-Plus', u'A plus (+)') key.set_metadata('Content-disposition', u'filename=Schöne Zeit.txt') key.set_metadata('Content-Encoding', 'gzip') key.set_metadata('Content-Language', 'de') key.set_metadata('Content-Type', 'application/pdf') self.assertEqual(key.content_type, 'application/pdf') key.set_metadata('X-Robots-Tag', 'all') key.set_metadata('Expires', u'Thu, 01 Dec 1994 16:00:00 GMT') key.set_contents_from_string('foo') check = self.bucket.get_key('test_header_encoding') remote_metadata = check._get_remote_metadata() # TODO: investigate whether encoding ' ' as '%20' makes sense self.assertIn( check.cache_control, ('public,%20max-age=500', 'public, max-age=500') ) self.assertIn(remote_metadata['cache-control'], ('public,%20max-age=500', 'public, max-age=500')) self.assertEqual(check.get_metadata('test-plus'), 'A plus (+)') self.assertEqual(check.content_disposition, 'filename=Sch%C3%B6ne Zeit.txt') self.assertEqual(remote_metadata['content-disposition'], 'filename=Sch%C3%B6ne Zeit.txt') self.assertEqual(check.content_encoding, 'gzip') self.assertEqual(remote_metadata['content-encoding'], 'gzip') self.assertEqual(check.content_language, 'de') self.assertEqual(remote_metadata['content-language'], 'de') self.assertEqual(check.content_type, 'application/pdf') self.assertEqual(remote_metadata['content-type'], 'application/pdf') self.assertEqual(check.x_robots_tag, 'all') self.assertEqual(remote_metadata['x-robots-tag'], 'all') self.assertEqual(check.expires, 'Thu, 01 Dec 1994 16:00:00 GMT') self.assertEqual(remote_metadata['expires'], 'Thu, 01 Dec 1994 16:00:00 GMT') expected = u'filename=Schöne Zeit.txt' if six.PY2: # Newer versions of python default to unicode strings, but python 2 # requires encoding to UTF-8 to compare the two properly expected = expected.encode('utf-8') self.assertEqual( urllib.parse.unquote(check.content_disposition), expected ) def test_set_contents_with_sse_c(self): content="01234567890123456789" # the plain text of customer key is "01testKeyToSSEC!" header = { "x-amz-server-side-encryption-customer-algorithm" : "AES256", "x-amz-server-side-encryption-customer-key" : "MAAxAHQAZQBzAHQASwBlAHkAVABvAFMAUwBFAEMAIQA=", "x-amz-server-side-encryption-customer-key-MD5" : "fUgCZDDh6bfEMuP2bN38mg==", } # upload and download content with AWS specified headers k = self.bucket.new_key("testkey_for_sse_c") k.set_contents_from_string(content, headers=header) kn = self.bucket.new_key("testkey_for_sse_c") ks = kn.get_contents_as_string(headers=header) self.assertEqual(ks, content.encode('utf-8')) class S3KeySigV4Test(unittest.TestCase): def setUp(self): self.conn = boto.s3.connect_to_region('eu-central-1') self.bucket_name = 'boto-sigv4-key-%d' % int(time.time()) self.bucket = self.conn.create_bucket(self.bucket_name, location='eu-central-1') def tearDown(self): for key in self.bucket: key.delete() self.bucket.delete() def test_put_get_with_non_string_headers_key(self): k = Key(self.bucket) k.key = 'foobar' body = 'This is a test of S3' # A content-length header will be added to this request since it # has a body. k.set_contents_from_string(body) # Set a header that has an integer. This checks for a bug where # the sigv4 signer assumes that all of the headers are strings. headers = {'Content-Length': 0} from_s3_key = self.bucket.get_key('foobar', headers=headers) self.assertEqual(from_s3_key.get_contents_as_string().decode('utf-8'), body) def test_head_put_get_with_non_ascii_key(self): k = Key(self.bucket) k.key = u'''pt-Olá_ch-你好_ko-안녕_ru-Здравствуйте%20,.<>~`!@#$%^&()_-+='"''' body = 'This is a test of S3' k.set_contents_from_string(body) from_s3_key = self.bucket.get_key(k.key, validate=True) self.assertEqual(from_s3_key.get_contents_as_string().decode('utf-8'), body) keys = self.bucket.get_all_keys(prefix=k.key, max_keys=1) self.assertEqual(1, len(keys)) class S3KeyVersionCopyTest(unittest.TestCase): def setUp(self): self.conn = S3Connection() self.bucket_name = 'boto-key-version-copy-%d' % int(time.time()) self.bucket = self.conn.create_bucket(self.bucket_name) self.bucket.configure_versioning(True) def tearDown(self): for key in self.bucket.list_versions(): key.delete() self.bucket.delete() def test_key_overwrite_and_copy(self): first_content = b"abcdefghijklm" second_content = b"nopqrstuvwxyz" k = Key(self.bucket, 'testkey') k.set_contents_from_string(first_content) # Wait for S3's eventual consistency (may not be necessary) while self.bucket.get_key('testkey') is None: time.sleep(5) # Get the first version_id first_key = self.bucket.get_key('testkey') first_version_id = first_key.version_id # Overwrite the key k = Key(self.bucket, 'testkey') k.set_contents_from_string(second_content) # Wait for eventual consistency while True: second_key = self.bucket.get_key('testkey') if second_key is None or second_key.version_id == first_version_id: time.sleep(5) else: break # Copy first key (no longer the current version) to a new key source_key = self.bucket.get_key('testkey', version_id=first_version_id) source_key.copy(self.bucket, 'copiedkey') while self.bucket.get_key('copiedkey') is None: time.sleep(5) copied_key = self.bucket.get_key('copiedkey') copied_key_contents = copied_key.get_contents_as_string() self.assertEqual(first_content, copied_key_contents) vendored/boto/tests/integration/s3/test_encryption.py 0000644 00000007230 15025013616 0017133 0 ustar 00 # Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2010, Eucalyptus Systems, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the S3 Encryption """ import unittest import time from boto.s3.connection import S3Connection from boto.exception import S3ResponseError json_policy = """{ "Version":"2008-10-17", "Id":"PutObjPolicy", "Statement":[{ "Sid":"DenyUnEncryptedObjectUploads", "Effect":"Deny", "Principal":{ "AWS":"*" }, "Action":"s3:PutObject", "Resource":"arn:aws:s3:::%s/*", "Condition":{ "StringNotEquals":{ "s3:x-amz-server-side-encryption":"AES256" } } } ] }""" class S3EncryptionTest (unittest.TestCase): s3 = True def test_1_versions(self): print('--- running S3Encryption tests ---') c = S3Connection() # create a new, empty bucket bucket_name = 'encryption-%d' % int(time.time()) bucket = c.create_bucket(bucket_name) # now try a get_bucket call and see if it's really there bucket = c.get_bucket(bucket_name) # create an unencrypted key k = bucket.new_key('foobar') s1 = 'This is unencrypted data' s2 = 'This is encrypted data' k.set_contents_from_string(s1) time.sleep(5) # now get the contents from s3 o = k.get_contents_as_string().decode('utf-8') # check to make sure content read from s3 is identical to original assert o == s1 # now overwrite that same key with encrypted data k.set_contents_from_string(s2, encrypt_key=True) time.sleep(5) # now retrieve the contents as a string and compare o = k.get_contents_as_string().decode('utf-8') assert o == s2 # now set bucket policy to require encrypted objects bucket.set_policy(json_policy % bucket.name) time.sleep(5) # now try to write unencrypted key write_failed = False try: k.set_contents_from_string(s1) except S3ResponseError: write_failed = True assert write_failed # now try to write unencrypted key write_failed = False try: k.set_contents_from_string(s1, encrypt_key=True) except S3ResponseError: write_failed = True assert not write_failed # Now do regular delete k.delete() time.sleep(5) # now delete bucket bucket.delete() print('--- tests completed ---') vendored/boto/tests/integration/s3/test_connect_to_region.py 0000644 00000006350 15025013616 0020441 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2014 Steven Richards <sbrichards@mit.edu> # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Unit test for passing in 'host' parameter and overriding the region See issue: #2522 """ from tests.compat import unittest from boto.s3.connection import S3Connection from boto.s3 import connect_to_region class S3SpecifyHost(unittest.TestCase): s3 = True def testWithNonAWSHost(self): connect_args = dict({'host':'www.not-a-website.com'}) connection = connect_to_region('us-east-1', **connect_args) self.assertEqual('www.not-a-website.com', connection.host) self.assertIsInstance(connection, S3Connection) def testSuccessWithHostOverrideRegion(self): connect_args = dict({'host':'s3.amazonaws.com'}) connection = connect_to_region('us-west-2', **connect_args) self.assertEqual('s3.amazonaws.com', connection.host) self.assertIsInstance(connection, S3Connection) def testSuccessWithDefaultUSWest1(self): connection = connect_to_region('us-west-2') self.assertEqual('s3-us-west-2.amazonaws.com', connection.host) self.assertIsInstance(connection, S3Connection) def testSuccessWithDefaultUSEast1(self): connection = connect_to_region('us-east-1') self.assertEqual('s3.amazonaws.com', connection.host) self.assertIsInstance(connection, S3Connection) def testSuccessWithDefaultEUCentral1(self): connection = connect_to_region('eu-central-1') self.assertEqual('s3.eu-central-1.amazonaws.com', connection.host) self.assertIsInstance(connection, S3Connection) def testDefaultWithInvalidHost(self): connect_args = dict({'host':''}) connection = connect_to_region('us-west-2', **connect_args) self.assertEqual('s3-us-west-2.amazonaws.com', connection.host) self.assertIsInstance(connection, S3Connection) def testDefaultWithInvalidHostNone(self): connect_args = dict({'host':None}) connection = connect_to_region('us-east-1', **connect_args) self.assertEqual('s3.amazonaws.com', connection.host) self.assertIsInstance(connection, S3Connection) def tearDown(self): self = connection = connect_args = None vendored/boto/tests/integration/s3/test_https_cert_validation.py 0000644 00000014034 15025013616 0021332 0 ustar 00 # Copyright 2011 Google Inc. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests to validate correct validation of SSL server certificates. Note that this test assumes two external dependencies are available: - A http proxy, which by default is assumed to be at host 'cache' and port 3128. This can be overridden with environment variables PROXY_HOST and PROXY_PORT, respectively. - An ssl-enabled web server that will return a valid certificate signed by one of the bundled CAs, and which can be reached by an alternate hostname that does not match the CN in that certificate. By default, this test uses host 'www' (without fully qualified domain). This can be overridden with environment variable INVALID_HOSTNAME_HOST. If no suitable host is already available, such a mapping can be established by temporarily adding an IP address for, say, www.google.com or www.amazon.com to /etc/hosts. """ import os import ssl import unittest import mock from nose.plugins.attrib import attr import boto from boto.pyami.config import Config from boto import exception, https_connection from boto.gs.connection import GSConnection from boto.s3.connection import S3Connection # File 'other_cacerts.txt' contains a valid CA certificate of a CA that is used # by neither S3 nor Google Cloud Storage. Validation against this CA cert should # result in a certificate error. DEFAULT_CA_CERTS_FILE = os.path.join( os.path.dirname(os.path.abspath(__file__ )), 'other_cacerts.txt') PROXY_HOST = os.environ.get('PROXY_HOST', 'cache') PROXY_PORT = os.environ.get('PROXY_PORT', '3128') # This test assumes that this host returns a certificate signed by one of the # trusted CAs, but with a Common Name that won't match host name 'www' (i.e., # the server should return a certificate with CN 'www.<somedomain>.com'). INVALID_HOSTNAME_HOST = os.environ.get('INVALID_HOSTNAME_HOST', 'www') @attr('notdefault', 'ssl') class CertValidationTest(unittest.TestCase): def setUp(self): self.config = Config() # Enable https_validate_certificates. self.config.add_section('Boto') self.config.setbool('Boto', 'https_validate_certificates', True) # Set up bogus credentials so that the auth module is willing to go # ahead and make a request; the request should fail with a service-level # error if it does get to the service (S3 or GS). self.config.add_section('Credentials') self.config.set('Credentials', 'gs_access_key_id', 'xyz') self.config.set('Credentials', 'gs_secret_access_key', 'xyz') self.config.set('Credentials', 'aws_access_key_id', 'xyz') self.config.set('Credentials', 'aws_secret_access_key', 'xyz') self._config_patch = mock.patch('boto.config', self.config) self._config_patch.start() def tearDown(self): self._config_patch.stop() def enableProxy(self): self.config.set('Boto', 'proxy', PROXY_HOST) self.config.set('Boto', 'proxy_port', PROXY_PORT) def assertConnectionThrows(self, connection_class, error): conn = connection_class('fake_id', 'fake_secret') self.assertRaises(error, conn.get_all_buckets) def do_test_valid_cert(self): # When connecting to actual servers with bundled root certificates, no # cert errors should be thrown; instead we will get "invalid # credentials" errors since the config used does not contain any # credentials. self.assertConnectionThrows(S3Connection, exception.S3ResponseError) self.assertConnectionThrows(GSConnection, exception.GSResponseError) def test_valid_cert(self): self.do_test_valid_cert() def test_valid_cert_with_proxy(self): self.enableProxy() self.do_test_valid_cert() def do_test_invalid_signature(self): self.config.set('Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) self.assertConnectionThrows(S3Connection, ssl.SSLError) self.assertConnectionThrows(GSConnection, ssl.SSLError) def test_invalid_signature(self): self.do_test_invalid_signature() def test_invalid_signature_with_proxy(self): self.enableProxy() self.do_test_invalid_signature() def do_test_invalid_host(self): self.config.set('Credentials', 'gs_host', INVALID_HOSTNAME_HOST) self.config.set('Credentials', 's3_host', INVALID_HOSTNAME_HOST) self.assertConnectionThrows(S3Connection, ssl.SSLError) self.assertConnectionThrows(GSConnection, ssl.SSLError) def do_test_invalid_host(self): self.config.set('Credentials', 'gs_host', INVALID_HOSTNAME_HOST) self.config.set('Credentials', 's3_host', INVALID_HOSTNAME_HOST) self.assertConnectionThrows( S3Connection, https_connection.InvalidCertificateException) self.assertConnectionThrows( GSConnection, https_connection.InvalidCertificateException) def test_invalid_host(self): self.do_test_invalid_host() def test_invalid_host_with_proxy(self): self.enableProxy() self.do_test_invalid_host() vendored/boto/tests/integration/s3/other_cacerts.txt 0000644 00000006725 15025013616 0016726 0 ustar 00 # Certifcate Authority certificates for validating SSL connections. # # This file contains PEM format certificates generated from # http://mxr.mozilla.org/seamonkey/source/security/nss/lib/ckfw/builtins/certdata.txt # # ***** BEGIN LICENSE BLOCK ***** # Version: MPL 1.1/GPL 2.0/LGPL 2.1 # # The contents of this file are subject to the Mozilla Public License Version # 1.1 (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # http://www.mozilla.org/MPL/ # # Software distributed under the License is distributed on an "AS IS" basis, # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License # for the specific language governing rights and limitations under the # License. # # The Original Code is the Netscape security libraries. # # The Initial Developer of the Original Code is # Netscape Communications Corporation. # Portions created by the Initial Developer are Copyright (C) 1994-2000 # the Initial Developer. All Rights Reserved. # # Contributor(s): # # Alternatively, the contents of this file may be used under the terms of # either the GNU General Public License Version 2 or later (the "GPL"), or # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), # in which case the provisions of the GPL or the LGPL are applicable instead # of those above. If you wish to allow use of your version of this file only # under the terms of either the GPL or the LGPL, and not to allow others to # use your version of this file under the terms of the MPL, indicate your # decision by deleting the provisions above and replace them with the notice # and other provisions required by the GPL or the LGPL. If you do not delete # the provisions above, a recipient may use your version of this file under # the terms of any one of the MPL, the GPL or the LGPL. # # ***** END LICENSE BLOCK ***** Comodo CA Limited, CN=Trusted Certificate Services ================================================== -----BEGIN CERTIFICATE----- MIIEQzCCAyugAwIBAgIBATANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJHQjEb MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDElMCMGA1UEAwwcVHJ1c3RlZCBDZXJ0 aWZpY2F0ZSBTZXJ2aWNlczAeFw0wNDAxMDEwMDAwMDBaFw0yODEyMzEyMzU5NTla MH8xCzAJBgNVBAYTAkdCMRswGQYDVQQIDBJHcmVhdGVyIE1hbmNoZXN0ZXIxEDAO BgNVBAcMB1NhbGZvcmQxGjAYBgNVBAoMEUNvbW9kbyBDQSBMaW1pdGVkMSUwIwYD VQQDDBxUcnVzdGVkIENlcnRpZmljYXRlIFNlcnZpY2VzMIIBIjANBgkqhkiG9w0B AQEFAAOCAQ8AMIIBCgKCAQEA33FvNlhTWvI2VFeAxHQIIO0Yfyod5jWaHiWsnOWW fnJSoBVC21ndZHoa0Lh73TkVvFVIxO06AOoxEbrycXQaZ7jPM8yoMa+j49d/vzMt TGo87IvDktJTdyR0nAducPy9C1t2ul/y/9c3S0pgePfw+spwtOpZqqPOSC+pw7IL fhdyFgymBwwbOM/JYrc/oJOlh0Hyt3BAd9i+FHzjqMB6juljatEPmsbS9Is6FARW 1O24zG71++IsWL1/T2sr92AkWCTOJu80kTrV44HQsvAEAtdbtz6SrGsSivnkBbA7 kUlcsutT6vifR4buv5XAwAaf0lteERv0xwQ1KdJVXOTt6wIDAQABo4HJMIHGMB0G A1UdDgQWBBTFe1i97doladL3WRaoszLAeydb9DAOBgNVHQ8BAf8EBAMCAQYwDwYD VR0TAQH/BAUwAwEB/zCBgwYDVR0fBHwwejA8oDqgOIY2aHR0cDovL2NybC5jb21v ZG9jYS5jb20vVHJ1c3RlZENlcnRpZmljYXRlU2VydmljZXMuY3JsMDqgOKA2hjRo dHRwOi8vY3JsLmNvbW9kby5uZXQvVHJ1c3RlZENlcnRpZmljYXRlU2VydmljZXMu Y3JsMA0GCSqGSIb3DQEBBQUAA4IBAQDIk4E7ibSvuIQSTI3S8NtwuleGFTQQuS9/ HrCoiWChisJ3DFBKmwCL2Iv0QeLQg4pKHBQGsKNoBXAxMKdTmw7pSqBYaWcOrp32 pSxBvzwGa+RZzG0Q8ZZvH9/0BAKkn0U+yNj6NkZEUD+Cl5EfKNsYEYwq5GWDVxIS jBc/lDb+XbDABHcTuPQV1T84zJQ6VdCsmPW6AF/ghhmBeC8owH7TzEIK9a5QoNE+ xqFx7D+gIIxmOom0jtTYsU0lR+4viMi14QVFwL4Ucd56/Y57fU0IlqUSc/Atyjcn dBInTMu2l+nZrghtWjlA3QVHdWpaIbOjGM9O9y5Xt5hwXsjEeLBi -----END CERTIFICATE----- vendored/boto/tests/integration/s3/test_multipart.py 0000644 00000020330 15025013616 0016756 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2011 Mitch Garnaat http://garnaat.org/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the S3 MultiPartUpload """ # Note: # Multipart uploads require at least one part. If you upload # multiple parts then all parts except the last part has to be # bigger than 5M. Hence we just use 1 part so we can keep # things small and still test logic. import os import unittest import time from boto.compat import StringIO import mock import boto from boto.s3.connection import S3Connection class S3MultiPartUploadTest(unittest.TestCase): s3 = True def setUp(self): self.conn = S3Connection(is_secure=False) self.bucket_name = 'multipart-%d' % int(time.time()) self.bucket = self.conn.create_bucket(self.bucket_name) def tearDown(self): for key in self.bucket: key.delete() self.bucket.delete() def test_abort(self): key_name = u"テスト" mpu = self.bucket.initiate_multipart_upload(key_name) mpu.cancel_upload() def test_complete_ascii(self): key_name = "test" mpu = self.bucket.initiate_multipart_upload(key_name) fp = StringIO("small file") mpu.upload_part_from_file(fp, part_num=1) fp.close() cmpu = mpu.complete_upload() self.assertEqual(cmpu.key_name, key_name) self.assertNotEqual(cmpu.etag, None) def test_complete_japanese(self): key_name = u"テスト" mpu = self.bucket.initiate_multipart_upload(key_name) fp = StringIO("small file") mpu.upload_part_from_file(fp, part_num=1) fp.close() cmpu = mpu.complete_upload() self.assertEqual(cmpu.key_name, key_name) self.assertNotEqual(cmpu.etag, None) def test_list_japanese(self): key_name = u"テスト" mpu = self.bucket.initiate_multipart_upload(key_name) rs = self.bucket.list_multipart_uploads() # New bucket, so only one upload expected lmpu = next(iter(rs)) self.assertEqual(lmpu.id, mpu.id) self.assertEqual(lmpu.key_name, key_name) # Abort using the one returned in the list lmpu.cancel_upload() def test_list_multipart_uploads(self): key_name = u"テスト" mpus = [] mpus.append(self.bucket.initiate_multipart_upload(key_name)) mpus.append(self.bucket.initiate_multipart_upload(key_name)) rs = self.bucket.list_multipart_uploads() # uploads (for a key) are returned in time initiated asc order for lmpu in rs: ompu = mpus.pop(0) self.assertEqual(lmpu.key_name, ompu.key_name) self.assertEqual(lmpu.id, ompu.id) self.assertEqual(0, len(mpus)) def test_get_all_multipart_uploads(self): key1 = 'a' key2 = 'b/c' mpu1 = self.bucket.initiate_multipart_upload(key1) mpu2 = self.bucket.initiate_multipart_upload(key2) rs = self.bucket.get_all_multipart_uploads(prefix='b/', delimiter='/') for lmpu in rs: # only expect upload for key2 (mpu2) returned self.assertEqual(lmpu.key_name, mpu2.key_name) self.assertEqual(lmpu.id, mpu2.id) def test_four_part_file(self): key_name = "k" contents = "01234567890123456789" sfp = StringIO(contents) # upload 20 bytes in 4 parts of 5 bytes each mpu = self.bucket.initiate_multipart_upload(key_name) mpu.upload_part_from_file(sfp, part_num=1, size=5) mpu.upload_part_from_file(sfp, part_num=2, size=5) mpu.upload_part_from_file(sfp, part_num=3, size=5) mpu.upload_part_from_file(sfp, part_num=4, size=5) sfp.close() etags = {} pn = 0 for part in mpu: pn += 1 self.assertEqual(5, part.size) etags[pn] = part.etag self.assertEqual(pn, 4) # etags for 01234 self.assertEqual(etags[1], etags[3]) # etags for 56789 self.assertEqual(etags[2], etags[4]) # etag 01234 != etag 56789 self.assertNotEqual(etags[1], etags[2]) # parts are too small to compete as each part must # be a min of 5MB so so we'll assume that is enough # testing and abort the upload. mpu.cancel_upload() # mpu.upload_part_from_file() now returns the uploaded part # which makes the etag available. Confirm the etag is # available and equal to the etag returned by the parts list. def test_etag_of_parts(self): key_name = "etagtest" mpu = self.bucket.initiate_multipart_upload(key_name) fp = StringIO("small file") # upload 2 parts and save each part uparts = [] uparts.append(mpu.upload_part_from_file(fp, part_num=1, size=5)) uparts.append(mpu.upload_part_from_file(fp, part_num=2)) fp.close() # compare uploaded parts etag to listed parts pn = 0 for lpart in mpu: self.assertEqual(uparts[pn].etag, lpart.etag) pn += 1 # Can't complete 2 small parts so just clean up. mpu.cancel_upload() class S3MultiPartUploadSigV4Test(unittest.TestCase): s3 = True def setUp(self): self.env_patch = mock.patch('os.environ', {'S3_USE_SIGV4': True}) self.env_patch.start() self.conn = boto.s3.connect_to_region('us-west-2') self.bucket_name = 'multipart-%d' % int(time.time()) self.bucket = self.conn.create_bucket(self.bucket_name, location='us-west-2') def tearDown(self): for key in self.bucket: key.delete() self.bucket.delete() self.env_patch.stop() def test_initiate_multipart(self): key_name = "multipart" multipart_upload = self.bucket.initiate_multipart_upload(key_name) multipart_uploads = self.bucket.get_all_multipart_uploads() for upload in multipart_uploads: # Check that the multipart upload was created. self.assertEqual(upload.key_name, multipart_upload.key_name) self.assertEqual(upload.id, multipart_upload.id) multipart_upload.cancel_upload() def test_upload_part_by_size(self): key_name = "k" contents = "01234567890123456789" sfp = StringIO(contents) # upload 20 bytes in 4 parts of 5 bytes each mpu = self.bucket.initiate_multipart_upload(key_name) mpu.upload_part_from_file(sfp, part_num=1, size=5) mpu.upload_part_from_file(sfp, part_num=2, size=5) mpu.upload_part_from_file(sfp, part_num=3, size=5) mpu.upload_part_from_file(sfp, part_num=4, size=5) sfp.close() etags = {} pn = 0 for part in mpu: pn += 1 self.assertEqual(5, part.size) etags[pn] = part.etag self.assertEqual(pn, 4) # etags for 01234 self.assertEqual(etags[1], etags[3]) # etags for 56789 self.assertEqual(etags[2], etags[4]) # etag 01234 != etag 56789 self.assertNotEqual(etags[1], etags[2]) # parts are too small to complete as each part must # be a min of 5MB so so we'll assume that is enough # testing and abort the upload. mpu.cancel_upload() vendored/boto/tests/integration/s3/test_versioning.py 0000644 00000014112 15025013616 0017121 0 ustar 00 # Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2010, Eucalyptus Systems, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the S3 Versioning. """ import unittest import time from boto.s3.connection import S3Connection from boto.exception import S3ResponseError from boto.s3.deletemarker import DeleteMarker from boto.compat import six class S3VersionTest (unittest.TestCase): def setUp(self): self.conn = S3Connection() self.bucket_name = 'version-%d' % int(time.time()) self.bucket = self.conn.create_bucket(self.bucket_name) def tearDown(self): for k in self.bucket.list_versions(): self.bucket.delete_key(k.name, version_id=k.version_id) self.bucket.delete() def test_1_versions(self): # check versioning off d = self.bucket.get_versioning_status() self.assertFalse('Versioning' in d) # enable versioning self.bucket.configure_versioning(versioning=True) d = self.bucket.get_versioning_status() self.assertEqual('Enabled', d['Versioning']) # create a new key in the versioned bucket k = self.bucket.new_key("foobar") s1 = 'This is v1' k.set_contents_from_string(s1) # remember the version id of this object v1 = k.version_id # now get the contents from s3 o1 = k.get_contents_as_string().decode('utf-8') # check to make sure content read from k is identical to original self.assertEqual(s1, o1) # now overwrite that same key with new data s2 = 'This is v2' k.set_contents_from_string(s2) v2 = k.version_id # now retrieve latest contents as a string and compare k2 = self.bucket.new_key("foobar") o2 = k2.get_contents_as_string().decode('utf-8') self.assertEqual(s2, o2) # next retrieve explicit versions and compare o1 = k.get_contents_as_string(version_id=v1).decode('utf-8') o2 = k.get_contents_as_string(version_id=v2).decode('utf-8') self.assertEqual(s1, o1) self.assertEqual(s2, o2) # Now list all versions and compare to what we have rs = self.bucket.get_all_versions() self.assertEqual(v2, rs[0].version_id) self.assertEqual(v1, rs[1].version_id) # Now do a regular list command and make sure only the new key shows up rs = self.bucket.get_all_keys() self.assertEqual(1, len(rs)) # Now do regular delete self.bucket.delete_key('foobar') # Now list versions and make sure old versions are there # plus the DeleteMarker which is latest. rs = self.bucket.get_all_versions() self.assertEqual(3, len(rs)) self.assertTrue(isinstance(rs[0], DeleteMarker)) # Now delete v1 of the key self.bucket.delete_key('foobar', version_id=v1) # Now list versions again and make sure v1 is not there rs = self.bucket.get_all_versions() versions = [k.version_id for k in rs] self.assertTrue(v1 not in versions) self.assertTrue(v2 in versions) # Now suspend Versioning on the bucket self.bucket.configure_versioning(False) # Allow time for the change to fully propagate. time.sleep(3) d = self.bucket.get_versioning_status() self.assertEqual('Suspended', d['Versioning']) def test_latest_version(self): self.bucket.configure_versioning(versioning=True) # add v1 of an object key_name = "key" kv1 = self.bucket.new_key(key_name) kv1.set_contents_from_string("v1") # read list which should contain latest v1 listed_kv1 = next(iter(self.bucket.get_all_versions())) self.assertEqual(listed_kv1.name, key_name) self.assertEqual(listed_kv1.version_id, kv1.version_id) self.assertEqual(listed_kv1.is_latest, True) # add v2 of the object kv2 = self.bucket.new_key(key_name) kv2.set_contents_from_string("v2") # read 2 versions, confirm v2 is latest i = iter(self.bucket.get_all_versions()) listed_kv2 = next(i) listed_kv1 = next(i) self.assertEqual(listed_kv2.version_id, kv2.version_id) self.assertEqual(listed_kv1.version_id, kv1.version_id) self.assertEqual(listed_kv2.is_latest, True) self.assertEqual(listed_kv1.is_latest, False) # delete key, which creates a delete marker as latest self.bucket.delete_key(key_name) i = iter(self.bucket.get_all_versions()) listed_kv3 = next(i) listed_kv2 = next(i) listed_kv1 = next(i) self.assertNotEqual(listed_kv3.version_id, None) self.assertEqual(listed_kv2.version_id, kv2.version_id) self.assertEqual(listed_kv1.version_id, kv1.version_id) self.assertEqual(listed_kv3.is_latest, True) self.assertEqual(listed_kv2.is_latest, False) self.assertEqual(listed_kv1.is_latest, False) vendored/boto/tests/integration/s3/test_connection.py 0000644 00000023521 15025013616 0017101 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the S3Connection """ import unittest import time import os import socket from boto.s3.connection import S3Connection from boto.s3.bucket import Bucket from boto.exception import S3PermissionsError, S3ResponseError from boto.compat import http_client, six, urlopen, urlsplit class S3ConnectionTest (unittest.TestCase): s3 = True def test_1_basic(self): print('--- running S3Connection tests ---') c = S3Connection() # create a new, empty bucket bucket_name = 'test-%d' % int(time.time()) bucket = c.create_bucket(bucket_name) # now try a get_bucket call and see if it's really there bucket = c.get_bucket(bucket_name) # test logging logging_bucket = c.create_bucket(bucket_name + '-log') logging_bucket.set_as_logging_target() bucket.enable_logging(target_bucket=logging_bucket, target_prefix=bucket.name) bucket.disable_logging() c.delete_bucket(logging_bucket) k = bucket.new_key('foobar') s1 = 'This is a test of file upload and download' s2 = 'This is a second string to test file upload and download' k.set_contents_from_string(s1) fp = open('foobar', 'wb') # now get the contents from s3 to a local file k.get_contents_to_file(fp) fp.close() fp = open('foobar') # check to make sure content read from s3 is identical to original assert s1 == fp.read(), 'corrupted file' fp.close() # test generated URLs url = k.generate_url(3600) file = urlopen(url) assert s1 == file.read().decode('utf-8'), 'invalid URL %s' % url url = k.generate_url(3600, force_http=True) file = urlopen(url) assert s1 == file.read().decode('utf-8'), 'invalid URL %s' % url url = k.generate_url(3600, force_http=True, headers={'x-amz-x-token': 'XYZ'}) file = urlopen(url) assert s1 == file.read().decode('utf-8'), 'invalid URL %s' % url rh = {'response-content-disposition': 'attachment; filename="foo.txt"'} url = k.generate_url(60, response_headers=rh) file = urlopen(url) assert s1 == file.read().decode('utf-8'), 'invalid URL %s' % url #test whether amperands and to-be-escaped characters work in header filename rh = {'response-content-disposition': 'attachment; filename="foo&z%20ar&ar&zar&bar.txt"'} url = k.generate_url(60, response_headers=rh, force_http=True) file = urlopen(url) assert s1 == file.read().decode('utf-8'), 'invalid URL %s' % url # overwrite foobar contents with a PUT url = k.generate_url(3600, 'PUT', force_http=True, policy='private', reduced_redundancy=True) up = urlsplit(url) con = http_client.HTTPConnection(up.hostname, up.port) con.request("PUT", up.path + '?' + up.query, body="hello there") resp = con.getresponse() self.assertEqual(200, resp.status) assert b"hello there" == k.get_contents_as_string() bucket.delete_key(k) # test a few variations on get_all_keys - first load some data # for the first one, let's override the content type phony_mimetype = 'application/x-boto-test' headers = {'Content-Type': phony_mimetype} k.name = 'foo/bar' k.set_contents_from_string(s1, headers) k.name = 'foo/bas' size = k.set_contents_from_filename('foobar') assert size == 42 k.name = 'foo/bat' k.set_contents_from_string(s1) k.name = 'fie/bar' k.set_contents_from_string(s1) k.name = 'fie/bas' k.set_contents_from_string(s1) k.name = 'fie/bat' k.set_contents_from_string(s1) # try resetting the contents to another value md5 = k.md5 k.set_contents_from_string(s2) assert k.md5 != md5 os.unlink('foobar') all = bucket.get_all_keys() assert len(all) == 6 rs = bucket.get_all_keys(prefix='foo') assert len(rs) == 3 rs = bucket.get_all_keys(prefix='', delimiter='/') assert len(rs) == 2 rs = bucket.get_all_keys(maxkeys=5) assert len(rs) == 5 # test the lookup method k = bucket.lookup('foo/bar') assert isinstance(k, bucket.key_class) assert k.content_type == phony_mimetype k = bucket.lookup('notthere') assert k == None # try some metadata stuff k = bucket.new_key('has_metadata') mdkey1 = 'meta1' mdval1 = 'This is the first metadata value' k.set_metadata(mdkey1, mdval1) mdkey2 = 'meta2' mdval2 = 'This is the second metadata value' k.set_metadata(mdkey2, mdval2) # try a unicode metadata value mdval3 = u'föö' mdkey3 = 'meta3' k.set_metadata(mdkey3, mdval3) k.set_contents_from_string(s1) k = bucket.lookup('has_metadata') assert k.get_metadata(mdkey1) == mdval1 assert k.get_metadata(mdkey2) == mdval2 assert k.get_metadata(mdkey3) == mdval3 k = bucket.new_key('has_metadata') k.get_contents_as_string() assert k.get_metadata(mdkey1) == mdval1 assert k.get_metadata(mdkey2) == mdval2 assert k.get_metadata(mdkey3) == mdval3 bucket.delete_key(k) # test list and iterator rs1 = bucket.list() num_iter = 0 for r in rs1: num_iter = num_iter + 1 rs = bucket.get_all_keys() num_keys = len(rs) assert num_iter == num_keys # try a key with a funny character k = bucket.new_key('testnewline\n') k.set_contents_from_string('This is a test') rs = bucket.get_all_keys() assert len(rs) == num_keys + 1 bucket.delete_key(k) rs = bucket.get_all_keys() assert len(rs) == num_keys # try some acl stuff bucket.set_acl('public-read') policy = bucket.get_acl() assert len(policy.acl.grants) == 2 bucket.set_acl('private') policy = bucket.get_acl() assert len(policy.acl.grants) == 1 k = bucket.lookup('foo/bar') k.set_acl('public-read') policy = k.get_acl() assert len(policy.acl.grants) == 2 k.set_acl('private') policy = k.get_acl() assert len(policy.acl.grants) == 1 # try the convenience methods for grants bucket.add_user_grant('FULL_CONTROL', 'c1e724fbfa0979a4448393c59a8c055011f739b6d102fb37a65f26414653cd67') try: bucket.add_email_grant('foobar', 'foo@bar.com') except S3PermissionsError: pass # now try to create an RRS key k = bucket.new_key('reduced_redundancy') k.set_contents_from_string('This key has reduced redundancy', reduced_redundancy=True) # now try to inject a response header data = k.get_contents_as_string(response_headers={'response-content-type' : 'foo/bar'}) assert k.content_type == 'foo/bar' # now delete all keys in bucket for k in bucket: if k.name == 'reduced_redundancy': assert k.storage_class == 'REDUCED_REDUNDANCY' bucket.delete_key(k) # now delete bucket time.sleep(5) c.delete_bucket(bucket) print('--- tests completed ---') def test_basic_anon(self): auth_con = S3Connection() # create a new, empty bucket bucket_name = 'test-%d' % int(time.time()) auth_bucket = auth_con.create_bucket(bucket_name) # try read the bucket anonymously anon_con = S3Connection(anon=True) anon_bucket = Bucket(anon_con, bucket_name) try: next(iter(anon_bucket.list())) self.fail("anon bucket list should fail") except S3ResponseError: pass # give bucket anon user access and anon read again auth_bucket.set_acl('public-read') time.sleep(10) # Was 5 secondes, turns out not enough try: next(iter(anon_bucket.list())) self.fail("not expecting contents") except S3ResponseError as e: self.fail("We should have public-read access, but received " "an error: %s" % e) except StopIteration: pass # cleanup auth_con.delete_bucket(auth_bucket) def test_error_code_populated(self): c = S3Connection() try: c.create_bucket('bad$bucket$name') except S3ResponseError as e: self.assertEqual(e.error_code, 'InvalidBucketName') except socket.gaierror: pass # This is also a possible result for an invalid bucket name else: self.fail("S3ResponseError not raised.") vendored/boto/tests/integration/s3/test_pool.py 0000644 00000020077 15025013616 0015716 0 ustar 00 # Copyright (c) 2011 Brian Beach # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some multi-threading tests of boto in a greenlet environment. """ from __future__ import print_function import boto import time import uuid from threading import Thread def spawn(function, *args, **kwargs): """ Spawns a new thread. API is the same as gevent.greenlet.Greenlet.spawn. """ t = Thread(target = function, args = args, kwargs = kwargs) t.start() return t def put_object(bucket, name): bucket.new_key(name).set_contents_from_string(name) def get_object(bucket, name): assert bucket.get_key(name).get_contents_as_string().decode('utf-8') == name def test_close_connections(): """ A test that exposes the problem where connections are returned to the connection pool (and closed) before the caller reads the response. I couldn't think of a way to test it without greenlets, so this test doesn't run as part of the standard test suite. That way, no more dependencies are added to the test suite. """ print("Running test_close_connections") # Connect to S3 s3 = boto.connect_s3() # Clean previous tests. for b in s3.get_all_buckets(): if b.name.startswith('test-'): for key in b.get_all_keys(): key.delete() b.delete() # Make a test bucket bucket = s3.create_bucket('test-%d' % int(time.time())) # Create 30 threads that each create an object in S3. The number # 30 is chosen because it is larger than the connection pool size # (20). names = [str(uuid.uuid4) for _ in range(30)] threads = [ spawn(put_object, bucket, name) for name in names ] for t in threads: t.join() # Create 30 threads to read the contents of the new objects. This # is where closing the connection early is a problem, because # there is a response that needs to be read, and it can't be read # if the connection has already been closed. threads = [ spawn(get_object, bucket, name) for name in names ] for t in threads: t.join() # test_reuse_connections needs to read a file that is big enough that # one read() call on the socket won't read the whole thing. BIG_SIZE = 10000 class WriteAndCount(object): """ A file-like object that counts the number of characters written. """ def __init__(self): self.size = 0 def write(self, data): self.size += len(data) time.sleep(0) # yield to other threads def read_big_object(s3, bucket, name, count): for _ in range(count): key = bucket.get_key(name) out = WriteAndCount() key.get_contents_to_file(out) if out.size != BIG_SIZE: print(out.size, BIG_SIZE) assert out.size == BIG_SIZE print(" pool size:", s3._pool.size()) class LittleQuerier(object): """ An object that manages a thread that keeps pulling down small objects from S3 and checking the answers until told to stop. """ def __init__(self, bucket, small_names): self.running = True self.bucket = bucket self.small_names = small_names self.thread = spawn(self.run) def stop(self): self.running = False self.thread.join() def run(self): count = 0 while self.running: i = count % 4 key = self.bucket.get_key(self.small_names[i]) expected = str(i) rh = { 'response-content-type' : 'small/' + str(i) } actual = key.get_contents_as_string(response_headers = rh).decode('utf-8') if expected != actual: print("AHA:", repr(expected), repr(actual)) assert expected == actual count += 1 def test_reuse_connections(): """ This test is an attempt to expose problems because of the fact that boto returns connections to the connection pool before reading the response. The strategy is to start a couple big reads from S3, where it will take time to read the response, and then start other requests that will reuse the same connection from the pool while the big response is still being read. The test passes because of an interesting combination of factors. I was expecting that it would fail because two threads would be reading the same connection at the same time. That doesn't happen because httplib catches the problem before it happens and raises an exception. Here's the sequence of events: - Thread 1: Send a request to read a big S3 object. - Thread 1: Returns connection to pool. - Thread 1: Start reading the body if the response. - Thread 2: Get the same connection from the pool. - Thread 2: Send another request on the same connection. - Thread 2: Try to read the response, but HTTPConnection.get_response notices that the previous response isn't done reading yet, and raises a ResponseNotReady exception. - Thread 2: _mexe catches the exception, does not return the connection to the pool, gets a new connection, and retries. - Thread 1: Finish reading the body of its response. - Server: Gets the second request on the connection, and sends a response. This response is ignored because the connection has been dropped on the client end. If you add a print statement in HTTPConnection.get_response at the point where it raises ResponseNotReady, and then run this test, you can see that it's happening. """ print("Running test_reuse_connections") # Connect to S3 s3 = boto.connect_s3() # Make a test bucket bucket = s3.create_bucket('test-%d' % int(time.time())) # Create some small objects in S3. small_names = [str(uuid.uuid4()) for _ in range(4)] for (i, name) in enumerate(small_names): bucket.new_key(name).set_contents_from_string(str(i)) # Wait, clean the connection pool, and make sure it's empty. print(" waiting for all connections to become stale") time.sleep(s3._pool.STALE_DURATION + 1) s3._pool.clean() assert s3._pool.size() == 0 print(" pool is empty") # Create a big object in S3. big_name = str(uuid.uuid4()) contents = "-" * BIG_SIZE bucket.new_key(big_name).set_contents_from_string(contents) # Start some threads to read it and check that they are reading # the correct thing. Each thread will read the object 40 times. threads = [ spawn(read_big_object, s3, bucket, big_name, 20) for _ in range(5) ] # Do some other things that may (incorrectly) re-use the same # connections while the big objects are being read. queriers = [ LittleQuerier(bucket, small_names) for _ in range(5) ] # Clean up. for t in threads: t.join() for q in queriers: q.stop() def main(): test_close_connections() test_reuse_connections() if __name__ == '__main__': main() vendored/boto/tests/integration/s3/mock_storage_service.py 0000644 00000056222 15025013616 0020104 0 ustar 00 # Copyright 2010 Google Inc. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Provides basic mocks of core storage service classes, for unit testing: ACL, Key, Bucket, Connection, and StorageUri. We implement a subset of the interfaces defined in the real boto classes, but don't handle most of the optional params (which we indicate with the constant "NOT_IMPL"). """ import copy import boto import base64 import re import six from hashlib import md5 from boto.utils import compute_md5 from boto.utils import find_matching_headers from boto.utils import merge_headers_by_name from boto.utils import write_to_fd from boto.s3.prefix import Prefix from boto.compat import six NOT_IMPL = None class MockAcl(object): def __init__(self, parent=NOT_IMPL): pass def startElement(self, name, attrs, connection): pass def endElement(self, name, value, connection): pass def to_xml(self): return '<mock_ACL_XML/>' class MockKey(object): def __init__(self, bucket=None, name=None): self.bucket = bucket self.name = name self.data = None self.etag = None self.size = None self.closed = True self.content_encoding = None self.content_language = None self.content_type = None self.last_modified = 'Wed, 06 Oct 2010 05:11:54 GMT' self.BufferSize = 8192 def __repr__(self): if self.bucket: return '<MockKey: %s,%s>' % (self.bucket.name, self.name) else: return '<MockKey: %s>' % self.name def get_contents_as_string(self, headers=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, torrent=NOT_IMPL, version_id=NOT_IMPL): return self.data def get_contents_to_file(self, fp, headers=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, torrent=NOT_IMPL, version_id=NOT_IMPL, res_download_handler=NOT_IMPL): data = six.ensure_binary(self.data) write_to_fd(fp, data) def get_file(self, fp, headers=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, torrent=NOT_IMPL, version_id=NOT_IMPL, override_num_retries=NOT_IMPL): data = six.ensure_binary(self.data) write_to_fd(fp, data) def _handle_headers(self, headers): if not headers: return if find_matching_headers('Content-Encoding', headers): self.content_encoding = merge_headers_by_name('Content-Encoding', headers) if find_matching_headers('Content-Type', headers): self.content_type = merge_headers_by_name('Content-Type', headers) if find_matching_headers('Content-Language', headers): self.content_language = merge_headers_by_name('Content-Language', headers) # Simplistic partial implementation for headers: Just supports range GETs # of flavor 'Range: bytes=xyz-'. def open_read(self, headers=None, query_args=NOT_IMPL, override_num_retries=NOT_IMPL): if self.closed: self.read_pos = 0 self.closed = False if headers and 'Range' in headers: match = re.match('bytes=([0-9]+)-$', headers['Range']) if match: self.read_pos = int(match.group(1)) def close(self, fast=NOT_IMPL): self.closed = True def read(self, size=0): self.open_read() if size == 0: data = self.data[self.read_pos:] self.read_pos = self.size else: data = self.data[self.read_pos:self.read_pos+size] self.read_pos += size if not data: self.close() return data def set_contents_from_file(self, fp, headers=None, replace=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, policy=NOT_IMPL, md5=NOT_IMPL, res_upload_handler=NOT_IMPL): self.data = fp.read() self.set_etag() self.size = len(self.data) self._handle_headers(headers) def set_contents_from_stream(self, fp, headers=None, replace=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, policy=NOT_IMPL, reduced_redundancy=NOT_IMPL, query_args=NOT_IMPL, size=NOT_IMPL): self.data = '' chunk = fp.read(self.BufferSize) while chunk: self.data += chunk chunk = fp.read(self.BufferSize) self.set_etag() self.size = len(self.data) self._handle_headers(headers) def set_contents_from_string(self, s, headers=NOT_IMPL, replace=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, policy=NOT_IMPL, md5=NOT_IMPL, reduced_redundancy=NOT_IMPL): self.data = copy.copy(s) self.set_etag() self.size = len(s) self._handle_headers(headers) def set_contents_from_filename(self, filename, headers=None, replace=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, policy=NOT_IMPL, md5=NOT_IMPL, res_upload_handler=NOT_IMPL): fp = open(filename, 'rb') self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy, md5, res_upload_handler) fp.close() def copy(self, dst_bucket_name, dst_key, metadata=NOT_IMPL, reduced_redundancy=NOT_IMPL, preserve_acl=NOT_IMPL): dst_bucket = self.bucket.connection.get_bucket(dst_bucket_name) return dst_bucket.copy_key(dst_key, self.bucket.name, self.name, metadata) @property def provider(self): provider = None if self.bucket and self.bucket.connection: provider = self.bucket.connection.provider return provider def set_etag(self): """ Set etag attribute by generating hex MD5 checksum on current contents of mock key. """ m = md5() if not isinstance(self.data, bytes): m.update(self.data.encode('utf-8')) else: m.update(self.data) hex_md5 = m.hexdigest() self.etag = hex_md5 def compute_md5(self, fp): """ :type fp: file :param fp: File pointer to the file to MD5 hash. The file pointer will be reset to the beginning of the file before the method returns. :rtype: tuple :return: A tuple containing the hex digest version of the MD5 hash as the first element and the base64 encoded version of the plain digest as the second element. """ tup = compute_md5(fp) # Returned values are MD5 hash, base64 encoded MD5 hash, and file size. # The internal implementation of compute_md5() needs to return the # file size but we don't want to return that value to the external # caller because it changes the class interface (i.e. it might # break some code) so we consume the third tuple value here and # return the remainder of the tuple to the caller, thereby preserving # the existing interface. self.size = tup[2] return tup[0:2] class MockBucket(object): def __init__(self, connection=None, name=None, key_class=NOT_IMPL): self.name = name self.keys = {} self.acls = {name: MockAcl()} # default object ACLs are one per bucket and not supported for keys self.def_acl = MockAcl() self.subresources = {} self.connection = connection self.logging = False def __repr__(self): return 'MockBucket: %s' % self.name def copy_key(self, new_key_name, src_bucket_name, src_key_name, metadata=NOT_IMPL, src_version_id=NOT_IMPL, storage_class=NOT_IMPL, preserve_acl=NOT_IMPL, encrypt_key=NOT_IMPL, headers=NOT_IMPL, query_args=NOT_IMPL): new_key = self.new_key(key_name=new_key_name) src_key = self.connection.get_bucket( src_bucket_name).get_key(src_key_name) new_key.data = copy.copy(src_key.data) new_key.size = len(new_key.data) return new_key def disable_logging(self): self.logging = False def enable_logging(self, target_bucket_prefix): self.logging = True def get_logging_config(self): return {"Logging": {}} def get_versioning_status(self, headers=NOT_IMPL): return False def get_acl(self, key_name='', headers=NOT_IMPL, version_id=NOT_IMPL): if key_name: # Return ACL for the key. return self.acls[key_name] else: # Return ACL for the bucket. return self.acls[self.name] def get_def_acl(self, key_name=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): # Return default ACL for the bucket. return self.def_acl def get_subresource(self, subresource, key_name=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): if subresource in self.subresources: return self.subresources[subresource] else: return '<Subresource/>' def get_tags(self): return [] def new_key(self, key_name=None): mock_key = MockKey(self, key_name) self.keys[key_name] = mock_key self.acls[key_name] = MockAcl() return mock_key def delete_key(self, key_name, headers=NOT_IMPL, version_id=NOT_IMPL, mfa_token=NOT_IMPL): if key_name not in self.keys: raise boto.exception.StorageResponseError(404, 'Not Found') del self.keys[key_name] def get_all_keys(self, headers=NOT_IMPL): return six.itervalues(self.keys) def get_key(self, key_name, headers=NOT_IMPL, version_id=NOT_IMPL): # Emulate behavior of boto when get_key called with non-existent key. if key_name not in self.keys: return None return self.keys[key_name] def list(self, prefix='', delimiter='', marker=NOT_IMPL, headers=NOT_IMPL): prefix = prefix or '' # Turn None into '' for prefix match. # Return list instead of using a generator so we don't get # 'dictionary changed size during iteration' error when performing # deletions while iterating (e.g., during test cleanup). result = [] key_name_set = set() for k in six.itervalues(self.keys): if k.name.startswith(prefix): k_name_past_prefix = k.name[len(prefix):] if delimiter: pos = k_name_past_prefix.find(delimiter) else: pos = -1 if (pos != -1): key_or_prefix = Prefix( bucket=self, name=k.name[:len(prefix)+pos+1]) else: key_or_prefix = MockKey(bucket=self, name=k.name) if key_or_prefix.name not in key_name_set: key_name_set.add(key_or_prefix.name) result.append(key_or_prefix) return result def set_acl(self, acl_or_str, key_name='', headers=NOT_IMPL, version_id=NOT_IMPL): # We only handle setting ACL XML here; if you pass a canned ACL # the get_acl call will just return that string name. if key_name: # Set ACL for the key. self.acls[key_name] = MockAcl(acl_or_str) else: # Set ACL for the bucket. self.acls[self.name] = MockAcl(acl_or_str) def set_def_acl(self, acl_or_str, key_name=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): # We only handle setting ACL XML here; if you pass a canned ACL # the get_acl call will just return that string name. # Set default ACL for the bucket. self.def_acl = acl_or_str def set_subresource(self, subresource, value, key_name=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): self.subresources[subresource] = value class MockProvider(object): def __init__(self, provider): self.provider = provider def get_provider_name(self): return self.provider class MockConnection(object): def __init__(self, aws_access_key_id=NOT_IMPL, aws_secret_access_key=NOT_IMPL, is_secure=NOT_IMPL, port=NOT_IMPL, proxy=NOT_IMPL, proxy_port=NOT_IMPL, proxy_user=NOT_IMPL, proxy_pass=NOT_IMPL, host=NOT_IMPL, debug=NOT_IMPL, https_connection_factory=NOT_IMPL, calling_format=NOT_IMPL, path=NOT_IMPL, provider='s3', bucket_class=NOT_IMPL): self.buckets = {} self.provider = MockProvider(provider) def create_bucket(self, bucket_name, headers=NOT_IMPL, location=NOT_IMPL, policy=NOT_IMPL, storage_class=NOT_IMPL): if bucket_name in self.buckets: raise boto.exception.StorageCreateError( 409, 'BucketAlreadyOwnedByYou', "<Message>Your previous request to create the named bucket " "succeeded and you already own it.</Message>") mock_bucket = MockBucket(name=bucket_name, connection=self) self.buckets[bucket_name] = mock_bucket return mock_bucket def delete_bucket(self, bucket, headers=NOT_IMPL): if bucket not in self.buckets: raise boto.exception.StorageResponseError( 404, 'NoSuchBucket', '<Message>no such bucket</Message>') del self.buckets[bucket] def get_bucket(self, bucket_name, validate=NOT_IMPL, headers=NOT_IMPL): if bucket_name not in self.buckets: raise boto.exception.StorageResponseError(404, 'NoSuchBucket', 'Not Found') return self.buckets[bucket_name] def get_all_buckets(self, headers=NOT_IMPL): return six.itervalues(self.buckets) # We only mock a single provider/connection. mock_connection = MockConnection() class MockBucketStorageUri(object): delim = '/' def __init__(self, scheme, bucket_name=None, object_name=None, debug=NOT_IMPL, suppress_consec_slashes=NOT_IMPL, version_id=None, generation=None, is_latest=False): self.scheme = scheme self.bucket_name = bucket_name self.object_name = object_name self.suppress_consec_slashes = suppress_consec_slashes if self.bucket_name and self.object_name: self.uri = ('%s://%s/%s' % (self.scheme, self.bucket_name, self.object_name)) elif self.bucket_name: self.uri = ('%s://%s/' % (self.scheme, self.bucket_name)) else: self.uri = ('%s://' % self.scheme) self.version_id = version_id self.generation = generation and int(generation) self.is_version_specific = (bool(self.generation) or bool(self.version_id)) self.is_latest = is_latest if bucket_name and object_name: self.versionless_uri = '%s://%s/%s' % (scheme, bucket_name, object_name) def __repr__(self): """Returns string representation of URI.""" return self.uri def acl_class(self): return MockAcl def canned_acls(self): return boto.provider.Provider('aws').canned_acls def clone_replace_name(self, new_name): return self.__class__(self.scheme, self.bucket_name, new_name) def clone_replace_key(self, key): return self.__class__( key.provider.get_provider_name(), bucket_name=key.bucket.name, object_name=key.name, suppress_consec_slashes=self.suppress_consec_slashes, version_id=getattr(key, 'version_id', None), generation=getattr(key, 'generation', None), is_latest=getattr(key, 'is_latest', None)) def connect(self, access_key_id=NOT_IMPL, secret_access_key=NOT_IMPL): return mock_connection def create_bucket(self, headers=NOT_IMPL, location=NOT_IMPL, policy=NOT_IMPL, storage_class=NOT_IMPL): return self.connect().create_bucket(self.bucket_name) def delete_bucket(self, headers=NOT_IMPL): return self.connect().delete_bucket(self.bucket_name) def get_versioning_config(self, headers=NOT_IMPL): self.get_bucket().get_versioning_status(headers) def has_version(self): return (issubclass(type(self), MockBucketStorageUri) and ((self.version_id is not None) or (self.generation is not None))) def delete_key(self, validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL, mfa_token=NOT_IMPL): self.get_bucket().delete_key(self.object_name) def disable_logging(self, validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): self.get_bucket().disable_logging() def enable_logging(self, target_bucket, target_prefix, validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): self.get_bucket().enable_logging(target_bucket) def get_logging_config(self, validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): return self.get_bucket().get_logging_config() def equals(self, uri): return self.uri == uri.uri def get_acl(self, validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): return self.get_bucket().get_acl(self.object_name) def get_def_acl(self, validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): return self.get_bucket().get_def_acl(self.object_name) def get_subresource(self, subresource, validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): return self.get_bucket().get_subresource(subresource, self.object_name) def get_all_buckets(self, headers=NOT_IMPL): return self.connect().get_all_buckets() def get_all_keys(self, validate=NOT_IMPL, headers=NOT_IMPL): return self.get_bucket().get_all_keys(self) def list_bucket(self, prefix='', delimiter='', headers=NOT_IMPL, all_versions=NOT_IMPL): return self.get_bucket().list(prefix=prefix, delimiter=delimiter) def get_bucket(self, validate=NOT_IMPL, headers=NOT_IMPL): return self.connect().get_bucket(self.bucket_name) def get_key(self, validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): return self.get_bucket().get_key(self.object_name) def is_file_uri(self): return False def is_cloud_uri(self): return True def names_container(self): return bool(not self.object_name) def names_singleton(self): return bool(self.object_name) def names_directory(self): return False def names_provider(self): return bool(not self.bucket_name) def names_bucket(self): return self.names_container() def names_file(self): return False def names_object(self): return not self.names_container() def is_stream(self): return False def new_key(self, validate=NOT_IMPL, headers=NOT_IMPL): bucket = self.get_bucket() return bucket.new_key(self.object_name) def set_acl(self, acl_or_str, key_name='', validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): self.get_bucket().set_acl(acl_or_str, key_name) def set_def_acl(self, acl_or_str, key_name=NOT_IMPL, validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): self.get_bucket().set_def_acl(acl_or_str) def set_subresource(self, subresource, value, validate=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): self.get_bucket().set_subresource(subresource, value, self.object_name) def copy_key(self, src_bucket_name, src_key_name, metadata=NOT_IMPL, src_version_id=NOT_IMPL, storage_class=NOT_IMPL, preserve_acl=NOT_IMPL, encrypt_key=NOT_IMPL, headers=NOT_IMPL, query_args=NOT_IMPL, src_generation=NOT_IMPL): dst_bucket = self.get_bucket() return dst_bucket.copy_key(new_key_name=self.object_name, src_bucket_name=src_bucket_name, src_key_name=src_key_name) def set_contents_from_string(self, s, headers=NOT_IMPL, replace=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, policy=NOT_IMPL, md5=NOT_IMPL, reduced_redundancy=NOT_IMPL): key = self.new_key() key.set_contents_from_string(s) def set_contents_from_file(self, fp, headers=None, replace=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, policy=NOT_IMPL, md5=NOT_IMPL, size=NOT_IMPL, rewind=NOT_IMPL, res_upload_handler=NOT_IMPL): key = self.new_key() return key.set_contents_from_file(fp, headers=headers) def set_contents_from_stream(self, fp, headers=NOT_IMPL, replace=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, policy=NOT_IMPL, reduced_redundancy=NOT_IMPL, query_args=NOT_IMPL, size=NOT_IMPL): dst_key.set_contents_from_stream(fp) def get_contents_to_file(self, fp, headers=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, torrent=NOT_IMPL, version_id=NOT_IMPL, res_download_handler=NOT_IMPL, response_headers=NOT_IMPL): key = self.get_key() key.get_contents_to_file(fp) def get_contents_to_stream(self, fp, headers=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, version_id=NOT_IMPL): key = self.get_key() return key.get_contents_to_file(fp) vendored/boto/tests/integration/s3/test_mfa.py 0000644 00000007055 15025013616 0015511 0 ustar 00 # Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2010, Eucalyptus Systems, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for S3 MfaDelete with versioning """ import unittest import time from nose.plugins.attrib import attr from boto.s3.connection import S3Connection from boto.exception import S3ResponseError from boto.s3.deletemarker import DeleteMarker @attr('notdefault', 's3mfa') class S3MFATest (unittest.TestCase): def setUp(self): self.conn = S3Connection() self.bucket_name = 'mfa-%d' % int(time.time()) self.bucket = self.conn.create_bucket(self.bucket_name) def tearDown(self): for k in self.bucket.list_versions(): self.bucket.delete_key(k.name, version_id=k.version_id) self.bucket.delete() def test_mfadel(self): # Enable Versioning with MfaDelete mfa_sn = raw_input('MFA S/N: ') mfa_code = raw_input('MFA Code: ') self.bucket.configure_versioning(True, mfa_delete=True, mfa_token=(mfa_sn, mfa_code)) # Check enabling mfa worked. i = 0 for i in range(1, 8): time.sleep(2**i) d = self.bucket.get_versioning_status() if d['Versioning'] == 'Enabled' and d['MfaDelete'] == 'Enabled': break self.assertEqual('Enabled', d['Versioning']) self.assertEqual('Enabled', d['MfaDelete']) # Add a key to the bucket k = self.bucket.new_key('foobar') s1 = 'This is v1' k.set_contents_from_string(s1) v1 = k.version_id # Now try to delete v1 without the MFA token try: self.bucket.delete_key('foobar', version_id=v1) self.fail("Must fail if not using MFA token") except S3ResponseError: pass # Now try delete again with the MFA token mfa_code = raw_input('MFA Code: ') self.bucket.delete_key('foobar', version_id=v1, mfa_token=(mfa_sn, mfa_code)) # Next suspend versioning and disable MfaDelete on the bucket mfa_code = raw_input('MFA Code: ') self.bucket.configure_versioning(False, mfa_delete=False, mfa_token=(mfa_sn, mfa_code)) # Lastly, check disabling mfa worked. i = 0 for i in range(1, 8): time.sleep(2**i) d = self.bucket.get_versioning_status() if d['Versioning'] == 'Suspended' and d['MfaDelete'] != 'Enabled': break self.assertEqual('Suspended', d['Versioning']) self.assertNotEqual('Enabled', d['MfaDelete']) vendored/boto/tests/integration/s3/test_cors.py 0000644 00000006044 15025013616 0015711 0 ustar 00 # Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2010, Eucalyptus Systems, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some integration tests for S3 CORS """ import unittest import time from boto.s3.connection import S3Connection from boto.exception import S3ResponseError from boto.s3.cors import CORSConfiguration class S3CORSTest (unittest.TestCase): s3 = True def setUp(self): self.conn = S3Connection() self.bucket_name = 'cors-%d' % int(time.time()) self.bucket = self.conn.create_bucket(self.bucket_name) def tearDown(self): self.bucket.delete() def test_cors(self): self.cfg = CORSConfiguration() self.cfg.add_rule(['PUT', 'POST', 'DELETE'], 'http://www.example.com', allowed_header='*', max_age_seconds=3000, expose_header='x-amz-server-side-encryption', id='foobar_rule') assert self.bucket.set_cors(self.cfg) time.sleep(5) cfg = self.bucket.get_cors() for i, rule in enumerate(cfg): self.assertEqual(rule.id, self.cfg[i].id) self.assertEqual(rule.max_age_seconds, self.cfg[i].max_age_seconds) methods = zip(rule.allowed_method, self.cfg[i].allowed_method) for v1, v2 in methods: self.assertEqual(v1, v2) origins = zip(rule.allowed_origin, self.cfg[i].allowed_origin) for v1, v2 in origins: self.assertEqual(v1, v2) headers = zip(rule.allowed_header, self.cfg[i].allowed_header) for v1, v2 in headers: self.assertEqual(v1, v2) headers = zip(rule.expose_header, self.cfg[i].expose_header) for v1, v2 in headers: self.assertEqual(v1, v2) self.bucket.delete_cors() time.sleep(5) try: self.bucket.get_cors() self.fail('CORS configuration should not be there') except S3ResponseError: pass vendored/boto/tests/integration/s3/test_multidelete.py 0000644 00000015162 15025013616 0017261 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2011 Mitch Garnaat http://garnaat.org/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the S3 MultiDelete """ import unittest import time from boto.s3.key import Key from boto.s3.deletemarker import DeleteMarker from boto.s3.prefix import Prefix from boto.s3.connection import S3Connection from boto.exception import S3ResponseError class S3MultiDeleteTest(unittest.TestCase): s3 = True def setUp(self): self.conn = S3Connection() self.bucket_name = 'multidelete-%d' % int(time.time()) self.bucket = self.conn.create_bucket(self.bucket_name) def tearDown(self): for key in self.bucket: key.delete() self.bucket.delete_keys(self.bucket.list_versions()) self.bucket.delete() def test_delete_nothing(self): result = self.bucket.delete_keys([]) self.assertEqual(len(result.deleted), 0) self.assertEqual(len(result.errors), 0) def test_delete_illegal(self): result = self.bucket.delete_keys([{"dict":"notallowed"}]) self.assertEqual(len(result.deleted), 0) self.assertEqual(len(result.errors), 1) def test_delete_mix(self): result = self.bucket.delete_keys(["king", ("mice", None), Key(name="regular"), Key(), Prefix(name="folder/"), DeleteMarker(name="deleted"), {"bad":"type"}]) self.assertEqual(len(result.deleted), 4) self.assertEqual(len(result.errors), 3) def test_delete_quietly(self): result = self.bucket.delete_keys(["king"], quiet=True) self.assertEqual(len(result.deleted), 0) self.assertEqual(len(result.errors), 0) def test_delete_must_escape(self): result = self.bucket.delete_keys([Key(name=">_<;")]) self.assertEqual(len(result.deleted), 1) self.assertEqual(len(result.errors), 0) def test_delete_unknown_version(self): no_ver = Key(name="no") no_ver.version_id = "version" result = self.bucket.delete_keys([no_ver]) self.assertEqual(len(result.deleted), 0) self.assertEqual(len(result.errors), 1) def test_delete_kanji(self): result = self.bucket.delete_keys([u"漢字", Key(name=u"日本語")]) self.assertEqual(len(result.deleted), 2) self.assertEqual(len(result.errors), 0) def test_delete_empty_by_list(self): result = self.bucket.delete_keys(self.bucket.list()) self.assertEqual(len(result.deleted), 0) self.assertEqual(len(result.errors), 0) def test_delete_kanji_by_list(self): for key_name in [u"漢字", u"日本語", u"テスト"]: key = self.bucket.new_key(key_name) key.set_contents_from_string('this is a test') result = self.bucket.delete_keys(self.bucket.list()) self.assertEqual(len(result.deleted), 3) self.assertEqual(len(result.errors), 0) def test_delete_with_prefixes(self): for key_name in ["a", "a/b", "b"]: key = self.bucket.new_key(key_name) key.set_contents_from_string('this is a test') # First delete all "files": "a" and "b" result = self.bucket.delete_keys(self.bucket.list(delimiter="/")) self.assertEqual(len(result.deleted), 2) # Using delimiter will cause 1 common prefix to be listed # which will be skipped as an error. self.assertEqual(len(result.errors), 1) self.assertEqual(result.errors[0].key, "a/") # Next delete any remaining objects: "a/b" result = self.bucket.delete_keys(self.bucket.list()) self.assertEqual(len(result.deleted), 1) self.assertEqual(len(result.errors), 0) self.assertEqual(result.deleted[0].key, "a/b") def test_delete_too_many_versions(self): # configure versioning first self.bucket.configure_versioning(True) # Add 1000 initial versions as DMs by deleting them :-) # Adding 1000 objects is painful otherwise... key_names = ['key-%03d' % i for i in range(0, 1000)] result = self.bucket.delete_keys(key_names) self.assertEqual(len(result.deleted) + len(result.errors), 1000) # delete them again to create 1000 more delete markers result = self.bucket.delete_keys(key_names) self.assertEqual(len(result.deleted) + len(result.errors), 1000) # Sometimes takes AWS sometime to settle time.sleep(10) # delete all versions to delete 2000 objects. # this tests the 1000 limit. result = self.bucket.delete_keys(self.bucket.list_versions()) self.assertEqual(len(result.deleted) + len(result.errors), 2000) def test_1(self): nkeys = 100 # create a bunch of keynames key_names = ['key-%03d' % i for i in range(0, nkeys)] # create the corresponding keys for key_name in key_names: key = self.bucket.new_key(key_name) key.set_contents_from_string('this is a test') # now count keys in bucket n = 0 for key in self.bucket: n += 1 self.assertEqual(n, nkeys) # now delete them all result = self.bucket.delete_keys(key_names) self.assertEqual(len(result.deleted), nkeys) self.assertEqual(len(result.errors), 0) time.sleep(5) # now count keys in bucket n = 0 for key in self.bucket: n += 1 self.assertEqual(n, 0) vendored/boto/tests/integration/s3/test_bucket.py 0000644 00000030362 15025013616 0016220 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2011 Mitch Garnaat http://garnaat.org/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the S3 Bucket """ from mock import patch, Mock import unittest import time from boto.exception import S3ResponseError from boto.s3.connection import S3Connection from boto.s3.bucketlogging import BucketLogging from boto.s3.lifecycle import Lifecycle from boto.s3.lifecycle import Transition from boto.s3.lifecycle import Expiration from boto.s3.lifecycle import Rule from boto.s3.acl import Grant from boto.s3.tagging import Tags, TagSet from boto.s3.website import RedirectLocation from boto.compat import unquote_str class S3BucketTest (unittest.TestCase): s3 = True def setUp(self): self.conn = S3Connection() self.bucket_name = 'bucket-%d' % int(time.time()) self.bucket = self.conn.create_bucket(self.bucket_name) def tearDown(self): for key in self.bucket: key.delete() self.bucket.delete() def test_next_marker(self): expected = ["a/", "b", "c"] for key_name in expected: key = self.bucket.new_key(key_name) key.set_contents_from_string(key_name) # Normal list of first 2 keys will have # no NextMarker set, so we use last key to iterate # last element will be "b" so no issue. rs = self.bucket.get_all_keys(max_keys=2) for element in rs: pass self.assertEqual(element.name, "b") self.assertEqual(rs.next_marker, None) # list using delimiter of first 2 keys will have # a NextMarker set (when truncated). As prefixes # are grouped together at the end, we get "a/" as # last element, but luckily we have next_marker. rs = self.bucket.get_all_keys(max_keys=2, delimiter="/") for element in rs: pass self.assertEqual(element.name, "a/") self.assertEqual(rs.next_marker, "b") # ensure bucket.list() still works by just # popping elements off the front of expected. rs = self.bucket.list() for element in rs: self.assertEqual(element.name, expected.pop(0)) self.assertEqual(expected, []) def test_list_with_url_encoding(self): expected = [u"α", u"β", u"γ"] for key_name in expected: key = self.bucket.new_key(key_name) key.set_contents_from_string(key_name) # ensure bucket.list() still works by just # popping elements off the front of expected. orig_getall = self.bucket._get_all getall = lambda *a, **k: orig_getall(*a, max_keys=2, **k) with patch.object(self.bucket, '_get_all', getall): rs = self.bucket.list(encoding_type="url") for element in rs: name = unquote_str(element.name) self.assertEqual(name, expected.pop(0)) self.assertEqual(expected, []) def test_logging(self): # use self.bucket as the target bucket so that teardown # will delete any log files that make it into the bucket # automatically and all we have to do is delete the # source bucket. sb_name = "src-" + self.bucket_name sb = self.conn.create_bucket(sb_name) # grant log write perms to target bucket using canned-acl self.bucket.set_acl("log-delivery-write") target_bucket = self.bucket_name target_prefix = u"jp/ログ/" # Check existing status is disabled bls = sb.get_logging_status() self.assertEqual(bls.target, None) # Create a logging status and grant auth users READ PERM authuri = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" authr = Grant(permission="READ", type="Group", uri=authuri) sb.enable_logging(target_bucket, target_prefix=target_prefix, grants=[authr]) # Check the status and confirm its set. bls = sb.get_logging_status() self.assertEqual(bls.target, target_bucket) self.assertEqual(bls.prefix, target_prefix) self.assertEqual(len(bls.grants), 1) self.assertEqual(bls.grants[0].type, "Group") self.assertEqual(bls.grants[0].uri, authuri) # finally delete the src bucket sb.delete() def test_tagging(self): tagging = """ <Tagging> <TagSet> <Tag> <Key>tagkey</Key> <Value>tagvalue</Value> </Tag> </TagSet> </Tagging> """ self.bucket.set_xml_tags(tagging) response = self.bucket.get_tags() self.assertEqual(response[0][0].key, 'tagkey') self.assertEqual(response[0][0].value, 'tagvalue') self.bucket.delete_tags() try: self.bucket.get_tags() except S3ResponseError as e: self.assertEqual(e.code, 'NoSuchTagSet') except Exception as e: self.fail("Wrong exception raised (expected S3ResponseError): %s" % e) else: self.fail("Expected S3ResponseError, but no exception raised.") def test_tagging_from_objects(self): """Create tags from python objects rather than raw xml.""" t = Tags() tag_set = TagSet() tag_set.add_tag('akey', 'avalue') tag_set.add_tag('anotherkey', 'anothervalue') t.add_tag_set(tag_set) self.bucket.set_tags(t) response = self.bucket.get_tags() tags = sorted(response[0], key=lambda tag: tag.key) self.assertEqual(tags[0].key, 'akey') self.assertEqual(tags[0].value, 'avalue') self.assertEqual(tags[1].key, 'anotherkey') self.assertEqual(tags[1].value, 'anothervalue') def test_website_configuration(self): response = self.bucket.configure_website('index.html') self.assertTrue(response) config = self.bucket.get_website_configuration() self.assertEqual(config, {'WebsiteConfiguration': {'IndexDocument': {'Suffix': 'index.html'}}}) config2, xml = self.bucket.get_website_configuration_with_xml() self.assertEqual(config, config2) self.assertTrue('<Suffix>index.html</Suffix>' in xml, xml) def test_website_redirect_all_requests(self): response = self.bucket.configure_website( redirect_all_requests_to=RedirectLocation('example.com')) config = self.bucket.get_website_configuration() self.assertEqual(config, { 'WebsiteConfiguration': { 'RedirectAllRequestsTo': { 'HostName': 'example.com'}}}) # Can configure the protocol as well. response = self.bucket.configure_website( redirect_all_requests_to=RedirectLocation('example.com', 'https')) config = self.bucket.get_website_configuration() self.assertEqual(config, { 'WebsiteConfiguration': {'RedirectAllRequestsTo': { 'HostName': 'example.com', 'Protocol': 'https', }}} ) def test_lifecycle(self): lifecycle = Lifecycle() lifecycle.add_rule('myid', '', 'Enabled', 30) self.assertTrue(self.bucket.configure_lifecycle(lifecycle)) response = self.bucket.get_lifecycle_config() self.assertEqual(len(response), 1) actual_lifecycle = response[0] self.assertEqual(actual_lifecycle.id, 'myid') self.assertEqual(actual_lifecycle.prefix, '') self.assertEqual(actual_lifecycle.status, 'Enabled') self.assertEqual(actual_lifecycle.transition, []) def test_lifecycle_with_glacier_transition(self): lifecycle = Lifecycle() transition = Transition(days=30, storage_class='GLACIER') rule = Rule('myid', prefix='', status='Enabled', expiration=None, transition=transition) lifecycle.append(rule) self.assertTrue(self.bucket.configure_lifecycle(lifecycle)) response = self.bucket.get_lifecycle_config() transition = response[0].transition self.assertEqual(transition.days, 30) self.assertEqual(transition.storage_class, 'GLACIER') self.assertEqual(transition.date, None) def test_lifecycle_multi(self): date = '2022-10-12T00:00:00.000Z' sc = 'GLACIER' lifecycle = Lifecycle() lifecycle.add_rule("1", "1/", "Enabled", 1) lifecycle.add_rule("2", "2/", "Enabled", Expiration(days=2)) lifecycle.add_rule("3", "3/", "Enabled", Expiration(date=date)) lifecycle.add_rule("4", "4/", "Enabled", None, Transition(days=4, storage_class=sc)) lifecycle.add_rule("5", "5/", "Enabled", None, Transition(date=date, storage_class=sc)) # set the lifecycle self.bucket.configure_lifecycle(lifecycle) # read the lifecycle back readlifecycle = self.bucket.get_lifecycle_config(); for rule in readlifecycle: if rule.id == "1": self.assertEqual(rule.prefix, "1/") self.assertEqual(rule.expiration.days, 1) elif rule.id == "2": self.assertEqual(rule.prefix, "2/") self.assertEqual(rule.expiration.days, 2) elif rule.id == "3": self.assertEqual(rule.prefix, "3/") self.assertEqual(rule.expiration.date, date) elif rule.id == "4": self.assertEqual(rule.prefix, "4/") self.assertEqual(rule.transition.days, 4) self.assertEqual(rule.transition.storage_class, sc) elif rule.id == "5": self.assertEqual(rule.prefix, "5/") self.assertEqual(rule.transition.date, date) self.assertEqual(rule.transition.storage_class, sc) else: self.fail("unexpected id %s" % rule.id) def test_lifecycle_jp(self): # test lifecycle with Japanese prefix name = "Japanese files" prefix = "日本語/" days = 30 lifecycle = Lifecycle() lifecycle.add_rule(name, prefix, "Enabled", days) # set the lifecycle self.bucket.configure_lifecycle(lifecycle) # read the lifecycle back readlifecycle = self.bucket.get_lifecycle_config(); for rule in readlifecycle: self.assertEqual(rule.id, name) self.assertEqual(rule.expiration.days, days) #Note: Boto seems correct? AWS seems broken? #self.assertEqual(rule.prefix, prefix) def test_lifecycle_with_defaults(self): lifecycle = Lifecycle() lifecycle.add_rule(expiration=30) self.assertTrue(self.bucket.configure_lifecycle(lifecycle)) response = self.bucket.get_lifecycle_config() self.assertEqual(len(response), 1) actual_lifecycle = response[0] self.assertNotEqual(len(actual_lifecycle.id), 0) self.assertEqual(actual_lifecycle.prefix, '') def test_lifecycle_rule_xml(self): # create a rule directly with id, prefix defaults rule = Rule(status='Enabled', expiration=30) s = rule.to_xml() # Confirm no ID is set in the rule. self.assertEqual(s.find("<ID>"), -1) # Confirm Prefix is '' and not set to 'None' self.assertNotEqual(s.find("<Prefix></Prefix>"), -1) vendored/boto/tests/integration/s3/__init__.py 0000644 00000002120 15025013616 0015432 0 ustar 00 # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/ec2containerservice/__init__.py 0000644 00000002142 15025013616 0021046 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/integration/ec2containerservice/test_ec2containerservice.py 0000644 00000003325 15025013616 0024307 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import boto from boto.ec2containerservice.exceptions import ClientException from tests.compat import unittest class TestEC2ContainerService(unittest.TestCase): def setUp(self): self.ecs = boto.connect_ec2containerservice() def test_list_clusters(self): response = self.ecs.list_clusters() self.assertIn('clusterArns', response['ListClustersResponse']['ListClustersResult']) def test_handle_not_found_exception(self): with self.assertRaises(ClientException): # Try to stop a task with an invalid arn. self.ecs.stop_task(task='foo') vendored/boto/tests/integration/beanstalk/test_wrapper.py 0000644 00000022106 15025013616 0020037 0 ustar 00 import random import time from functools import partial from tests.compat import unittest from boto.beanstalk.wrapper import Layer1Wrapper import boto.beanstalk.response as response class BasicSuite(unittest.TestCase): def setUp(self): self.random_id = str(random.randint(1, 1000000)) self.app_name = 'app-' + self.random_id self.app_version = 'version-' + self.random_id self.template = 'template-' + self.random_id self.environment = 'environment-' + self.random_id self.beanstalk = Layer1Wrapper() class MiscSuite(BasicSuite): def test_check_dns_availability(self): result = self.beanstalk.check_dns_availability('amazon') self.assertIsInstance(result, response.CheckDNSAvailabilityResponse, 'incorrect response object returned') self.assertFalse(result.available) class TestApplicationObjects(BasicSuite): def create_application(self): # This method is used for any API calls that require an application # object. This also adds a cleanup step to automatically delete the # app when the test is finished. No assertions are performed # here. If you want to validate create_application, don't use this # method. self.beanstalk.create_application(application_name=self.app_name) self.addCleanup(partial(self.beanstalk.delete_application, application_name=self.app_name)) def test_create_delete_application_version(self): # This will create an app, create an app version, delete the app # version, and delete the app. For each API call we check that the # return type is what we expect and that a few attributes have the # correct values. app_result = self.beanstalk.create_application(application_name=self.app_name) self.assertIsInstance(app_result, response.CreateApplicationResponse) self.assertEqual(app_result.application.application_name, self.app_name) version_result = self.beanstalk.create_application_version( application_name=self.app_name, version_label=self.app_version) self.assertIsInstance(version_result, response.CreateApplicationVersionResponse) self.assertEqual(version_result.application_version.version_label, self.app_version) result = self.beanstalk.delete_application_version( application_name=self.app_name, version_label=self.app_version) self.assertIsInstance(result, response.DeleteApplicationVersionResponse) result = self.beanstalk.delete_application( application_name=self.app_name ) self.assertIsInstance(result, response.DeleteApplicationResponse) def test_create_configuration_template(self): self.create_application() result = self.beanstalk.create_configuration_template( application_name=self.app_name, template_name=self.template, solution_stack_name='32bit Amazon Linux running Tomcat 6') self.assertIsInstance( result, response.CreateConfigurationTemplateResponse) self.assertEqual(result.solution_stack_name, '32bit Amazon Linux running Tomcat 6') def test_create_storage_location(self): result = self.beanstalk.create_storage_location() self.assertIsInstance(result, response.CreateStorageLocationResponse) def test_update_application(self): self.create_application() result = self.beanstalk.update_application(application_name=self.app_name) self.assertIsInstance(result, response.UpdateApplicationResponse) def test_update_application_version(self): self.create_application() self.beanstalk.create_application_version( application_name=self.app_name, version_label=self.app_version) result = self.beanstalk.update_application_version( application_name=self.app_name, version_label=self.app_version) self.assertIsInstance( result, response.UpdateApplicationVersionResponse) class GetSuite(BasicSuite): def test_describe_applications(self): result = self.beanstalk.describe_applications() self.assertIsInstance(result, response.DescribeApplicationsResponse) def test_describe_application_versions(self): result = self.beanstalk.describe_application_versions() self.assertIsInstance(result, response.DescribeApplicationVersionsResponse) def test_describe_configuration_options(self): result = self.beanstalk.describe_configuration_options() self.assertIsInstance(result, response.DescribeConfigurationOptionsResponse) def test_12_describe_environments(self): result = self.beanstalk.describe_environments() self.assertIsInstance( result, response.DescribeEnvironmentsResponse) def test_14_describe_events(self): result = self.beanstalk.describe_events() self.assertIsInstance(result, response.DescribeEventsResponse) def test_15_list_available_solution_stacks(self): result = self.beanstalk.list_available_solution_stacks() self.assertIsInstance( result, response.ListAvailableSolutionStacksResponse) self.assertIn('32bit Amazon Linux running Tomcat 6', result.solution_stacks) class TestsWithEnvironment(unittest.TestCase): @classmethod def setUpClass(cls): cls.random_id = str(random.randint(1, 1000000)) cls.app_name = 'app-' + cls.random_id cls.environment = 'environment-' + cls.random_id cls.template = 'template-' + cls.random_id cls.beanstalk = Layer1Wrapper() cls.beanstalk.create_application(application_name=cls.app_name) cls.beanstalk.create_configuration_template( application_name=cls.app_name, template_name=cls.template, solution_stack_name='32bit Amazon Linux running Tomcat 6') cls.app_version = 'version-' + cls.random_id cls.beanstalk.create_application_version( application_name=cls.app_name, version_label=cls.app_version) cls.beanstalk.create_environment(cls.app_name, cls.environment, template_name=cls.template) cls.wait_for_env(cls.environment) @classmethod def tearDownClass(cls): cls.beanstalk.delete_application(application_name=cls.app_name, terminate_env_by_force=True) cls.wait_for_env(cls.environment, 'Terminated') @classmethod def wait_for_env(cls, env_name, status='Ready'): while not cls.env_ready(env_name, status): time.sleep(15) @classmethod def env_ready(cls, env_name, desired_status): result = cls.beanstalk.describe_environments( application_name=cls.app_name, environment_names=env_name) status = result.environments[0].status return status == desired_status def test_describe_environment_resources(self): result = self.beanstalk.describe_environment_resources( environment_name=self.environment) self.assertIsInstance( result, response.DescribeEnvironmentResourcesResponse) def test_describe_configuration_settings(self): result = self.beanstalk.describe_configuration_settings( application_name=self.app_name, environment_name=self.environment) self.assertIsInstance( result, response.DescribeConfigurationSettingsResponse) def test_request_environment_info(self): result = self.beanstalk.request_environment_info( environment_name=self.environment, info_type='tail') self.assertIsInstance(result, response.RequestEnvironmentInfoResponse) self.wait_for_env(self.environment) result = self.beanstalk.retrieve_environment_info( environment_name=self.environment, info_type='tail') self.assertIsInstance(result, response.RetrieveEnvironmentInfoResponse) def test_rebuild_environment(self): result = self.beanstalk.rebuild_environment( environment_name=self.environment) self.assertIsInstance(result, response.RebuildEnvironmentResponse) self.wait_for_env(self.environment) def test_restart_app_server(self): result = self.beanstalk.restart_app_server( environment_name=self.environment) self.assertIsInstance(result, response.RestartAppServerResponse) self.wait_for_env(self.environment) def test_update_configuration_template(self): result = self.beanstalk.update_configuration_template( application_name=self.app_name, template_name=self.template) self.assertIsInstance( result, response.UpdateConfigurationTemplateResponse) def test_update_environment(self): result = self.beanstalk.update_environment( environment_name=self.environment) self.assertIsInstance(result, response.UpdateEnvironmentResponse) self.wait_for_env(self.environment) if __name__ == '__main__': unittest.main() vendored/boto/tests/integration/iam/test_cert_verification.py 0000644 00000003006 15025013616 0020656 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.iam class IAMCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): iam = True regions = boto.iam.regions() def sample_service_call(self, conn): conn.get_all_users() vendored/boto/tests/integration/iam/test_policy.py 0000644 00000007546 15025013616 0016473 0 ustar 00 # Copyright (c) 2015 Shaun Brady. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import boto import time import json from tests.compat import unittest class TestIAMPolicy(unittest.TestCase): iam = True def test_policy_actions(self): # Test managed policy create/attach/detach/delete iam = boto.connect_iam() time_suffix = time.time() rolename = 'boto-test-role-%d' % time_suffix groupname = 'boto-test-group-%d' % time_suffix username = 'boto-test-user-%d' % time_suffix policyname = 'TestPolicyName-%d' % time_suffix iam.create_role(rolename) iam.create_group(groupname) iam.create_user(username) policy_doc = { "Version": "2012-10-17", "Id": "TestPermission", "Statement": [ { "Sid": "TestSid", "Action": "s3:*", "Effect": "Deny", "Resource": "arn:aws:s3:::*" } ] } policy_json = json.dumps(policy_doc) # Create policy policy = iam.create_policy(policyname, policy_json) # Get it back, verify it is the same policy_copy = iam.get_policy(policy.arn) if not policy_copy.arn == policy.arn: raise Exception("Policies not equal.") # Show that policy is not attached result = iam.list_entities_for_policy(policy.arn)[ 'list_entities_for_policy_response'][ 'list_entities_for_policy_result'] if not len(result['policy_roles']) == 0: raise Exception("Roles when not expected") if not len(result['policy_groups']) == 0: raise Exception("Groups when not expected") if not len(result['policy_users']) == 0: raise Exception("Users when not expected") # Attach the policy iam.attach_role_policy(policy.arn, rolename) iam.attach_group_policy(policy.arn, groupname) iam.attach_user_policy(policy.arn, username) # Show that policy is indeed attached result = iam.list_entities_for_policy(policy.arn)[ 'list_entities_for_policy_response'][ 'list_entities_for_policy_result'] if not len(result['policy_roles']) == 1: raise Exception("Roles expected") if not len(result['policy_groups']) == 1: raise Exception("Groups expected") if not len(result['policy_users']) == 1: raise Exception("Users expected") # Detach the policy iam.detach_role_policy(policy.arn, rolename) iam.detach_group_policy(policy.arn, groupname) iam.detach_user_policy(policy.arn, username) # Clean up iam.delete_policy(policy.arn) iam.delete_role(rolename) iam.delete_user(username) iam.delete_group(groupname) vendored/boto/tests/integration/iam/test_connection.py 0000644 00000003322 15025013616 0017317 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import boto import time from tests.compat import unittest class TestIAM(unittest.TestCase): iam = True def test_group_users(self): # A very basic test to create a group, a user, add the user # to the group and then delete everything iam = boto.connect_iam() name = 'boto-test-%d' % time.time() username = 'boto-test-user-%d' % time.time() iam.create_group(name) iam.create_user(username) iam.add_user_to_group(name, username) iam.remove_user_from_group(name, username) iam.delete_user(username) iam.delete_group(name) vendored/boto/tests/integration/iam/__init__.py 0000644 00000002112 15025013616 0015654 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/iam/test_password_policy.py 0000644 00000007547 15025013616 0020416 0 ustar 00 # Copyright (c) 2014 Rocket Internet AG. # Luca Bruno <luca.bruno@rocket-internet.de> # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import boto import time from tests.compat import unittest class IAMAccountPasswordPolicy(unittest.TestCase): iam = True def test_password_policy(self): # A series of tests to check the password policy API iam = boto.connect_iam() # First preserve what is the current password policy try: initial_policy_result = iam.get_account_password_policy() except boto.exception.BotoServerError as srv_error: initial_policy = None if srv_error.status != 404: raise srv_error # Update the policy and check it back test_min_length = 88 iam.update_account_password_policy(minimum_password_length=test_min_length) new_policy = iam.get_account_password_policy() new_min_length = new_policy['get_account_password_policy_response']\ ['get_account_password_policy_result']['password_policy']\ ['minimum_password_length'] if test_min_length != int(new_min_length): raise Exception("Failed to update account password policy") # Delete the policy and check the correct deletion test_policy = '' iam.delete_account_password_policy() try: test_policy = iam.get_account_password_policy() except boto.exception.BotoServerError as srv_error: test_policy = None if srv_error.status != 404: raise srv_error if test_policy is not None: raise Exception("Failed to delete account password policy") # Restore initial account password policy if initial_policy: p = initial_policy['get_account_password_policy_response']\ ['get_account_password_policy_result']['password_policy'] iam.update_account_password_policy(minimum_password_length=int(p['minimum_password_length']), allow_users_to_change_password=bool(p['allow_users_to_change_password']), hard_expiry=bool(p['hard_expiry']), max_password_age=int(p['max_password_age']), password_reuse_prevention=int(p['password_reuse_prevention']), require_lowercase_characters=bool(p['require_lowercase_characters']), require_numbers=bool(p['require_numbers']), require_symbols=bool(p['require_symbols']), require_uppercase_characters=bool(p['require_uppercase_characters'])) vendored/boto/tests/integration/support/test_cert_verification.py 0000644 00000002633 15025013616 0021631 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import unittest from tests.integration import ServiceCertVerificationTest import boto.support class SupportCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): support = True regions = boto.support.regions() def sample_service_call(self, conn): conn.describe_services() vendored/boto/tests/integration/support/test_layer1.py 0000644 00000005725 15025013616 0017334 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import unittest import time from boto.support.layer1 import SupportConnection from boto.support import exceptions class TestSupportLayer1Management(unittest.TestCase): support = True def setUp(self): self.api = SupportConnection() self.wait_time = 5 def test_as_much_as_possible_before_teardown(self): cases = self.api.describe_cases() preexisting_count = len(cases.get('cases', [])) services = self.api.describe_services() self.assertTrue('services' in services) service_codes = [serv['code'] for serv in services['services']] self.assertTrue('amazon-cloudsearch' in service_codes) severity = self.api.describe_severity_levels() self.assertTrue('severityLevels' in severity) severity_codes = [sev['code'] for sev in severity['severityLevels']] self.assertTrue('low' in severity_codes) case_1 = self.api.create_case( subject='TEST: I am a test case.', service_code='amazon-cloudsearch', category_code='other', communication_body="This is a test problem", severity_code='low', language='en' ) time.sleep(self.wait_time) case_id = case_1['caseId'] new_cases = self.api.describe_cases() self.assertTrue(len(new_cases['cases']) > preexisting_count) result = self.api.add_communication_to_case( communication_body="This is a test solution.", case_id=case_id ) self.assertTrue(result.get('result', False)) time.sleep(self.wait_time) final_cases = self.api.describe_cases(case_id_list=[case_id]) comms = final_cases['cases'][0]['recentCommunications']\ ['communications'] self.assertEqual(len(comms), 2) close_result = self.api.resolve_case(case_id=case_id) vendored/boto/tests/integration/support/__init__.py 0000644 00000000000 15025013616 0016614 0 ustar 00 vendored/boto/tests/integration/sqs/test_cert_verification.py 0000644 00000002777 15025013616 0020734 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on SQS endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.sqs class SQSCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): sqs = True regions = boto.sqs.regions() def sample_service_call(self, conn): conn.get_all_queues() vendored/boto/tests/integration/sqs/test_connection.py 0000644 00000025705 15025013616 0017370 0 ustar 00 # Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2010, Eucalyptus Systems, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the SQSConnection """ import time from threading import Timer from tests.unit import unittest from boto.sqs.connection import SQSConnection from boto.sqs.message import Message from boto.sqs.message import MHMessage from boto.exception import SQSError class SQSConnectionTest(unittest.TestCase): sqs = True def test_1_basic(self): print('--- running SQSConnection tests ---') c = SQSConnection() rs = c.get_all_queues() num_queues = 0 for q in rs: num_queues += 1 # try illegal name try: queue = c.create_queue('bad*queue*name') self.fail('queue name should have been bad') except SQSError: pass # now create one that should work and should be unique (i.e. a new one) queue_name = 'test%d' % int(time.time()) timeout = 60 queue_1 = c.create_queue(queue_name, timeout) self.addCleanup(c.delete_queue, queue_1, True) time.sleep(60) rs = c.get_all_queues() i = 0 for q in rs: i += 1 assert i == num_queues + 1 assert queue_1.count_slow() == 0 # check the visibility timeout t = queue_1.get_timeout() assert t == timeout, '%d != %d' % (t, timeout) # now try to get queue attributes a = q.get_attributes() assert 'ApproximateNumberOfMessages' in a assert 'VisibilityTimeout' in a a = q.get_attributes('ApproximateNumberOfMessages') assert 'ApproximateNumberOfMessages' in a assert 'VisibilityTimeout' not in a a = q.get_attributes('VisibilityTimeout') assert 'ApproximateNumberOfMessages' not in a assert 'VisibilityTimeout' in a # now change the visibility timeout timeout = 45 queue_1.set_timeout(timeout) time.sleep(60) t = queue_1.get_timeout() assert t == timeout, '%d != %d' % (t, timeout) # now add a message message_body = 'This is a test\n' message = queue_1.new_message(message_body) queue_1.write(message) time.sleep(60) assert queue_1.count_slow() == 1 time.sleep(90) # now read the message from the queue with a 10 second timeout message = queue_1.read(visibility_timeout=10) assert message assert message.get_body() == message_body # now immediately try another read, shouldn't find anything message = queue_1.read() assert message == None # now wait 30 seconds and try again time.sleep(30) message = queue_1.read() assert message # now delete the message queue_1.delete_message(message) time.sleep(30) assert queue_1.count_slow() == 0 # try a batch write num_msgs = 10 msgs = [(i, 'This is message %d' % i, 0) for i in range(num_msgs)] queue_1.write_batch(msgs) # try to delete all of the messages using batch delete deleted = 0 while deleted < num_msgs: time.sleep(5) msgs = queue_1.get_messages(num_msgs) if msgs: br = queue_1.delete_message_batch(msgs) deleted += len(br.results) # try a batch write with message attributes num_msgs = 10 attrs = { 'foo': { 'data_type': 'String', 'string_value': 'Hello, World!' }, } msgs = [(i, 'This is message %d' % i, 0, attrs) for i in range(num_msgs)] queue_1.write_batch(msgs) # create another queue so we can test force deletion # we will also test MHMessage with this queue queue_name = 'test%d' % int(time.time()) timeout = 60 queue_2 = c.create_queue(queue_name, timeout) self.addCleanup(c.delete_queue, queue_2, True) queue_2.set_message_class(MHMessage) time.sleep(30) # now add a couple of messages message = queue_2.new_message() message['foo'] = 'bar' queue_2.write(message) message_body = {'fie': 'baz', 'foo': 'bar'} message = queue_2.new_message(body=message_body) queue_2.write(message) time.sleep(30) m = queue_2.read() assert m['foo'] == 'bar' print('--- tests completed ---') def test_sqs_timeout(self): c = SQSConnection() queue_name = 'test_sqs_timeout_%s' % int(time.time()) queue = c.create_queue(queue_name) self.addCleanup(c.delete_queue, queue, True) start = time.time() poll_seconds = 2 response = queue.read(visibility_timeout=None, wait_time_seconds=poll_seconds) total_time = time.time() - start self.assertTrue(total_time > poll_seconds, "SQS queue did not block for at least %s seconds: %s" % (poll_seconds, total_time)) self.assertIsNone(response) # Now that there's an element in the queue, we should not block for 2 # seconds. c.send_message(queue, 'test message') start = time.time() poll_seconds = 2 message = c.receive_message( queue, number_messages=1, visibility_timeout=None, attributes=None, wait_time_seconds=poll_seconds)[0] total_time = time.time() - start self.assertTrue(total_time < poll_seconds, "SQS queue blocked longer than %s seconds: %s" % (poll_seconds, total_time)) self.assertEqual(message.get_body(), 'test message') attrs = c.get_queue_attributes(queue, 'ReceiveMessageWaitTimeSeconds') self.assertEqual(attrs['ReceiveMessageWaitTimeSeconds'], '0') def test_sqs_longpoll(self): c = SQSConnection() queue_name = 'test_sqs_longpoll_%s' % int(time.time()) queue = c.create_queue(queue_name) self.addCleanup(c.delete_queue, queue, True) messages = [] # The basic idea is to spawn a timer thread that will put something # on the queue in 5 seconds and verify that our long polling client # sees the message after waiting for approximately that long. def send_message(): messages.append( queue.write(queue.new_message('this is a test message'))) t = Timer(5.0, send_message) t.start() self.addCleanup(t.join) start = time.time() response = queue.read(wait_time_seconds=10) end = time.time() t.join() self.assertEqual(response.id, messages[0].id) self.assertEqual(response.get_body(), messages[0].get_body()) # The timer thread should send the message in 5 seconds, so # we're giving +- 1 second for the total time the queue # was blocked on the read call. self.assertTrue(4.0 <= (end - start) <= 6.0) def test_queue_deletion_affects_full_queues(self): conn = SQSConnection() initial_count = len(conn.get_all_queues()) empty = conn.create_queue('empty%d' % int(time.time())) full = conn.create_queue('full%d' % int(time.time())) time.sleep(60) # Make sure they're both around. self.assertEqual(len(conn.get_all_queues()), initial_count + 2) # Put a message in the full queue. m1 = Message() m1.set_body('This is a test message.') full.write(m1) self.assertEqual(full.count(), 1) self.assertTrue(conn.delete_queue(empty)) # Here's the regression for the docs. SQS will delete a queue with # messages in it, no ``force_deletion`` needed. self.assertTrue(conn.delete_queue(full)) # Wait long enough for SQS to finally remove the queues. time.sleep(90) self.assertEqual(len(conn.get_all_queues()), initial_count) def test_get_messages_attributes(self): conn = SQSConnection() current_timestamp = int(time.time()) test = self.create_temp_queue(conn) time.sleep(65) # Put a message in the queue. self.put_queue_message(test) self.assertEqual(test.count(), 1) # Check all attributes. msgs = test.get_messages( num_messages=1, attributes='All' ) for msg in msgs: self.assertEqual(msg.attributes['ApproximateReceiveCount'], '1') first_rec = msg.attributes['ApproximateFirstReceiveTimestamp'] first_rec = int(first_rec) / 1000 self.assertTrue(first_rec >= current_timestamp) # Put another message in the queue. self.put_queue_message(test) self.assertEqual(test.count(), 1) # Check a specific attribute. msgs = test.get_messages( num_messages=1, attributes='ApproximateReceiveCount' ) for msg in msgs: self.assertEqual(msg.attributes['ApproximateReceiveCount'], '1') with self.assertRaises(KeyError): msg.attributes['ApproximateFirstReceiveTimestamp'] def test_queue_purge(self): conn = SQSConnection() test = self.create_temp_queue(conn) time.sleep(65) # Put some messages in the queue. for x in range(0, 4): self.put_queue_message(test) self.assertEqual(test.count(), 4) # Now purge the queue conn.purge_queue(test) # Now assert queue count is 0 self.assertEqual(test.count(), 0) def create_temp_queue(self, conn): current_timestamp = int(time.time()) queue_name = 'test%d' % int(time.time()) test = conn.create_queue(queue_name) self.addCleanup(conn.delete_queue, test) return test def put_queue_message(self, queue): m1 = Message() m1.set_body('This is a test message.') queue.write(m1) vendored/boto/tests/integration/sqs/test_bigmessage.py 0000644 00000005200 15025013616 0017323 0 ustar 00 # Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2010, Eucalyptus Systems, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the SQSConnection """ import time from threading import Timer from tests.unit import unittest import boto from boto.compat import StringIO from boto.sqs.bigmessage import BigMessage from boto.exception import SQSError class TestBigMessage(unittest.TestCase): sqs = True def test_1_basic(self): c = boto.connect_sqs() # create a queue so we can test BigMessage queue_name = 'test%d' % int(time.time()) timeout = 60 queue = c.create_queue(queue_name, timeout) self.addCleanup(c.delete_queue, queue, True) queue.set_message_class(BigMessage) # create a bucket with the same name to store the message in s3 = boto.connect_s3() bucket = s3.create_bucket(queue_name) self.addCleanup(s3.delete_bucket, queue_name) time.sleep(30) # now add a message msg_body = 'This is a test of the big message' fp = StringIO(msg_body) s3_url = 's3://%s' % queue_name message = queue.new_message(fp, s3_url=s3_url) queue.write(message) time.sleep(30) s3_object_name = message.s3_url.split('/')[-1] # Make sure msg body is in bucket self.assertTrue(bucket.lookup(s3_object_name)) m = queue.read() self.assertEqual(m.get_body().decode('utf-8'), msg_body) m.delete() time.sleep(30) # Make sure msg is deleted from bucket self.assertIsNone(bucket.lookup(s3_object_name)) vendored/boto/tests/integration/sqs/__init__.py 0000644 00000002120 15025013616 0015713 0 ustar 00 # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/swf/test_cert_verification.py 0000644 00000003021 15025013616 0020704 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.swf class SWFCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): swf = True regions = boto.swf.regions() def sample_service_call(self, conn): conn.list_domains('REGISTERED') vendored/boto/tests/integration/swf/test_layer1_workflow_execution.py 0000644 00000015322 15025013616 0022426 0 ustar 00 """ Tests for Layer1 of Simple Workflow """ import time import uuid import json import traceback from boto.swf.layer1_decisions import Layer1Decisions from tests.integration.swf.test_layer1 import SimpleWorkflowLayer1TestBase class SwfL1WorkflowExecutionTest(SimpleWorkflowLayer1TestBase): """ test a simple workflow execution """ swf = True def run_decider(self): """ run one iteration of a simple decision engine """ # Poll for a decision task. tries = 0 while True: dtask = self.conn.poll_for_decision_task(self._domain, self._task_list, reverse_order=True) if dtask.get('taskToken') is not None: # This means a real decision task has arrived. break time.sleep(2) tries += 1 if tries > 10: # Give up if it's taking too long. Probably # means something is broken somewhere else. assert False, 'no decision task occurred' # Get the most recent interesting event. ignorable = ( 'DecisionTaskScheduled', 'DecisionTaskStarted', 'DecisionTaskTimedOut', ) event = None for tevent in dtask['events']: if tevent['eventType'] not in ignorable: event = tevent break # Construct the decision response. decisions = Layer1Decisions() if event['eventType'] == 'WorkflowExecutionStarted': activity_id = str(uuid.uuid1()) decisions.schedule_activity_task(activity_id, self._activity_type_name, self._activity_type_version, task_list=self._task_list, input=event['workflowExecutionStartedEventAttributes']['input']) elif event['eventType'] == 'ActivityTaskCompleted': decisions.complete_workflow_execution( result=event['activityTaskCompletedEventAttributes']['result']) elif event['eventType'] == 'ActivityTaskFailed': decisions.fail_workflow_execution( reason=event['activityTaskFailedEventAttributes']['reason'], details=event['activityTaskFailedEventAttributes']['details']) else: decisions.fail_workflow_execution( reason='unhandled decision task type; %r' % (event['eventType'],)) # Send the decision response. r = self.conn.respond_decision_task_completed(dtask['taskToken'], decisions=decisions._data, execution_context=None) assert r is None def run_worker(self): """ run one iteration of a simple worker engine """ # Poll for an activity task. tries = 0 while True: atask = self.conn.poll_for_activity_task(self._domain, self._task_list, identity='test worker') if atask.get('activityId') is not None: # This means a real activity task has arrived. break time.sleep(2) tries += 1 if tries > 10: # Give up if it's taking too long. Probably # means something is broken somewhere else. assert False, 'no activity task occurred' # Do the work or catch a "work exception." reason = None try: result = json.dumps(sum(json.loads(atask['input']))) except: reason = 'an exception was raised' details = traceback.format_exc() if reason is None: r = self.conn.respond_activity_task_completed( atask['taskToken'], result) else: r = self.conn.respond_activity_task_failed( atask['taskToken'], reason=reason, details=details) assert r is None def test_workflow_execution(self): # Start a workflow execution whose activity task will succeed. workflow_id = 'wfid-%.2f' % (time.time(),) r = self.conn.start_workflow_execution(self._domain, workflow_id, self._workflow_type_name, self._workflow_type_version, execution_start_to_close_timeout='20', input='[600, 15]') # Need the run_id to lookup the execution history later. run_id = r['runId'] # Move the workflow execution forward by having the # decider schedule an activity task. self.run_decider() # Run the worker to handle the scheduled activity task. self.run_worker() # Complete the workflow execution by having the # decider close it down. self.run_decider() # Check that the result was stored in the execution history. r = self.conn.get_workflow_execution_history(self._domain, run_id, workflow_id, reverse_order=True)['events'][0] result = r['workflowExecutionCompletedEventAttributes']['result'] assert json.loads(result) == 615 def test_failed_workflow_execution(self): # Start a workflow execution whose activity task will fail. workflow_id = 'wfid-%.2f' % (time.time(),) r = self.conn.start_workflow_execution(self._domain, workflow_id, self._workflow_type_name, self._workflow_type_version, execution_start_to_close_timeout='20', input='[600, "s"]') # Need the run_id to lookup the execution history later. run_id = r['runId'] # Move the workflow execution forward by having the # decider schedule an activity task. self.run_decider() # Run the worker to handle the scheduled activity task. self.run_worker() # Complete the workflow execution by having the # decider close it down. self.run_decider() # Check that the failure was stored in the execution history. r = self.conn.get_workflow_execution_history(self._domain, run_id, workflow_id, reverse_order=True)['events'][0] reason = r['workflowExecutionFailedEventAttributes']['reason'] assert reason == 'an exception was raised' vendored/boto/tests/integration/swf/test_layer1.py 0000644 00000024767 15025013616 0016426 0 ustar 00 """ Tests for Layer1 of Simple Workflow """ import os import unittest import time from boto.swf.layer1 import Layer1 from boto.swf import exceptions as swf_exceptions # A standard AWS account is permitted a maximum of 100 of SWF domains, # registered or deprecated. Deleting deprecated domains on demand does # not appear possible. Therefore, these tests reuse a default or # user-named testing domain. This is named by the user via the environment # variable BOTO_SWF_UNITTEST_DOMAIN, if available. Otherwise the default # testing domain is literally "boto-swf-unittest-domain". Do not use # the testing domain for other purposes. BOTO_SWF_UNITTEST_DOMAIN = os.environ.get("BOTO_SWF_UNITTEST_DOMAIN", "boto-swf-unittest-domain") # A standard domain can have a maxiumum of 10,000 workflow types and # activity types, registered or deprecated. Therefore, eventually any # tests which register new workflow types or activity types would begin # to fail with LimitExceeded. Instead of generating new workflow types # and activity types, these tests reuse the existing types. # The consequence of the limits and inability to delete deprecated # domains, workflow types, and activity types is that the tests in # this module will not test for the three register actions: # * register_domain # * register_workflow_type # * register_activity_type # Instead, the setUp of the TestCase create a domain, workflow type, # and activity type, expecting that they may already exist, and the # tests themselves test other things. # If you really want to re-test the register_* functions in their # ability to create things (rather than just reporting that they # already exist), you'll need to use a new BOTO_SWF_UNITTEST_DOMAIN. # But, beware that once you hit 100 domains, you are cannot create any # more, delete existing ones, or rename existing ones. # Some API calls establish resources, but these resources are not instantly # available to the next API call. For testing purposes, it is necessary to # have a short pause to avoid having tests fail for invalid reasons. PAUSE_SECONDS = 4 class SimpleWorkflowLayer1TestBase(unittest.TestCase): """ There are at least two test cases which share this setUp/tearDown and the class-based parameter definitions: * SimpleWorkflowLayer1Test * tests.swf.test_layer1_workflow_execution.SwfL1WorkflowExecutionTest """ swf = True # Some params used throughout the tests... # Domain registration params... _domain = BOTO_SWF_UNITTEST_DOMAIN _workflow_execution_retention_period_in_days = 'NONE' _domain_description = 'test workflow domain' # Type registration params used for workflow type and activity type... _task_list = 'tasklist1' # Workflow type registration params... _workflow_type_name = 'wft1' _workflow_type_version = '1' _workflow_type_description = 'wft1 description' _default_child_policy = 'REQUEST_CANCEL' _default_execution_start_to_close_timeout = '600' _default_task_start_to_close_timeout = '60' # Activity type registration params... _activity_type_name = 'at1' _activity_type_version = '1' _activity_type_description = 'at1 description' _default_task_heartbeat_timeout = '30' _default_task_schedule_to_close_timeout = '90' _default_task_schedule_to_start_timeout = '10' _default_task_start_to_close_timeout = '30' def setUp(self): # Create a Layer1 connection for testing. # Tester needs boto config or keys in environment variables. self.conn = Layer1() # Register a domain. Expect None (success) or # SWFDomainAlreadyExistsError. try: r = self.conn.register_domain(self._domain, self._workflow_execution_retention_period_in_days, description=self._domain_description) assert r is None time.sleep(PAUSE_SECONDS) except swf_exceptions.SWFDomainAlreadyExistsError: pass # Register a workflow type. Expect None (success) or # SWFTypeAlreadyExistsError. try: r = self.conn.register_workflow_type(self._domain, self._workflow_type_name, self._workflow_type_version, task_list=self._task_list, default_child_policy=self._default_child_policy, default_execution_start_to_close_timeout= self._default_execution_start_to_close_timeout, default_task_start_to_close_timeout= self._default_task_start_to_close_timeout, description=self._workflow_type_description) assert r is None time.sleep(PAUSE_SECONDS) except swf_exceptions.SWFTypeAlreadyExistsError: pass # Register an activity type. Expect None (success) or # SWFTypeAlreadyExistsError. try: r = self.conn.register_activity_type(self._domain, self._activity_type_name, self._activity_type_version, task_list=self._task_list, default_task_heartbeat_timeout= self._default_task_heartbeat_timeout, default_task_schedule_to_close_timeout= self._default_task_schedule_to_close_timeout, default_task_schedule_to_start_timeout= self._default_task_schedule_to_start_timeout, default_task_start_to_close_timeout= self._default_task_start_to_close_timeout, description=self._activity_type_description) assert r is None time.sleep(PAUSE_SECONDS) except swf_exceptions.SWFTypeAlreadyExistsError: pass def tearDown(self): # Delete what we can... pass class SimpleWorkflowLayer1Test(SimpleWorkflowLayer1TestBase): def test_list_domains(self): # Find the domain. r = self.conn.list_domains('REGISTERED') found = None for info in r['domainInfos']: if info['name'] == self._domain: found = info break self.assertNotEqual(found, None, 'list_domains; test domain not found') # Validate some properties. self.assertEqual(found['description'], self._domain_description, 'list_domains; description does not match') self.assertEqual(found['status'], 'REGISTERED', 'list_domains; status does not match') def test_list_workflow_types(self): # Find the workflow type. r = self.conn.list_workflow_types(self._domain, 'REGISTERED') found = None for info in r['typeInfos']: if ( info['workflowType']['name'] == self._workflow_type_name and info['workflowType']['version'] == self._workflow_type_version ): found = info break self.assertNotEqual(found, None, 'list_workflow_types; test type not found') # Validate some properties. self.assertEqual(found['description'], self._workflow_type_description, 'list_workflow_types; description does not match') self.assertEqual(found['status'], 'REGISTERED', 'list_workflow_types; status does not match') def test_list_activity_types(self): # Find the activity type. r = self.conn.list_activity_types(self._domain, 'REGISTERED') found = None for info in r['typeInfos']: if info['activityType']['name'] == self._activity_type_name: found = info break self.assertNotEqual(found, None, 'list_activity_types; test type not found') # Validate some properties. self.assertEqual(found['description'], self._activity_type_description, 'list_activity_types; description does not match') self.assertEqual(found['status'], 'REGISTERED', 'list_activity_types; status does not match') def test_list_closed_workflow_executions(self): # Test various legal ways to call function. latest_date = time.time() oldest_date = time.time() - 3600 # With startTimeFilter... self.conn.list_closed_workflow_executions(self._domain, start_latest_date=latest_date, start_oldest_date=oldest_date) # With closeTimeFilter... self.conn.list_closed_workflow_executions(self._domain, close_latest_date=latest_date, close_oldest_date=oldest_date) # With closeStatusFilter... self.conn.list_closed_workflow_executions(self._domain, close_latest_date=latest_date, close_oldest_date=oldest_date, close_status='COMPLETED') # With tagFilter... self.conn.list_closed_workflow_executions(self._domain, close_latest_date=latest_date, close_oldest_date=oldest_date, tag='ig') # With executionFilter... self.conn.list_closed_workflow_executions(self._domain, close_latest_date=latest_date, close_oldest_date=oldest_date, workflow_id='ig') # With typeFilter... self.conn.list_closed_workflow_executions(self._domain, close_latest_date=latest_date, close_oldest_date=oldest_date, workflow_name='ig', workflow_version='ig') # With reverseOrder... self.conn.list_closed_workflow_executions(self._domain, close_latest_date=latest_date, close_oldest_date=oldest_date, reverse_order=True) def test_list_open_workflow_executions(self): # Test various legal ways to call function. latest_date = time.time() oldest_date = time.time() - 3600 # With required params only... self.conn.list_closed_workflow_executions(self._domain, latest_date, oldest_date) # With tagFilter... self.conn.list_closed_workflow_executions(self._domain, latest_date, oldest_date, tag='ig') # With executionFilter... self.conn.list_closed_workflow_executions(self._domain, latest_date, oldest_date, workflow_id='ig') # With typeFilter... self.conn.list_closed_workflow_executions(self._domain, latest_date, oldest_date, workflow_name='ig', workflow_version='ig') # With reverseOrder... self.conn.list_closed_workflow_executions(self._domain, latest_date, oldest_date, reverse_order=True) vendored/boto/tests/integration/swf/__init__.py 0000644 00000000000 15025013616 0015677 0 ustar 00 vendored/boto/tests/integration/cloudsearch/test_cert_verification.py 0000644 00000003051 15025013616 0022404 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.cloudsearch class CloudSearchCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): cloudsearch = True regions = boto.cloudsearch.regions() def sample_service_call(self, conn): conn.describe_domains() vendored/boto/tests/integration/cloudsearch/test_layers.py 0000644 00000005266 15025013616 0020216 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests for Layer1 of Cloudsearch """ import time from tests.unit import unittest from boto.cloudsearch.layer1 import Layer1 from boto.cloudsearch.layer2 import Layer2 from boto.regioninfo import RegionInfo class CloudSearchLayer1Test(unittest.TestCase): cloudsearch = True def setUp(self): super(CloudSearchLayer1Test, self).setUp() self.layer1 = Layer1() self.domain_name = 'test-%d' % int(time.time()) def test_create_domain(self): resp = self.layer1.create_domain(self.domain_name) self.addCleanup(self.layer1.delete_domain, self.domain_name) self.assertTrue(resp.get('created', False)) class CloudSearchLayer2Test(unittest.TestCase): cloudsearch = True def setUp(self): super(CloudSearchLayer2Test, self).setUp() self.layer2 = Layer2() self.domain_name = 'test-%d' % int(time.time()) def test_create_domain(self): domain = self.layer2.create_domain(self.domain_name) self.addCleanup(domain.delete) self.assertTrue(domain.created, False) self.assertEqual(domain.domain_name, self.domain_name) self.assertEqual(domain.num_searchable_docs, 0) def test_initialization_regression(self): us_west_2 = RegionInfo( name='us-west-2', endpoint='cloudsearch.us-west-2.amazonaws.com' ) self.layer2 = Layer2( region=us_west_2, host='cloudsearch.us-west-2.amazonaws.com' ) self.assertEqual( self.layer2.layer1.host, 'cloudsearch.us-west-2.amazonaws.com' ) vendored/boto/tests/integration/cloudsearch/__init__.py 0000644 00000002227 15025013616 0017411 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All Rights Reserved # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/mws/__init__.py 0000644 00000000000 15025013616 0015706 0 ustar 00 vendored/boto/tests/integration/mws/test.py 0000644 00000011112 15025013616 0015134 0 ustar 00 #!/usr/bin/env python from __future__ import print_function import sys import os import os.path from datetime import datetime, timedelta simple = os.environ.get('MWS_MERCHANT', None) if not simple: print(""" Please set the MWS_MERCHANT environmental variable to your Merchant or SellerId to enable MWS tests. """) advanced = False isolator = True if __name__ == "__main__": devpath = os.path.relpath(os.path.join('..', '..', '..'), start=os.path.dirname(__file__)) sys.path = [devpath] + sys.path advanced = simple and True or False if advanced: print('>>> advanced MWS tests; using local boto sources') from boto.mws.connection import MWSConnection from tests.compat import unittest class MWSTestCase(unittest.TestCase): def setUp(self): self.mws = MWSConnection(Merchant=simple, debug=0) @unittest.skipUnless(simple and isolator, "skipping simple test") def test_feedlist(self): self.mws.get_feed_submission_list() @unittest.skipUnless(simple and isolator, "skipping simple test") def test_inbound_status(self): response = self.mws.get_inbound_service_status() status = response.GetServiceStatusResult.Status self.assertIn(status, ('GREEN', 'GREEN_I', 'YELLOW', 'RED')) @property def marketplace(self): try: return self._marketplace except AttributeError: response = self.mws.list_marketplace_participations() result = response.ListMarketplaceParticipationsResult self._marketplace = result.ListMarketplaces.Marketplace[0] return self.marketplace @property def marketplace_id(self): return self.marketplace.MarketplaceId @unittest.skipUnless(simple and isolator, "skipping simple test") def test_marketplace_participations(self): response = self.mws.list_marketplace_participations() result = response.ListMarketplaceParticipationsResult self.assertTrue(result.ListMarketplaces.Marketplace[0].MarketplaceId) @unittest.skipUnless(simple and isolator, "skipping simple test") def test_get_product_categories_for_asin(self): asin = '144930544X' response = self.mws.get_product_categories_for_asin( MarketplaceId=self.marketplace_id, ASIN=asin) self.assertEqual(len(response._result.Self), 3) categoryids = [x.ProductCategoryId for x in response._result.Self] self.assertSequenceEqual(categoryids, ['285856', '21', '491314']) @unittest.skipUnless(simple and isolator, "skipping simple test") def test_list_matching_products(self): response = self.mws.list_matching_products( MarketplaceId=self.marketplace_id, Query='boto') products = response._result.Products self.assertTrue(len(products)) @unittest.skipUnless(simple and isolator, "skipping simple test") def test_get_matching_product(self): asin = 'B001UDRNHO' response = self.mws.get_matching_product( MarketplaceId=self.marketplace_id, ASINList=[asin]) attributes = response._result[0].Product.AttributeSets.ItemAttributes self.assertEqual(attributes[0].Label, 'Serengeti') @unittest.skipUnless(simple and isolator, "skipping simple test") def test_get_matching_product_for_id(self): asins = ['B001UDRNHO', '144930544X'] response = self.mws.get_matching_product_for_id( MarketplaceId=self.marketplace_id, IdType='ASIN', IdList=asins) self.assertEqual(len(response._result), 2) for result in response._result: self.assertEqual(len(result.Products.Product), 1) @unittest.skipUnless(simple and isolator, "skipping simple test") def test_get_lowest_offer_listings_for_asin(self): asin = '144930544X' response = self.mws.get_lowest_offer_listings_for_asin( MarketplaceId=self.marketplace_id, ItemCondition='New', ASINList=[asin]) listings = response._result[0].Product.LowestOfferListings self.assertTrue(len(listings.LowestOfferListing)) @unittest.skipUnless(simple and isolator, "skipping simple test") def test_list_inventory_supply(self): asof = (datetime.today() - timedelta(days=30)).isoformat() response = self.mws.list_inventory_supply(QueryStartDateTime=asof, ResponseGroup='Basic') self.assertTrue(hasattr(response._result, 'InventorySupplyList')) if __name__ == "__main__": unittest.main() vendored/boto/tests/integration/kms/test_kms.py 0000644 00000003205 15025013616 0015776 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import boto from boto.kms.exceptions import NotFoundException from tests.compat import unittest class TestKMS(unittest.TestCase): def setUp(self): self.kms = boto.connect_kms() def test_list_keys(self): response = self.kms.list_keys() self.assertIn('Keys', response) def test_handle_not_found_exception(self): with self.assertRaises(NotFoundException): # Describe some key that does not exists self.kms.describe_key( key_id='nonexistant_key', ) vendored/boto/tests/integration/cloudformation/test_cert_verification.py 0000644 00000003064 15025013616 0023141 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.cloudformation class CloudFormationCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): cloudformation = True regions = boto.cloudformation.regions() def sample_service_call(self, conn): conn.describe_stacks() vendored/boto/tests/integration/cloudformation/test_connection.py 0000644 00000012437 15025013616 0021605 0 ustar 00 #!/usr/bin/env python import time import json from tests.unit import unittest from boto.cloudformation.connection import CloudFormationConnection BASIC_EC2_TEMPLATE = { "AWSTemplateFormatVersion": "2010-09-09", "Description": "AWS CloudFormation Sample Template EC2InstanceSample", "Parameters": { "Parameter1": { "Description": "Test Parameter 1", "Type": "String" }, "Parameter2": { "Description": "Test Parameter 2", "Type": "String" } }, "Mappings": { "RegionMap": { "us-east-1": { "AMI": "ami-7f418316" } } }, "Resources": { "Ec2Instance": { "Type": "AWS::EC2::Instance", "Properties": { "ImageId": { "Fn::FindInMap": [ "RegionMap", { "Ref": "AWS::Region" }, "AMI" ] }, "UserData": { "Fn::Base64": { "Fn::Join":[ "", [{"Ref": "Parameter1"}, {"Ref": "Parameter2"}] ] } } } } }, "Outputs": { "InstanceId": { "Description": "InstanceId of the newly created EC2 instance", "Value": { "Ref": "Ec2Instance" } }, "AZ": { "Description": "Availability Zone of the newly created EC2 instance", "Value": { "Fn::GetAtt": [ "Ec2Instance", "AvailabilityZone" ] } }, "PublicIP": { "Description": "Public IP address of the newly created EC2 instance", "Value": { "Fn::GetAtt": [ "Ec2Instance", "PublicIp" ] } }, "PrivateIP": { "Description": "Private IP address of the newly created EC2 instance", "Value": { "Fn::GetAtt": [ "Ec2Instance", "PrivateIp" ] } }, "PublicDNS": { "Description": "Public DNSName of the newly created EC2 instance", "Value": { "Fn::GetAtt": [ "Ec2Instance", "PublicDnsName" ] } }, "PrivateDNS": { "Description": "Private DNSName of the newly created EC2 instance", "Value": { "Fn::GetAtt": [ "Ec2Instance", "PrivateDnsName" ] } } } } class TestCloudformationConnection(unittest.TestCase): def setUp(self): self.connection = CloudFormationConnection() self.stack_name = 'testcfnstack' + str(int(time.time())) def test_large_template_stack_size(self): # See https://github.com/boto/boto/issues/1037 body = self.connection.create_stack( self.stack_name, template_body=json.dumps(BASIC_EC2_TEMPLATE), parameters=[('Parameter1', 'initial_value'), ('Parameter2', 'initial_value')]) self.addCleanup(self.connection.delete_stack, self.stack_name) # A newly created stack should have events events = self.connection.describe_stack_events(self.stack_name) self.assertTrue(events) # No policy should be set on the stack by default policy = self.connection.get_stack_policy(self.stack_name) self.assertEqual(None, policy) # Our new stack should show up in the stack list stacks = self.connection.describe_stacks(self.stack_name) stack = stacks[0] self.assertEqual(self.stack_name, stack.stack_name) params = [(p.key, p.value) for p in stack.parameters] self.assertEqual([('Parameter1', 'initial_value'), ('Parameter2', 'initial_value')], params) for _ in range(30): stack.update() if stack.stack_status.find("PROGRESS") == -1: break time.sleep(5) body = self.connection.update_stack( self.stack_name, template_body=json.dumps(BASIC_EC2_TEMPLATE), parameters=[('Parameter1', '', True), ('Parameter2', 'updated_value')]) stacks = self.connection.describe_stacks(self.stack_name) stack = stacks[0] params = [(p.key, p.value) for p in stacks[0].parameters] self.assertEqual([('Parameter1', 'initial_value'), ('Parameter2', 'updated_value')], params) # Waiting for the update to complete to unblock the delete_stack in the # cleanup. for _ in range(30): stack.update() if stack.stack_status.find("PROGRESS") == -1: break time.sleep(5) if __name__ == '__main__': unittest.main() vendored/boto/tests/integration/cloudformation/__init__.py 0000644 00000002227 15025013616 0020142 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All Rights Reserved # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/elasticache/test_layer1.py 0000644 00000005250 15025013616 0020056 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import time from tests.unit import unittest from boto.elasticache import layer1 from boto.exception import BotoServerError class TestElastiCacheConnection(unittest.TestCase): def setUp(self): self.elasticache = layer1.ElastiCacheConnection() def wait_until_cluster_available(self, cluster_id): timeout = time.time() + 600 while time.time() < timeout: response = self.elasticache.describe_cache_clusters(cluster_id) status = (response['DescribeCacheClustersResponse'] ['DescribeCacheClustersResult'] ['CacheClusters'][0]['CacheClusterStatus']) if status == 'available': break time.sleep(5) else: self.fail('Timeout waiting for cache cluster %r' 'to become available.' % cluster_id) def test_create_delete_cache_cluster(self): cluster_id = 'cluster-id2' self.elasticache.create_cache_cluster( cluster_id, 1, 'cache.t1.micro', 'memcached') self.wait_until_cluster_available(cluster_id) self.elasticache.delete_cache_cluster(cluster_id) timeout = time.time() + 600 while time.time() < timeout: try: self.elasticache.describe_cache_clusters(cluster_id) except BotoServerError: break time.sleep(5) else: self.fail('Timeout waiting for cache cluster %s' 'to be deleted.' % cluster_id) if __name__ == '__main__': unittest.main() vendored/boto/tests/integration/elasticache/__init__.py 0000644 00000000000 15025013616 0017345 0 ustar 00 vendored/boto/tests/integration/ses/test_cert_verification.py 0000644 00000003026 15025013616 0020704 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.ses class SESCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): ses = True regions = boto.ses.regions() def sample_service_call(self, conn): conn.list_verified_email_addresses() vendored/boto/tests/integration/ses/test_connection.py 0000644 00000002727 15025013616 0017353 0 ustar 00 from tests.unit import unittest from boto.ses.connection import SESConnection from boto.ses import exceptions class SESConnectionTest(unittest.TestCase): ses = True def setUp(self): self.ses = SESConnection() def test_get_dkim_attributes(self): response = self.ses.get_identity_dkim_attributes(['example.com']) # Verify we get the structure we expect, we don't care about the # values. self.assertTrue('GetIdentityDkimAttributesResponse' in response) self.assertTrue('GetIdentityDkimAttributesResult' in response['GetIdentityDkimAttributesResponse']) self.assertTrue( 'DkimAttributes' in response['GetIdentityDkimAttributesResponse'] ['GetIdentityDkimAttributesResult']) def test_set_identity_dkim_enabled(self): # This api call should fail because have not verified the domain, # so we can test that it at least fails we we expect. with self.assertRaises(exceptions.SESIdentityNotVerifiedError): self.ses.set_identity_dkim_enabled('example.com', True) def test_verify_domain_dkim(self): # This api call should fail because have not confirmed the domain, # so we can test that it at least fails we we expect. with self.assertRaises(exceptions.SESDomainNotConfirmedError): self.ses.verify_domain_dkim('example.com') if __name__ == '__main__': unittest.main() vendored/boto/tests/integration/ses/__init__.py 0000644 00000000000 15025013616 0015672 0 ustar 00 vendored/boto/tests/integration/gs/test_generation_conditionals.py 0000644 00000033723 15025013616 0021734 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2013, Google, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Integration tests for GS versioning support.""" import os import tempfile from xml import sax from six import StringIO from boto import handler from boto.exception import GSResponseError from boto.gs.acl import ACL from tests.integration.gs.testcase import GSTestCase # HTTP Error returned when a generation precondition fails. VERSION_MISMATCH = "412" class GSGenerationConditionalsTest(GSTestCase): def testConditionalSetContentsFromFile(self): b = self._MakeBucket() k = b.new_key("foo") s1 = "test1" fp = StringIO.StringIO(s1) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_contents_from_file(fp, if_generation=999) fp = StringIO.StringIO(s1) k.set_contents_from_file(fp, if_generation=0) g1 = k.generation s2 = "test2" fp = StringIO.StringIO(s2) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_contents_from_file(fp, if_generation=int(g1)+1) fp = StringIO.StringIO(s2) k.set_contents_from_file(fp, if_generation=g1) self.assertEqual(k.get_contents_as_string(), s2) def testConditionalSetContentsFromString(self): b = self._MakeBucket() k = b.new_key("foo") s1 = "test1" with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_contents_from_string(s1, if_generation=999) k.set_contents_from_string(s1, if_generation=0) g1 = k.generation s2 = "test2" with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_contents_from_string(s2, if_generation=int(g1)+1) k.set_contents_from_string(s2, if_generation=g1) self.assertEqual(k.get_contents_as_string(), s2) def testConditionalSetContentsFromFilename(self): s1 = "test1" s2 = "test2" f1 = tempfile.NamedTemporaryFile(prefix="boto-gs-test", delete=False) f2 = tempfile.NamedTemporaryFile(prefix="boto-gs-test", delete=False) fname1 = f1.name fname2 = f2.name f1.write(s1) f1.close() f2.write(s2) f2.close() try: b = self._MakeBucket() k = b.new_key("foo") with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_contents_from_filename(fname1, if_generation=999) k.set_contents_from_filename(fname1, if_generation=0) g1 = k.generation with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_contents_from_filename(fname2, if_generation=int(g1)+1) k.set_contents_from_filename(fname2, if_generation=g1) self.assertEqual(k.get_contents_as_string(), s2) finally: os.remove(fname1) os.remove(fname2) def testBucketConditionalSetAcl(self): b = self._MakeVersionedBucket() k = b.new_key("foo") s1 = "test1" k.set_contents_from_string(s1) g1 = k.generation mg1 = k.metageneration self.assertEqual(str(mg1), "1") b.set_acl("public-read", key_name="foo") k = b.get_key("foo") g2 = k.generation mg2 = k.metageneration self.assertEqual(g2, g1) self.assertGreater(mg2, mg1) with self.assertRaisesRegex(ValueError, ("Received if_metageneration " "argument with no " "if_generation argument")): b.set_acl("bucket-owner-full-control", key_name="foo", if_metageneration=123) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): b.set_acl("bucket-owner-full-control", key_name="foo", if_generation=int(g2) + 1) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): b.set_acl("bucket-owner-full-control", key_name="foo", if_generation=g2, if_metageneration=int(mg2) + 1) b.set_acl("bucket-owner-full-control", key_name="foo", if_generation=g2) k = b.get_key("foo") g3 = k.generation mg3 = k.metageneration self.assertEqual(g3, g2) self.assertGreater(mg3, mg2) b.set_acl("public-read", key_name="foo", if_generation=g3, if_metageneration=mg3) def testConditionalSetContentsFromStream(self): b = self._MakeBucket() k = b.new_key("foo") s1 = "test1" fp = StringIO.StringIO(s1) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_contents_from_stream(fp, if_generation=999) fp = StringIO.StringIO(s1) k.set_contents_from_stream(fp, if_generation=0) g1 = k.generation k = b.get_key("foo") s2 = "test2" fp = StringIO.StringIO(s2) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_contents_from_stream(fp, if_generation=int(g1)+1) fp = StringIO.StringIO(s2) k.set_contents_from_stream(fp, if_generation=g1) self.assertEqual(k.get_contents_as_string(), s2) def testBucketConditionalSetCannedAcl(self): b = self._MakeVersionedBucket() k = b.new_key("foo") s1 = "test1" k.set_contents_from_string(s1) g1 = k.generation mg1 = k.metageneration self.assertEqual(str(mg1), "1") b.set_canned_acl("public-read", key_name="foo") k = b.get_key("foo") g2 = k.generation mg2 = k.metageneration self.assertEqual(g2, g1) self.assertGreater(mg2, mg1) with self.assertRaisesRegex(ValueError, ("Received if_metageneration " "argument with no " "if_generation argument")): b.set_canned_acl("bucket-owner-full-control", key_name="foo", if_metageneration=123) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): b.set_canned_acl("bucket-owner-full-control", key_name="foo", if_generation=int(g2) + 1) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): b.set_canned_acl("bucket-owner-full-control", key_name="foo", if_generation=g2, if_metageneration=int(mg2) + 1) b.set_canned_acl("bucket-owner-full-control", key_name="foo", if_generation=g2) k = b.get_key("foo") g3 = k.generation mg3 = k.metageneration self.assertEqual(g3, g2) self.assertGreater(mg3, mg2) b.set_canned_acl("public-read", key_name="foo", if_generation=g3, if_metageneration=mg3) def testBucketConditionalSetXmlAcl(self): b = self._MakeVersionedBucket() k = b.new_key("foo") s1 = "test1" k.set_contents_from_string(s1) g1 = k.generation mg1 = k.metageneration self.assertEqual(str(mg1), "1") acl_xml = ( '<ACCESSControlList><EntrIes><Entry>' + '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' + '</Entry></EntrIes></ACCESSControlList>') acl = ACL() h = handler.XmlHandler(acl, b) sax.parseString(acl_xml, h) acl = acl.to_xml() b.set_xml_acl(acl, key_name="foo") k = b.get_key("foo") g2 = k.generation mg2 = k.metageneration self.assertEqual(g2, g1) self.assertGreater(mg2, mg1) with self.assertRaisesRegex(ValueError, ("Received if_metageneration " "argument with no " "if_generation argument")): b.set_xml_acl(acl, key_name="foo", if_metageneration=123) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): b.set_xml_acl(acl, key_name="foo", if_generation=int(g2) + 1) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): b.set_xml_acl(acl, key_name="foo", if_generation=g2, if_metageneration=int(mg2) + 1) b.set_xml_acl(acl, key_name="foo", if_generation=g2) k = b.get_key("foo") g3 = k.generation mg3 = k.metageneration self.assertEqual(g3, g2) self.assertGreater(mg3, mg2) b.set_xml_acl(acl, key_name="foo", if_generation=g3, if_metageneration=mg3) def testObjectConditionalSetAcl(self): b = self._MakeVersionedBucket() k = b.new_key("foo") k.set_contents_from_string("test1") g1 = k.generation mg1 = k.metageneration self.assertEqual(str(mg1), "1") k.set_acl("public-read") k = b.get_key("foo") g2 = k.generation mg2 = k.metageneration self.assertEqual(g2, g1) self.assertGreater(mg2, mg1) with self.assertRaisesRegex(ValueError, ("Received if_metageneration " "argument with no " "if_generation argument")): k.set_acl("bucket-owner-full-control", if_metageneration=123) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_acl("bucket-owner-full-control", if_generation=int(g2) + 1) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_acl("bucket-owner-full-control", if_generation=g2, if_metageneration=int(mg2) + 1) k.set_acl("bucket-owner-full-control", if_generation=g2) k = b.get_key("foo") g3 = k.generation mg3 = k.metageneration self.assertEqual(g3, g2) self.assertGreater(mg3, mg2) k.set_acl("public-read", if_generation=g3, if_metageneration=mg3) def testObjectConditionalSetCannedAcl(self): b = self._MakeVersionedBucket() k = b.new_key("foo") k.set_contents_from_string("test1") g1 = k.generation mg1 = k.metageneration self.assertEqual(str(mg1), "1") k.set_canned_acl("public-read") k = b.get_key("foo") g2 = k.generation mg2 = k.metageneration self.assertEqual(g2, g1) self.assertGreater(mg2, mg1) with self.assertRaisesRegex(ValueError, ("Received if_metageneration " "argument with no " "if_generation argument")): k.set_canned_acl("bucket-owner-full-control", if_metageneration=123) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_canned_acl("bucket-owner-full-control", if_generation=int(g2) + 1) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_canned_acl("bucket-owner-full-control", if_generation=g2, if_metageneration=int(mg2) + 1) k.set_canned_acl("bucket-owner-full-control", if_generation=g2) k = b.get_key("foo") g3 = k.generation mg3 = k.metageneration self.assertEqual(g3, g2) self.assertGreater(mg3, mg2) k.set_canned_acl("public-read", if_generation=g3, if_metageneration=mg3) def testObjectConditionalSetXmlAcl(self): b = self._MakeVersionedBucket() k = b.new_key("foo") s1 = "test1" k.set_contents_from_string(s1) g1 = k.generation mg1 = k.metageneration self.assertEqual(str(mg1), "1") acl_xml = ( '<ACCESSControlList><EntrIes><Entry>' + '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' + '</Entry></EntrIes></ACCESSControlList>') acl = ACL() h = handler.XmlHandler(acl, b) sax.parseString(acl_xml, h) acl = acl.to_xml() k.set_xml_acl(acl) k = b.get_key("foo") g2 = k.generation mg2 = k.metageneration self.assertEqual(g2, g1) self.assertGreater(mg2, mg1) with self.assertRaisesRegex(ValueError, ("Received if_metageneration " "argument with no " "if_generation argument")): k.set_xml_acl(acl, if_metageneration=123) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_xml_acl(acl, if_generation=int(g2) + 1) with self.assertRaisesRegex(GSResponseError, VERSION_MISMATCH): k.set_xml_acl(acl, if_generation=g2, if_metageneration=int(mg2) + 1) k.set_xml_acl(acl, if_generation=g2) k = b.get_key("foo") g3 = k.generation mg3 = k.metageneration self.assertEqual(g3, g2) self.assertGreater(mg3, mg2) k.set_xml_acl(acl, if_generation=g3, if_metageneration=mg3) vendored/boto/tests/integration/gs/test_resumable_downloads.py 0000644 00000037466 15025013616 0021074 0 ustar 00 # Copyright 2010 Google Inc. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests of resumable downloads. """ import errno import os import re import boto from boto.s3.resumable_download_handler import get_cur_file_size from boto.s3.resumable_download_handler import ResumableDownloadHandler from boto.exception import ResumableTransferDisposition from boto.exception import ResumableDownloadException from .cb_test_harness import CallbackTestHarness from tests.integration.gs.testcase import GSTestCase SMALL_KEY_SIZE = 2 * 1024 # 2 KB. LARGE_KEY_SIZE = 500 * 1024 # 500 KB. class ResumableDownloadTests(GSTestCase): """Resumable download test suite.""" def make_small_key(self): small_src_key_as_string = os.urandom(SMALL_KEY_SIZE) small_src_key = self._MakeKey(data=small_src_key_as_string) return small_src_key_as_string, small_src_key def make_tracker_file(self, tmpdir=None): if not tmpdir: tmpdir = self._MakeTempDir() tracker_file = os.path.join(tmpdir, 'tracker') return tracker_file def make_dst_fp(self, tmpdir=None): if not tmpdir: tmpdir = self._MakeTempDir() dst_file = os.path.join(tmpdir, 'dstfile') return open(dst_file, 'w') def test_non_resumable_download(self): """ Tests that non-resumable downloads work """ dst_fp = self.make_dst_fp() small_src_key_as_string, small_src_key = self.make_small_key() small_src_key.get_contents_to_file(dst_fp) self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string()) def test_download_without_persistent_tracker(self): """ Tests a single resumable download, with no tracker persistence """ res_download_handler = ResumableDownloadHandler() dst_fp = self.make_dst_fp() small_src_key_as_string, small_src_key = self.make_small_key() small_src_key.get_contents_to_file( dst_fp, res_download_handler=res_download_handler) self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string()) def test_failed_download_with_persistent_tracker(self): """ Tests that failed resumable download leaves a correct tracker file """ harness = CallbackTestHarness() tmpdir = self._MakeTempDir() tracker_file_name = self.make_tracker_file(tmpdir) dst_fp = self.make_dst_fp(tmpdir) res_download_handler = ResumableDownloadHandler( tracker_file_name=tracker_file_name, num_retries=0) small_src_key_as_string, small_src_key = self.make_small_key() try: small_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) self.fail('Did not get expected ResumableDownloadException') except ResumableDownloadException as e: # We'll get a ResumableDownloadException at this point because # of CallbackTestHarness (above). Check that the tracker file was # created correctly. self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) self.assertTrue(os.path.exists(tracker_file_name)) f = open(tracker_file_name) etag_line = f.readline() self.assertEqual(etag_line.rstrip('\n'), small_src_key.etag.strip('"\'')) def test_retryable_exception_recovery(self): """ Tests handling of a retryable exception """ # Test one of the RETRYABLE_EXCEPTIONS. exception = ResumableDownloadHandler.RETRYABLE_EXCEPTIONS[0] harness = CallbackTestHarness(exception=exception) res_download_handler = ResumableDownloadHandler(num_retries=1) dst_fp = self.make_dst_fp() small_src_key_as_string, small_src_key = self.make_small_key() small_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string()) def test_broken_pipe_recovery(self): """ Tests handling of a Broken Pipe (which interacts with an httplib bug) """ exception = IOError(errno.EPIPE, "Broken pipe") harness = CallbackTestHarness(exception=exception) res_download_handler = ResumableDownloadHandler(num_retries=1) dst_fp = self.make_dst_fp() small_src_key_as_string, small_src_key = self.make_small_key() small_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string()) def test_non_retryable_exception_handling(self): """ Tests resumable download that fails with a non-retryable exception """ harness = CallbackTestHarness( exception=OSError(errno.EACCES, 'Permission denied')) res_download_handler = ResumableDownloadHandler(num_retries=1) dst_fp = self.make_dst_fp() small_src_key_as_string, small_src_key = self.make_small_key() try: small_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) self.fail('Did not get expected OSError') except OSError as e: # Ensure the error was re-raised. self.assertEqual(e.errno, 13) def test_failed_and_restarted_download_with_persistent_tracker(self): """ Tests resumable download that fails once and then completes, with tracker file """ harness = CallbackTestHarness() tmpdir = self._MakeTempDir() tracker_file_name = self.make_tracker_file(tmpdir) dst_fp = self.make_dst_fp(tmpdir) small_src_key_as_string, small_src_key = self.make_small_key() res_download_handler = ResumableDownloadHandler( tracker_file_name=tracker_file_name, num_retries=1) small_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string()) # Ensure tracker file deleted. self.assertFalse(os.path.exists(tracker_file_name)) def test_multiple_in_process_failures_then_succeed(self): """ Tests resumable download that fails twice in one process, then completes """ res_download_handler = ResumableDownloadHandler(num_retries=3) dst_fp = self.make_dst_fp() small_src_key_as_string, small_src_key = self.make_small_key() small_src_key.get_contents_to_file( dst_fp, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string()) def test_multiple_in_process_failures_then_succeed_with_tracker_file(self): """ Tests resumable download that fails completely in one process, then when restarted completes, using a tracker file """ # Set up test harness that causes more failures than a single # ResumableDownloadHandler instance will handle, writing enough data # before the first failure that some of it survives that process run. harness = CallbackTestHarness( fail_after_n_bytes=LARGE_KEY_SIZE/2, num_times_to_fail=2) larger_src_key_as_string = os.urandom(LARGE_KEY_SIZE) larger_src_key = self._MakeKey(data=larger_src_key_as_string) tmpdir = self._MakeTempDir() tracker_file_name = self.make_tracker_file(tmpdir) dst_fp = self.make_dst_fp(tmpdir) res_download_handler = ResumableDownloadHandler( tracker_file_name=tracker_file_name, num_retries=0) try: larger_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) self.fail('Did not get expected ResumableDownloadException') except ResumableDownloadException as e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) # Ensure a tracker file survived. self.assertTrue(os.path.exists(tracker_file_name)) # Try it one more time; this time should succeed. larger_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) self.assertEqual(LARGE_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(larger_src_key_as_string, larger_src_key.get_contents_as_string()) self.assertFalse(os.path.exists(tracker_file_name)) # Ensure some of the file was downloaded both before and after failure. self.assertTrue( len(harness.transferred_seq_before_first_failure) > 1 and len(harness.transferred_seq_after_first_failure) > 1) def test_download_with_inital_partial_download_before_failure(self): """ Tests resumable download that successfully downloads some content before it fails, then restarts and completes """ # Set up harness to fail download after several hundred KB so download # server will have saved something before we retry. harness = CallbackTestHarness( fail_after_n_bytes=LARGE_KEY_SIZE/2) larger_src_key_as_string = os.urandom(LARGE_KEY_SIZE) larger_src_key = self._MakeKey(data=larger_src_key_as_string) res_download_handler = ResumableDownloadHandler(num_retries=1) dst_fp = self.make_dst_fp() larger_src_key.get_contents_to_file( dst_fp, cb=harness.call, res_download_handler=res_download_handler) # Ensure downloaded object has correct content. self.assertEqual(LARGE_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(larger_src_key_as_string, larger_src_key.get_contents_as_string()) # Ensure some of the file was downloaded both before and after failure. self.assertTrue( len(harness.transferred_seq_before_first_failure) > 1 and len(harness.transferred_seq_after_first_failure) > 1) def test_zero_length_object_download(self): """ Tests downloading a zero-length object (exercises boundary conditions). """ res_download_handler = ResumableDownloadHandler() dst_fp = self.make_dst_fp() k = self._MakeKey() k.get_contents_to_file(dst_fp, res_download_handler=res_download_handler) self.assertEqual(0, get_cur_file_size(dst_fp)) def test_download_with_invalid_tracker_etag(self): """ Tests resumable download with a tracker file containing an invalid etag """ tmp_dir = self._MakeTempDir() dst_fp = self.make_dst_fp(tmp_dir) small_src_key_as_string, small_src_key = self.make_small_key() invalid_etag_tracker_file_name = os.path.join(tmp_dir, 'invalid_etag_tracker') f = open(invalid_etag_tracker_file_name, 'w') f.write('3.14159\n') f.close() res_download_handler = ResumableDownloadHandler( tracker_file_name=invalid_etag_tracker_file_name) # An error should be printed about the invalid tracker, but then it # should run the update successfully. small_src_key.get_contents_to_file( dst_fp, res_download_handler=res_download_handler) self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string()) def test_download_with_inconsistent_etag_in_tracker(self): """ Tests resumable download with an inconsistent etag in tracker file """ tmp_dir = self._MakeTempDir() dst_fp = self.make_dst_fp(tmp_dir) small_src_key_as_string, small_src_key = self.make_small_key() inconsistent_etag_tracker_file_name = os.path.join(tmp_dir, 'inconsistent_etag_tracker') f = open(inconsistent_etag_tracker_file_name, 'w') good_etag = small_src_key.etag.strip('"\'') new_val_as_list = [] for c in reversed(good_etag): new_val_as_list.append(c) f.write('%s\n' % ''.join(new_val_as_list)) f.close() res_download_handler = ResumableDownloadHandler( tracker_file_name=inconsistent_etag_tracker_file_name) # An error should be printed about the expired tracker, but then it # should run the update successfully. small_src_key.get_contents_to_file( dst_fp, res_download_handler=res_download_handler) self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp)) self.assertEqual(small_src_key_as_string, small_src_key.get_contents_as_string()) def test_download_with_unwritable_tracker_file(self): """ Tests resumable download with an unwritable tracker file """ # Make dir where tracker_file lives temporarily unwritable. tmp_dir = self._MakeTempDir() tracker_file_name = os.path.join(tmp_dir, 'tracker') save_mod = os.stat(tmp_dir).st_mode try: os.chmod(tmp_dir, 0) res_download_handler = ResumableDownloadHandler( tracker_file_name=tracker_file_name) except ResumableDownloadException as e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) self.assertNotEqual( e.message.find('Couldn\'t write URI tracker file'), -1) finally: # Restore original protection of dir where tracker_file lives. os.chmod(tmp_dir, save_mod) vendored/boto/tests/integration/gs/testcase.py 0000644 00000011565 15025013616 0015607 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2013, Google, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Base TestCase class for gs integration tests.""" import shutil import tempfile import time from boto.exception import GSResponseError from boto.gs.connection import GSConnection from tests.integration.gs import util from tests.integration.gs.util import retry from tests.unit import unittest @unittest.skipUnless(util.has_google_credentials(), "Google credentials are required to run the Google " "Cloud Storage tests. Update your boto.cfg to run " "these tests.") class GSTestCase(unittest.TestCase): gs = True def setUp(self): self._conn = GSConnection() self._buckets = [] self._tempdirs = [] # Retry with an exponential backoff if a server error is received. This # ensures that we try *really* hard to clean up after ourselves. @retry(GSResponseError) def tearDown(self): while len(self._tempdirs): tmpdir = self._tempdirs.pop() shutil.rmtree(tmpdir, ignore_errors=True) while(len(self._buckets)): b = self._buckets[-1] try: bucket = self._conn.get_bucket(b) while len(list(bucket.list_versions())) > 0: for k in bucket.list_versions(): try: bucket.delete_key(k.name, generation=k.generation) except GSResponseError as e: if e.status != 404: raise bucket.delete() except GSResponseError as e: if e.status != 404: raise self._buckets.pop() def _GetConnection(self): """Returns the GSConnection object used to connect to GCS.""" return self._conn def _MakeTempName(self): """Creates and returns a temporary name for testing that is likely to be unique.""" return "boto-gs-test-%s" % repr(time.time()).replace(".", "-") def _MakeBucketName(self): """Creates and returns a temporary bucket name for testing that is likely to be unique.""" b = self._MakeTempName() self._buckets.append(b) return b def _MakeBucket(self): """Creates and returns temporary bucket for testing. After the test, the contents of the bucket and the bucket itself will be deleted.""" b = self._conn.create_bucket(self._MakeBucketName()) return b def _MakeKey(self, data='', bucket=None, set_contents=True): """Creates and returns a Key with provided data. If no bucket is given, a temporary bucket is created.""" if data and not set_contents: # The data and set_contents parameters are mutually exclusive. raise ValueError('MakeKey called with a non-empty data parameter ' 'but set_contents was set to False.') if not bucket: bucket = self._MakeBucket() key_name = self._MakeTempName() k = bucket.new_key(key_name) if set_contents: k.set_contents_from_string(data) return k def _MakeVersionedBucket(self): """Creates and returns temporary versioned bucket for testing. After the test, the contents of the bucket and the bucket itself will be deleted.""" b = self._MakeBucket() b.configure_versioning(True) time.sleep(30) # Ensure versioning config propagates. return b def _MakeTempDir(self): """Creates and returns a temporary directory on disk. After the test, the contents of the directory and the directory itself will be deleted.""" tmpdir = tempfile.mkdtemp(prefix=self._MakeTempName()) self._tempdirs.append(tmpdir) return tmpdir vendored/boto/tests/integration/gs/cb_test_harness.py 0000644 00000006621 15025013616 0017137 0 ustar 00 # Copyright 2010 Google Inc. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Test harness that allows us to raise exceptions, change file content, and record the byte transfer callback sequence, to test various resumable upload and download cases. The 'call' method of this harness can be passed as the 'cb' parameter to boto.s3.Key.send_file() and boto.s3.Key.get_file(), allowing testing of various file upload/download conditions. """ import socket import time class CallbackTestHarness(object): def __init__(self, fail_after_n_bytes=0, num_times_to_fail=1, exception=socket.error('mock socket error', 0), fp_to_change=None, fp_change_pos=None, delay_after_change=None): self.fail_after_n_bytes = fail_after_n_bytes self.num_times_to_fail = num_times_to_fail self.exception = exception # If fp_to_change and fp_change_pos are specified, 3 bytes will be # written at that position just before the first exception is thrown. self.fp_to_change = fp_to_change self.fp_change_pos = fp_change_pos self.delay_after_change = delay_after_change self.num_failures = 0 self.transferred_seq_before_first_failure = [] self.transferred_seq_after_first_failure = [] def call(self, total_bytes_transferred, unused_total_size): """ To use this test harness, pass the 'call' method of the instantiated object as the cb param to the set_contents_from_file() or get_contents_to_file() call. """ # Record transfer sequence to allow verification. if self.num_failures: self.transferred_seq_after_first_failure.append( total_bytes_transferred) else: self.transferred_seq_before_first_failure.append( total_bytes_transferred) if (total_bytes_transferred >= self.fail_after_n_bytes and self.num_failures < self.num_times_to_fail): self.num_failures += 1 if self.fp_to_change and self.fp_change_pos is not None: cur_pos = self.fp_to_change.tell() self.fp_to_change.seek(self.fp_change_pos) self.fp_to_change.write('abc') self.fp_to_change.seek(cur_pos) if self.delay_after_change: time.sleep(self.delay_after_change) self.called = True raise self.exception vendored/boto/tests/integration/gs/util.py 0000644 00000006254 15025013616 0014750 0 ustar 00 # Copyright (c) 2012, Google, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import time from boto.provider import Provider _HAS_GOOGLE_CREDENTIALS = None def has_google_credentials(): global _HAS_GOOGLE_CREDENTIALS if _HAS_GOOGLE_CREDENTIALS is None: provider = Provider('google') if (provider.get_access_key() is None or provider.get_secret_key() is None): _HAS_GOOGLE_CREDENTIALS = False else: _HAS_GOOGLE_CREDENTIALS = True return _HAS_GOOGLE_CREDENTIALS def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None): """Retry calling the decorated function using an exponential backoff. Taken from: https://github.com/saltycrane/retry-decorator Licensed under BSD: https://github.com/saltycrane/retry-decorator/blob/master/LICENSE :param ExceptionToCheck: the exception to check. may be a tuple of exceptions to check :type ExceptionToCheck: Exception or tuple :param tries: number of times to try (not retry) before giving up :type tries: int :param delay: initial delay between retries in seconds :type delay: int :param backoff: backoff multiplier e.g. value of 2 will double the delay each retry :type backoff: int :param logger: logger to use. If None, print :type logger: logging.Logger instance """ def deco_retry(f): def f_retry(*args, **kwargs): mtries, mdelay = tries, delay try_one_last_time = True while mtries > 1: try: return f(*args, **kwargs) try_one_last_time = False break except ExceptionToCheck as e: msg = "%s, Retrying in %d seconds..." % (str(e), mdelay) if logger: logger.warning(msg) else: print(msg) time.sleep(mdelay) mtries -= 1 mdelay *= backoff if try_one_last_time: return f(*args, **kwargs) return return f_retry # true decorator return deco_retry vendored/boto/tests/integration/gs/test_versioning.py 0000644 00000023052 15025013616 0017210 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2012, Google, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Integration tests for GS versioning support.""" from xml import sax from boto import handler from boto.gs import acl from tests.integration.gs.testcase import GSTestCase class GSVersioningTest(GSTestCase): def testVersioningToggle(self): b = self._MakeBucket() self.assertFalse(b.get_versioning_status()) b.configure_versioning(True) self.assertTrue(b.get_versioning_status()) b.configure_versioning(False) self.assertFalse(b.get_versioning_status()) def testDeleteVersionedKey(self): b = self._MakeVersionedBucket() k = b.new_key("foo") s1 = "test1" k.set_contents_from_string(s1) k = b.get_key("foo") g1 = k.generation s2 = "test2" k.set_contents_from_string(s2) k = b.get_key("foo") g2 = k.generation versions = list(b.list_versions()) self.assertEqual(len(versions), 2) self.assertEqual(versions[0].name, "foo") self.assertEqual(versions[1].name, "foo") generations = [k.generation for k in versions] self.assertIn(g1, generations) self.assertIn(g2, generations) # Delete "current" version and make sure that version is no longer # visible from a basic GET call. b.delete_key("foo", generation=None) self.assertIsNone(b.get_key("foo")) # Both old versions should still be there when listed using the versions # query parameter. versions = list(b.list_versions()) self.assertEqual(len(versions), 2) self.assertEqual(versions[0].name, "foo") self.assertEqual(versions[1].name, "foo") generations = [k.generation for k in versions] self.assertIn(g1, generations) self.assertIn(g2, generations) # Delete generation 2 and make sure it's gone. b.delete_key("foo", generation=g2) versions = list(b.list_versions()) self.assertEqual(len(versions), 1) self.assertEqual(versions[0].name, "foo") self.assertEqual(versions[0].generation, g1) # Delete generation 1 and make sure it's gone. b.delete_key("foo", generation=g1) versions = list(b.list_versions()) self.assertEqual(len(versions), 0) def testGetVersionedKey(self): b = self._MakeVersionedBucket() k = b.new_key("foo") s1 = "test1" k.set_contents_from_string(s1) k = b.get_key("foo") g1 = k.generation o1 = k.get_contents_as_string() self.assertEqual(o1, s1) s2 = "test2" k.set_contents_from_string(s2) k = b.get_key("foo") g2 = k.generation self.assertNotEqual(g2, g1) o2 = k.get_contents_as_string() self.assertEqual(o2, s2) k = b.get_key("foo", generation=g1) self.assertEqual(k.get_contents_as_string(), s1) k = b.get_key("foo", generation=g2) self.assertEqual(k.get_contents_as_string(), s2) def testVersionedBucketCannedAcl(self): b = self._MakeVersionedBucket() k = b.new_key("foo") s1 = "test1" k.set_contents_from_string(s1) k = b.get_key("foo") g1 = k.generation s2 = "test2" k.set_contents_from_string(s2) k = b.get_key("foo") g2 = k.generation acl1g1 = b.get_acl("foo", generation=g1) acl1g2 = b.get_acl("foo", generation=g2) owner1g1 = acl1g1.owner.id owner1g2 = acl1g2.owner.id self.assertEqual(owner1g1, owner1g2) entries1g1 = acl1g1.entries.entry_list entries1g2 = acl1g2.entries.entry_list self.assertEqual(len(entries1g1), len(entries1g2)) b.set_acl("public-read", key_name="foo", generation=g1) acl2g1 = b.get_acl("foo", generation=g1) acl2g2 = b.get_acl("foo", generation=g2) entries2g1 = acl2g1.entries.entry_list entries2g2 = acl2g2.entries.entry_list self.assertEqual(len(entries2g2), len(entries1g2)) public_read_entries1 = [e for e in entries2g1 if e.permission == "READ" and e.scope.type == acl.ALL_USERS] public_read_entries2 = [e for e in entries2g2 if e.permission == "READ" and e.scope.type == acl.ALL_USERS] self.assertEqual(len(public_read_entries1), 1) self.assertEqual(len(public_read_entries2), 0) def testVersionedBucketXmlAcl(self): b = self._MakeVersionedBucket() k = b.new_key("foo") s1 = "test1" k.set_contents_from_string(s1) k = b.get_key("foo") g1 = k.generation s2 = "test2" k.set_contents_from_string(s2) k = b.get_key("foo") g2 = k.generation acl1g1 = b.get_acl("foo", generation=g1) acl1g2 = b.get_acl("foo", generation=g2) owner1g1 = acl1g1.owner.id owner1g2 = acl1g2.owner.id self.assertEqual(owner1g1, owner1g2) entries1g1 = acl1g1.entries.entry_list entries1g2 = acl1g2.entries.entry_list self.assertEqual(len(entries1g1), len(entries1g2)) acl_xml = ( '<ACCESSControlList><EntrIes><Entry>' + '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' + '</Entry></EntrIes></ACCESSControlList>') aclo = acl.ACL() h = handler.XmlHandler(aclo, b) sax.parseString(acl_xml, h) b.set_acl(aclo, key_name="foo", generation=g1) acl2g1 = b.get_acl("foo", generation=g1) acl2g2 = b.get_acl("foo", generation=g2) entries2g1 = acl2g1.entries.entry_list entries2g2 = acl2g2.entries.entry_list self.assertEqual(len(entries2g2), len(entries1g2)) public_read_entries1 = [e for e in entries2g1 if e.permission == "READ" and e.scope.type == acl.ALL_USERS] public_read_entries2 = [e for e in entries2g2 if e.permission == "READ" and e.scope.type == acl.ALL_USERS] self.assertEqual(len(public_read_entries1), 1) self.assertEqual(len(public_read_entries2), 0) def testVersionedObjectCannedAcl(self): b = self._MakeVersionedBucket() k = b.new_key("foo") s1 = "test1" k.set_contents_from_string(s1) k = b.get_key("foo") g1 = k.generation s2 = "test2" k.set_contents_from_string(s2) k = b.get_key("foo") g2 = k.generation acl1g1 = b.get_acl("foo", generation=g1) acl1g2 = b.get_acl("foo", generation=g2) owner1g1 = acl1g1.owner.id owner1g2 = acl1g2.owner.id self.assertEqual(owner1g1, owner1g2) entries1g1 = acl1g1.entries.entry_list entries1g2 = acl1g2.entries.entry_list self.assertEqual(len(entries1g1), len(entries1g2)) b.set_acl("public-read", key_name="foo", generation=g1) acl2g1 = b.get_acl("foo", generation=g1) acl2g2 = b.get_acl("foo", generation=g2) entries2g1 = acl2g1.entries.entry_list entries2g2 = acl2g2.entries.entry_list self.assertEqual(len(entries2g2), len(entries1g2)) public_read_entries1 = [e for e in entries2g1 if e.permission == "READ" and e.scope.type == acl.ALL_USERS] public_read_entries2 = [e for e in entries2g2 if e.permission == "READ" and e.scope.type == acl.ALL_USERS] self.assertEqual(len(public_read_entries1), 1) self.assertEqual(len(public_read_entries2), 0) def testCopyVersionedKey(self): b = self._MakeVersionedBucket() k = b.new_key("foo") s1 = "test1" k.set_contents_from_string(s1) k = b.get_key("foo") g1 = k.generation s2 = "test2" k.set_contents_from_string(s2) b2 = self._MakeVersionedBucket() b2.copy_key("foo2", b.name, "foo", src_generation=g1) k2 = b2.get_key("foo2") s3 = k2.get_contents_as_string() self.assertEqual(s3, s1) def testKeyGenerationUpdatesOnSet(self): b = self._MakeVersionedBucket() k = b.new_key("foo") self.assertIsNone(k.generation) k.set_contents_from_string("test1") g1 = k.generation self.assertRegex(g1, r'[0-9]+') self.assertEqual(k.metageneration, '1') k.set_contents_from_string("test2") g2 = k.generation self.assertNotEqual(g1, g2) self.assertRegex(g2, r'[0-9]+') self.assertGreater(int(g2), int(g1)) self.assertEqual(k.metageneration, '1') vendored/boto/tests/integration/gs/test_basic.py 0000644 00000053245 15025013616 0016115 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2010, Eucalyptus Systems, Inc. # Copyright (c) 2011, Nexenta Systems, Inc. # Copyright (c) 2012, Google, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some integration tests for the GSConnection """ import os import re import urllib import xml.sax from six import StringIO from boto import handler from boto import storage_uri from boto.gs.acl import ACL from boto.gs.cors import Cors from boto.gs.lifecycle import LifecycleConfig from tests.integration.gs.testcase import GSTestCase CORS_EMPTY = '<CorsConfig></CorsConfig>' CORS_DOC = ('<CorsConfig><Cors><Origins><Origin>origin1.example.com' '</Origin><Origin>origin2.example.com</Origin></Origins>' '<Methods><Method>GET</Method><Method>PUT</Method>' '<Method>POST</Method></Methods><ResponseHeaders>' '<ResponseHeader>foo</ResponseHeader>' '<ResponseHeader>bar</ResponseHeader></ResponseHeaders>' '</Cors></CorsConfig>') ENCRYPTION_CONFIG_WITH_KEY = ( '<?xml version="1.0" encoding="UTF-8"?>\n' '<EncryptionConfiguration>' '<DefaultKmsKeyName>%s</DefaultKmsKeyName>' '</EncryptionConfiguration>') LIFECYCLE_EMPTY = ('<?xml version="1.0" encoding="UTF-8"?>' '<LifecycleConfiguration></LifecycleConfiguration>') LIFECYCLE_DOC = ('<?xml version="1.0" encoding="UTF-8"?>' '<LifecycleConfiguration><Rule>' '<Action><Delete/></Action>' '<Condition>''<IsLive>true</IsLive>' '<MatchesStorageClass>STANDARD</MatchesStorageClass>' '<Age>365</Age>' '<CreatedBefore>2013-01-15</CreatedBefore>' '<NumberOfNewerVersions>3</NumberOfNewerVersions>' '</Condition></Rule><Rule>' '<Action><SetStorageClass>NEARLINE</SetStorageClass></Action>' '<Condition><Age>366</Age>' '</Condition></Rule></LifecycleConfiguration>') LIFECYCLE_CONDITIONS_FOR_DELETE_RULE = { 'Age': '365', 'CreatedBefore': '2013-01-15', 'NumberOfNewerVersions': '3', 'IsLive': 'true', 'MatchesStorageClass': ['STANDARD']} LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE = {'Age': '366'} BILLING_EMPTY = {'BillingConfiguration': {}} BILLING_ENABLED = {'BillingConfiguration': {'RequesterPays': 'Enabled'}} BILLING_DISABLED = {'BillingConfiguration': {'RequesterPays': 'Disabled'}} # Regexp for matching project-private default object ACL. PROJECT_PRIVATE_RE = (r'\s*<AccessControlList>\s*<Entries>\s*<Entry>' r'\s*<Scope type="GroupById">\s*<ID>[-a-zA-Z0-9]+</ID>' r'\s*(<Name>[^<]+</Name>)?\s*</Scope>' r'\s*<Permission>FULL_CONTROL</Permission>\s*</Entry>\s*<Entry>' r'\s*<Scope type="GroupById">\s*<ID>[-a-zA-Z0-9]+</ID>' r'\s*(<Name>[^<]+</Name>)?\s*</Scope>' r'\s*<Permission>FULL_CONTROL</Permission>\s*</Entry>\s*<Entry>' r'\s*<Scope type="GroupById">\s*<ID>[-a-zA-Z0-9]+</ID>' r'\s*(<Name>[^<]+</Name>)?\s*</Scope>' r'\s*<Permission>READ</Permission>\s*</Entry>\s*</Entries>' r'\s*</AccessControlList>\s*') class GSBasicTest(GSTestCase): """Tests some basic GCS functionality.""" def test_read_write(self): """Tests basic read/write to keys.""" bucket = self._MakeBucket() bucket_name = bucket.name # now try a get_bucket call and see if it's really there bucket = self._GetConnection().get_bucket(bucket_name) key_name = 'foobar' k = bucket.new_key(key_name) s1 = 'This is a test of file upload and download' k.set_contents_from_string(s1) tmpdir = self._MakeTempDir() fpath = os.path.join(tmpdir, key_name) fp = open(fpath, 'wb') # now get the contents from gcs to a local file k.get_contents_to_file(fp) fp.close() fp = open(fpath) # check to make sure content read from gcs is identical to original self.assertEqual(s1, fp.read()) fp.close() # Use generate_url to get the contents url = self._conn.generate_url(900, 'GET', bucket=bucket.name, key=key_name) f = urllib.urlopen(url) self.assertEqual(s1, f.read()) f.close() # check to make sure set_contents_from_file is working sfp = StringIO.StringIO('foo') k.set_contents_from_file(sfp) self.assertEqual(k.get_contents_as_string(), 'foo') sfp2 = StringIO.StringIO('foo2') k.set_contents_from_file(sfp2) self.assertEqual(k.get_contents_as_string(), 'foo2') def test_get_all_keys(self): """Tests get_all_keys.""" phony_mimetype = 'application/x-boto-test' headers = {'Content-Type': phony_mimetype} tmpdir = self._MakeTempDir() fpath = os.path.join(tmpdir, 'foobar1') fpath2 = os.path.join(tmpdir, 'foobar') with open(fpath2, 'w') as f: f.write('test-data') bucket = self._MakeBucket() # First load some data for the first one, overriding content type. k = bucket.new_key('foobar') s1 = 'test-contents' s2 = 'test-contents2' k.name = 'foo/bar' k.set_contents_from_string(s1, headers) k.name = 'foo/bas' k.set_contents_from_filename(fpath2) k.name = 'foo/bat' k.set_contents_from_string(s1) k.name = 'fie/bar' k.set_contents_from_string(s1) k.name = 'fie/bas' k.set_contents_from_string(s1) k.name = 'fie/bat' k.set_contents_from_string(s1) # try resetting the contents to another value md5 = k.md5 k.set_contents_from_string(s2) self.assertNotEqual(k.md5, md5) fp2 = open(fpath2, 'rb') k.md5 = None k.base64md5 = None k.set_contents_from_stream(fp2) fp = open(fpath, 'wb') k.get_contents_to_file(fp) fp.close() fp2.seek(0, 0) fp = open(fpath, 'rb') self.assertEqual(fp2.read(), fp.read()) fp.close() fp2.close() all = bucket.get_all_keys() self.assertEqual(len(all), 6) rs = bucket.get_all_keys(prefix='foo') self.assertEqual(len(rs), 3) rs = bucket.get_all_keys(prefix='', delimiter='/') self.assertEqual(len(rs), 2) rs = bucket.get_all_keys(maxkeys=5) self.assertEqual(len(rs), 5) def test_bucket_lookup(self): """Test the bucket lookup method.""" bucket = self._MakeBucket() k = bucket.new_key('foo/bar') phony_mimetype = 'application/x-boto-test' headers = {'Content-Type': phony_mimetype} k.set_contents_from_string('testdata', headers) k = bucket.lookup('foo/bar') self.assertIsInstance(k, bucket.key_class) self.assertEqual(k.content_type, phony_mimetype) k = bucket.lookup('notthere') self.assertIsNone(k) def test_metadata(self): """Test key metadata operations.""" bucket = self._MakeBucket() k = self._MakeKey(bucket=bucket) key_name = k.name s1 = 'This is a test of file upload and download' mdkey1 = 'meta1' mdval1 = 'This is the first metadata value' k.set_metadata(mdkey1, mdval1) mdkey2 = 'meta2' mdval2 = 'This is the second metadata value' k.set_metadata(mdkey2, mdval2) # Test unicode character. mdval3 = u'föö' mdkey3 = 'meta3' k.set_metadata(mdkey3, mdval3) k.set_contents_from_string(s1) k = bucket.lookup(key_name) self.assertEqual(k.get_metadata(mdkey1), mdval1) self.assertEqual(k.get_metadata(mdkey2), mdval2) self.assertEqual(k.get_metadata(mdkey3), mdval3) k = bucket.new_key(key_name) k.get_contents_as_string() self.assertEqual(k.get_metadata(mdkey1), mdval1) self.assertEqual(k.get_metadata(mdkey2), mdval2) self.assertEqual(k.get_metadata(mdkey3), mdval3) def test_list_iterator(self): """Test list and iterator.""" bucket = self._MakeBucket() num_iter = len([k for k in bucket.list()]) rs = bucket.get_all_keys() num_keys = len(rs) self.assertEqual(num_iter, num_keys) def test_acl(self): """Test bucket and key ACLs.""" bucket = self._MakeBucket() # try some acl stuff bucket.set_acl('public-read') acl = bucket.get_acl() self.assertEqual(len(acl.entries.entry_list), 2) bucket.set_acl('private') acl = bucket.get_acl() self.assertEqual(len(acl.entries.entry_list), 1) k = self._MakeKey(bucket=bucket) k.set_acl('public-read') acl = k.get_acl() self.assertEqual(len(acl.entries.entry_list), 2) k.set_acl('private') acl = k.get_acl() self.assertEqual(len(acl.entries.entry_list), 1) # Test case-insensitivity of XML ACL parsing. acl_xml = ( '<ACCESSControlList><EntrIes><Entry>' + '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' + '</Entry></EntrIes></ACCESSControlList>') acl = ACL() h = handler.XmlHandler(acl, bucket) xml.sax.parseString(acl_xml, h) bucket.set_acl(acl) self.assertEqual(len(acl.entries.entry_list), 1) aclstr = k.get_xml_acl() self.assertGreater(aclstr.count('/Entry', 1), 0) def test_logging(self): """Test set/get raw logging subresource.""" bucket = self._MakeBucket() empty_logging_str="<?xml version='1.0' encoding='UTF-8'?><Logging/>" logging_str = ( "<?xml version='1.0' encoding='UTF-8'?><Logging>" "<LogBucket>log-bucket</LogBucket>" + "<LogObjectPrefix>example</LogObjectPrefix>" + "</Logging>") bucket.set_subresource('logging', logging_str) self.assertEqual(bucket.get_subresource('logging'), logging_str) # try disable/enable logging bucket.disable_logging() self.assertEqual(bucket.get_subresource('logging'), empty_logging_str) bucket.enable_logging('log-bucket', 'example') self.assertEqual(bucket.get_subresource('logging'), logging_str) def test_copy_key(self): """Test copying a key from one bucket to another.""" # create two new, empty buckets bucket1 = self._MakeBucket() bucket2 = self._MakeBucket() bucket_name_1 = bucket1.name bucket_name_2 = bucket2.name # verify buckets got created bucket1 = self._GetConnection().get_bucket(bucket_name_1) bucket2 = self._GetConnection().get_bucket(bucket_name_2) # create a key in bucket1 and give it some content key_name = 'foobar' k1 = bucket1.new_key(key_name) self.assertIsInstance(k1, bucket1.key_class) k1.name = key_name s = 'This is a test.' k1.set_contents_from_string(s) # copy the new key from bucket1 to bucket2 k1.copy(bucket_name_2, key_name) # now copy the contents from bucket2 to a local file k2 = bucket2.lookup(key_name) self.assertIsInstance(k2, bucket2.key_class) tmpdir = self._MakeTempDir() fpath = os.path.join(tmpdir, 'foobar') fp = open(fpath, 'wb') k2.get_contents_to_file(fp) fp.close() fp = open(fpath) # check to make sure content read is identical to original self.assertEqual(s, fp.read()) fp.close() # delete keys bucket1.delete_key(k1) bucket2.delete_key(k2) def test_default_object_acls(self): """Test default object acls.""" # create a new bucket bucket = self._MakeBucket() # get default acl and make sure it's project-private acl = bucket.get_def_acl() self.assertIsNotNone(re.search(PROJECT_PRIVATE_RE, acl.to_xml())) # set default acl to a canned acl and verify it gets set bucket.set_def_acl('public-read') acl = bucket.get_def_acl() # save public-read acl for later test public_read_acl = acl self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' '</Entry></Entries></AccessControlList>')) # back to private acl bucket.set_def_acl('private') acl = bucket.get_def_acl() self.assertEqual(acl.to_xml(), '<AccessControlList></AccessControlList>') # set default acl to an xml acl and verify it gets set bucket.set_def_acl(public_read_acl) acl = bucket.get_def_acl() self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' '</Entry></Entries></AccessControlList>')) # back to private acl bucket.set_def_acl('private') acl = bucket.get_def_acl() self.assertEqual(acl.to_xml(), '<AccessControlList></AccessControlList>') def test_default_object_acls_storage_uri(self): """Test default object acls using storage_uri.""" # create a new bucket bucket = self._MakeBucket() bucket_name = bucket.name uri = storage_uri('gs://' + bucket_name) # get default acl and make sure it's project-private acl = uri.get_def_acl() self.assertIsNotNone( re.search(PROJECT_PRIVATE_RE, acl.to_xml()), 'PROJECT_PRIVATE_RE not found in ACL XML:\n' + acl.to_xml()) # set default acl to a canned acl and verify it gets set uri.set_def_acl('public-read') acl = uri.get_def_acl() # save public-read acl for later test public_read_acl = acl self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' '</Entry></Entries></AccessControlList>')) # back to private acl uri.set_def_acl('private') acl = uri.get_def_acl() self.assertEqual(acl.to_xml(), '<AccessControlList></AccessControlList>') # set default acl to an xml acl and verify it gets set uri.set_def_acl(public_read_acl) acl = uri.get_def_acl() self.assertEqual(acl.to_xml(), ('<AccessControlList><Entries><Entry>' '<Scope type="AllUsers"></Scope><Permission>READ</Permission>' '</Entry></Entries></AccessControlList>')) # back to private acl uri.set_def_acl('private') acl = uri.get_def_acl() self.assertEqual(acl.to_xml(), '<AccessControlList></AccessControlList>') def test_cors_xml_bucket(self): """Test setting and getting of CORS XML documents on Bucket.""" # create a new bucket bucket = self._MakeBucket() bucket_name = bucket.name # now call get_bucket to see if it's really there bucket = self._GetConnection().get_bucket(bucket_name) # get new bucket cors and make sure it's empty cors = re.sub(r'\s', '', bucket.get_cors().to_xml()) self.assertEqual(cors, CORS_EMPTY) # set cors document on new bucket bucket.set_cors(CORS_DOC) cors = re.sub(r'\s', '', bucket.get_cors().to_xml()) self.assertEqual(cors, CORS_DOC) def test_cors_xml_storage_uri(self): """Test setting and getting of CORS XML documents with storage_uri.""" # create a new bucket bucket = self._MakeBucket() bucket_name = bucket.name uri = storage_uri('gs://' + bucket_name) # get new bucket cors and make sure it's empty cors = re.sub(r'\s', '', uri.get_cors().to_xml()) self.assertEqual(cors, CORS_EMPTY) # set cors document on new bucket cors_obj = Cors() h = handler.XmlHandler(cors_obj, None) xml.sax.parseString(CORS_DOC, h) uri.set_cors(cors_obj) cors = re.sub(r'\s', '', uri.get_cors().to_xml()) self.assertEqual(cors, CORS_DOC) def test_lifecycle_config_bucket(self): """Test setting and getting of lifecycle config on Bucket.""" # create a new bucket bucket = self._MakeBucket() bucket_name = bucket.name # now call get_bucket to see if it's really there bucket = self._GetConnection().get_bucket(bucket_name) # get lifecycle config and make sure it's empty xml = bucket.get_lifecycle_config().to_xml() self.assertEqual(xml, LIFECYCLE_EMPTY) # set lifecycle config lifecycle_config = LifecycleConfig() lifecycle_config.add_rule( 'Delete', None, LIFECYCLE_CONDITIONS_FOR_DELETE_RULE) lifecycle_config.add_rule( 'SetStorageClass', 'NEARLINE', LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE) bucket.configure_lifecycle(lifecycle_config) xml = bucket.get_lifecycle_config().to_xml() self.assertEqual(xml, LIFECYCLE_DOC) def test_lifecycle_config_storage_uri(self): """Test setting and getting of lifecycle config with storage_uri.""" # create a new bucket bucket = self._MakeBucket() bucket_name = bucket.name uri = storage_uri('gs://' + bucket_name) # get lifecycle config and make sure it's empty xml = uri.get_lifecycle_config().to_xml() self.assertEqual(xml, LIFECYCLE_EMPTY) # set lifecycle config lifecycle_config = LifecycleConfig() lifecycle_config.add_rule( 'Delete', None, LIFECYCLE_CONDITIONS_FOR_DELETE_RULE) lifecycle_config.add_rule( 'SetStorageClass', 'NEARLINE', LIFECYCLE_CONDITIONS_FOR_SET_STORAGE_CLASS_RULE) uri.configure_lifecycle(lifecycle_config) xml = uri.get_lifecycle_config().to_xml() self.assertEqual(xml, LIFECYCLE_DOC) def test_billing_config_bucket(self): """Test setting and getting of billing config on Bucket.""" # create a new bucket bucket = self._MakeBucket() bucket_name = bucket.name # get billing config and make sure it's empty billing = bucket.get_billing_config() self.assertEqual(billing, BILLING_EMPTY) # set requester pays to enabled bucket.configure_billing(requester_pays=True) billing = bucket.get_billing_config() self.assertEqual(billing, BILLING_ENABLED) # set requester pays to disabled bucket.configure_billing(requester_pays=False) billing = bucket.get_billing_config() self.assertEqual(billing, BILLING_DISABLED) def test_billing_config_storage_uri(self): """Test setting and getting of billing config with storage_uri.""" # create a new bucket bucket = self._MakeBucket() bucket_name = bucket.name uri = storage_uri('gs://' + bucket_name) # get billing config and make sure it's empty billing = uri.get_billing_config() self.assertEqual(billing, BILLING_EMPTY) # set requester pays to enabled uri.configure_billing(requester_pays=True) billing = uri.get_billing_config() self.assertEqual(billing, BILLING_ENABLED) # set requester pays to disabled uri.configure_billing(requester_pays=False) billing = uri.get_billing_config() self.assertEqual(billing, BILLING_DISABLED) def test_encryption_config_bucket(self): """Test setting and getting of EncryptionConfig on gs Bucket objects.""" # Create a new bucket. bucket = self._MakeBucket() bucket_name = bucket.name # Get EncryptionConfig and make sure it's empty. encryption_config = bucket.get_encryption_config() self.assertIsNone(encryption_config.default_kms_key_name) # Testing set functionality would require having an existing Cloud KMS # key. Since we can't hardcode a key name or dynamically create one, we # only test here that we're creating the correct XML document to send to # GCS. xmldoc = bucket._construct_encryption_config_xml( default_kms_key_name='dummykey') self.assertEqual(xmldoc, ENCRYPTION_CONFIG_WITH_KEY % 'dummykey') # Test that setting an empty encryption config works. bucket.set_encryption_config() def test_encryption_config_storage_uri(self): """Test setting and getting of EncryptionConfig with storage_uri.""" # Create a new bucket. bucket = self._MakeBucket() bucket_name = bucket.name uri = storage_uri('gs://' + bucket_name) # Get EncryptionConfig and make sure it's empty. encryption_config = uri.get_encryption_config() self.assertIsNone(encryption_config.default_kms_key_name) # Test that setting an empty encryption config works. uri.set_encryption_config() vendored/boto/tests/integration/gs/test_resumable_uploads.py 0000644 00000062334 15025013616 0020541 0 ustar 00 # Copyright 2010 Google Inc. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests of Google Cloud Storage resumable uploads. """ import errno import random import os import time from six import StringIO import boto from boto import storage_uri from boto.gs.resumable_upload_handler import ResumableUploadHandler from boto.exception import InvalidUriError from boto.exception import ResumableTransferDisposition from boto.exception import ResumableUploadException from .cb_test_harness import CallbackTestHarness from tests.integration.gs.testcase import GSTestCase SMALL_KEY_SIZE = 2 * 1024 # 2 KB. LARGE_KEY_SIZE = 500 * 1024 # 500 KB. LARGEST_KEY_SIZE = 1024 * 1024 # 1 MB. class ResumableUploadTests(GSTestCase): """Resumable upload test suite.""" def build_input_file(self, size): buf = [] # I manually construct the random data here instead of calling # os.urandom() because I want to constrain the range of data (in # this case to 0'..'9') so the test # code can easily overwrite part of the StringIO file with # known-to-be-different values. for i in range(size): buf.append(str(random.randint(0, 9))) file_as_string = ''.join(buf) return (file_as_string, StringIO.StringIO(file_as_string)) def make_small_file(self): return self.build_input_file(SMALL_KEY_SIZE) def make_large_file(self): return self.build_input_file(LARGE_KEY_SIZE) def make_tracker_file(self, tmpdir=None): if not tmpdir: tmpdir = self._MakeTempDir() tracker_file = os.path.join(tmpdir, 'tracker') return tracker_file def test_non_resumable_upload(self): """ Tests that non-resumable uploads work """ small_src_file_as_string, small_src_file = self.make_small_file() # Seek to end incase its the first test. small_src_file.seek(0, os.SEEK_END) dst_key = self._MakeKey(set_contents=False) try: dst_key.set_contents_from_file(small_src_file) self.fail("should fail as need to rewind the filepointer") except AttributeError: pass # Now try calling with a proper rewind. dst_key.set_contents_from_file(small_src_file, rewind=True) self.assertEqual(SMALL_KEY_SIZE, dst_key.size) self.assertEqual(small_src_file_as_string, dst_key.get_contents_as_string()) def test_upload_without_persistent_tracker(self): """ Tests a single resumable upload, with no tracker URI persistence """ res_upload_handler = ResumableUploadHandler() small_src_file_as_string, small_src_file = self.make_small_file() small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) dst_key.set_contents_from_file( small_src_file, res_upload_handler=res_upload_handler) self.assertEqual(SMALL_KEY_SIZE, dst_key.size) self.assertEqual(small_src_file_as_string, dst_key.get_contents_as_string()) def test_failed_upload_with_persistent_tracker(self): """ Tests that failed resumable upload leaves a correct tracker URI file """ harness = CallbackTestHarness() tracker_file_name = self.make_tracker_file() res_upload_handler = ResumableUploadHandler( tracker_file_name=tracker_file_name, num_retries=0) small_src_file_as_string, small_src_file = self.make_small_file() small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) try: dst_key.set_contents_from_file( small_src_file, cb=harness.call, res_upload_handler=res_upload_handler) self.fail('Did not get expected ResumableUploadException') except ResumableUploadException as e: # We'll get a ResumableUploadException at this point because # of CallbackTestHarness (above). Check that the tracker file was # created correctly. self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) self.assertTrue(os.path.exists(tracker_file_name)) f = open(tracker_file_name) uri_from_file = f.readline().strip() f.close() self.assertEqual(uri_from_file, res_upload_handler.get_tracker_uri()) def test_retryable_exception_recovery(self): """ Tests handling of a retryable exception """ # Test one of the RETRYABLE_EXCEPTIONS. exception = ResumableUploadHandler.RETRYABLE_EXCEPTIONS[0] harness = CallbackTestHarness(exception=exception) res_upload_handler = ResumableUploadHandler(num_retries=1) small_src_file_as_string, small_src_file = self.make_small_file() small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) dst_key.set_contents_from_file( small_src_file, cb=harness.call, res_upload_handler=res_upload_handler) # Ensure uploaded object has correct content. self.assertEqual(SMALL_KEY_SIZE, dst_key.size) self.assertEqual(small_src_file_as_string, dst_key.get_contents_as_string()) def test_broken_pipe_recovery(self): """ Tests handling of a Broken Pipe (which interacts with an httplib bug) """ exception = IOError(errno.EPIPE, "Broken pipe") harness = CallbackTestHarness(exception=exception) res_upload_handler = ResumableUploadHandler(num_retries=1) small_src_file_as_string, small_src_file = self.make_small_file() small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) dst_key.set_contents_from_file( small_src_file, cb=harness.call, res_upload_handler=res_upload_handler) # Ensure uploaded object has correct content. self.assertEqual(SMALL_KEY_SIZE, dst_key.size) self.assertEqual(small_src_file_as_string, dst_key.get_contents_as_string()) def test_non_retryable_exception_handling(self): """ Tests a resumable upload that fails with a non-retryable exception """ harness = CallbackTestHarness( exception=OSError(errno.EACCES, 'Permission denied')) res_upload_handler = ResumableUploadHandler(num_retries=1) small_src_file_as_string, small_src_file = self.make_small_file() small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) try: dst_key.set_contents_from_file( small_src_file, cb=harness.call, res_upload_handler=res_upload_handler) self.fail('Did not get expected OSError') except OSError as e: # Ensure the error was re-raised. self.assertEqual(e.errno, 13) def test_failed_and_restarted_upload_with_persistent_tracker(self): """ Tests resumable upload that fails once and then completes, with tracker file """ harness = CallbackTestHarness() tracker_file_name = self.make_tracker_file() res_upload_handler = ResumableUploadHandler( tracker_file_name=tracker_file_name, num_retries=1) small_src_file_as_string, small_src_file = self.make_small_file() small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) dst_key.set_contents_from_file( small_src_file, cb=harness.call, res_upload_handler=res_upload_handler) # Ensure uploaded object has correct content. self.assertEqual(SMALL_KEY_SIZE, dst_key.size) self.assertEqual(small_src_file_as_string, dst_key.get_contents_as_string()) # Ensure tracker file deleted. self.assertFalse(os.path.exists(tracker_file_name)) def test_multiple_in_process_failures_then_succeed(self): """ Tests resumable upload that fails twice in one process, then completes """ res_upload_handler = ResumableUploadHandler(num_retries=3) small_src_file_as_string, small_src_file = self.make_small_file() small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) dst_key.set_contents_from_file( small_src_file, res_upload_handler=res_upload_handler) # Ensure uploaded object has correct content. self.assertEqual(SMALL_KEY_SIZE, dst_key.size) self.assertEqual(small_src_file_as_string, dst_key.get_contents_as_string()) def test_multiple_in_process_failures_then_succeed_with_tracker_file(self): """ Tests resumable upload that fails completely in one process, then when restarted completes, using a tracker file """ # Set up test harness that causes more failures than a single # ResumableUploadHandler instance will handle, writing enough data # before the first failure that some of it survives that process run. harness = CallbackTestHarness( fail_after_n_bytes=LARGE_KEY_SIZE/2, num_times_to_fail=2) tracker_file_name = self.make_tracker_file() res_upload_handler = ResumableUploadHandler( tracker_file_name=tracker_file_name, num_retries=1) larger_src_file_as_string, larger_src_file = self.make_large_file() larger_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) try: dst_key.set_contents_from_file( larger_src_file, cb=harness.call, res_upload_handler=res_upload_handler) self.fail('Did not get expected ResumableUploadException') except ResumableUploadException as e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) # Ensure a tracker file survived. self.assertTrue(os.path.exists(tracker_file_name)) # Try it one more time; this time should succeed. larger_src_file.seek(0) dst_key.set_contents_from_file( larger_src_file, cb=harness.call, res_upload_handler=res_upload_handler) self.assertEqual(LARGE_KEY_SIZE, dst_key.size) self.assertEqual(larger_src_file_as_string, dst_key.get_contents_as_string()) self.assertFalse(os.path.exists(tracker_file_name)) # Ensure some of the file was uploaded both before and after failure. self.assertTrue(len(harness.transferred_seq_before_first_failure) > 1 and len(harness.transferred_seq_after_first_failure) > 1) def test_upload_with_inital_partial_upload_before_failure(self): """ Tests resumable upload that successfully uploads some content before it fails, then restarts and completes """ # Set up harness to fail upload after several hundred KB so upload # server will have saved something before we retry. harness = CallbackTestHarness( fail_after_n_bytes=LARGE_KEY_SIZE/2) res_upload_handler = ResumableUploadHandler(num_retries=1) larger_src_file_as_string, larger_src_file = self.make_large_file() larger_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) dst_key.set_contents_from_file( larger_src_file, cb=harness.call, res_upload_handler=res_upload_handler) # Ensure uploaded object has correct content. self.assertEqual(LARGE_KEY_SIZE, dst_key.size) self.assertEqual(larger_src_file_as_string, dst_key.get_contents_as_string()) # Ensure some of the file was uploaded both before and after failure. self.assertTrue(len(harness.transferred_seq_before_first_failure) > 1 and len(harness.transferred_seq_after_first_failure) > 1) def test_empty_file_upload(self): """ Tests uploading an empty file (exercises boundary conditions). """ res_upload_handler = ResumableUploadHandler() empty_src_file = StringIO.StringIO('') empty_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) dst_key.set_contents_from_file( empty_src_file, res_upload_handler=res_upload_handler) self.assertEqual(0, dst_key.size) def test_upload_retains_metadata(self): """ Tests that resumable upload correctly sets passed metadata """ res_upload_handler = ResumableUploadHandler() headers = {'Content-Type' : 'text/plain', 'x-goog-meta-abc' : 'my meta', 'x-goog-acl' : 'public-read'} small_src_file_as_string, small_src_file = self.make_small_file() small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) dst_key.set_contents_from_file( small_src_file, headers=headers, res_upload_handler=res_upload_handler) self.assertEqual(SMALL_KEY_SIZE, dst_key.size) self.assertEqual(small_src_file_as_string, dst_key.get_contents_as_string()) dst_key.open_read() self.assertEqual('text/plain', dst_key.content_type) self.assertTrue('abc' in dst_key.metadata) self.assertEqual('my meta', str(dst_key.metadata['abc'])) acl = dst_key.get_acl() for entry in acl.entries.entry_list: if str(entry.scope) == '<AllUsers>': self.assertEqual('READ', str(acl.entries.entry_list[1].permission)) return self.fail('No <AllUsers> scope found') def test_upload_with_file_size_change_between_starts(self): """ Tests resumable upload on a file that changes sizes between initial upload start and restart """ harness = CallbackTestHarness( fail_after_n_bytes=LARGE_KEY_SIZE/2) tracker_file_name = self.make_tracker_file() # Set up first process' ResumableUploadHandler not to do any # retries (initial upload request will establish expected size to # upload server). res_upload_handler = ResumableUploadHandler( tracker_file_name=tracker_file_name, num_retries=0) larger_src_file_as_string, larger_src_file = self.make_large_file() larger_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) try: dst_key.set_contents_from_file( larger_src_file, cb=harness.call, res_upload_handler=res_upload_handler) self.fail('Did not get expected ResumableUploadException') except ResumableUploadException as e: # First abort (from harness-forced failure) should be # ABORT_CUR_PROCESS. self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS) # Ensure a tracker file survived. self.assertTrue(os.path.exists(tracker_file_name)) # Try it again, this time with different size source file. # Wait 1 second between retry attempts, to give upload server a # chance to save state so it can respond to changed file size with # 500 response in the next attempt. time.sleep(1) try: largest_src_file = self.build_input_file(LARGEST_KEY_SIZE)[1] largest_src_file.seek(0) dst_key.set_contents_from_file( largest_src_file, res_upload_handler=res_upload_handler) self.fail('Did not get expected ResumableUploadException') except ResumableUploadException as e: # This abort should be a hard abort (file size changing during # transfer). self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) self.assertNotEqual(e.message.find('file size changed'), -1, e.message) def test_upload_with_file_size_change_during_upload(self): """ Tests resumable upload on a file that changes sizes while upload in progress """ # Create a file we can change during the upload. test_file_size = 500 * 1024 # 500 KB. test_file = self.build_input_file(test_file_size)[1] harness = CallbackTestHarness(fp_to_change=test_file, fp_change_pos=test_file_size) res_upload_handler = ResumableUploadHandler(num_retries=1) dst_key = self._MakeKey(set_contents=False) try: dst_key.set_contents_from_file( test_file, cb=harness.call, res_upload_handler=res_upload_handler) self.fail('Did not get expected ResumableUploadException') except ResumableUploadException as e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) self.assertNotEqual( e.message.find('File changed during upload'), -1) def test_upload_with_file_content_change_during_upload(self): """ Tests resumable upload on a file that changes one byte of content (so, size stays the same) while upload in progress. """ def Execute(): res_upload_handler = ResumableUploadHandler(num_retries=1) dst_key = self._MakeKey(set_contents=False) bucket_uri = storage_uri('gs://' + dst_key.bucket.name) dst_key_uri = bucket_uri.clone_replace_name(dst_key.name) try: dst_key.set_contents_from_file( test_file, cb=harness.call, res_upload_handler=res_upload_handler) return False except ResumableUploadException as e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) # Ensure the file size didn't change. test_file.seek(0, os.SEEK_END) self.assertEqual(test_file_size, test_file.tell()) self.assertNotEqual( e.message.find('md5 signature doesn\'t match etag'), -1) # Ensure the bad data wasn't left around. try: dst_key_uri.get_key() self.fail('Did not get expected InvalidUriError') except InvalidUriError as e: pass return True test_file_size = 500 * 1024 # 500 KB # The sizes of all the blocks written, except the final block, must be a # multiple of 256K bytes. We need to trigger a failure after the first # 256K bytes have been uploaded so that at least one block of data is # written on the server. # See https://developers.google.com/storage/docs/concepts-techniques#resumable # for more information about chunking of uploads. n_bytes = 300 * 1024 # 300 KB delay = 0 # First, try the test without a delay. If that fails, try it with a # 15-second delay. The first attempt may fail to recognize that the # server has a block if the server hasn't yet committed that block # when we resume the transfer. This would cause a restarted upload # instead of a resumed upload. for attempt in range(2): test_file = self.build_input_file(test_file_size)[1] harness = CallbackTestHarness( fail_after_n_bytes=n_bytes, fp_to_change=test_file, # Write to byte 1, as the CallbackTestHarness writes # 3 bytes. This will result in the data on the server # being different than the local file. fp_change_pos=1, delay_after_change=delay) if Execute(): break if (attempt == 0 and 0 in harness.transferred_seq_after_first_failure): # We can confirm the upload was restarted instead of resumed # by determining if there is an entry of 0 in the # transferred_seq_after_first_failure list. # In that case, try again with a 15 second delay. delay = 15 continue self.fail('Did not get expected ResumableUploadException') def test_upload_with_content_length_header_set(self): """ Tests resumable upload on a file when the user supplies a Content-Length header. This is used by gsutil, for example, to set the content length when gzipping a file. """ res_upload_handler = ResumableUploadHandler() small_src_file_as_string, small_src_file = self.make_small_file() small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) try: dst_key.set_contents_from_file( small_src_file, res_upload_handler=res_upload_handler, headers={'Content-Length' : SMALL_KEY_SIZE}) self.fail('Did not get expected ResumableUploadException') except ResumableUploadException as e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) self.assertNotEqual( e.message.find('Attempt to specify Content-Length header'), -1) def test_upload_with_syntactically_invalid_tracker_uri(self): """ Tests resumable upload with a syntactically invalid tracker URI """ tmp_dir = self._MakeTempDir() syntactically_invalid_tracker_file_name = os.path.join(tmp_dir, 'synt_invalid_uri_tracker') with open(syntactically_invalid_tracker_file_name, 'w') as f: f.write('ftp://example.com') res_upload_handler = ResumableUploadHandler( tracker_file_name=syntactically_invalid_tracker_file_name) small_src_file_as_string, small_src_file = self.make_small_file() # An error should be printed about the invalid URI, but then it # should run the update successfully. small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) dst_key.set_contents_from_file( small_src_file, res_upload_handler=res_upload_handler) self.assertEqual(SMALL_KEY_SIZE, dst_key.size) self.assertEqual(small_src_file_as_string, dst_key.get_contents_as_string()) def test_upload_with_invalid_upload_id_in_tracker_file(self): """ Tests resumable upload with invalid upload ID """ invalid_upload_id = ('http://pub.storage.googleapis.com/?upload_id=' 'AyzB2Uo74W4EYxyi5dp_-r68jz8rtbvshsv4TX7srJVkJ57CxTY5Dw2') tmpdir = self._MakeTempDir() invalid_upload_id_tracker_file_name = os.path.join(tmpdir, 'invalid_upload_id_tracker') with open(invalid_upload_id_tracker_file_name, 'w') as f: f.write(invalid_upload_id) res_upload_handler = ResumableUploadHandler( tracker_file_name=invalid_upload_id_tracker_file_name) small_src_file_as_string, small_src_file = self.make_small_file() # An error should occur, but then the tracker URI should be # regenerated and the the update should succeed. small_src_file.seek(0) dst_key = self._MakeKey(set_contents=False) dst_key.set_contents_from_file( small_src_file, res_upload_handler=res_upload_handler) self.assertEqual(SMALL_KEY_SIZE, dst_key.size) self.assertEqual(small_src_file_as_string, dst_key.get_contents_as_string()) self.assertNotEqual(invalid_upload_id, res_upload_handler.get_tracker_uri()) def test_upload_with_unwritable_tracker_file(self): """ Tests resumable upload with an unwritable tracker file """ # Make dir where tracker_file lives temporarily unwritable. tmp_dir = self._MakeTempDir() tracker_file_name = self.make_tracker_file(tmp_dir) save_mod = os.stat(tmp_dir).st_mode try: os.chmod(tmp_dir, 0) res_upload_handler = ResumableUploadHandler( tracker_file_name=tracker_file_name) except ResumableUploadException as e: self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT) self.assertNotEqual( e.message.find('Couldn\'t write URI tracker file'), -1) finally: # Restore original protection of dir where tracker_file lives. os.chmod(tmp_dir, save_mod) vendored/boto/tests/integration/gs/test_storage_uri.py 0000644 00000014551 15025013616 0017354 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2013, Google, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Integration tests for StorageUri interface.""" import binascii import re from six import StringIO from boto import storage_uri from boto.exception import BotoClientError from boto.gs.acl import SupportedPermissions as perms from tests.integration.gs.testcase import GSTestCase class GSStorageUriTest(GSTestCase): def testHasVersion(self): uri = storage_uri("gs://bucket/obj") self.assertFalse(uri.has_version()) uri.version_id = "versionid" self.assertTrue(uri.has_version()) uri = storage_uri("gs://bucket/obj") # Generation triggers versioning. uri.generation = 12345 self.assertTrue(uri.has_version()) uri.generation = None self.assertFalse(uri.has_version()) # Zero-generation counts as a version. uri = storage_uri("gs://bucket/obj") uri.generation = 0 self.assertTrue(uri.has_version()) def testCloneReplaceKey(self): b = self._MakeBucket() k = b.new_key("obj") k.set_contents_from_string("stringdata") orig_uri = storage_uri("gs://%s/" % b.name) uri = orig_uri.clone_replace_key(k) self.assertTrue(uri.has_version()) self.assertRegex(str(uri.generation), r"[0-9]+") def testSetAclXml(self): """Ensures that calls to the set_xml_acl functions succeed.""" b = self._MakeBucket() k = b.new_key("obj") k.set_contents_from_string("stringdata") bucket_uri = storage_uri("gs://%s/" % b.name) # Get a valid ACL for an object. bucket_uri.object_name = "obj" bucket_acl = bucket_uri.get_acl() bucket_uri.object_name = None # Add a permission to the ACL. all_users_read_permission = ("<Entry><Scope type='AllUsers'/>" "<Permission>READ</Permission></Entry>") acl_string = re.sub(r"</Entries>", all_users_read_permission + "</Entries>", bucket_acl.to_xml()) # Test-generated owner IDs are not currently valid for buckets acl_no_owner_string = re.sub(r"<Owner>.*</Owner>", "", acl_string) # Set ACL on an object. bucket_uri.set_xml_acl(acl_string, "obj") # Set ACL on a bucket. bucket_uri.set_xml_acl(acl_no_owner_string) # Set the default ACL for a bucket. bucket_uri.set_def_xml_acl(acl_no_owner_string) # Verify all the ACLs were successfully applied. new_obj_acl_string = k.get_acl().to_xml() new_bucket_acl_string = bucket_uri.get_acl().to_xml() new_bucket_def_acl_string = bucket_uri.get_def_acl().to_xml() self.assertRegex(new_obj_acl_string, r"AllUsers") self.assertRegex(new_bucket_acl_string, r"AllUsers") self.assertRegex(new_bucket_def_acl_string, r"AllUsers") def testPropertiesUpdated(self): b = self._MakeBucket() bucket_uri = storage_uri("gs://%s" % b.name) key_uri = bucket_uri.clone_replace_name("obj") key_uri.set_contents_from_string("data1") self.assertRegex(str(key_uri.generation), r"[0-9]+") k = b.get_key("obj") self.assertEqual(k.generation, key_uri.generation) self.assertEqual(k.get_contents_as_string(), "data1") key_uri.set_contents_from_stream(StringIO.StringIO("data2")) self.assertRegex(str(key_uri.generation), r"[0-9]+") self.assertGreater(key_uri.generation, k.generation) k = b.get_key("obj") self.assertEqual(k.generation, key_uri.generation) self.assertEqual(k.get_contents_as_string(), "data2") key_uri.set_contents_from_file(StringIO.StringIO("data3")) self.assertRegex(str(key_uri.generation), r"[0-9]+") self.assertGreater(key_uri.generation, k.generation) k = b.get_key("obj") self.assertEqual(k.generation, key_uri.generation) self.assertEqual(k.get_contents_as_string(), "data3") def testCompose(self): data1 = 'hello ' data2 = 'world!' expected_crc = 1238062967 b = self._MakeBucket() bucket_uri = storage_uri("gs://%s" % b.name) key_uri1 = bucket_uri.clone_replace_name("component1") key_uri1.set_contents_from_string(data1) key_uri2 = bucket_uri.clone_replace_name("component2") key_uri2.set_contents_from_string(data2) # Simple compose. key_uri_composite = bucket_uri.clone_replace_name("composite") components = [key_uri1, key_uri2] key_uri_composite.compose(components, content_type='text/plain') self.assertEqual(key_uri_composite.get_contents_as_string(), data1 + data2) composite_key = key_uri_composite.get_key() cloud_crc32c = binascii.hexlify( composite_key.cloud_hashes['crc32c']) self.assertEqual(cloud_crc32c, hex(expected_crc)[2:]) self.assertEqual(composite_key.content_type, 'text/plain') # Compose disallowed between buckets. key_uri1.bucket_name += '2' try: key_uri_composite.compose(components) self.fail('Composing between buckets didn\'t fail as expected.') except BotoClientError as err: self.assertEqual( err.reason, 'GCS does not support inter-bucket composing') vendored/boto/tests/integration/gs/__init__.py 0000644 00000000000 15025013616 0015511 0 ustar 00 vendored/boto/tests/integration/storage_uri/test_storage_uri.py 0000644 00000004463 15025013616 0021267 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for StorageUri """ from tests.unit import unittest import time import boto from boto.s3.connection import S3Connection, Location class StorageUriTest(unittest.TestCase): s3 = True def nuke_bucket(self, bucket): for key in bucket: key.delete() bucket.delete() def test_storage_uri_regionless(self): # First, create a bucket in a different region. conn = S3Connection( host='s3-us-west-2.amazonaws.com' ) bucket_name = 'keytest-%d' % int(time.time()) bucket = conn.create_bucket(bucket_name, location=Location.USWest2) self.addCleanup(self.nuke_bucket, bucket) # Now use ``storage_uri`` to try to make a new key. # This would throw a 301 exception. suri = boto.storage_uri('s3://%s/test' % bucket_name) the_key = suri.new_key() the_key.key = 'Test301' the_key.set_contents_from_string( 'This should store in a different region.' ) # Check it a different way. alt_conn = boto.connect_s3(host='s3-us-west-2.amazonaws.com') alt_bucket = alt_conn.get_bucket(bucket_name) alt_key = alt_bucket.get_key('Test301') vendored/boto/tests/integration/storage_uri/__init__.py 0000644 00000000000 15025013616 0017423 0 ustar 00 vendored/boto/tests/integration/redshift/test_cert_verification.py 0000644 00000002647 15025013616 0021732 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import unittest from tests.integration import ServiceCertVerificationTest import boto.redshift class RedshiftCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): redshift = True regions = boto.redshift.regions() def sample_service_call(self, conn): conn.describe_cluster_versions() vendored/boto/tests/integration/redshift/test_layer1.py 0000644 00000012410 15025013616 0017415 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import unittest import time from nose.plugins.attrib import attr from boto.redshift.layer1 import RedshiftConnection from boto.redshift.exceptions import ClusterNotFoundFault from boto.redshift.exceptions import ResizeNotFoundFault class TestRedshiftLayer1Management(unittest.TestCase): redshift = True def setUp(self): self.api = RedshiftConnection() self.cluster_prefix = 'boto-redshift-cluster-%s' self.node_type = 'dw.hs1.xlarge' self.master_username = 'mrtest' self.master_password = 'P4ssword' self.db_name = 'simon' # Redshift was taking ~20 minutes to bring clusters up in testing. self.wait_time = 60 * 20 def cluster_id(self): # This need to be unique per-test method. return self.cluster_prefix % str(int(time.time())) def create_cluster(self): cluster_id = self.cluster_id() self.api.create_cluster( cluster_id, self.node_type, self.master_username, self.master_password, db_name=self.db_name, number_of_nodes=3 ) # Wait for it to come up. time.sleep(self.wait_time) self.addCleanup(self.delete_cluster_the_slow_way, cluster_id) return cluster_id def delete_cluster_the_slow_way(self, cluster_id): # Because there might be other operations in progress. :( time.sleep(self.wait_time) self.api.delete_cluster(cluster_id, skip_final_cluster_snapshot=True) @attr('notdefault') def test_create_delete_cluster(self): cluster_id = self.cluster_id() self.api.create_cluster( cluster_id, self.node_type, self.master_username, self.master_password, db_name=self.db_name, number_of_nodes=3 ) # Wait for it to come up. time.sleep(self.wait_time) self.api.delete_cluster(cluster_id, skip_final_cluster_snapshot=True) @attr('notdefault') def test_as_much_as_possible_before_teardown(self): # Per @garnaat, for the sake of suite time, we'll test as much as we # can before we teardown. # Test a non-existent cluster ID. with self.assertRaises(ClusterNotFoundFault): self.api.describe_clusters('badpipelineid') # Now create the cluster & move on. cluster_id = self.create_cluster() # Test never resized. with self.assertRaises(ResizeNotFoundFault): self.api.describe_resize(cluster_id) # The cluster shows up in describe_clusters clusters = self.api.describe_clusters()['DescribeClustersResponse']\ ['DescribeClustersResult']\ ['Clusters'] cluster_ids = [c['ClusterIdentifier'] for c in clusters] self.assertIn(cluster_id, cluster_ids) # The cluster shows up in describe_clusters w/ id response = self.api.describe_clusters(cluster_id) self.assertEqual(response['DescribeClustersResponse']\ ['DescribeClustersResult']['Clusters'][0]\ ['ClusterIdentifier'], cluster_id) snapshot_id = "snap-%s" % cluster_id # Test creating a snapshot. response = self.api.create_cluster_snapshot(snapshot_id, cluster_id) self.assertEqual(response['CreateClusterSnapshotResponse']\ ['CreateClusterSnapshotResult']['Snapshot']\ ['SnapshotIdentifier'], snapshot_id) self.assertEqual(response['CreateClusterSnapshotResponse']\ ['CreateClusterSnapshotResult']['Snapshot']\ ['Status'], 'creating') self.addCleanup(self.api.delete_cluster_snapshot, snapshot_id) # More waiting. :( time.sleep(self.wait_time) # Describe the snapshots. response = self.api.describe_cluster_snapshots( cluster_identifier=cluster_id ) snap = response['DescribeClusterSnapshotsResponse']\ ['DescribeClusterSnapshotsResult']['Snapshots'][-1] self.assertEqual(snap['SnapshotType'], 'manual') self.assertEqual(snap['DBName'], self.db_name) vendored/boto/tests/integration/redshift/__init__.py 0000644 00000000000 15025013616 0016710 0 ustar 00 vendored/boto/tests/integration/sns/test_cert_verification.py 0000644 00000002777 15025013616 0020731 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on SQS endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.sns class SNSCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): sns = True regions = boto.sns.regions() def sample_service_call(self, conn): conn.get_all_topics() vendored/boto/tests/integration/sns/test_sns_sqs_subscription.py 0000644 00000007374 15025013616 0021525 0 ustar 00 # Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2010, Eucalyptus Systems, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Unit tests for subscribing SQS queues to SNS topics. """ import hashlib import time from tests.unit import unittest from boto.compat import json from boto.sqs.connection import SQSConnection from boto.sns.connection import SNSConnection class SNSSubcribeSQSTest(unittest.TestCase): sqs = True sns = True def setUp(self): self.sqsc = SQSConnection() self.snsc = SNSConnection() def get_policy_statements(self, queue): attrs = queue.get_attributes('Policy') policy = json.loads(attrs.get('Policy', "{}")) return policy.get('Statement', {}) def test_correct_sid(self): now = time.time() topic_name = queue_name = "test_correct_sid%d" % (now) timeout = 60 queue = self.sqsc.create_queue(queue_name, timeout) self.addCleanup(self.sqsc.delete_queue, queue, True) queue_arn = queue.arn topic = self.snsc.create_topic(topic_name) topic_arn = topic['CreateTopicResponse']['CreateTopicResult']\ ['TopicArn'] self.addCleanup(self.snsc.delete_topic, topic_arn) expected_sid = hashlib.md5((topic_arn + queue_arn).encode('utf-8')).hexdigest() resp = self.snsc.subscribe_sqs_queue(topic_arn, queue) found_expected_sid = False statements = self.get_policy_statements(queue) for statement in statements: if statement['Sid'] == expected_sid: found_expected_sid = True break self.assertTrue(found_expected_sid) def test_idempotent_subscribe(self): now = time.time() topic_name = queue_name = "test_idempotent_subscribe%d" % (now) timeout = 60 queue = self.sqsc.create_queue(queue_name, timeout) self.addCleanup(self.sqsc.delete_queue, queue, True) initial_statements = self.get_policy_statements(queue) queue_arn = queue.arn topic = self.snsc.create_topic(topic_name) topic_arn = topic['CreateTopicResponse']['CreateTopicResult']\ ['TopicArn'] self.addCleanup(self.snsc.delete_topic, topic_arn) resp = self.snsc.subscribe_sqs_queue(topic_arn, queue) time.sleep(3) first_subscribe_statements = self.get_policy_statements(queue) self.assertEqual(len(first_subscribe_statements), len(initial_statements) + 1) resp2 = self.snsc.subscribe_sqs_queue(topic_arn, queue) time.sleep(3) second_subscribe_statements = self.get_policy_statements(queue) self.assertEqual(len(second_subscribe_statements), len(first_subscribe_statements)) vendored/boto/tests/integration/sns/test_connection.py 0000644 00000005031 15025013616 0017353 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. from tests.compat import mock, unittest from boto.compat import http_client from boto.sns import connect_to_region class StubResponse(object): status = 403 reason = 'nopenopenope' def getheader(self, val): return b'' def getheaders(self): return b'' def read(self): return b'' class TestSNSConnection(unittest.TestCase): sns = True def setUp(self): self.connection = connect_to_region('us-west-2') def test_list_platform_applications(self): response = self.connection.list_platform_applications() def test_forced_host(self): # This test asserts that the ``Host`` header is correctly set. # On Python 2.5(.6), not having this in place would cause any SigV4 # calls to fail, due to a signature mismatch (the port would be present # when it shouldn't be). https = http_client.HTTPConnection mpo = mock.patch.object with mpo(https, 'request') as mock_request: with mpo(https, 'getresponse', return_value=StubResponse()): with self.assertRaises(self.connection.ResponseError): self.connection.list_platform_applications() # Now, assert that the ``Host`` was there & correct. call = mock_request.call_args_list[0] headers = call[0][3] self.assertTrue('Host' in headers) self.assertEqual(headers['Host'], 'sns.us-west-2.amazonaws.com') vendored/boto/tests/integration/sns/__init__.py 0000644 00000002120 15025013616 0015710 0 ustar 00 # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/ec2/test_cert_verification.py 0000644 00000003015 15025013616 0020561 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.ec2 class EC2CertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): ec2 = True regions = boto.ec2.regions() def sample_service_call(self, conn): conn.get_all_reservations() vendored/boto/tests/integration/ec2/cloudwatch/test_cert_verification.py 0000644 00000003075 15025013616 0022724 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ from tests.integration import ServiceCertVerificationTest import boto.ec2.cloudwatch from tests.compat import unittest class CloudWatchCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): cloudwatch = True regions = boto.ec2.cloudwatch.regions() def sample_service_call(self, conn): conn.describe_alarms() vendored/boto/tests/integration/ec2/cloudwatch/test_connection.py 0000644 00000026725 15025013616 0021373 0 ustar 00 # Copyright (c) 2010 Hunter Blanks http://artifex.org/~hblanks/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Initial, and very limited, unit tests for CloudWatchConnection. """ import datetime from boto.ec2.cloudwatch import CloudWatchConnection from tests.compat import unittest, OrderedDict # HTTP response body for CloudWatchConnection.describe_alarms DESCRIBE_ALARMS_BODY = """<DescribeAlarmsResponse xmlns="http://monitoring.amazonaws.com/doc/2010-08-01/"> <DescribeAlarmsResult> <NextToken>mynexttoken</NextToken> <MetricAlarms> <member> <StateUpdatedTimestamp>2011-11-18T23:43:59.111Z</StateUpdatedTimestamp> <InsufficientDataActions/> <StateReasonData>{"version":"1.0","queryDate":"2011-11-18T23:43:59.089+0000","startDate":"2011-11-18T23:30:00.000+0000","statistic":"Maximum","period":60,"recentDatapoints":[1.0,null,null,null,null,null,null,null,null,null,1.0],"threshold":1.0}</StateReasonData> <AlarmArn>arn:aws:cloudwatch:us-east-1:1234:alarm:FancyAlarm</AlarmArn> <AlarmConfigurationUpdatedTimestamp>2011-11-18T23:43:58.489Z</AlarmConfigurationUpdatedTimestamp> <AlarmName>FancyAlarm</AlarmName> <StateValue>OK</StateValue> <Period>60</Period> <OKActions/> <ActionsEnabled>true</ActionsEnabled> <Namespace>AcmeCo/Cronjobs</Namespace> <EvaluationPeriods>15</EvaluationPeriods> <Threshold>1.0</Threshold> <Statistic>Maximum</Statistic> <AlarmActions> <member>arn:aws:sns:us-east-1:1234:Alerts</member> </AlarmActions> <StateReason>Threshold Crossed: 2 datapoints were not less than the threshold (1.0). The most recent datapoints: [1.0, 1.0].</StateReason> <Dimensions> <member> <Name>Job</Name> <Value>ANiceCronJob</Value> </member> </Dimensions> <ComparisonOperator>LessThanThreshold</ComparisonOperator> <MetricName>Success</MetricName> </member> <member> <StateUpdatedTimestamp>2011-11-19T08:09:20.655Z</StateUpdatedTimestamp> <InsufficientDataActions/> <StateReasonData>{"version":"1.0","queryDate":"2011-11-19T08:09:20.633+0000","startDate":"2011-11-19T08:07:00.000+0000","statistic":"Maximum","period":60,"recentDatapoints":[1.0],"threshold":1.0}</StateReasonData> <AlarmArn>arn:aws:cloudwatch:us-east-1:1234:alarm:SuprtFancyAlarm</AlarmArn> <AlarmConfigurationUpdatedTimestamp>2011-11-19T16:20:19.687Z</AlarmConfigurationUpdatedTimestamp> <AlarmName>SuperFancyAlarm</AlarmName> <StateValue>OK</StateValue> <Period>60</Period> <OKActions/> <ActionsEnabled>true</ActionsEnabled> <Namespace>AcmeCo/CronJobs</Namespace> <EvaluationPeriods>60</EvaluationPeriods> <Threshold>1.0</Threshold> <Statistic>Maximum</Statistic> <AlarmActions> <member>arn:aws:sns:us-east-1:1234:alerts</member> </AlarmActions> <StateReason>Threshold Crossed: 1 datapoint (1.0) was not less than the threshold (1.0).</StateReason> <Dimensions> <member> <Name>Job</Name> <Value>ABadCronJob</Value> </member> </Dimensions> <ComparisonOperator>GreaterThanThreshold</ComparisonOperator> <MetricName>Success</MetricName> </member> </MetricAlarms> </DescribeAlarmsResult> <ResponseMetadata> <RequestId>f621311-1463-11e1-95c3-312389123</RequestId> </ResponseMetadata> </DescribeAlarmsResponse>""" class CloudWatchConnectionTest(unittest.TestCase): ec2 = True def test_build_list_params(self): c = CloudWatchConnection() params = {} c.build_list_params( params, ['thing1', 'thing2', 'thing3'], 'ThingName%d') expected_params = { 'ThingName1': 'thing1', 'ThingName2': 'thing2', 'ThingName3': 'thing3' } self.assertEqual(params, expected_params) def test_build_put_params_one(self): c = CloudWatchConnection() params = {} c.build_put_params(params, name="N", value=1, dimensions={"D": "V"}) expected_params = { 'MetricData.member.1.MetricName': 'N', 'MetricData.member.1.Value': 1, 'MetricData.member.1.Dimensions.member.1.Name': 'D', 'MetricData.member.1.Dimensions.member.1.Value': 'V', } self.assertEqual(params, expected_params) def test_build_put_params_multiple_metrics(self): c = CloudWatchConnection() params = {} c.build_put_params(params, name=["N", "M"], value=[1, 2], dimensions={"D": "V"}) expected_params = { 'MetricData.member.1.MetricName': 'N', 'MetricData.member.1.Value': 1, 'MetricData.member.1.Dimensions.member.1.Name': 'D', 'MetricData.member.1.Dimensions.member.1.Value': 'V', 'MetricData.member.2.MetricName': 'M', 'MetricData.member.2.Value': 2, 'MetricData.member.2.Dimensions.member.1.Name': 'D', 'MetricData.member.2.Dimensions.member.1.Value': 'V', } self.assertEqual(params, expected_params) def test_build_put_params_multiple_dimensions(self): c = CloudWatchConnection() params = {} c.build_put_params(params, name="N", value=[1, 2], dimensions=[{"D": "V"}, {"D": "W"}]) expected_params = { 'MetricData.member.1.MetricName': 'N', 'MetricData.member.1.Value': 1, 'MetricData.member.1.Dimensions.member.1.Name': 'D', 'MetricData.member.1.Dimensions.member.1.Value': 'V', 'MetricData.member.2.MetricName': 'N', 'MetricData.member.2.Value': 2, 'MetricData.member.2.Dimensions.member.1.Name': 'D', 'MetricData.member.2.Dimensions.member.1.Value': 'W', } self.assertEqual(params, expected_params) def test_build_put_params_multiple_parameter_dimension(self): self.maxDiff = None c = CloudWatchConnection() params = {} dimensions = [OrderedDict((("D1", "V"), ("D2", "W")))] c.build_put_params(params, name="N", value=[1], dimensions=dimensions) expected_params = { 'MetricData.member.1.MetricName': 'N', 'MetricData.member.1.Value': 1, 'MetricData.member.1.Dimensions.member.1.Name': 'D1', 'MetricData.member.1.Dimensions.member.1.Value': 'V', 'MetricData.member.1.Dimensions.member.2.Name': 'D2', 'MetricData.member.1.Dimensions.member.2.Value': 'W', } self.assertEqual(params, expected_params) def test_build_get_params_multiple_parameter_dimension1(self): self.maxDiff = None c = CloudWatchConnection() params = {} dimensions = OrderedDict((("D1", "V"), ("D2", "W"))) c.build_dimension_param(dimensions, params) expected_params = { 'Dimensions.member.1.Name': 'D1', 'Dimensions.member.1.Value': 'V', 'Dimensions.member.2.Name': 'D2', 'Dimensions.member.2.Value': 'W', } self.assertEqual(params, expected_params) def test_build_get_params_multiple_parameter_dimension2(self): self.maxDiff = None c = CloudWatchConnection() params = {} dimensions = OrderedDict((("D1", ["V1", "V2"]), ("D2", "W"), ("D3", None))) c.build_dimension_param(dimensions, params) expected_params = { 'Dimensions.member.1.Name': 'D1', 'Dimensions.member.1.Value': 'V1', 'Dimensions.member.2.Name': 'D1', 'Dimensions.member.2.Value': 'V2', 'Dimensions.member.3.Name': 'D2', 'Dimensions.member.3.Value': 'W', 'Dimensions.member.4.Name': 'D3', } self.assertEqual(params, expected_params) def test_build_put_params_invalid(self): c = CloudWatchConnection() params = {} try: c.build_put_params(params, name=["N", "M"], value=[1, 2, 3]) except: pass else: self.fail("Should not accept lists of different lengths.") def test_get_metric_statistics(self): c = CloudWatchConnection() m = c.list_metrics()[0] end = datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None) start = end - datetime.timedelta(hours=24 * 14) c.get_metric_statistics( 3600 * 24, start, end, m.name, m.namespace, ['Average', 'Sum']) def test_put_metric_data(self): c = CloudWatchConnection() now = datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None) name, namespace = 'unit-test-metric', 'boto-unit-test' c.put_metric_data(namespace, name, 5, now, 'Bytes') # Uncomment the following lines for a slower but more thorough # test. (Hurrah for eventual consistency...) # # metric = Metric(connection=c) # metric.name = name # metric.namespace = namespace # time.sleep(60) # l = metric.query( # now - datetime.timedelta(seconds=60), # datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None), # 'Average') # assert l # for row in l: # self.assertEqual(row['Unit'], 'Bytes') # self.assertEqual(row['Average'], 5.0) def test_describe_alarms(self): c = CloudWatchConnection() def make_request(*args, **kwargs): class Body(object): def __init__(self): self.status = 200 def read(self): return DESCRIBE_ALARMS_BODY return Body() c.make_request = make_request alarms = c.describe_alarms() self.assertEqual(alarms.next_token, 'mynexttoken') self.assertEqual(alarms[0].name, 'FancyAlarm') self.assertEqual(alarms[0].comparison, '<') self.assertEqual(alarms[0].dimensions, {u'Job': [u'ANiceCronJob']}) self.assertEqual(alarms[1].name, 'SuperFancyAlarm') self.assertEqual(alarms[1].comparison, '>') self.assertEqual(alarms[1].dimensions, {u'Job': [u'ABadCronJob']}) if __name__ == '__main__': unittest.main() vendored/boto/tests/integration/ec2/cloudwatch/__init__.py 0000644 00000002117 15025013616 0017721 0 ustar 00 # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/ec2/test_connection.py 0000644 00000021622 15025013616 0017225 0 ustar 00 # Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2009, Eucalyptus Systems, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the EC2Connection """ import unittest import time import telnetlib import socket from nose.plugins.attrib import attr from boto.ec2.connection import EC2Connection from boto.exception import EC2ResponseError import boto.ec2 class EC2ConnectionTest(unittest.TestCase): ec2 = True @attr('notdefault') def test_launch_permissions(self): # this is my user_id, if you want to run these tests you should # replace this with yours or they won't work user_id = '963068290131' print('--- running EC2Connection tests ---') c = EC2Connection() # get list of private AMI's rs = c.get_all_images(owners=[user_id]) assert len(rs) > 0 # now pick the first one image = rs[0] # temporarily make this image runnable by everyone status = image.set_launch_permissions(group_names=['all']) assert status d = image.get_launch_permissions() assert 'groups' in d assert len(d['groups']) > 0 # now remove that permission status = image.remove_launch_permissions(group_names=['all']) assert status time.sleep(10) d = image.get_launch_permissions() assert 'groups' not in d def test_1_basic(self): # create 2 new security groups c = EC2Connection() group1_name = 'test-%d' % int(time.time()) group_desc = 'This is a security group created during unit testing' group1 = c.create_security_group(group1_name, group_desc) time.sleep(2) group2_name = 'test-%d' % int(time.time()) group_desc = 'This is a security group created during unit testing' group2 = c.create_security_group(group2_name, group_desc) # now get a listing of all security groups and look for our new one rs = c.get_all_security_groups() found = False for g in rs: if g.name == group1_name: found = True assert found # now pass arg to filter results to only our new group rs = c.get_all_security_groups([group1_name]) assert len(rs) == 1 # try some group to group authorizations/revocations # first try the old style status = c.authorize_security_group(group1.name, group2.name, group2.owner_id) assert status status = c.revoke_security_group(group1.name, group2.name, group2.owner_id) assert status # now try specifying a specific port status = c.authorize_security_group(group1.name, group2.name, group2.owner_id, 'tcp', 22, 22) assert status status = c.revoke_security_group(group1.name, group2.name, group2.owner_id, 'tcp', 22, 22) assert status # now delete the second security group status = c.delete_security_group(group2_name) # now make sure it's really gone rs = c.get_all_security_groups() found = False for g in rs: if g.name == group2_name: found = True assert not found group = group1 # now try to launch apache image with our new security group rs = c.get_all_images() img_loc = 'ec2-public-images/fedora-core4-apache.manifest.xml' for image in rs: if image.location == img_loc: break reservation = image.run(security_groups=[group.name]) instance = reservation.instances[0] while instance.state != 'running': print('\tinstance is %s' % instance.state) time.sleep(30) instance.update() # instance in now running, try to telnet to port 80 t = telnetlib.Telnet() try: t.open(instance.dns_name, 80) except socket.error: pass # now open up port 80 and try again, it should work group.authorize('tcp', 80, 80, '0.0.0.0/0') t.open(instance.dns_name, 80) t.close() # now revoke authorization and try again group.revoke('tcp', 80, 80, '0.0.0.0/0') try: t.open(instance.dns_name, 80) except socket.error: pass # now kill the instance and delete the security group instance.terminate() # check that state and previous_state have updated assert instance.state == 'shutting-down' assert instance.state_code == 32 assert instance.previous_state == 'running' assert instance.previous_state_code == 16 # unfortunately, I can't delete the sg within this script #sg.delete() # create a new key pair key_name = 'test-%d' % int(time.time()) status = c.create_key_pair(key_name) assert status # now get a listing of all key pairs and look for our new one rs = c.get_all_key_pairs() found = False for k in rs: if k.name == key_name: found = True assert found # now pass arg to filter results to only our new key pair rs = c.get_all_key_pairs([key_name]) assert len(rs) == 1 key_pair = rs[0] # now delete the key pair status = c.delete_key_pair(key_name) # now make sure it's really gone rs = c.get_all_key_pairs() found = False for k in rs: if k.name == key_name: found = True assert not found # short test around Paid AMI capability demo_paid_ami_id = 'ami-bd9d78d4' demo_paid_ami_product_code = 'A79EC0DB' l = c.get_all_images([demo_paid_ami_id]) assert len(l) == 1 assert len(l[0].product_codes) == 1 assert l[0].product_codes[0] == demo_paid_ami_product_code print('--- tests completed ---') def test_dry_run(self): c = EC2Connection() dry_run_msg = 'Request would have succeeded, but DryRun flag is set.' try: rs = c.get_all_images(dry_run=True) self.fail("Should have gotten an exception") except EC2ResponseError as e: self.assertTrue(dry_run_msg in str(e)) try: rs = c.run_instances( image_id='ami-a0cd60c9', instance_type='m1.small', dry_run=True ) self.fail("Should have gotten an exception") except EC2ResponseError as e: self.assertTrue(dry_run_msg in str(e)) # Need an actual instance for the rest of this... rs = c.run_instances( image_id='ami-a0cd60c9', instance_type='m1.small' ) time.sleep(120) try: rs = c.stop_instances( instance_ids=[rs.instances[0].id], dry_run=True ) self.fail("Should have gotten an exception") except EC2ResponseError as e: self.assertTrue(dry_run_msg in str(e)) try: rs = c.terminate_instances( instance_ids=[rs.instances[0].id], dry_run=True ) self.fail("Should have gotten an exception") except EC2ResponseError as e: self.assertTrue(dry_run_msg in str(e)) # And kill it. rs.instances[0].terminate() def test_can_get_all_instances_sigv4(self): connection = boto.ec2.connect_to_region('eu-central-1') self.assertTrue(isinstance(connection.get_all_instances(), list)) vendored/boto/tests/integration/ec2/autoscale/test_cert_verification.py 0000644 00000003047 15025013616 0022546 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.ec2.autoscale class AutoscaleCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): autoscale = True regions = boto.ec2.autoscale.regions() def sample_service_call(self, conn): conn.get_all_groups() vendored/boto/tests/integration/ec2/autoscale/test_connection.py 0000644 00000014640 15025013616 0021207 0 ustar 00 # Copyright (c) 2011 Reza Lotun http://reza.lotun.name # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the AutoscaleConnection """ import time from boto.ec2.autoscale import AutoScaleConnection from boto.ec2.autoscale.activity import Activity from boto.ec2.autoscale.group import AutoScalingGroup, ProcessType from boto.ec2.autoscale.launchconfig import LaunchConfiguration from boto.ec2.autoscale.policy import AdjustmentType, MetricCollectionTypes, ScalingPolicy from boto.ec2.autoscale.scheduled import ScheduledUpdateGroupAction from boto.ec2.autoscale.instance import Instance from boto.ec2.autoscale.tag import Tag from tests.compat import unittest class AutoscaleConnectionTest(unittest.TestCase): ec2 = True autoscale = True def test_basic(self): # NB: as it says on the tin these are really basic tests that only # (lightly) exercise read-only behaviour - and that's only if you # have any autoscale groups to introspect. It's useful, however, to # catch simple errors print('--- running %s tests ---' % self.__class__.__name__) c = AutoScaleConnection() self.assertTrue(repr(c).startswith('AutoScaleConnection')) groups = c.get_all_groups() for group in groups: self.assertIsInstance(group, AutoScalingGroup) # get activities activities = group.get_activities() for activity in activities: self.assertIsInstance(activity, Activity) # get launch configs configs = c.get_all_launch_configurations() for config in configs: self.assertIsInstance(config, LaunchConfiguration) # get policies policies = c.get_all_policies() for policy in policies: self.assertIsInstance(policy, ScalingPolicy) # get scheduled actions actions = c.get_all_scheduled_actions() for action in actions: self.assertIsInstance(action, ScheduledUpdateGroupAction) # get instances instances = c.get_all_autoscaling_instances() for instance in instances: self.assertIsInstance(instance, Instance) # get all scaling process types ptypes = c.get_all_scaling_process_types() for ptype in ptypes: self.assertTrue(ptype, ProcessType) # get adjustment types adjustments = c.get_all_adjustment_types() for adjustment in adjustments: self.assertIsInstance(adjustment, AdjustmentType) # get metrics collection types types = c.get_all_metric_collection_types() self.assertIsInstance(types, MetricCollectionTypes) # create the simplest possible AutoScale group # first create the launch configuration time_string = '%d' % int(time.time()) lc_name = 'lc-%s' % time_string lc = LaunchConfiguration(name=lc_name, image_id='ami-2272864b', instance_type='t1.micro') c.create_launch_configuration(lc) found = False lcs = c.get_all_launch_configurations() for lc in lcs: if lc.name == lc_name: found = True break assert found # now create autoscaling group group_name = 'group-%s' % time_string group = AutoScalingGroup(name=group_name, launch_config=lc, availability_zones=['us-east-1a'], min_size=1, max_size=1) c.create_auto_scaling_group(group) found = False groups = c.get_all_groups() for group in groups: if group.name == group_name: found = True break assert found # now create a tag tag = Tag(key='foo', value='bar', resource_id=group_name, propagate_at_launch=True) c.create_or_update_tags([tag]) found = False tags = c.get_all_tags() for tag in tags: if tag.resource_id == group_name and tag.key == 'foo': found = True break assert found c.delete_tags([tag]) # shutdown instances and wait for them to disappear group.shutdown_instances() instances = True while instances: time.sleep(5) groups = c.get_all_groups() for group in groups: if group.name == group_name: if not group.instances: instances = False group.delete() lc.delete() found = True while found: found = False time.sleep(5) tags = c.get_all_tags() for tag in tags: if tag.resource_id == group_name and tag.key == 'foo': found = True assert not found print('--- tests completed ---') def test_ebs_optimized_regression(self): c = AutoScaleConnection() time_string = '%d' % int(time.time()) lc_name = 'lc-%s' % time_string lc = LaunchConfiguration( name=lc_name, image_id='ami-2272864b', instance_type='t1.micro', ebs_optimized=True ) # This failed due to the difference between native Python ``True/False`` # & the expected string variants. c.create_launch_configuration(lc) self.addCleanup(c.delete_launch_configuration, lc_name) vendored/boto/tests/integration/ec2/autoscale/__init__.py 0000644 00000002112 15025013616 0017537 0 ustar 00 # Copyright (c) 2011 Reza Lotun http://reza.lotun.name # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/ec2/elb/test_cert_verification.py 0000644 00000003051 15025013616 0021323 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ from tests.integration import ServiceCertVerificationTest from tests.compat import unittest import boto.ec2.elb class ELBCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): elb = True regions = boto.ec2.elb.regions() def sample_service_call(self, conn): conn.get_all_load_balancers() vendored/boto/tests/integration/ec2/elb/test_connection.py 0000644 00000030412 15025013616 0017764 0 ustar 00 # Copyright (c) 2010 Hunter Blanks http://artifex.org/~hblanks/ # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Initial, and very limited, unit tests for ELBConnection. """ import boto import time from tests.compat import unittest from boto.ec2.elb import ELBConnection import boto.ec2.elb class ELBConnectionTest(unittest.TestCase): ec2 = True def setUp(self): """Creates a named load balancer that can be safely deleted at the end of each test""" self.conn = ELBConnection() self.name = 'elb-boto-unit-test' self.availability_zones = ['us-east-1a'] self.listeners = [(80, 8000, 'HTTP')] self.balancer = self.conn.create_load_balancer( self.name, self.availability_zones, self.listeners) # S3 bucket for log tests self.s3 = boto.connect_s3() self.timestamp = str(int(time.time())) self.bucket_name = 'boto-elb-%s' % self.timestamp self.bucket = self.s3.create_bucket(self.bucket_name) self.bucket.set_canned_acl('public-read-write') self.addCleanup(self.cleanup_bucket, self.bucket) def cleanup_bucket(self, bucket): for key in bucket.get_all_keys(): key.delete() bucket.delete() def tearDown(self): """ Deletes the test load balancer after every test. It does not delete EVERY load balancer in your account""" self.balancer.delete() def test_build_list_params(self): params = {} self.conn.build_list_params( params, ['thing1', 'thing2', 'thing3'], 'ThingName%d') expected_params = { 'ThingName1': 'thing1', 'ThingName2': 'thing2', 'ThingName3': 'thing3' } self.assertEqual(params, expected_params) # TODO: for these next tests, consider sleeping until our load # balancer comes up, then testing for connectivity to # balancer.dns_name, along the lines of the existing EC2 unit tests. def test_create_load_balancer(self): self.assertEqual(self.balancer.name, self.name) self.assertEqual(self.balancer.availability_zones, self.availability_zones) self.assertEqual(self.balancer.listeners, self.listeners) balancers = self.conn.get_all_load_balancers() self.assertEqual([lb.name for lb in balancers], [self.name]) def test_create_load_balancer_listeners(self): more_listeners = [(443, 8001, 'HTTP')] self.conn.create_load_balancer_listeners(self.name, more_listeners) balancers = self.conn.get_all_load_balancers() self.assertEqual([lb.name for lb in balancers], [self.name]) self.assertEqual( sorted(l.get_tuple() for l in balancers[0].listeners), sorted(self.listeners + more_listeners) ) def test_delete_load_balancer_listeners(self): mod_listeners = [(80, 8000, 'HTTP'), (443, 8001, 'HTTP')] mod_name = self.name + "-mod" self.mod_balancer = self.conn.create_load_balancer( mod_name, self.availability_zones, mod_listeners) mod_balancers = self.conn.get_all_load_balancers( load_balancer_names=[mod_name]) self.assertEqual([lb.name for lb in mod_balancers], [mod_name]) self.assertEqual( sorted([l.get_tuple() for l in mod_balancers[0].listeners]), sorted(mod_listeners)) self.conn.delete_load_balancer_listeners(self.mod_balancer.name, [443]) mod_balancers = self.conn.get_all_load_balancers( load_balancer_names=[mod_name]) self.assertEqual([lb.name for lb in mod_balancers], [mod_name]) self.assertEqual([l.get_tuple() for l in mod_balancers[0].listeners], mod_listeners[:1]) self.mod_balancer.delete() def test_create_load_balancer_listeners_with_policies(self): more_listeners = [(443, 8001, 'HTTP')] self.conn.create_load_balancer_listeners(self.name, more_listeners) lb_policy_name = 'lb-policy' self.conn.create_lb_cookie_stickiness_policy( 1000, self.name, lb_policy_name) self.conn.set_lb_policies_of_listener( self.name, self.listeners[0][0], lb_policy_name) app_policy_name = 'app-policy' self.conn.create_app_cookie_stickiness_policy( 'appcookie', self.name, app_policy_name) self.conn.set_lb_policies_of_listener( self.name, more_listeners[0][0], app_policy_name) balancers = self.conn.get_all_load_balancers( load_balancer_names=[self.name]) self.assertEqual([lb.name for lb in balancers], [self.name]) self.assertEqual( sorted(l.get_tuple() for l in balancers[0].listeners), sorted(self.listeners + more_listeners) ) # Policy names should be checked here once they are supported # in the Listener object. def test_create_load_balancer_backend_with_policies(self): other_policy_name = 'enable-proxy-protocol' backend_port = 8081 self.conn.create_lb_policy( self.name, other_policy_name, 'ProxyProtocolPolicyType', {'ProxyProtocol': True}) self.conn.set_lb_policies_of_backend_server( self.name, backend_port, [other_policy_name]) balancers = self.conn.get_all_load_balancers( load_balancer_names=[self.name]) self.assertEqual([lb.name for lb in balancers], [self.name]) self.assertEqual(len(balancers[0].policies.other_policies), 1) self.assertEqual(balancers[0].policies.other_policies[0].policy_name, other_policy_name) self.assertEqual(len(balancers[0].backends), 1) self.assertEqual(balancers[0].backends[0].instance_port, backend_port) self.assertEqual(balancers[0].backends[0].policies[0].policy_name, other_policy_name) self.conn.set_lb_policies_of_backend_server(self.name, backend_port, []) balancers = self.conn.get_all_load_balancers( load_balancer_names=[self.name]) self.assertEqual([lb.name for lb in balancers], [self.name]) self.assertEqual(len(balancers[0].policies.other_policies), 1) self.assertEqual(len(balancers[0].backends), 0) def test_create_load_balancer_complex_listeners(self): complex_listeners = [ (8080, 80, 'HTTP', 'HTTP'), (2525, 25, 'TCP', 'TCP'), ] self.conn.create_load_balancer_listeners( self.name, complex_listeners=complex_listeners ) balancers = self.conn.get_all_load_balancers( load_balancer_names=[self.name] ) self.assertEqual([lb.name for lb in balancers], [self.name]) self.assertEqual( sorted(l.get_complex_tuple() for l in balancers[0].listeners), # We need an extra 'HTTP' here over what ``self.listeners`` uses. sorted([(80, 8000, 'HTTP', 'HTTP')] + complex_listeners) ) def test_load_balancer_access_log(self): attributes = self.balancer.get_attributes() self.assertEqual(False, attributes.access_log.enabled) attributes.access_log.enabled = True attributes.access_log.s3_bucket_name = self.bucket_name attributes.access_log.s3_bucket_prefix = 'access-logs' attributes.access_log.emit_interval = 5 self.conn.modify_lb_attribute(self.balancer.name, 'accessLog', attributes.access_log) new_attributes = self.balancer.get_attributes() self.assertEqual(True, new_attributes.access_log.enabled) self.assertEqual(self.bucket_name, new_attributes.access_log.s3_bucket_name) self.assertEqual('access-logs', new_attributes.access_log.s3_bucket_prefix) self.assertEqual(5, new_attributes.access_log.emit_interval) def test_load_balancer_get_attributes(self): attributes = self.balancer.get_attributes() connection_draining = self.conn.get_lb_attribute(self.balancer.name, 'ConnectionDraining') self.assertEqual(connection_draining.enabled, attributes.connection_draining.enabled) self.assertEqual(connection_draining.timeout, attributes.connection_draining.timeout) access_log = self.conn.get_lb_attribute(self.balancer.name, 'AccessLog') self.assertEqual(access_log.enabled, attributes.access_log.enabled) self.assertEqual(access_log.s3_bucket_name, attributes.access_log.s3_bucket_name) self.assertEqual(access_log.s3_bucket_prefix, attributes.access_log.s3_bucket_prefix) self.assertEqual(access_log.emit_interval, attributes.access_log.emit_interval) cross_zone_load_balancing = self.conn.get_lb_attribute( self.balancer.name, 'CrossZoneLoadBalancing') self.assertEqual(cross_zone_load_balancing, attributes.cross_zone_load_balancing.enabled) def change_and_verify_load_balancer_connection_draining( self, enabled, timeout=None): attributes = self.balancer.get_attributes() attributes.connection_draining.enabled = enabled if timeout is not None: attributes.connection_draining.timeout = timeout self.conn.modify_lb_attribute( self.balancer.name, 'ConnectionDraining', attributes.connection_draining) attributes = self.balancer.get_attributes() self.assertEqual(enabled, attributes.connection_draining.enabled) if timeout is not None: self.assertEqual(timeout, attributes.connection_draining.timeout) def test_load_balancer_connection_draining_config(self): self.change_and_verify_load_balancer_connection_draining(True, 128) self.change_and_verify_load_balancer_connection_draining(True, 256) self.change_and_verify_load_balancer_connection_draining(False) self.change_and_verify_load_balancer_connection_draining(True, 64) def test_set_load_balancer_policies_of_listeners(self): more_listeners = [(443, 8001, 'HTTP')] self.conn.create_load_balancer_listeners(self.name, more_listeners) lb_policy_name = 'lb-policy' self.conn.create_lb_cookie_stickiness_policy( 1000, self.name, lb_policy_name ) self.conn.set_lb_policies_of_listener( self.name, self.listeners[0][0], lb_policy_name ) # Try to remove the policy by passing empty list. # http://docs.aws.amazon.com/ElasticLoadBalancing/latest/APIReference/API_SetLoadBalancerPoliciesOfListener.html # documents this as the way to remove policies. self.conn.set_lb_policies_of_listener( self.name, self.listeners[0][0], [] ) def test_can_make_sigv4_call(self): connection = boto.ec2.elb.connect_to_region('eu-central-1') lbs = connection.get_all_load_balancers() self.assertTrue(isinstance(lbs, list)) if __name__ == '__main__': unittest.main() vendored/boto/tests/integration/ec2/elb/__init__.py 0000644 00000002117 15025013616 0016326 0 ustar 00 # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/ec2/vpc/test_connection.py 0000644 00000020653 15025013616 0020020 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import time import boto from boto.compat import six from tests.compat import unittest from boto.ec2.networkinterface import NetworkInterfaceCollection from boto.ec2.networkinterface import NetworkInterfaceSpecification from boto.ec2.networkinterface import PrivateIPAddress class TestVPCConnection(unittest.TestCase): def setUp(self): # Registry of instances to be removed self.instances = [] # Registry for cleaning up the vpc after all instances are terminated # in the format [ ( func, (arg1, ... argn) ) ] self.post_terminate_cleanups = [] self.api = boto.connect_vpc() self.vpc = self.api.create_vpc('10.0.0.0/16') # Need time for the VPC to be in place. :/ time.sleep(5) self.subnet = self.api.create_subnet(self.vpc.id, '10.0.0.0/24') # Register the subnet to be deleted after instance termination self.post_terminate_cleanups.append((self.api.delete_subnet, (self.subnet.id,))) # Need time for the subnet to be in place. time.sleep(10) def post_terminate_cleanup(self): """Helper to run clean up tasks after instances are removed.""" for fn, args in self.post_terminate_cleanups: fn(*args) # Give things time to catch up each time time.sleep(10) # Now finally delete the vpc if self.vpc: self.api.delete_vpc(self.vpc.id) def terminate_instances(self): """Helper to remove all instances and kick off additional cleanup once they are terminated. """ for instance in self.instances: self.terminate_instance(instance) self.post_terminate_cleanup() def terminate_instance(self, instance): instance.terminate() for i in six.moves.range(300): instance.update() if instance.state == 'terminated': # Give it a litle more time to settle. time.sleep(30) return else: time.sleep(10) def delete_elastic_ip(self, eip): # Fetch a new copy of the eip so we're up to date new_eip = self.api.get_all_addresses([eip.public_ip])[0] if new_eip.association_id: new_eip.disassociate() new_eip.release() time.sleep(10) def test_multi_ip_create(self): interface = NetworkInterfaceSpecification( device_index=0, subnet_id=self.subnet.id, private_ip_address='10.0.0.21', description="This is a test interface using boto.", delete_on_termination=True, private_ip_addresses=[ PrivateIPAddress(private_ip_address='10.0.0.22', primary=False), PrivateIPAddress(private_ip_address='10.0.0.23', primary=False), PrivateIPAddress(private_ip_address='10.0.0.24', primary=False)]) interfaces = NetworkInterfaceCollection(interface) reservation = self.api.run_instances(image_id='ami-a0cd60c9', instance_type='m1.small', network_interfaces=interfaces) # Give it a few seconds to start up. time.sleep(10) instance = reservation.instances[0] self.addCleanup(self.terminate_instance, instance) retrieved = self.api.get_all_reservations(instance_ids=[instance.id]) self.assertEqual(len(retrieved), 1) retrieved_instances = retrieved[0].instances self.assertEqual(len(retrieved_instances), 1) retrieved_instance = retrieved_instances[0] self.assertEqual(len(retrieved_instance.interfaces), 1) interface = retrieved_instance.interfaces[0] private_ip_addresses = interface.private_ip_addresses self.assertEqual(len(private_ip_addresses), 4) self.assertEqual(private_ip_addresses[0].private_ip_address, '10.0.0.21') self.assertEqual(private_ip_addresses[0].primary, True) self.assertEqual(private_ip_addresses[1].private_ip_address, '10.0.0.22') self.assertEqual(private_ip_addresses[2].private_ip_address, '10.0.0.23') self.assertEqual(private_ip_addresses[3].private_ip_address, '10.0.0.24') def test_associate_public_ip(self): # Supplying basically nothing ought to work. interface = NetworkInterfaceSpecification( associate_public_ip_address=True, subnet_id=self.subnet.id, # Just for testing. delete_on_termination=True ) interfaces = NetworkInterfaceCollection(interface) reservation = self.api.run_instances( image_id='ami-a0cd60c9', instance_type='m1.small', network_interfaces=interfaces ) instance = reservation.instances[0] self.instances.append(instance) self.addCleanup(self.terminate_instances) # Give it a **LONG** time to start up. # Because the public IP won't be there right away. time.sleep(60) retrieved = self.api.get_all_reservations( instance_ids=[ instance.id ] ) self.assertEqual(len(retrieved), 1) retrieved_instances = retrieved[0].instances self.assertEqual(len(retrieved_instances), 1) retrieved_instance = retrieved_instances[0] self.assertEqual(len(retrieved_instance.interfaces), 1) interface = retrieved_instance.interfaces[0] # There ought to be a public IP there. # We can't reason about the IP itself, so just make sure it vaguely # resembles an IP (& isn't empty/``None``)... self.assertTrue(interface.publicIp.count('.') >= 3) def test_associate_elastic_ip(self): interface = NetworkInterfaceSpecification( associate_public_ip_address=False, subnet_id=self.subnet.id, # Just for testing. delete_on_termination=True ) interfaces = NetworkInterfaceCollection(interface) reservation = self.api.run_instances( image_id='ami-a0cd60c9', instance_type='m1.small', network_interfaces=interfaces ) instance = reservation.instances[0] # Register instance to be removed self.instances.append(instance) # Add terminate instances helper as cleanup command self.addCleanup(self.terminate_instances) # Create an internet gateway so we can attach an eip igw = self.api.create_internet_gateway() # Wait on gateway before attaching time.sleep(5) # Attach and register clean up tasks self.api.attach_internet_gateway(igw.id, self.vpc.id) self.post_terminate_cleanups.append((self.api.detach_internet_gateway, (igw.id, self.vpc.id))) self.post_terminate_cleanups.append((self.api.delete_internet_gateway, (igw.id,))) # Allocate an elastic ip to this vpc eip = self.api.allocate_address('vpc') self.post_terminate_cleanups.append((self.delete_elastic_ip, (eip,))) # Wait on instance and eip then try to associate directly to instance time.sleep(60) eip.associate(instance.id) if __name__ == '__main__': unittest.main() vendored/boto/tests/integration/ec2/vpc/__init__.py 0000644 00000000000 15025013616 0016341 0 ustar 00 vendored/boto/tests/integration/ec2/__init__.py 0000644 00000002117 15025013616 0015564 0 ustar 00 # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/cloudtrail/test_cert_verification.py 0000644 00000002755 15025013616 0022264 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.cloudtrail class CloudTrailCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): cloudtrail = True regions = boto.cloudtrail.regions() def sample_service_call(self, conn): conn.describe_trails() vendored/boto/tests/integration/cloudtrail/test_cloudtrail.py 0000644 00000005761 15025013616 0020727 0 ustar 00 import boto from time import time from tests.compat import unittest DEFAULT_S3_POLICY = """{ "Version": "2012-10-17", "Statement": [ { "Sid": "AWSCloudTrailAclCheck20131101", "Effect": "Allow", "Principal": { "AWS": [ "arn:aws:iam::086441151436:root", "arn:aws:iam::113285607260:root" ] }, "Action": "s3:GetBucketAcl", "Resource": "arn:aws:s3:::<BucketName>" }, { "Sid": "AWSCloudTrailWrite20131101", "Effect": "Allow", "Principal": { "AWS": [ "arn:aws:iam::086441151436:root", "arn:aws:iam::113285607260:root" ] }, "Action": "s3:PutObject", "Resource": "arn:aws:s3:::<BucketName>/<Prefix>/AWSLogs/<CustomerAccountID>/*", "Condition": { "StringEquals": { "s3:x-amz-acl": "bucket-owner-full-control" } } } ] }""" class TestCloudTrail(unittest.TestCase): def test_cloudtrail(self): cloudtrail = boto.connect_cloudtrail() # Don't delete existing customer data! res = cloudtrail.describe_trails() if len(res['trailList']): self.fail('A trail already exists on this account!') # Who am I? iam = boto.connect_iam() response = iam.get_user() account_id = response['get_user_response']['get_user_result'] \ ['user']['user_id'] # Setup a new bucket s3 = boto.connect_s3() bucket_name = 'cloudtrail-integ-{0}'.format(time()) policy = DEFAULT_S3_POLICY.replace('<BucketName>', bucket_name)\ .replace('<CustomerAccountID>', account_id)\ .replace('<Prefix>/', '') b = s3.create_bucket(bucket_name) b.set_policy(policy) # Setup CloudTrail cloudtrail.create_trail(trail={'Name': 'test', 'S3BucketName': bucket_name}) cloudtrail.update_trail(trail={'Name': 'test', 'IncludeGlobalServiceEvents': False}) trails = cloudtrail.describe_trails() self.assertEqual('test', trails['trailList'][0]['Name']) self.assertFalse(trails['trailList'][0]['IncludeGlobalServiceEvents']) cloudtrail.start_logging(name='test') status = cloudtrail.get_trail_status(name='test') self.assertTrue(status['IsLogging']) cloudtrail.stop_logging(name='test') status = cloudtrail.get_trail_status(name='test') self.assertFalse(status['IsLogging']) # Clean up cloudtrail.delete_trail(name='test') for key in b.list(): key.delete() s3.delete_bucket(bucket_name) vendored/boto/tests/integration/cloudtrail/__init__.py 0000644 00000000000 15025013616 0017242 0 ustar 00 vendored/boto/tests/integration/awslambda/test_awslambda.py 0000644 00000003204 15025013616 0020277 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import boto from boto.awslambda.exceptions import ResourceNotFoundException from tests.compat import unittest class TestAWSLambda(unittest.TestCase): def setUp(self): self.awslambda = boto.connect_awslambda() def test_list_functions(self): response = self.awslambda.list_functions() self.assertIn('Functions', response) def test_resource_not_found_exceptions(self): with self.assertRaises(ResourceNotFoundException): self.awslambda.get_function(function_name='non-existant-function') vendored/boto/tests/integration/awslambda/__init__.py 0000644 00000002143 15025013616 0017045 0 ustar 00 # Copyright (c) 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/integration/cloudsearch2/test_cert_verification.py 0000644 00000003053 15025013617 0022471 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.cloudsearch2 class CloudSearchCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): cloudsearch = True regions = boto.cloudsearch2.regions() def sample_service_call(self, conn): conn.describe_domains() vendored/boto/tests/integration/cloudsearch2/test_layers.py 0000644 00000005436 15025013617 0020300 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests for Layer1 of Cloudsearch """ import time from tests.unit import unittest from boto.cloudsearch2.layer1 import CloudSearchConnection from boto.cloudsearch2.layer2 import Layer2 from boto.regioninfo import RegionInfo class CloudSearchLayer1Test(unittest.TestCase): cloudsearch = True def setUp(self): super(CloudSearchLayer1Test, self).setUp() self.layer1 = CloudSearchConnection() self.domain_name = 'test-%d' % int(time.time()) def test_create_domain(self): resp = self.layer1.create_domain(self.domain_name) resp = (resp['CreateDomainResponse'] ['CreateDomainResult'] ['DomainStatus']) self.addCleanup(self.layer1.delete_domain, self.domain_name) self.assertTrue(resp.get('Created', False)) class CloudSearchLayer2Test(unittest.TestCase): cloudsearch = True def setUp(self): super(CloudSearchLayer2Test, self).setUp() self.layer2 = Layer2() self.domain_name = 'test-%d' % int(time.time()) def test_create_domain(self): domain = self.layer2.create_domain(self.domain_name) self.addCleanup(domain.delete) self.assertTrue(domain.created, False) self.assertEqual(domain.domain_name, self.domain_name) def test_initialization_regression(self): us_west_2 = RegionInfo( name='us-west-2', endpoint='cloudsearch.us-west-2.amazonaws.com' ) self.layer2 = Layer2( region=us_west_2, host='cloudsearch.us-west-2.amazonaws.com' ) self.assertEqual( self.layer2.layer1.host, 'cloudsearch.us-west-2.amazonaws.com' ) vendored/boto/tests/integration/cloudsearch2/__init__.py 0000644 00000002227 15025013617 0017474 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All Rights Reserved # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/__init__.py 0000644 00000004444 15025013617 0015121 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Base class to make checking the certs easier. """ # We subclass from ``object`` instead of ``TestCase`` here so that this doesn't # add noise to the test suite (otherwise these no-ops would run on every # import). class ServiceCertVerificationTest(object): ssl = True # SUBCLASSES MUST OVERRIDE THIS! # Something like ``boto.sqs.regions()``... regions = [] def test_certs(self): self.assertTrue(len(self.regions) > 0) for region in self.regions: special_access_required = False for snippet in ('gov', 'cn-'): if snippet in region.name: special_access_required = True break try: c = region.connect() self.sample_service_call(c) except: # This is bad (because the SSL cert failed). Re-raise the # exception. if not special_access_required: raise def sample_service_call(self, conn): """ Subclasses should override this method to do a service call that will always succeed (like fetch a list, even if it's empty). """ pass vendored/boto/tests/integration/sdb/test_cert_verification.py 0000644 00000003010 15025013617 0020654 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.sdb class SDBCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): sdb = True regions = boto.sdb.regions() def sample_service_call(self, conn): conn.get_all_domains() vendored/boto/tests/integration/sdb/test_connection.py 0000644 00000010340 15025013617 0017320 0 ustar 00 # Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2010, Eucalyptus Systems, Inc. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Some unit tests for the SDBConnection """ import unittest import time from boto.sdb.connection import SDBConnection from boto.exception import SDBResponseError class SDBConnectionTest (unittest.TestCase): sdb = True def test_1_basic(self): print('--- running SDBConnection tests ---') c = SDBConnection() rs = c.get_all_domains() num_domains = len(rs) # try illegal name try: domain = c.create_domain('bad:domain:name') except SDBResponseError: pass # now create one that should work and should be unique (i.e. a new one) domain_name = 'test%d' % int(time.time()) domain = c.create_domain(domain_name) rs = c.get_all_domains() assert len(rs) == num_domains + 1 # now let's a couple of items and attributes item_1 = 'item1' same_value = 'same_value' attrs_1 = {'name1': same_value, 'name2': 'diff_value_1'} domain.put_attributes(item_1, attrs_1) item_2 = 'item2' attrs_2 = {'name1': same_value, 'name2': 'diff_value_2'} domain.put_attributes(item_2, attrs_2) # try to get the attributes and see if they match item = domain.get_attributes(item_1, consistent_read=True) assert len(item.keys()) == len(attrs_1.keys()) assert item['name1'] == attrs_1['name1'] assert item['name2'] == attrs_1['name2'] # try a search or two query = 'select * from %s where name1="%s"' % (domain_name, same_value) rs = domain.select(query, consistent_read=True) n = 0 for item in rs: n += 1 assert n == 2 query = 'select * from %s where name2="diff_value_2"' % domain_name rs = domain.select(query, consistent_read=True) n = 0 for item in rs: n += 1 assert n == 1 # delete all attributes associated with item_1 stat = domain.delete_attributes(item_1) assert stat # now try a batch put operation on the domain item3 = {'name3_1': 'value3_1', 'name3_2': 'value3_2', 'name3_3': ['value3_3_1', 'value3_3_2']} item4 = {'name4_1': 'value4_1', 'name4_2': ['value4_2_1', 'value4_2_2'], 'name4_3': 'value4_3'} items = {'item3': item3, 'item4': item4} domain.batch_put_attributes(items) item = domain.get_attributes('item3', consistent_read=True) assert item['name3_2'] == 'value3_2' # now try a batch delete operation (variation #1) items = {'item3': item3} stat = domain.batch_delete_attributes(items) item = domain.get_attributes('item3', consistent_read=True) assert not item # now try a batch delete operation (variation #2) stat = domain.batch_delete_attributes({'item4': None}) item = domain.get_attributes('item4', consistent_read=True) assert not item # now delete the domain stat = c.delete_domain(domain) assert stat print('--- tests completed ---') vendored/boto/tests/integration/sdb/__init__.py 0000644 00000002120 15025013617 0015656 0 ustar 00 # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/dynamodb2/test_cert_verification.py 0000644 00000003034 15025013617 0021771 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.dynamodb2 class DynamoDB2CertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): dynamodb2 = True regions = boto.dynamodb2.regions() def sample_service_call(self, conn): conn.list_tables() vendored/boto/tests/integration/dynamodb2/forum_test_data.json 0000644 00000003660 15025013617 0020741 0 ustar 00 [ { "thread": "Favorite chiptune band?", "posted_by": "joe", "posted_on": "2013-12-24T12:30:54", "body": "Forum poll: What's your favorite chiptune band & why?" }, { "thread": "Favorite chiptune band?", "posted_by": "jane", "posted_on": "2013-12-24T12:35:40", "body": "I'd definitely go with POWERLIFTER. Love the use of LSDJ & vocals." }, { "thread": "Favorite chiptune band?", "posted_by": "joe", "posted_on": "2013-12-24T13:45:30", "body": "Hm, I hadn't heard of them before. Will give a listen.\n\nMy favorite is definitely D&D Sluggers so far." }, { "thread": "Favorite chiptune band?", "posted_by": "joe", "posted_on": "2013-12-24T14:15:14", "body": "Oh man, POWERLIFTER is really good. Do they have any more albums than the first one?" }, { "thread": "Favorite chiptune band?", "posted_by": "jane", "posted_on": "2013-12-24T14:25:33", "body": "Yeah, check out their site. The second album has been out for a bit & is just as good." }, { "thread": "Help with compression?", "posted_by": "jane", "posted_on": "2013-12-24T14:26:51", "body": "I'm working on my latest & having some trouble. I've got compression on my drum track but I still can't keep the drum for muddying the bass line without losing clarity on the hats. :( Help?" }, { "thread": "Favorite chiptune band?", "posted_by": "joe", "posted_on": "2013-12-24T15:22:22", "body": "Thanks for the tip! I'll have to check it out!" }, { "thread": "Help with compression?", "posted_by": "joe", "posted_on": "2013-12-24T15:26:06", "body": "Have you tried using side-chaining the compression? That'll allow the bass' input to control the volume of the drums based on when it's playing." } ] vendored/boto/tests/integration/dynamodb2/test_highlevel.py 0000644 00000063331 15025013617 0020247 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests for DynamoDB v2 high-level abstractions. """ import os import time from tests.unit import unittest from boto.dynamodb2 import exceptions from boto.dynamodb2.fields import (HashKey, RangeKey, KeysOnlyIndex, GlobalKeysOnlyIndex, GlobalIncludeIndex, GlobalAllIndex) from boto.dynamodb2.items import Item from boto.dynamodb2.table import Table from boto.dynamodb2.types import NUMBER, STRING try: import json except ImportError: import simplejson as json class DynamoDBv2Test(unittest.TestCase): dynamodb = True def test_integration(self): # Test creating a full table with all options specified. users = Table.create('users', schema=[ HashKey('username'), RangeKey('friend_count', data_type=NUMBER) ], throughput={ 'read': 5, 'write': 5, }, indexes=[ KeysOnlyIndex('LastNameIndex', parts=[ HashKey('username'), RangeKey('last_name') ]), ]) self.addCleanup(users.delete) self.assertEqual(len(users.schema), 2) self.assertEqual(users.throughput['read'], 5) # Wait for it. time.sleep(60) # Make sure things line up if we're introspecting the table. users_hit_api = Table('users') users_hit_api.describe() self.assertEqual(len(users.schema), len(users_hit_api.schema)) self.assertEqual(users.throughput, users_hit_api.throughput) self.assertEqual(len(users.indexes), len(users_hit_api.indexes)) # Test putting some items individually. users.put_item(data={ 'username': 'johndoe', 'first_name': 'John', 'last_name': 'Doe', 'friend_count': 4 }) users.put_item(data={ 'username': 'alice', 'first_name': 'Alice', 'last_name': 'Expert', 'friend_count': 2 }) time.sleep(5) # Test batch writing. with users.batch_write() as batch: batch.put_item({ 'username': 'jane', 'first_name': 'Jane', 'last_name': 'Doe', 'friend_count': 3 }) batch.delete_item(username='alice', friend_count=2) batch.put_item({ 'username': 'bob', 'first_name': 'Bob', 'last_name': 'Smith', 'friend_count': 1 }) time.sleep(5) # Does it exist? It should? self.assertTrue(users.has_item(username='jane', friend_count=3)) # But this shouldn't be there... self.assertFalse(users.has_item( username='mrcarmichaeljones', friend_count=72948 )) # Test getting an item & updating it. # This is the "safe" variant (only write if there have been no # changes). jane = users.get_item(username='jane', friend_count=3) self.assertEqual(jane['first_name'], 'Jane') jane['last_name'] = 'Doh' self.assertTrue(jane.save()) # Test strongly consistent getting of an item. # Additionally, test the overwrite behavior. client_1_jane = users.get_item( username='jane', friend_count=3, consistent=True ) self.assertEqual(jane['first_name'], 'Jane') client_2_jane = users.get_item( username='jane', friend_count=3, consistent=True ) self.assertEqual(jane['first_name'], 'Jane') # Write & assert the ``first_name`` is gone, then... del client_1_jane['first_name'] self.assertTrue(client_1_jane.save()) check_name = users.get_item( username='jane', friend_count=3, consistent=True ) self.assertEqual(check_name['first_name'], None) # ...overwrite the data with what's in memory. client_2_jane['first_name'] = 'Joan' # Now a write that fails due to default expectations... self.assertRaises(exceptions.JSONResponseError, client_2_jane.save) # ... so we force an overwrite. self.assertTrue(client_2_jane.save(overwrite=True)) check_name_again = users.get_item( username='jane', friend_count=3, consistent=True ) self.assertEqual(check_name_again['first_name'], 'Joan') # Reset it. jane['username'] = 'jane' jane['first_name'] = 'Jane' jane['last_name'] = 'Doe' jane['friend_count'] = 3 self.assertTrue(jane.save(overwrite=True)) # Test the partial update behavior. client_3_jane = users.get_item( username='jane', friend_count=3, consistent=True ) client_4_jane = users.get_item( username='jane', friend_count=3, consistent=True ) client_3_jane['favorite_band'] = 'Feed Me' # No ``overwrite`` needed due to new data. self.assertTrue(client_3_jane.save()) # Expectations are only checked on the ``first_name``, so what wouldn't # have succeeded by default does succeed here. client_4_jane['first_name'] = 'Jacqueline' self.assertTrue(client_4_jane.partial_save()) partial_jane = users.get_item( username='jane', friend_count=3, consistent=True ) self.assertEqual(partial_jane['favorite_band'], 'Feed Me') self.assertEqual(partial_jane['first_name'], 'Jacqueline') # Reset it. jane['username'] = 'jane' jane['first_name'] = 'Jane' jane['last_name'] = 'Doe' jane['friend_count'] = 3 self.assertTrue(jane.save(overwrite=True)) # Ensure that partial saves of a brand-new object work. sadie = Item(users, data={ 'username': 'sadie', 'first_name': 'Sadie', 'favorite_band': 'Zedd', 'friend_count': 7 }) self.assertTrue(sadie.partial_save()) serverside_sadie = users.get_item( username='sadie', friend_count=7, consistent=True ) self.assertEqual(serverside_sadie['first_name'], 'Sadie') # Test the eventually consistent query. results = users.query_2( username__eq='johndoe', last_name__eq='Doe', index='LastNameIndex', attributes=('username',), reverse=True ) for res in results: self.assertTrue(res['username'] in ['johndoe',]) self.assertEqual(list(res.keys()), ['username']) # Ensure that queries with attributes don't return the hash key. results = users.query_2( username__eq='johndoe', friend_count__eq=4, attributes=('first_name',) ) for res in results: self.assertEqual(res['first_name'], 'John') self.assertEqual(list(res.keys()), ['first_name']) # Test the strongly consistent query. c_results = users.query_2( username__eq='johndoe', last_name__eq='Doe', index='LastNameIndex', reverse=True, consistent=True ) for res in c_results: self.assertEqual(res['username'], 'johndoe') # Test a query with query filters results = users.query_2( username__eq='johndoe', query_filter={ 'first_name__beginswith': 'J' }, attributes=('first_name',) ) for res in results: self.assertTrue(res['first_name'] in ['John']) # Test scans without filters. all_users = users.scan(limit=7) self.assertEqual(next(all_users)['username'], 'bob') self.assertEqual(next(all_users)['username'], 'jane') self.assertEqual(next(all_users)['username'], 'johndoe') # Test scans with a filter. filtered_users = users.scan(limit=2, username__beginswith='j') self.assertEqual(next(filtered_users)['username'], 'jane') self.assertEqual(next(filtered_users)['username'], 'johndoe') # Test deleting a single item. johndoe = users.get_item(username='johndoe', friend_count=4) johndoe.delete() # Set batch get limit to ensure keys with no results are # handled correctly. users.max_batch_get = 2 # Test the eventually consistent batch get. results = users.batch_get(keys=[ {'username': 'noone', 'friend_count': 4}, {'username': 'nothere', 'friend_count': 10}, {'username': 'bob', 'friend_count': 1}, {'username': 'jane', 'friend_count': 3} ]) batch_users = [] for res in results: batch_users.append(res) self.assertIn(res['first_name'], ['Bob', 'Jane']) self.assertEqual(len(batch_users), 2) # Test the strongly consistent batch get. c_results = users.batch_get(keys=[ {'username': 'bob', 'friend_count': 1}, {'username': 'jane', 'friend_count': 3} ], consistent=True) c_batch_users = [] for res in c_results: c_batch_users.append(res) self.assertTrue(res['first_name'] in ['Bob', 'Jane']) self.assertEqual(len(c_batch_users), 2) # Test count, but in a weak fashion. Because lag time. self.assertTrue(users.count() > -1) # Test query count count = users.query_count( username__eq='bob', ) self.assertEqual(count, 1) # Test without LSIs (describe calls shouldn't fail). admins = Table.create('admins', schema=[ HashKey('username') ]) self.addCleanup(admins.delete) time.sleep(60) admins.describe() self.assertEqual(admins.throughput['read'], 5) self.assertEqual(admins.indexes, []) # A single query term should fail on a table with *ONLY* a HashKey. self.assertRaises( exceptions.QueryError, admins.query, username__eq='johndoe' ) # But it shouldn't break on more complex tables. res = users.query_2(username__eq='johndoe') # Test putting with/without sets. mau5_created = users.put_item(data={ 'username': 'mau5', 'first_name': 'dead', 'last_name': 'mau5', 'friend_count': 2, 'friends': set(['skrill', 'penny']), }) self.assertTrue(mau5_created) penny_created = users.put_item(data={ 'username': 'penny', 'first_name': 'Penny', 'friend_count': 0, 'friends': set([]), }) self.assertTrue(penny_created) # Test attributes. mau5 = users.get_item( username='mau5', friend_count=2, attributes=['username', 'first_name'] ) self.assertEqual(mau5['username'], 'mau5') self.assertEqual(mau5['first_name'], 'dead') self.assertTrue('last_name' not in mau5) def test_unprocessed_batch_writes(self): # Create a very limited table w/ low throughput. users = Table.create('slow_users', schema=[ HashKey('user_id'), ], throughput={ 'read': 1, 'write': 1, }) self.addCleanup(users.delete) # Wait for it. time.sleep(60) with users.batch_write() as batch: for i in range(500): batch.put_item(data={ 'user_id': str(i), 'name': 'Droid #{0}'.format(i), }) # Before ``__exit__`` runs, we should have a bunch of unprocessed # items. self.assertTrue(len(batch._unprocessed) > 0) # Post-__exit__, they should all be gone. self.assertEqual(len(batch._unprocessed), 0) def test_gsi(self): users = Table.create('gsi_users', schema=[ HashKey('user_id'), ], throughput={ 'read': 5, 'write': 3, }, global_indexes=[ GlobalKeysOnlyIndex('StuffIndex', parts=[ HashKey('user_id') ], throughput={ 'read': 2, 'write': 1, }), ]) self.addCleanup(users.delete) # Wait for it. time.sleep(60) users.update( throughput={ 'read': 3, 'write': 4 }, global_indexes={ 'StuffIndex': { 'read': 1, 'write': 2 } } ) # Wait again for the changes to finish propagating. time.sleep(150) def test_gsi_with_just_hash_key(self): # GSI allows for querying off of different keys. This is behavior we # previously disallowed (due to standard & LSI queries). # See https://forums.aws.amazon.com/thread.jspa?threadID=146212&tstart=0 users = Table.create('gsi_query_users', schema=[ HashKey('user_id') ], throughput={ 'read': 5, 'write': 3, }, global_indexes=[ GlobalIncludeIndex('UsernameIndex', parts=[ HashKey('username'), ], includes=['user_id', 'username'], throughput={ 'read': 3, 'write': 1, }) ]) self.addCleanup(users.delete) # Wait for it. time.sleep(60) users.put_item(data={ 'user_id': '7', 'username': 'johndoe', 'first_name': 'John', 'last_name': 'Doe', }) users.put_item(data={ 'user_id': '24', 'username': 'alice', 'first_name': 'Alice', 'last_name': 'Expert', }) users.put_item(data={ 'user_id': '35', 'username': 'jane', 'first_name': 'Jane', 'last_name': 'Doe', }) # Try the main key. Should be fine. rs = users.query_2( user_id__eq='24' ) results = sorted([user['username'] for user in rs]) self.assertEqual(results, ['alice']) # Now try the GSI. Also should work. rs = users.query_2( username__eq='johndoe', index='UsernameIndex' ) results = sorted([user['username'] for user in rs]) self.assertEqual(results, ['johndoe']) def test_query_with_limits(self): # Per the DDB team, it's recommended to do many smaller gets with a # reduced page size. # Clamp down the page size while ensuring that the correct number of # results are still returned. posts = Table.create('posts', schema=[ HashKey('thread'), RangeKey('posted_on') ], throughput={ 'read': 5, 'write': 5, }) self.addCleanup(posts.delete) # Wait for it. time.sleep(60) # Add some data. test_data_path = os.path.join( os.path.dirname(__file__), 'forum_test_data.json' ) with open(test_data_path, 'r') as test_data: data = json.load(test_data) with posts.batch_write() as batch: for post in data: batch.put_item(post) time.sleep(5) # Test the reduced page size. results = posts.query_2( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00', max_page_size=2 ) all_posts = list(results) self.assertEqual( [post['posted_by'] for post in all_posts], ['joe', 'jane', 'joe', 'joe', 'jane', 'joe'] ) self.assertTrue(results._fetches >= 3) def test_query_with_reverse(self): posts = Table.create('more-posts', schema=[ HashKey('thread'), RangeKey('posted_on') ], throughput={ 'read': 5, 'write': 5, }) self.addCleanup(posts.delete) # Wait for it. time.sleep(60) # Add some data. test_data_path = os.path.join( os.path.dirname(__file__), 'forum_test_data.json' ) with open(test_data_path, 'r') as test_data: data = json.load(test_data) with posts.batch_write() as batch: for post in data: batch.put_item(post) time.sleep(5) # Test the default order (ascending). results = posts.query_2( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00' ) self.assertEqual( [post['posted_on'] for post in results], [ '2013-12-24T12:30:54', '2013-12-24T12:35:40', '2013-12-24T13:45:30', '2013-12-24T14:15:14', '2013-12-24T14:25:33', '2013-12-24T15:22:22', ] ) # Test the explicit ascending order. results = posts.query_2( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00', reverse=False ) self.assertEqual( [post['posted_on'] for post in results], [ '2013-12-24T12:30:54', '2013-12-24T12:35:40', '2013-12-24T13:45:30', '2013-12-24T14:15:14', '2013-12-24T14:25:33', '2013-12-24T15:22:22', ] ) # Test the explicit descending order. results = posts.query_2( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00', reverse=True ) self.assertEqual( [post['posted_on'] for post in results], [ '2013-12-24T15:22:22', '2013-12-24T14:25:33', '2013-12-24T14:15:14', '2013-12-24T13:45:30', '2013-12-24T12:35:40', '2013-12-24T12:30:54', ] ) # Test the old, broken style. results = posts.query( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00' ) self.assertEqual( [post['posted_on'] for post in results], [ '2013-12-24T15:22:22', '2013-12-24T14:25:33', '2013-12-24T14:15:14', '2013-12-24T13:45:30', '2013-12-24T12:35:40', '2013-12-24T12:30:54', ] ) results = posts.query( thread__eq='Favorite chiptune band?', posted_on__gte='2013-12-24T00:00:00', reverse=True ) self.assertEqual( [post['posted_on'] for post in results], [ '2013-12-24T12:30:54', '2013-12-24T12:35:40', '2013-12-24T13:45:30', '2013-12-24T14:15:14', '2013-12-24T14:25:33', '2013-12-24T15:22:22', ] ) def test_query_after_describe_with_gsi(self): # Create a table to using gsi to reproduce the error mentioned on issue # https://github.com/boto/boto/issues/2828 users = Table.create('more_gsi_query_users', schema=[ HashKey('user_id') ], throughput={ 'read': 5, 'write': 5 }, global_indexes=[ GlobalAllIndex('EmailGSIIndex', parts=[ HashKey('email') ], throughput={ 'read': 1, 'write': 1 }) ]) # Add this function to be called after tearDown() self.addCleanup(users.delete) # Wait for it. time.sleep(60) # populate a couple of items in it users.put_item(data={ 'user_id': '7', 'username': 'johndoe', 'first_name': 'John', 'last_name': 'Doe', 'email': 'johndoe@johndoe.com', }) users.put_item(data={ 'user_id': '24', 'username': 'alice', 'first_name': 'Alice', 'last_name': 'Expert', 'email': 'alice@alice.com', }) users.put_item(data={ 'user_id': '35', 'username': 'jane', 'first_name': 'Jane', 'last_name': 'Doe', 'email': 'jane@jane.com', }) # Try the GSI. it should work. rs = users.query_2( email__eq='johndoe@johndoe.com', index='EmailGSIIndex' ) for rs_item in rs: self.assertEqual(rs_item['username'], ['johndoe']) # The issue arises when we're introspecting the table and try to # query_2 after call describe method users_hit_api = Table('more_gsi_query_users') users_hit_api.describe() # Try the GSI. This is what os going wrong on #2828 issue. It should # work fine now. rs = users_hit_api.query_2( email__eq='johndoe@johndoe.com', index='EmailGSIIndex' ) for rs_item in rs: self.assertEqual(rs_item['username'], ['johndoe']) def test_update_table_online_indexing_support(self): # Create a table using gsi to test the DynamoDB online indexing support # https://github.com/boto/boto/pull/2925 users = Table.create('online_indexing_support_users', schema=[ HashKey('user_id') ], throughput={ 'read': 5, 'write': 5 }, global_indexes=[ GlobalAllIndex('EmailGSIIndex', parts=[ HashKey('email') ], throughput={ 'read': 2, 'write': 2 }) ]) # Add this function to be called after tearDown() self.addCleanup(users.delete) # Wait for it. time.sleep(60) # Fetch fresh table desc from DynamoDB users.describe() # Assert if everything is fine so far self.assertEqual(len(users.global_indexes), 1) self.assertEqual(users.global_indexes[0].throughput['read'], 2) self.assertEqual(users.global_indexes[0].throughput['write'], 2) # Update a GSI throughput. it should work. users.update_global_secondary_index(global_indexes={ 'EmailGSIIndex': { 'read': 2, 'write': 1, } }) # Wait for it. time.sleep(60) # Fetch fresh table desc from DynamoDB users.describe() # Assert if everything is fine so far self.assertEqual(len(users.global_indexes), 1) self.assertEqual(users.global_indexes[0].throughput['read'], 2) self.assertEqual(users.global_indexes[0].throughput['write'], 1) # Update a GSI throughput using the old fashion way for compatibility # purposes. it should work. users.update(global_indexes={ 'EmailGSIIndex': { 'read': 3, 'write': 2, } }) # Wait for it. time.sleep(60) # Fetch fresh table desc from DynamoDB users.describe() # Assert if everything is fine so far self.assertEqual(len(users.global_indexes), 1) self.assertEqual(users.global_indexes[0].throughput['read'], 3) self.assertEqual(users.global_indexes[0].throughput['write'], 2) # Delete a GSI. it should work. users.delete_global_secondary_index('EmailGSIIndex') # Wait for it. time.sleep(60) # Fetch fresh table desc from DynamoDB users.describe() # Assert if everything is fine so far self.assertEqual(len(users.global_indexes), 0) # Create a GSI. it should work. users.create_global_secondary_index( global_index=GlobalAllIndex( 'AddressGSIIndex', parts=[ HashKey('address', data_type=STRING) ], throughput={ 'read': 1, 'write': 1, }) ) # Wait for it. This operation usually takes much longer than the others time.sleep(60*10) # Fetch fresh table desc from DynamoDB users.describe() # Assert if everything is fine so far self.assertEqual(len(users.global_indexes), 1) self.assertEqual(users.global_indexes[0].throughput['read'], 1) self.assertEqual(users.global_indexes[0].throughput['write'], 1) vendored/boto/tests/integration/dynamodb2/test_layer1.py 0000644 00000031314 15025013617 0017471 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Tests for Layer1 of DynamoDB v2 """ import time from tests.unit import unittest from boto.dynamodb2 import exceptions from boto.dynamodb2.layer1 import DynamoDBConnection class DynamoDBv2Layer1Test(unittest.TestCase): dynamodb = True def setUp(self): self.dynamodb = DynamoDBConnection() self.table_name = 'test-%d' % int(time.time()) self.hash_key_name = 'username' self.hash_key_type = 'S' self.range_key_name = 'date_joined' self.range_key_type = 'N' self.read_units = 5 self.write_units = 5 self.attributes = [ { 'AttributeName': self.hash_key_name, 'AttributeType': self.hash_key_type, }, { 'AttributeName': self.range_key_name, 'AttributeType': self.range_key_type, } ] self.schema = [ { 'AttributeName': self.hash_key_name, 'KeyType': 'HASH', }, { 'AttributeName': self.range_key_name, 'KeyType': 'RANGE', }, ] self.provisioned_throughput = { 'ReadCapacityUnits': self.read_units, 'WriteCapacityUnits': self.write_units, } self.lsi = [ { 'IndexName': 'MostRecentIndex', 'KeySchema': [ { 'AttributeName': self.hash_key_name, 'KeyType': 'HASH', }, { 'AttributeName': self.range_key_name, 'KeyType': 'RANGE', }, ], 'Projection': { 'ProjectionType': 'KEYS_ONLY', } } ] def create_table(self, table_name, attributes, schema, provisioned_throughput, lsi=None, wait=True): # Note: This is a slightly different ordering that makes less sense. result = self.dynamodb.create_table( attributes, table_name, schema, provisioned_throughput, local_secondary_indexes=lsi ) self.addCleanup(self.dynamodb.delete_table, table_name) if wait: while True: description = self.dynamodb.describe_table(table_name) if description['Table']['TableStatus'].lower() == 'active': return result else: time.sleep(5) else: return result def test_integrated(self): result = self.create_table( self.table_name, self.attributes, self.schema, self.provisioned_throughput, self.lsi ) self.assertEqual( result['TableDescription']['TableName'], self.table_name ) description = self.dynamodb.describe_table(self.table_name) self.assertEqual(description['Table']['ItemCount'], 0) # Create some records. record_1_data = { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056668'}, 'friend_count': {'N': '3'}, 'friends': {'SS': ['alice', 'bob', 'jane']}, } r1_result = self.dynamodb.put_item(self.table_name, record_1_data) # Get the data. record_1 = self.dynamodb.get_item(self.table_name, key={ 'username': {'S': 'johndoe'}, 'date_joined': {'N': '1366056668'}, }, consistent_read=True) self.assertEqual(record_1['Item']['username']['S'], 'johndoe') self.assertEqual(record_1['Item']['first_name']['S'], 'John') self.assertEqual(record_1['Item']['friends']['SS'], [ 'alice', 'bob', 'jane' ]) # Now in a batch. self.dynamodb.batch_write_item({ self.table_name: [ { 'PutRequest': { 'Item': { 'username': {'S': 'jane'}, 'first_name': {'S': 'Jane'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056789'}, 'friend_count': {'N': '1'}, 'friends': {'SS': ['johndoe']}, }, }, }, ] }) # Now a query. lsi_results = self.dynamodb.query( self.table_name, index_name='MostRecentIndex', key_conditions={ 'username': { 'AttributeValueList': [ {'S': 'johndoe'}, ], 'ComparisonOperator': 'EQ', }, }, consistent_read=True ) self.assertEqual(lsi_results['Count'], 1) results = self.dynamodb.query(self.table_name, key_conditions={ 'username': { 'AttributeValueList': [ {'S': 'jane'}, ], 'ComparisonOperator': 'EQ', }, 'date_joined': { 'AttributeValueList': [ {'N': '1366050000'} ], 'ComparisonOperator': 'GT', } }, consistent_read=True) self.assertEqual(results['Count'], 1) # Now a scan. results = self.dynamodb.scan(self.table_name) self.assertEqual(results['Count'], 2) s_items = sorted([res['username']['S'] for res in results['Items']]) self.assertEqual(s_items, ['jane', 'johndoe']) self.dynamodb.delete_item(self.table_name, key={ 'username': {'S': 'johndoe'}, 'date_joined': {'N': '1366056668'}, }) results = self.dynamodb.scan(self.table_name) self.assertEqual(results['Count'], 1) # Parallel scan (minus client-side threading). self.dynamodb.batch_write_item({ self.table_name: [ { 'PutRequest': { 'Item': { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'Johann'}, 'last_name': {'S': 'Does'}, 'date_joined': {'N': '1366058000'}, 'friend_count': {'N': '1'}, 'friends': {'SS': ['jane']}, }, }, 'PutRequest': { 'Item': { 'username': {'S': 'alice'}, 'first_name': {'S': 'Alice'}, 'last_name': {'S': 'Expert'}, 'date_joined': {'N': '1366056800'}, 'friend_count': {'N': '2'}, 'friends': {'SS': ['johndoe', 'jane']}, }, }, }, ] }) time.sleep(20) results = self.dynamodb.scan(self.table_name, segment=0, total_segments=2) self.assertTrue(results['Count'] in [1, 2]) results = self.dynamodb.scan(self.table_name, segment=1, total_segments=2) self.assertTrue(results['Count'] in [1, 2]) def test_without_range_key(self): result = self.create_table( self.table_name, [ { 'AttributeName': self.hash_key_name, 'AttributeType': self.hash_key_type, }, ], [ { 'AttributeName': self.hash_key_name, 'KeyType': 'HASH', }, ], self.provisioned_throughput ) self.assertEqual( result['TableDescription']['TableName'], self.table_name ) description = self.dynamodb.describe_table(self.table_name) self.assertEqual(description['Table']['ItemCount'], 0) # Create some records. record_1_data = { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056668'}, 'friend_count': {'N': '3'}, 'friends': {'SS': ['alice', 'bob', 'jane']}, } r1_result = self.dynamodb.put_item(self.table_name, record_1_data) # Now try a range-less get. johndoe = self.dynamodb.get_item(self.table_name, key={ 'username': {'S': 'johndoe'}, }, consistent_read=True) self.assertEqual(johndoe['Item']['username']['S'], 'johndoe') self.assertEqual(johndoe['Item']['first_name']['S'], 'John') self.assertEqual(johndoe['Item']['friends']['SS'], [ 'alice', 'bob', 'jane' ]) def test_throughput_exceeded_regression(self): tiny_tablename = 'TinyThroughput' tiny = self.create_table( tiny_tablename, self.attributes, self.schema, { 'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1, } ) self.dynamodb.put_item(tiny_tablename, { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056668'}, }) self.dynamodb.put_item(tiny_tablename, { 'username': {'S': 'jane'}, 'first_name': {'S': 'Jane'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056669'}, }) self.dynamodb.put_item(tiny_tablename, { 'username': {'S': 'alice'}, 'first_name': {'S': 'Alice'}, 'last_name': {'S': 'Expert'}, 'date_joined': {'N': '1366057000'}, }) time.sleep(20) for i in range(100): # This would cause an exception due to a non-existant instance variable. self.dynamodb.scan(tiny_tablename) def test_recursive(self): result = self.create_table( self.table_name, self.attributes, self.schema, self.provisioned_throughput, self.lsi ) self.assertEqual( result['TableDescription']['TableName'], self.table_name ) description = self.dynamodb.describe_table(self.table_name) self.assertEqual(description['Table']['ItemCount'], 0) # Create some records with one being a recursive shape. record_1_data = { 'username': {'S': 'johndoe'}, 'first_name': {'S': 'John'}, 'last_name': {'S': 'Doe'}, 'date_joined': {'N': '1366056668'}, 'friend_count': {'N': '3'}, 'friend_data': {'M': {'username': {'S': 'alice'}, 'friend_count': {'N': '4'}}} } r1_result = self.dynamodb.put_item(self.table_name, record_1_data) # Get the data. record_1 = self.dynamodb.get_item(self.table_name, key={ 'username': {'S': 'johndoe'}, 'date_joined': {'N': '1366056668'}, }, consistent_read=True) self.assertEqual(record_1['Item']['username']['S'], 'johndoe') self.assertEqual(record_1['Item']['first_name']['S'], 'John') recursive_data = record_1['Item']['friend_data']['M'] self.assertEqual(recursive_data['username']['S'], 'alice') self.assertEqual(recursive_data['friend_count']['N'], '4') vendored/boto/tests/integration/dynamodb2/__init__.py 0000644 00000000000 15025013617 0016760 0 ustar 00 vendored/boto/tests/integration/directconnect/test_directconnect.py 0000644 00000003042 15025013617 0022062 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. # All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import boto from tests.compat import unittest class DirectConnectTest(unittest.TestCase): """ A very basic test to make sure signatures and basic calls work. """ def test_basic(self): conn = boto.connect_directconnect() response = conn.describe_connections() self.assertTrue(response) self.assertTrue('connections' in response) self.assertIsInstance(response['connections'], list) vendored/boto/tests/integration/directconnect/__init__.py 0000644 00000000000 15025013617 0017725 0 ustar 00 vendored/boto/tests/integration/logs/test_cert_verification.py 0000644 00000002745 15025013617 0021066 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ from tests.compat import unittest from tests.integration import ServiceCertVerificationTest import boto.logs class CloudWatchLogsCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): regions = boto.logs.regions() def sample_service_call(self, conn): conn.describe_log_groups() vendored/boto/tests/integration/logs/test_layer1.py 0000644 00000003426 15025013617 0016561 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import boto from tests.compat import unittest class TestCloudWatchLogs(unittest.TestCase): def setUp(self): self.logs = boto.connect_logs() def test_logs(self): logs = self.logs response = logs.describe_log_groups(log_group_name_prefix='test') self.assertIsInstance(response['logGroups'], list) mfilter = '[ip, id, user, ..., status_code=500, size]' sample = [ '127.0.0.1 - frank "GET /apache_pb.gif HTTP/1.0" 200 1534', '127.0.0.1 - frank "GET /apache_pb.gif HTTP/1.0" 500 5324', ] response = logs.test_metric_filter(mfilter, sample) self.assertEqual(len(response['matches']), 1) vendored/boto/tests/integration/logs/__init__.py 0000644 00000000000 15025013617 0016045 0 ustar 00 vendored/boto/tests/integration/emr/test_cert_verification.py 0000644 00000003006 15025013617 0020674 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all EMR endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.emr class EMRCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): emr = True regions = boto.emr.regions() def sample_service_call(self, conn): conn.describe_jobflows() vendored/boto/tests/integration/emr/__init__.py 0000644 00000002120 15025013617 0015671 0 ustar 00 # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/rds/test_cert_verification.py 0000644 00000003014 15025013617 0020700 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.rds class RDSCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): rds = True regions = boto.rds.regions() def sample_service_call(self, conn): conn.get_all_dbinstances() vendored/boto/tests/integration/rds/test_promote_modify.py 0000644 00000012471 15025013617 0020244 0 ustar 00 # Author: Bruce Pennypacker # # Create a temporary RDS database instance, then create a read-replica of the # instance. Once the replica is available, promote it and verify that the # promotion succeeds, then rename it. Delete the databases upon completion i # of the tests. # # For each step (creating the databases, promoting, etc) we loop for up # to 15 minutes to wait for the instance to become available. It should # never take that long for any of the steps to complete. """ Check that promotion of read replicas and renaming instances works as expected """ import unittest import time from boto.rds import RDSConnection class PromoteReadReplicaTest(unittest.TestCase): rds = True def setUp(self): self.conn = RDSConnection() self.masterDB_name = "boto-db-%s" % str(int(time.time())) self.replicaDB_name = "replica-%s" % self.masterDB_name self.renamedDB_name = "renamed-replica-%s" % self.masterDB_name def tearDown(self): instances = self.conn.get_all_dbinstances() for db in [self.masterDB_name, self.replicaDB_name, self.renamedDB_name]: for i in instances: if i.id == db: self.conn.delete_dbinstance(db, skip_final_snapshot=True) def test_promote(self): print('--- running RDS promotion & renaming tests ---') self.masterDB = self.conn.create_dbinstance(self.masterDB_name, 5, 'db.t1.micro', 'root', 'bototestpw') # Wait up to 15 minutes for the masterDB to become available print('--- waiting for "%s" to become available ---' % self.masterDB_name) wait_timeout = time.time() + (15 * 60) time.sleep(60) instances = self.conn.get_all_dbinstances(self.masterDB_name) inst = instances[0] while wait_timeout > time.time() and inst.status != 'available': time.sleep(15) instances = self.conn.get_all_dbinstances(self.masterDB_name) inst = instances[0] self.assertTrue(inst.status == 'available') self.replicaDB = self.conn.create_dbinstance_read_replica(self.replicaDB_name, self.masterDB_name) # Wait up to 15 minutes for the replicaDB to become available print('--- waiting for "%s" to become available ---' % self.replicaDB_name) wait_timeout = time.time() + (15 * 60) time.sleep(60) instances = self.conn.get_all_dbinstances(self.replicaDB_name) inst = instances[0] while wait_timeout > time.time() and inst.status != 'available': time.sleep(15) instances = self.conn.get_all_dbinstances(self.replicaDB_name) inst = instances[0] self.assertTrue(inst.status == 'available') # Promote the replicaDB and wait for it to become available self.replicaDB = self.conn.promote_read_replica(self.replicaDB_name) # Wait up to 15 minutes for the replicaDB to become available print('--- waiting for "%s" to be promoted and available ---' % self.replicaDB_name) wait_timeout = time.time() + (15 * 60) time.sleep(60) instances = self.conn.get_all_dbinstances(self.replicaDB_name) inst = instances[0] while wait_timeout > time.time() and inst.status != 'available': time.sleep(15) instances = self.conn.get_all_dbinstances(self.replicaDB_name) inst = instances[0] # Verify that the replica is now a standalone instance and no longer # functioning as a read replica self.assertTrue(inst) self.assertTrue(inst.status == 'available') self.assertFalse(inst.status_infos) # Verify that the master no longer has any read replicas instances = self.conn.get_all_dbinstances(self.masterDB_name) inst = instances[0] self.assertFalse(inst.read_replica_dbinstance_identifiers) print('--- renaming "%s" to "%s" ---' % ( self.replicaDB_name, self.renamedDB_name )) self.renamedDB = self.conn.modify_dbinstance(self.replicaDB_name, new_instance_id=self.renamedDB_name, apply_immediately=True) # Wait up to 15 minutes for the masterDB to become available print('--- waiting for "%s" to exist ---' % self.renamedDB_name) wait_timeout = time.time() + (15 * 60) time.sleep(60) # Wait up to 15 minutes until the new name shows up in the instance table found = False while found == False and wait_timeout > time.time(): instances = self.conn.get_all_dbinstances() for i in instances: if i.id == self.renamedDB_name: found = True if found == False: time.sleep(15) self.assertTrue(found) print('--- waiting for "%s" to become available ---' % self.renamedDB_name) instances = self.conn.get_all_dbinstances(self.renamedDB_name) inst = instances[0] # Now wait for the renamed instance to become available while wait_timeout > time.time() and inst.status != 'available': time.sleep(15) instances = self.conn.get_all_dbinstances(self.renamedDB_name) inst = instances[0] self.assertTrue(inst.status == 'available') # Since the replica DB was renamed... self.replicaDB = None print('--- tests completed ---') vendored/boto/tests/integration/rds/test_db_subnet_group.py 0000644 00000007401 15025013617 0020366 0 ustar 00 # Copyright (c) 2013 Franc Carter franc.carter@gmail.com # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that db_subnet_groups behave sanely """ import time import unittest import boto.rds from boto.vpc import VPCConnection from boto.rds import RDSConnection def _is_ok(subnet_group, vpc_id, description, subnets): if subnet_group.vpc_id != vpc_id: print('vpc_id is ',subnet_group.vpc_id, 'but should be ', vpc_id) return 0 if subnet_group.description != description: print("description is '"+subnet_group.description+"' but should be '"+description+"'") return 0 if set(subnet_group.subnet_ids) != set(subnets): subnets_are = ','.join(subnet_group.subnet_ids) should_be = ','.join(subnets) print("subnets are "+subnets_are+" but should be "+should_be) return 0 return 1 class DbSubnetGroupTest(unittest.TestCase): rds = True def test_db_subnet_group(self): vpc_api = VPCConnection() rds_api = RDSConnection() vpc = vpc_api.create_vpc('10.0.0.0/16') az_list = vpc_api.get_all_zones(filters={'state':'available'}) subnet = list() n = 0; for az in az_list: try: subnet.append(vpc_api.create_subnet(vpc.id, '10.0.'+str(n)+'.0/24',availability_zone=az.name)) n = n+1 except: pass grp_name = 'db_subnet_group'+str(int(time.time())) subnet_group = rds_api.create_db_subnet_group(grp_name, grp_name, [subnet[0].id,subnet[1].id]) if not _is_ok(subnet_group, vpc.id, grp_name, [subnet[0].id,subnet[1].id]): raise Exception("create_db_subnet_group returned bad values") rds_api.modify_db_subnet_group(grp_name, description='new description') subnet_grps = rds_api.get_all_db_subnet_groups(name=grp_name) if not _is_ok(subnet_grps[0], vpc.id, 'new description', [subnet[0].id,subnet[1].id]): raise Exception("modifying the subnet group desciption returned bad values") rds_api.modify_db_subnet_group(grp_name, subnet_ids=[subnet[1].id,subnet[2].id]) subnet_grps = rds_api.get_all_db_subnet_groups(name=grp_name) if not _is_ok(subnet_grps[0], vpc.id, 'new description', [subnet[1].id,subnet[2].id]): raise Exception("modifying the subnet group subnets returned bad values") rds_api.delete_db_subnet_group(subnet_group.name) try: rds_api.get_all_db_subnet_groups(name=grp_name) raise Exception(subnet_group.name+" still accessible after delete_db_subnet_group") except: pass while n > 0: n = n-1 vpc_api.delete_subnet(subnet[n].id) vpc_api.delete_vpc(vpc.id) vendored/boto/tests/integration/rds/__init__.py 0000644 00000002227 15025013617 0015706 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All Rights Reserved # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. vendored/boto/tests/integration/elastictranscoder/test_cert_verification.py 0000644 00000002722 15025013617 0023626 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.integration import ServiceCertVerificationTest import boto.elastictranscoder from tests.compat import unittest class ElasticTranscoderCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): elastictranscoder = True regions = boto.elastictranscoder.regions() def sample_service_call(self, conn): conn.list_pipelines() vendored/boto/tests/integration/elastictranscoder/test_layer1.py 0000644 00000011570 15025013617 0021325 0 ustar 00 # Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import time from boto.elastictranscoder.layer1 import ElasticTranscoderConnection from boto.elastictranscoder.exceptions import ValidationException from tests.compat import unittest import boto.s3 import boto.sns import boto.iam import boto.sns class TestETSLayer1PipelineManagement(unittest.TestCase): def setUp(self): self.api = ElasticTranscoderConnection() self.s3 = boto.connect_s3() self.sns = boto.connect_sns() self.iam = boto.connect_iam() self.sns = boto.connect_sns() self.timestamp = str(int(time.time())) self.input_bucket = 'boto-pipeline-%s' % self.timestamp self.output_bucket = 'boto-pipeline-out-%s' % self.timestamp self.role_name = 'boto-ets-role-%s' % self.timestamp self.pipeline_name = 'boto-pipeline-%s' % self.timestamp self.s3.create_bucket(self.input_bucket) self.s3.create_bucket(self.output_bucket) self.addCleanup(self.s3.delete_bucket, self.input_bucket) self.addCleanup(self.s3.delete_bucket, self.output_bucket) self.role = self.iam.create_role(self.role_name) self.role_arn = self.role['create_role_response']['create_role_result']\ ['role']['arn'] self.addCleanup(self.iam.delete_role, self.role_name) def create_pipeline(self): pipeline = self.api.create_pipeline( self.pipeline_name, self.input_bucket, self.output_bucket, self.role_arn, {'Progressing': '', 'Completed': '', 'Warning': '', 'Error': ''}) pipeline_id = pipeline['Pipeline']['Id'] self.addCleanup(self.api.delete_pipeline, pipeline_id) return pipeline_id def test_create_delete_pipeline(self): pipeline = self.api.create_pipeline( self.pipeline_name, self.input_bucket, self.output_bucket, self.role_arn, {'Progressing': '', 'Completed': '', 'Warning': '', 'Error': ''}) pipeline_id = pipeline['Pipeline']['Id'] self.api.delete_pipeline(pipeline_id) def test_can_retrieve_pipeline_information(self): pipeline_id = self.create_pipeline() # The pipeline shows up in list_pipelines pipelines = self.api.list_pipelines()['Pipelines'] pipeline_names = [p['Name'] for p in pipelines] self.assertIn(self.pipeline_name, pipeline_names) # The pipeline shows up in read_pipeline response = self.api.read_pipeline(pipeline_id) self.assertEqual(response['Pipeline']['Id'], pipeline_id) def test_update_pipeline(self): pipeline_id = self.create_pipeline() self.api.update_pipeline_status(pipeline_id, 'Paused') response = self.api.read_pipeline(pipeline_id) self.assertEqual(response['Pipeline']['Status'], 'Paused') def test_update_pipeline_notification(self): pipeline_id = self.create_pipeline() response = self.sns.create_topic('pipeline-errors') topic_arn = response['CreateTopicResponse']['CreateTopicResult']\ ['TopicArn'] self.addCleanup(self.sns.delete_topic, topic_arn) self.api.update_pipeline_notifications( pipeline_id, {'Progressing': '', 'Completed': '', 'Warning': '', 'Error': topic_arn}) response = self.api.read_pipeline(pipeline_id) self.assertEqual(response['Pipeline']['Notifications']['Error'], topic_arn) def test_list_jobs_by_pipeline(self): pipeline_id = self.create_pipeline() response = self.api.list_jobs_by_pipeline(pipeline_id) self.assertEqual(response['Jobs'], []) def test_proper_error_when_pipeline_does_not_exist(self): with self.assertRaises(ValidationException): self.api.read_pipeline('badpipelineid') vendored/boto/tests/integration/elastictranscoder/__init__.py 0000644 00000000000 15025013617 0020612 0 ustar 00 vendored/boto/tests/integration/glacier/test_cert_verification.py 0000644 00000003024 15025013617 0021517 0 ustar 00 # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Check that all of the certs on all service endpoints validate. """ import unittest from tests.integration import ServiceCertVerificationTest import boto.glacier class GlacierCertVerificationTest(unittest.TestCase, ServiceCertVerificationTest): glacier = True regions = boto.glacier.regions() def sample_service_call(self, conn): conn.list_vaults() vendored/boto/tests/integration/glacier/test_layer2.py 0000644 00000003751 15025013617 0017225 0 ustar 00 # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import time from tests.unit import unittest from boto.glacier.layer2 import Layer1, Layer2 class TestGlacierLayer2(unittest.TestCase): glacier = True def setUp(self): self.layer2 = Layer2() self.vault_name = 'testvault%s' % int(time.time()) def test_create_delete_vault(self): vault = self.layer2.create_vault(self.vault_name) retrieved_vault = self.layer2.get_vault(self.vault_name) self.layer2.delete_vault(self.vault_name) self.assertEqual(vault.name, retrieved_vault.name) self.assertEqual(vault.arn, retrieved_vault.arn) self.assertEqual(vault.creation_date, retrieved_vault.creation_date) self.assertEqual(vault.last_inventory_date, retrieved_vault.last_inventory_date) self.assertEqual(vault.number_of_archives, retrieved_vault.number_of_archives) vendored/boto/tests/integration/glacier/test_layer1.py 0000644 00000003754 15025013617 0017227 0 ustar 00 # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from tests.unit import unittest from boto.glacier.layer1 import Layer1 class TestGlacierLayer1(unittest.TestCase): glacier = True def delete_vault(self, vault_name): pass def test_initialiate_multipart_upload(self): # Create a vault, initiate a multipart upload, # then cancel it. glacier = Layer1() glacier.create_vault('l1testvault') self.addCleanup(glacier.delete_vault, 'l1testvault') upload_id = glacier.initiate_multipart_upload('l1testvault', 4 * 1024 * 1024, 'double spaces here')['UploadId'] self.addCleanup(glacier.abort_multipart_upload, 'l1testvault', upload_id) response = glacier.list_multipart_uploads('l1testvault')['UploadsList'] self.assertEqual(len(response), 1) self.assertEqual(response[0]['MultipartUploadId'], upload_id) vendored/boto/tests/integration/glacier/__init__.py 0000644 00000002154 15025013617 0016523 0 ustar 00 # -*- coding: utf-8 -*- # Copyright (c) 2012 Thomas Parslow http://almostobsolete.net/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/tests/test.py 0000644 00000005552 15025013617 0012017 0 ustar 00 #!/usr/bin/env python # Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. from __future__ import print_function import argparse import os import sys from nose.core import run def main(): description = ("Runs boto unit and/or integration tests. " "Arguments will be passed on to nosetests. " "See nosetests --help for more information.") parser = argparse.ArgumentParser(description=description) parser.add_argument('-t', '--service-tests', action="append", default=[], help="Run tests for a given service. This will " "run any test tagged with the specified value, " "e.g -t s3 -t ec2") known_args, remaining_args = parser.parse_known_args() attribute_args = [] for service_attribute in known_args.service_tests: attribute_args.extend(['-a', '!notdefault,' + service_attribute]) if not attribute_args: # If the user did not specify any filtering criteria, we at least # will filter out any test tagged 'notdefault'. attribute_args = ['-a', '!notdefault'] # Set default tests used by e.g. tox. For Py2 this means all unit # tests, while for Py3 it's just whitelisted ones. if 'default' in remaining_args: # Run from the base project directory os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) for i, arg in enumerate(remaining_args): if arg == 'default': remaining_args[i] = 'tests/unit' all_args = [__file__] + attribute_args + remaining_args print("nose command:", ' '.join(all_args)) if run(argv=all_args): # run will return True is all the tests pass. We want # this to equal a 0 rc return 0 else: return 1 if __name__ == "__main__": sys.exit(main()) vendored/boto/tests/compat.py 0000644 00000003030 15025013617 0012310 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # # Use unittest2 for older versions of Python try: import unittest2 as unittest except ImportError: import unittest # Use thirdt party ordereddict for older versions of Python try: from collections import OrderedDict except ImportError: from ordereddict import OrderedDict # Use standard unittest.mock if possible. (mock doesn't support Python 3.4) try: from unittest import mock except ImportError: import mock vendored/boto/tests/mturk/create_hit_from_hit_type.doctest 0000644 00000006330 15025013617 0020251 0 ustar 00 >>> import uuid >>> import datetime >>> from _init_environment import MTurkConnection, mturk_host >>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer >>> >>> conn = MTurkConnection(host=mturk_host) >>> keywords=['boto', 'test', 'doctest'] >>> hit_type_rs = conn.register_hit_type('Boto Test HIT type', ... 'HIT Type for testing Boto', ... 0.12, ... 60*6, ... keywords=keywords, ... approval_delay=60*60) # this was a valid request >>> hit_type_rs.status True # the HIT Type Id is a unicode string >>> hit_type_id = hit_type_rs.HITTypeId >>> hit_type_id # doctest: +ELLIPSIS u'...' # create content for a question >>> qn_content = QuestionContent() >>> qn_content.append_field('Title', 'Boto question content create_hit_from_hit_type') >>> qn_content.append_field('Text', 'What is a boto create_hit_from_hit_type?') # create the question specification >>> qn = Question(identifier=str(uuid.uuid4()), ... content=qn_content, ... answer_spec=AnswerSpecification(FreeTextAnswer())) # now, create the actual HIT for the question using the HIT type # NOTE - the response_groups are specified to get back additional information for testing >>> create_hit_rs = conn.create_hit(hit_type=hit_type_rs.HITTypeId, ... question=qn, ... lifetime=60*65, ... max_assignments=2, ... annotation='An annotation from boto create_hit_from_hit_type test', ... response_groups=['Minimal', ... 'HITDetail', ... 'HITQuestion', ... 'HITAssignmentSummary',]) # this is a valid request >>> create_hit_rs.status True >>> len(create_hit_rs) 1 >>> hit = create_hit_rs[0] # for the requested hit type id >>> hit.HITTypeId == hit_type_id True # with the correct number of maximum assignments >>> hit.MaxAssignments u'2' # and the approval delay >>> hit.AutoApprovalDelayInSeconds u'3600' # expiration should be very close to now + the lifetime in seconds >>> expected_datetime = datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None) + datetime.timedelta(seconds=3900) >>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') >>> delta = expected_datetime - expiration_datetime >>> abs(delta).seconds < 5 True # duration is as specified for the HIT type >>> hit.AssignmentDurationInSeconds u'360' # the reward has been set correctly >>> float(hit.Amount) == 0.12 True >>> hit.FormattedPrice u'$0.12' # only US currency supported at present >>> hit.CurrencyCode u'USD' # title is the HIT type title >>> hit.Title u'Boto Test HIT type' # title is the HIT type description >>> hit.Description u'HIT Type for testing Boto' # annotation is correct >>> hit.RequesterAnnotation u'An annotation from boto create_hit_from_hit_type test' # not reviewed yet >>> hit.HITReviewStatus u'NotReviewed' vendored/boto/tests/mturk/hit_persistence.py 0000644 00000001353 15025013617 0015365 0 ustar 00 import unittest import pickle from .common import MTurkCommon class TestHITPersistence(MTurkCommon): def create_hit_result(self): return self.conn.create_hit( question=self.get_question(), **self.get_hit_params() ) def test_pickle_hit_result(self): result = self.create_hit_result() new_result = pickle.loads(pickle.dumps(result)) def test_pickle_deserialized_version(self): """ It seems the technique used to store and reload the object must result in an equivalent object, or subsequent pickles may fail. This tests a double-pickle to elicit that error. """ result = self.create_hit_result() new_result = pickle.loads(pickle.dumps(result)) pickle.dumps(new_result) if __name__ == '__main__': unittest.main() vendored/boto/tests/mturk/common.py 0000644 00000003400 15025013617 0013460 0 ustar 00 import unittest import uuid import datetime from boto.mturk.question import ( Question, QuestionContent, AnswerSpecification, FreeTextAnswer, ) from ._init_environment import SetHostMTurkConnection, config_environment class MTurkCommon(unittest.TestCase): def setUp(self): config_environment() self.conn = SetHostMTurkConnection() @staticmethod def get_question(): # create content for a question qn_content = QuestionContent() qn_content.append_field('Title', 'Boto no hit type question content') qn_content.append_field('Text', 'What is a boto no hit type?') # create the question specification qn = Question( identifier=str(uuid.uuid4()), content=qn_content, answer_spec=AnswerSpecification(FreeTextAnswer())) return qn @staticmethod def get_hit_params(): return dict( lifetime=datetime.timedelta(minutes=65), max_assignments=2, title='Boto create_hit title', description='Boto create_hit description', keywords=['boto', 'test'], reward=0.23, duration=datetime.timedelta(minutes=6), approval_delay=60*60, annotation='An annotation from boto create_hit test', response_groups=['Minimal', 'HITDetail', 'HITQuestion', 'HITAssignmentSummary',], ) vendored/boto/tests/mturk/all_tests.py 0000644 00000001113 15025013617 0014161 0 ustar 00 import unittest import doctest from glob import glob from .create_hit_test import * from .create_hit_with_qualifications import * from .create_hit_external import * from .create_hit_with_qualifications import * from .hit_persistence import * doctest_suite = doctest.DocFileSuite( *glob('*.doctest'), **{'optionflags': doctest.REPORT_ONLY_FIRST_FAILURE} ) class Program(unittest.TestProgram): def runTests(self, *args, **kwargs): self.test = unittest.TestSuite([self.test, doctest_suite]) super(Program, self).runTests(*args, **kwargs) if __name__ == '__main__': Program() vendored/boto/tests/mturk/create_hit_test.py 0000644 00000000771 15025013617 0015346 0 ustar 00 import unittest import os from boto.mturk.question import QuestionForm from .common import MTurkCommon class TestHITCreation(MTurkCommon): def testCallCreateHitWithOneQuestion(self): create_hit_rs = self.conn.create_hit( question=self.get_question(), **self.get_hit_params() ) def testCallCreateHitWithQuestionForm(self): create_hit_rs = self.conn.create_hit( questions=QuestionForm([self.get_question()]), **self.get_hit_params() ) if __name__ == '__main__': unittest.main() vendored/boto/tests/mturk/_init_environment.py 0000644 00000001646 15025013617 0015730 0 ustar 00 import os import functools live_connection = False mturk_host = 'mechanicalturk.sandbox.amazonaws.com' external_url = 'http://www.example.com/' SetHostMTurkConnection = None def config_environment(): global SetHostMTurkConnection try: local = os.path.join(os.path.dirname(__file__), 'local.py') execfile(local) except: pass if live_connection: #TODO: you must set the auth credentials to something valid from boto.mturk.connection import MTurkConnection else: # Here the credentials must be set, but it doesn't matter what # they're set to. os.environ.setdefault('AWS_ACCESS_KEY_ID', 'foo') os.environ.setdefault('AWS_SECRET_ACCESS_KEY', 'bar') from mocks import MTurkConnection SetHostMTurkConnection = functools.partial(MTurkConnection, host=mturk_host) vendored/boto/tests/mturk/search_hits.doctest 0000644 00000000616 15025013617 0015507 0 ustar 00 >>> from _init_environment import MTurkConnection, mturk_host >>> conn = MTurkConnection(host=mturk_host) # should have some HIT's returned by a search (but only if your account has existing HIT's) >>> search_rs = conn.search_hits() # this is a valid request >>> search_rs.status True >>> len(search_rs) > 1 True >>> search_rs # doctest: +ELLIPSIS [<boto.mturk.connection.HIT instance at ...] vendored/boto/tests/mturk/support.py 0000644 00000000224 15025013617 0013705 0 ustar 00 import sys # use unittest2 under Python 2.6 and earlier. if sys.version_info >= (2, 7): import unittest else: import unittest2 as unittest vendored/boto/tests/mturk/.gitignore 0000644 00000000012 15025013617 0013602 0 ustar 00 local.py vendored/boto/tests/mturk/create_hit_external.py 0000644 00000001705 15025013617 0016207 0 ustar 00 import unittest import uuid import datetime from boto.mturk.question import ExternalQuestion from ._init_environment import SetHostMTurkConnection, external_url, \ config_environment class Test(unittest.TestCase): def setUp(self): config_environment() def test_create_hit_external(self): q = ExternalQuestion(external_url=external_url, frame_height=800) conn = SetHostMTurkConnection() keywords=['boto', 'test', 'doctest'] create_hit_rs = conn.create_hit(question=q, lifetime=60*65, max_assignments=2, title="Boto External Question Test", keywords=keywords, reward = 0.05, duration=60*6, approval_delay=60*60, annotation='An annotation from boto external question test', response_groups=['Minimal', 'HITDetail', 'HITQuestion', 'HITAssignmentSummary',]) assert(create_hit_rs.status == True) if __name__ == "__main__": unittest.main() vendored/boto/tests/mturk/reviewable_hits.doctest 0000644 00000007577 15025013617 0016404 0 ustar 00 >>> import uuid >>> import datetime >>> from _init_environment import MTurkConnection, mturk_host >>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer >>> conn = MTurkConnection(host=mturk_host) # create content for a question >>> qn_content = QuestionContent() >>> qn_content.append_field('Title', 'Boto no hit type question content') >>> qn_content.append_field('Text', 'What is a boto no hit type?') # create the question specification >>> qn = Question(identifier=str(uuid.uuid4()), ... content=qn_content, ... answer_spec=AnswerSpecification(FreeTextAnswer())) # now, create the actual HIT for the question without using a HIT type # NOTE - the response_groups are specified to get back additional information for testing >>> keywords=['boto', 'test', 'doctest'] >>> create_hit_rs = conn.create_hit(question=qn, ... lifetime=60*65, ... max_assignments=1, ... title='Boto Hit to be Reviewed', ... description='Boto reviewable_hits description', ... keywords=keywords, ... reward=0.23, ... duration=60*6, ... approval_delay=60*60, ... annotation='An annotation from boto create_hit test', ... response_groups=['Minimal', ... 'HITDetail', ... 'HITQuestion', ... 'HITAssignmentSummary',]) # this is a valid request >>> create_hit_rs.status True >>> len(create_hit_rs) 1 >>> hit = create_hit_rs[0] # for the requested hit type id # the HIT Type Id is a unicode string >>> hit_type_id = hit.HITTypeId >>> hit_type_id # doctest: +ELLIPSIS u'...' >>> from selenium_support import complete_hit, has_selenium >>> if has_selenium(): complete_hit(hit_type_id, response='reviewable_hits_test') >>> import time Give mechanical turk some time to process the hit >>> if has_selenium(): time.sleep(10) # should have some reviewable HIT's returned, especially if returning all HIT type's # NOTE: but only if your account has existing HIT's in the reviewable state >>> reviewable_rs = conn.get_reviewable_hits() # this is a valid request >>> reviewable_rs.status True >>> len(reviewable_rs) >= 1 True # should contain at least one HIT object >>> reviewable_rs # doctest: +ELLIPSIS [<boto.mturk.connection.HIT instance at ...] >>> hit_id = reviewable_rs[0].HITId # check that we can retrieve the assignments for a HIT >>> assignments_rs = conn.get_assignments(hit_id) # this is a valid request >>> assignments_rs.status True >>> int(assignments_rs.NumResults) >= 1 True >>> len(assignments_rs) == int(assignments_rs.NumResults) True >>> int(assignments_rs.PageNumber) 1 >>> int(assignments_rs.TotalNumResults) >= 1 True # should contain at least one Assignment object >>> assignments_rs # doctest: +ELLIPSIS [<boto.mturk.connection.Assignment instance at ...] # should have returned assignments for the requested HIT id >>> assignment = assignments_rs[0] >>> assignment.HITId == hit_id True # should have a valid status >>> assignment.AssignmentStatus in ['Submitted', 'Approved', 'Rejected'] True # should have returned at least one answer >>> len(assignment.answers) > 0 True # should contain at least one set of QuestionFormAnswer objects >>> assignment.answers # doctest: +ELLIPSIS [[<boto.mturk.connection.QuestionFormAnswer instance at ...]] >>> answer = assignment.answers[0][0] # the answer should have exactly one field >>> len(answer.fields) 1 >>> qid, text = answer.fields[0] >>> text # doctest: +ELLIPSIS u'...' # question identifier should be a unicode string >>> qid # doctest: +ELLIPSIS u'...' vendored/boto/tests/mturk/create_free_text_question_regex.doctest 0000644 00000006517 15025013617 0021652 0 ustar 00 >>> import uuid >>> import datetime >>> from _init_environment import MTurkConnection, mturk_host >>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer, RegExConstraint >>> conn = MTurkConnection(host=mturk_host) # create content for a question >>> qn_content = QuestionContent() >>> qn_content.append_field('Title', 'Boto no hit type question content') >>> qn_content.append_field('Text', 'What is a boto no hit type?') # create a free text answer that is not quite so free! >>> constraints = [ ... RegExConstraint( ... "^[12][0-9]{3}-[01]?\d-[0-3]?\d$", ... error_text="You must enter a date with the format yyyy-mm-dd.", ... flags='i', ... )] >>> ft_answer = FreeTextAnswer(constraints=constraints, ... default="This is not a valid format") # create the question specification >>> qn = Question(identifier=str(uuid.uuid4()), ... content=qn_content, ... answer_spec=AnswerSpecification(ft_answer)) # now, create the actual HIT for the question without using a HIT type # NOTE - the response_groups are specified to get back additional information for testing >>> keywords=['boto', 'test', 'doctest'] >>> create_hit_rs = conn.create_hit(question=qn, ... lifetime=60*65, ... max_assignments=2, ... title='Boto create_hit title', ... description='Boto create_hit description', ... keywords=keywords, ... reward=0.23, ... duration=60*6, ... approval_delay=60*60, ... annotation='An annotation from boto create_hit test', ... response_groups=['Minimal', ... 'HITDetail', ... 'HITQuestion', ... 'HITAssignmentSummary',]) # this is a valid request >>> create_hit_rs.status True # for the requested hit type id # the HIT Type Id is a unicode string >>> len(create_hit_rs) 1 >>> hit = create_hit_rs[0] >>> hit_type_id = hit.HITTypeId >>> hit_type_id # doctest: +ELLIPSIS u'...' >>> hit.MaxAssignments u'2' >>> hit.AutoApprovalDelayInSeconds u'3600' # expiration should be very close to now + the lifetime in seconds >>> expected_datetime = datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None) + datetime.timedelta(seconds=3900) >>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') >>> delta = expected_datetime - expiration_datetime >>> abs(delta).seconds < 5 True # duration is as specified for the HIT type >>> hit.AssignmentDurationInSeconds u'360' # the reward has been set correctly (allow for float error here) >>> int(float(hit.Amount) * 100) 23 >>> hit.FormattedPrice u'$0.23' # only US currency supported at present >>> hit.CurrencyCode u'USD' # title is the HIT type title >>> hit.Title u'Boto create_hit title' # title is the HIT type description >>> hit.Description u'Boto create_hit description' # annotation is correct >>> hit.RequesterAnnotation u'An annotation from boto create_hit test' >>> hit.HITReviewStatus u'NotReviewed' vendored/boto/tests/mturk/test_disable_hit.py 0000644 00000000472 15025013617 0015504 0 ustar 00 from tests.mturk.support import unittest from .common import MTurkCommon from boto.mturk.connection import MTurkRequestError class TestDisableHITs(MTurkCommon): def test_disable_invalid_hit(self): self.assertRaises(MTurkRequestError, self.conn.disable_hit, 'foo') if __name__ == '__main__': unittest.main() vendored/boto/tests/mturk/cleanup_tests.py 0000644 00000002723 15025013617 0015050 0 ustar 00 import itertools from ._init_environment import SetHostMTurkConnection from ._init_environment import config_environment def description_filter(substring): return lambda hit: substring in hit.Title def disable_hit(hit): return conn.disable_hit(hit.HITId) def dispose_hit(hit): # assignments must be first approved or rejected for assignment in conn.get_assignments(hit.HITId): if assignment.AssignmentStatus == 'Submitted': conn.approve_assignment(assignment.AssignmentId) return conn.dispose_hit(hit.HITId) def cleanup(): """Remove any boto test related HIT's""" config_environment() global conn conn = SetHostMTurkConnection() is_boto = description_filter('Boto') print('getting hits...') all_hits = list(conn.get_all_hits()) is_reviewable = lambda hit: hit.HITStatus == 'Reviewable' is_not_reviewable = lambda hit: not is_reviewable(hit) hits_to_process = filter(is_boto, all_hits) hits_to_disable = filter(is_not_reviewable, hits_to_process) hits_to_dispose = filter(is_reviewable, hits_to_process) print('disabling/disposing %d/%d hits' % (len(hits_to_disable), len(hits_to_dispose))) map(disable_hit, hits_to_disable) map(dispose_hit, hits_to_dispose) total_hits = len(all_hits) hits_processed = len(hits_to_process) skipped = total_hits - hits_processed fmt = 'Processed: %(total_hits)d HITs, disabled/disposed: %(hits_processed)d, skipped: %(skipped)d' print(fmt % vars()) if __name__ == '__main__': cleanup() vendored/boto/tests/mturk/mocks.py 0000644 00000000656 15025013617 0013316 0 ustar 00 from boto.mturk.connection import MTurkConnection as RealMTurkConnection class MTurkConnection(RealMTurkConnection): """ Mock MTurkConnection that doesn't connect, but instead just prepares the request and captures information about its usage. """ def _process_request(self, *args, **kwargs): saved_args = self.__dict__.setdefault('_mock_saved_args', dict()) saved_args['_process_request'] = (args, kwargs) vendored/boto/tests/mturk/create_hit_with_qualifications.py 0000644 00000001663 15025013617 0020436 0 ustar 00 from boto.mturk.connection import MTurkConnection from boto.mturk.question import ExternalQuestion from boto.mturk.qualification import Qualifications, PercentAssignmentsApprovedRequirement def test(): q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800) conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') keywords=['boto', 'test', 'doctest'] qualifications = Qualifications() qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95")) create_hit_rs = conn.create_hit(question=q, lifetime=60*65, max_assignments=2, title="Boto External Question Test", keywords=keywords, reward = 0.05, duration=60*6, approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications) assert(create_hit_rs.status == True) print(create_hit_rs.HITTypeId) if __name__ == "__main__": test() vendored/boto/tests/mturk/run-doctest.py 0000644 00000000406 15025013617 0014442 0 ustar 00 import argparse import doctest parser = argparse.ArgumentParser( description="Run a test by name" ) parser.add_argument('test_name') args = parser.parse_args() doctest.testfile( args.test_name, optionflags=doctest.REPORT_ONLY_FIRST_FAILURE ) vendored/boto/tests/mturk/__init__.py 0000644 00000000000 15025013617 0013720 0 ustar 00 vendored/boto/tests/mturk/selenium_support.py 0000644 00000003317 15025013617 0015614 0 ustar 00 from __future__ import absolute_import from boto.mturk.test.support import unittest sel_args = ('localhost', 4444, '*chrome', 'https://workersandbox.mturk.com') class SeleniumFailed(object): def __init__(self, message): self.message = message def __nonzero__(self): return False def has_selenium(): try: from selenium import selenium globals().update(selenium=selenium) sel = selenium(*sel_args) # a little trick to see if the server is responding try: sel.do_command('shutdown', '') except Exception as e: if not 'Server Exception' in str(e): raise result = True except ImportError: result = SeleniumFailed('selenium RC not installed') except Exception: msg = 'Error occurred initializing selenium: %s' % e result = SeleniumFailed(msg) # overwrite has_selenium, so the same result is returned every time globals().update(has_selenium=lambda: result) return result identity = lambda x: x def skip_unless_has_selenium(): res = has_selenium() if not res: return unittest.skip(res.message) return identity def complete_hit(hit_type_id, response='Some Response'): verificationErrors = [] sel = selenium(*sel_args) sel.start() sel.open("/mturk/welcome") sel.click("lnkWorkerSignin") sel.wait_for_page_to_load("30000") sel.type("email", "boto.tester@example.com") sel.type("password", "BotoTest") sel.click("Continue") sel.wait_for_page_to_load("30000") sel.open("/mturk/preview?groupId={hit_type_id}".format(**vars())) sel.click("/accept") sel.wait_for_page_to_load("30000") sel.type("Answer_1_FreeText", response) sel.click("//div[5]/table/tbody/tr[2]/td[1]/input") sel.wait_for_page_to_load("30000") sel.click("link=Sign Out") sel.wait_for_page_to_load("30000") sel.stop() vendored/boto/tests/mturk/create_hit_binary.doctest 0000644 00000006164 15025013617 0016672 0 ustar 00 >>> import uuid >>> import datetime >>> from _init_environment import MTurkConnection, mturk_host >>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer, Binary >>> conn = MTurkConnection(host=mturk_host) # create content for a question >>> qn_content = QuestionContent() >>> qn_content.append_field('Title','Boto no hit type question content') >>> qn_content.append_field('Text', 'What is a boto binary hit type?') >>> binary_content = Binary('image', 'jpeg', 'http://www.example.com/test1.jpg', alt_text='image is missing') >>> qn_content.append(binary_content) # create the question specification >>> qn = Question(identifier=str(uuid.uuid4()), ... content=qn_content, ... answer_spec=AnswerSpecification(FreeTextAnswer())) # now, create the actual HIT for the question without using a HIT type # NOTE - the response_groups are specified to get back additional information for testing >>> keywords=['boto', 'test', 'doctest'] >>> lifetime = datetime.timedelta(minutes=65) >>> create_hit_rs = conn.create_hit(question=qn, ... lifetime=lifetime, ... max_assignments=2, ... title='Boto create_hit title', ... description='Boto create_hit description', ... keywords=keywords, ... reward=0.23, ... duration=60*6, ... approval_delay=60*60, ... annotation='An annotation from boto create_hit test', ... response_groups=['Minimal', ... 'HITDetail', ... 'HITQuestion', ... 'HITAssignmentSummary',]) # this is a valid request >>> create_hit_rs.status True >>> len(create_hit_rs) 1 >>> hit = create_hit_rs[0] # for the requested hit type id # the HIT Type Id is a unicode string >>> hit_type_id = hit.HITTypeId >>> hit_type_id # doctest: +ELLIPSIS u'...' >>> hit.MaxAssignments u'2' >>> hit.AutoApprovalDelayInSeconds u'3600' # expiration should be very close to now + the lifetime >>> expected_datetime = datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None) + lifetime >>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') >>> delta = expected_datetime - expiration_datetime >>> abs(delta).seconds < 5 True # duration is as specified for the HIT type >>> hit.AssignmentDurationInSeconds u'360' # the reward has been set correctly (allow for float error here) >>> int(float(hit.Amount) * 100) 23 >>> hit.FormattedPrice u'$0.23' # only US currency supported at present >>> hit.CurrencyCode u'USD' # title is the HIT type title >>> hit.Title u'Boto create_hit title' # title is the HIT type description >>> hit.Description u'Boto create_hit description' # annotation is correct >>> hit.RequesterAnnotation u'An annotation from boto create_hit test' >>> hit.HITReviewStatus u'NotReviewed' vendored/boto/tests/mturk/create_hit.doctest 0000644 00000005725 15025013617 0015330 0 ustar 00 >>> import uuid >>> import datetime >>> from _init_environment import MTurkConnection, mturk_host >>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer >>> conn = MTurkConnection(host=mturk_host) # create content for a question >>> qn_content = QuestionContent() >>> qn_content.append_field('Title', 'Boto no hit type question content') >>> qn_content.append_field('Text', 'What is a boto no hit type?') # create the question specification >>> qn = Question(identifier=str(uuid.uuid4()), ... content=qn_content, ... answer_spec=AnswerSpecification(FreeTextAnswer())) # now, create the actual HIT for the question without using a HIT type # NOTE - the response_groups are specified to get back additional information for testing >>> keywords=['boto', 'test', 'doctest'] >>> lifetime = datetime.timedelta(minutes=65) >>> create_hit_rs = conn.create_hit(question=qn, ... lifetime=lifetime, ... max_assignments=2, ... title='Boto create_hit title', ... description='Boto create_hit description', ... keywords=keywords, ... reward=0.23, ... duration=60*6, ... approval_delay=60*60, ... annotation='An annotation from boto create_hit test', ... response_groups=['Minimal', ... 'HITDetail', ... 'HITQuestion', ... 'HITAssignmentSummary',]) # this is a valid request >>> create_hit_rs.status True >>> len(create_hit_rs) 1 >>> hit = create_hit_rs[0] # for the requested hit type id # the HIT Type Id is a unicode string >>> hit_type_id = hit.HITTypeId >>> hit_type_id # doctest: +ELLIPSIS u'...' >>> hit.MaxAssignments u'2' >>> hit.AutoApprovalDelayInSeconds u'3600' # expiration should be very close to now + the lifetime >>> expected_datetime = datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None) + lifetime >>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') >>> delta = expected_datetime - expiration_datetime >>> abs(delta).seconds < 5 True # duration is as specified for the HIT type >>> hit.AssignmentDurationInSeconds u'360' # the reward has been set correctly (allow for float error here) >>> int(float(hit.Amount) * 100) 23 >>> hit.FormattedPrice u'$0.23' # only US currency supported at present >>> hit.CurrencyCode u'USD' # title is the HIT type title >>> hit.Title u'Boto create_hit title' # title is the HIT type description >>> hit.Description u'Boto create_hit description' # annotation is correct >>> hit.RequesterAnnotation u'An annotation from boto create_hit test' >>> hit.HITReviewStatus u'NotReviewed' vendored/boto/CONTRIBUTING 0000644 00000006241 15025013617 0011152 0 ustar 00 ============ Contributing ============ For more information, please see the official contribution docs at http://docs.pythonboto.org/en/latest/contributing.html. Contributing Code ================= * A good patch: * is clear. * works across all supported versions of Python. * follows the existing style of the code base (PEP-8). * has comments included as needed. * A test case that demonstrates the previous flaw that now passes with the included patch. * If it adds/changes a public API, it must also include documentation for those changes. * Must be appropriately licensed (New BSD). Reporting An Issue/Feature ========================== * Check to see if there's an existing issue/pull request for the bug/feature. All issues are at https://github.com/boto/boto/issues and pull reqs are at https://github.com/boto/boto/pulls. * If there isn't an existing issue there, please file an issue. The ideal report includes: * A description of the problem/suggestion. * How to recreate the bug. * If relevant, including the versions of your: * Python interpreter * boto * Optionally of the other dependencies involved * If possible, create a pull request with a (failing) test case demonstrating what's wrong. This makes the process for fixing bugs quicker & gets issues resolved sooner. Maintenance =========== You do not have to be labeled as a "maintainer" to be able to help with the triaging, resolving, and reviewing of boto issues and pull requests. These are the processes that the maintainers follow, that you can also follow to help speed up the resolution of an issue or pull request: Pull Requests ------------- There are some key points that are needed to be met before a pull request can be merged: * All tests must pass for all python versions. * All pull requests require tests that either test the new feature or test that the specific bug is fixed. Pull requests for minor things like adding a new region or fixing a typo do not need tests. * Must follow PEP8 conventions. * All changes must be backwards compatible. * If the pull request is for a feature, make sure a link to the corresponding API is provided in order to easily check that the feature matches up with the service's API. The best way to help with pull requests is to comment on pull requests by noting if any of these key points are missing, it will both help get feedback sooner to the issuer of the pull request and make it easier to determine for an individual with write permissions to the repository if a pull request is ready to be merged. Issues ------ Here are the best ways to help with open issues: * If there is an issue without a set of instructions on how to reproduce the bug, feel free to try to reproduce the issue, comment with the minimal amount of steps to reproduce the bug (a code snippet would be ideal). If there is not a set of steps that can be made to reproduce the issue, at least make sure there are debug logs that capture the unexpected behavior. * Consolidate all related issue to one issues by closing out related issues and linking them to the single issue that outlines the general issue. * Submit pull requests for open issues. vendored/boto/boto/contrib/ymlmessage.py 0000644 00000003535 15025013617 0014446 0 ustar 00 # Copyright (c) 2006,2007 Chris Moyer # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ This module was contributed by Chris Moyer. It provides a subclass of the SQS Message class that supports YAML as the body of the message. This module requires the yaml module. """ from boto.sqs.message import Message import yaml class YAMLMessage(Message): """ The YAMLMessage class provides a YAML compatible message. Encoding and decoding are handled automaticaly. Access this message data like such: m.data = [ 1, 2, 3] m.data[0] # Returns 1 This depends on the PyYAML package """ def __init__(self, queue=None, body='', xml_attrs=None): self.data = None super(YAMLMessage, self).__init__(queue, body) def set_body(self, body): self.data = yaml.safe_load(body) def get_body(self): return yaml.dump(self.data) vendored/boto/boto/contrib/__init__.py 0000644 00000002121 15025013617 0014025 0 ustar 00 # Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/boto/auth_handler.py 0000644 00000004020 15025013617 0013264 0 ustar 00 # Copyright 2010 Google Inc. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """ Defines an interface which all Auth handlers need to implement. """ from boto.plugin import Plugin class NotReadyToAuthenticate(Exception): pass class AuthHandler(Plugin): capability = [] def __init__(self, host, config, provider): """Constructs the handlers. :type host: string :param host: The host to which the request is being sent. :type config: boto.pyami.Config :param config: Boto configuration. :type provider: boto.provider.Provider :param provider: Provider details. Raises: NotReadyToAuthenticate: if this handler is not willing to authenticate for the given provider and config. """ pass def add_auth(self, http_request): """Invoked to add authentication details to request. :type http_request: boto.connection.HTTPRequest :param http_request: HTTP request that needs to be authenticated. """ pass vendored/boto/boto/__pycache__/regioninfo.cpython-39.pyc 0000644 00000016141 15025013617 0017263 0 ustar 00 a sFg% � @ s� d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlmZ i Z dd� Z dd � Z d d � Z d d � Z e fdd�Zd dd�Zd dd�Zd dd�Zd dd�Zdd� ZG dd� de�ZdS ) � N)�json)�BotoClientError)�BotoEndpointResolver)�StaticEndpointBuilderc C s t | �S )�� Loads a given JSON file & returns it. :param path: The path to the JSON file :type path: string :returns: The loaded data )�_load_json_file)�path� r �2/opt/gsutil/gslib/vendored/boto/boto/regioninfo.py�load_endpoint_json# s r c C s8 t | d��}t�|�W d � S 1 s*0 Y dS )r �rN)�openr �load)r Zendpoints_filer r r r / s r c C s0 |� � D ]"\}}| �|i � | | �|� q| S )aW Given an existing set of endpoint data, this will deep-update it with any similarly structured data in the additions. :param defaults: The existing endpoints data :type defaults: dict :param defaults: The additional endpoints data :type defaults: dict :returns: The modified endpoints data :rtype: dict )�items� setdefault�update)�defaultsZ additionsZserviceZ region_infor r r �merge_endpoints< s r c C sX t � } d}tj�d�r"tjd }n tj�dd�r>tj�dd�}|rTt|�}t| |�} | S )a� Actually load the region/endpoint information from the JSON files. By default, this loads from the default included ``boto/endpoints.json`` file. Users can override/extend this by supplying either a ``BOTO_ENDPOINTS`` environment variable or a ``endpoints_path`` config variable, either of which should be an absolute path to the user's JSON file. :returns: The endpoints data :rtype: dict NZBOTO_ENDPOINTS�BotoZendpoints_path)�_load_builtin_endpoints�os�environ�get�boto�configr r )� endpointsZadditional_path� additionalr r r � load_regionsU s r c C s8 | r| S t tj�}t|�}t|�}|�� }| �|� | S )z1Loads the builtin endpoints in the legacy format.)r r �ENDPOINTS_PATHr r Zbuild_static_endpointsr )�_cacher �resolverZbuilderr r r r v s r c C s\ t � }| |vrtd| ��|du r&t}g }|�| i ��� D ] \}}|�||||d�� q:|S )ae Given a service name (like ``ec2``), returns a list of ``RegionInfo`` objects for that service. This leverages the ``endpoints.json`` file (+ optional user overrides) to configure/construct all the objects. :param service_name: The name of the service to construct the ``RegionInfo`` objects for. Ex: ``ec2``, ``s3``, ``sns``, etc. :type service_name: string :param region_cls: (Optional) The class to use when constructing. By default, this is ``RegionInfo``. :type region_cls: class :param connection_cls: (Optional) The connection class for the ``RegionInfo`` object. Providing this allows the ``connect`` method on the ``RegionInfo`` to work. Default is ``None`` (no connection). :type connection_cls: class :returns: A list of configured ``RegionInfo`` objects :rtype: list z$Service '%s' not found in endpoints.N��name�endpoint�connection_cls)r r � RegionInfor r �append)� service_name� region_clsr$ r Z region_objs� region_namer# r r r � get_regions� s" ���r* c K sR |du r t }t| |||�}|du r6t� r6t| |||�}|du rBdS |jf i |��S )a( Create a connection class for a given service in a given region. :param service_name: The name of the service to construct the ``RegionInfo`` object for, e.g. ``ec2``, ``s3``, etc. :type service_name: str :param region_name: The name of the region to connect to, e.g. ``us-west-2``, ``eu-central-1``, etc. :type region_name: str :param region_cls: (Optional) The class to use when constructing. By default, this is ``RegionInfo``. :type region_cls: class :param connection_cls: (Optional) The connection class for the ``RegionInfo`` object. Providing this allows the ``connect`` method on the ``RegionInfo`` to work. Default is ``None`` (no connection). :type connection_cls: class :returns: A configured connection class. N)r% � _get_region�_use_endpoint_heuristics�_get_region_with_heuristics�connect)r' r) r( r$ � kw_params�regionr r r r. � s �r. c C s( t | ||�D ]}|j|kr | S q dS )z8Finds the region by searching through the known regions.N)r* r" )r' r) r( r$ r0 r r r r+ � s r+ c C s, t tj�}t|�}|�| |�}||||d�S )z4Finds the region using known regions and heuristics.r! )r r r r Zresolve_hostname)r' r) r( r$ r r �hostnamer r r r- � s �r- c C s. t j�dd��� } tj�ddd�}| dkp,|S )NZ BOTO_USE_ENDPOINT_HEURISTICS�falser Zuse_endpoint_heuristicsF�true)r r r �lowerr r �getbool)Zenv_varZ config_varr r r r, � s r, c @ s: e Zd ZdZd dd�Zdd� Zdd� Zd d � Zd d � ZdS )r% z" Represents an AWS Region Nc C s || _ || _|| _|| _d S �N)� connectionr" r# r$ )�selfr7 r" r# r$ r r r �__init__ s zRegionInfo.__init__c C s d| j S )Nz RegionInfo:%s)r" )r8 r r r �__repr__ s zRegionInfo.__repr__c C s d S r6 r )r8 r" �attrsr7 r r r � startElement s zRegionInfo.startElementc C s0 |dkr|| _ n |dkr || _n t| ||� d S )NZ regionNameZregionEndpoint)r" r# �setattr)r8 r" �valuer7 r r r � endElement s zRegionInfo.endElementc K s | j r| j f d| i|��S dS )a� Connect to this Region's endpoint. Returns an connection object pointing to the endpoint associated with this region. You may pass any of the arguments accepted by the connection class's constructor as keyword arguments and they will be passed along to the connection object. :rtype: Connection object :return: The connection to this regions endpoint r0 N)r$ )r8 r/ r r r r. s zRegionInfo.connect)NNNN) �__name__� __module__� __qualname__�__doc__r9 r: r< r? r. r r r r r% � s � r% )NN)NN)NN)NN)r r � boto.compatr �boto.exceptionr Zboto.endpointsr r Z_endpoints_cacher r r r r r* r. r+ r- r, �objectr% r r r r �<module> s. ! 0 � & � � vendored/boto/boto/__pycache__/storage_uri.cpython-39.pyc 0000644 00000075147 15025013617 0017462 0 ustar 00 a sFg� � @ sx d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlmZ G dd� de �Z G dd� de �Z G d d � d e �Z dS ) � N)� DeleteMarker��BotoClientError)�InvalidUriErrorc @ s� e Zd ZdZdZdZi Zdd� Zdd� Zdd� Z d d � Z d d � Z d d� Z dd� Z d.dd�Zdd� Zd/dd�Zd0dd�Zd1dd �Zd2d d �Zd3d d �Zd4d!d"�Zd5d#d$�Zd6d&d'�Zd7d(d)�Zd*d+� Zd,d-� ZdS )8� StorageUriz� Base class for representing storage provider-independent bucket and object name with a shorthand URI-like syntax. This is an abstract class: the constructor cannot be called (throws an exception if you try). Nc C s t d��dS )zBUncallable constructor on abstract base StorageUri class. z0Attempt to instantiate abstract StorageUri classNr ��self� r �3/opt/gsutil/gslib/vendored/boto/boto/storage_uri.py�__init__6 s zStorageUri.__init__c C s | j S )z%Returns string representation of URI.��urir r r r �__repr__< s zStorageUri.__repr__c C s | j |j kS )z#Returns true if two URIs are equal.r )r r r r r �equals@ s zStorageUri.equalsc C s* |d u r&t d�t�d||f d����d S )N� z�Attempt to get %s for "%s" failed. This can happen if the URI refers to a non-existent object or if you meant to operate on a directory (e.g., leaving off -R option on gsutil cp, mv, or ls of a bucket)�P )r �join�textwrap�wrap)r �resp�levelr r r r �check_responseD s ��zStorageUri.check_responsec C s* t t| �t�r&| js&td|| jf ��d S )Nz%s on bucket-less URI (%s))� issubclass�type�BucketStorageUri� bucket_namer r �r � function_namer r r �_check_bucket_uriL s �z StorageUri._check_bucket_uric C s* t t| �t�r&| js&td|| jf ��d S )Nz%s on object-less URI (%s))r r r � object_namer r r r r r �_check_object_uriQ s �z StorageUri._check_object_uric K s4 |D ]*}|| rt j�d||t|| �f � qd S )Nz$Warning: %s ignores argument: %s=%s )�sys�stderr�write�str)r r �args�argr r r �_warn_about_argsV s ��zStorageUri._warn_about_argsc K s t | jp d�}t| d�r*d|vr*| j|d< |�|� | js�| j| jv rV| j| j | _n�| jdkr�ddlm } |||fi |��| _| j| j| j< nj| jdkr�ddl m } |||fi |��| _| j| j| j< n0| jdkr�dd l m } || �| _ntd | j ��| j| j_| jS ) a� Opens a connection to appropriate provider, depending on provider portion of URI. Requires Credentials defined in boto config file (see boto/pyami/config.py). @type storage_uri: StorageUri @param storage_uri: StorageUri specifying a bucket or a bucket+object @rtype: L{AWSAuthConnection<boto.gs.connection.AWSAuthConnection>} @return: A connection to storage service provider of the given URI. r �suppress_consec_slashes�s3r )� S3Connection�gs)� GSConnection�file)�FileConnectionzUnrecognized scheme "%s")�dict�connection_args�hasattrr( �update� connection�scheme� provider_pool�boto.s3.connectionr* �boto.gs.connectionr, Zboto.file.connectionr. r �debug)r Z access_key_id�secret_access_key�kwargsr0 r* r, r. r r r �connect] sD �� �� �� � zStorageUri.connectc C s" t t| �t�o | jd up | jd uS �N)r r r � version_id� generationr r r r � has_version� s �zStorageUri.has_versionFc C s( | � d� | �||�}|�| j|||�S )N� delete_key)r � get_bucketr@ r �r �validate�headersr= Z mfa_token�bucketr r r r@ � s �zStorageUri.delete_key� c C sH | � d� | j|d�}|r4dd� |j|||d�D �S |j|||d�S d S )N� list_bucket�rD c s s | ]}t |t�s|V qd S r< )� isinstancer )�.0�vr r r � <genexpr>� s �z)StorageUri.list_bucket.<locals>.<genexpr>)�prefix� delimiterrD )r rA Z list_versions�list)r rM rN rD Z all_versionsrE r r r rG � s � �zStorageUri.list_bucketc C s | � ||�}|�|�S r< )rA � get_all_keys)r rC rD rM rE r r r rP � s zStorageUri.get_all_keysc C s6 | � d� | �� }|�| j||�}| �|d| j� |S )NrA rE )r r; rA r r r )r rC rD �connrE r r r rA � s zStorageUri.get_bucketc C s: | � d� | �||�}|�| j||�}| �|d| j� |S )N�get_key�key)r rA rR r r r �r rC rD r= rE rS r r r rR � s zStorageUri.get_keyc C s" | � d� | �||�}|�| j�S )N�new_key)r rA rU r �r rC rD rE r r r rU � s zStorageUri.new_keyc C sD | � d� | jddd� | �d |�}| �|d| j� |j|||d�S )NrR F)rC rS �r= )r r' rR r r �get_contents_to_file)r �fprD r= rS r r r �get_contents_to_stream� s z!StorageUri.get_contents_to_stream� c C sd | � d� | �d |�} | �| d| j� | rH| j||||||||| d� n| �||||||||� d S )NrX rS )� hash_algs)r rR r r rX ) r rY rD �cb�num_cb�torrentr= Zres_download_handlerZresponse_headersr\ rS r r r rX � s ��z StorageUri.get_contents_to_filec C s8 | � d� | �||�}| �|d| j� |�|||||�S )N�get_contents_as_stringrS )r rR r r r` )r rC rD r] r^ r_ r= rS r r r r` � s �z!StorageUri.get_contents_as_stringc C s$ | � � }|jj}| �|d| j� |S )N� acl_class)r; �providerra r r )r rQ ra r r r ra � s zStorageUri.acl_classc C s$ | � � }|jj}| �|d| j� |S )N� canned_acls)r; rb rc r r )r rQ rc r r r rc � s zStorageUri.canned_acls)NN)FNNN)rF rF NF)FNN)FN)FNN)FN)NN)NNr[ FNNNN)FNNr[ FN)�__name__� __module__� __qualname__�__doc__r3 r0 r5 r r r r r r r' r; r? r@ rG rP rA rR rU rZ rX r` ra rc r r r r r sD ) � � � � r c @ s^ e Zd ZdZdZeg �Zd�dd�Zd d � Zd d � Z d d� Z d�dd�Z d�dd�Z dd� Z dd� Zd�dd�Zd�dd�Zd�dd �Zd�d d �Zd�d d �Zd�d!d"�Zd�d#d$�Zd�d%d&�Zd�d'd(�Zd�d)d*�Zd�d+d,�Zd�d-d.�Zd/d0� Zd1d2� Z d3d4� Z d5d6� Z d7d8� Z d9d:� Z d;d<� Z!d=d>� Z"d?d@� Z#dAdB� Z$d�dDdE�Z%d�dFdG�Z&d�dHdI�Z'dJdK� Z(d�dLdM�Z)d�dNdO�Z*d�dPdQ�Z+d�dRdS�Z,d�dTdU�Z-d�dVdW�Z.d�dXdY�Z/d�d[d\�Z0d�d]d^�Z1d�d_d`�Z2d�dbdc�Z3d�ddde�Z4d�dfdg�Z5d�dhdi�Z6d�djdk�Z7d�dldm�Z8d�dndo�Z9d�dpdq�Z:d�drds�Z;d�dtdu�Z<d�dvdw�Z=d�dxdy�Z>d�dzd{�Z?d�d|d}�Z@d�d~d�ZAd�d�d��ZBd�d�d��ZCdS )�r z� StorageUri subclass that handles bucket storage providers. Callers should instantiate this class by calling boto.storage_uri(). �/Nr TFc C sb || _ || _|| _|| _|r"|| _|| _|| _|o8t|�| _| | _ t | j�pRt |�| _ | � � dS )a� Instantiate a BucketStorageUri from scheme,bucket,object tuple. @type scheme: string @param scheme: URI scheme naming the storage provider (gs, s3, etc.) @type bucket_name: string @param bucket_name: bucket name @type object_name: string @param object_name: object name, excluding generation/version. @type debug: int @param debug: debug level to pass in to connection (range 0..2) @type connection_args: map @param connection_args: optional map containing args to be passed to {S3,GS}Connection constructor (e.g., to override https_connection_factory). @param suppress_consec_slashes: If provided, controls whether consecutive slashes will be suppressed in key paths. @param version_id: Object version id (S3-specific). @param generation: Object generation number (GCS-specific). @param is_latest: boolean indicating that a versioned object is the current version After instantiation the components are available in the following fields: scheme, bucket_name, object_name, version_id, generation, is_latest, versionless_uri, version_specific_uri, uri. Note: If instantiated without version info, the string representation for a URI stays versionless; similarly, if instantiated with version info, the string representation for a URI stays version-specific. If you call one of the uri.set_contents_from_xyz() methods, a specific object version will be created, and its version-specific URI string can be retrieved from version_specific_uri even if the URI was instantiated without version info. N) r4 r r r8 r0 r( r= �intr> � is_latest�bool�is_version_specific�_build_uri_strings) r r4 r r r8 r0 r( r= r> rj r r r r � s $zBucketStorageUri.__init__c C s� | j rn| jrnd| j| j | jf | _| jr<d| j| jf | _n| jrTd| j| jf | _| jrd| j| _q�| j| _n&| j r�d| j| j f | _n d| j | _d S )Nz %s://%s/%sz%s#%sz%s://%s/z%s://) r r r4 Zversionless_urir> Zversion_specific_urir= rl r r r r r rm s$ �� � z#BucketStorageUri._build_uri_stringsc C s4 | � t|dd �t|dd �t|dd �t|dd �� d S )Nr= r> rj �md5)�_update_from_values�getattr)r rS r r r �_update_from_key/ s �z!BucketStorageUri._update_from_keyc C s$ || _ || _|| _| �� || _d S r< )r= r> rj rm rn )r r= r> rj rn r r r ro 6 s z$BucketStorageUri._update_from_valuesc C st | � d� | �||�}| �� jdkr>|j| j||p6| jd�}n"| �� jdkr`|j| j|| jd�}| �|d| j � |S )NrR �awsrW �google�r> rS ) r rA � get_provider�namerR r r= r> r r rT r r r rR = s � �zBucketStorageUri.get_keyc C sf | � d� | �||�}| �� jdkr@|p,| j}|�| j|||�S | �� jdkrb|j| j|| jd�S d S )Nr@ rr rs rt )r rA ru rv r= r@ r r> rB r r r r@ I s � �zBucketStorageUri.delete_keyc C s$ | � d� t| j| j|| j| jd�S )z�Instantiate a BucketStorageUri from the current BucketStorageUri, but replacing the object_name. @type new_name: string @param new_name: new object name �clone_replace_name)r r r8 r( )r r r4 r r8 r( �r �new_namer r r rw U s �z#BucketStorageUri.clone_replace_namec C sn | � d� d}d}d}t|d�r&|j}t|d�r6|j}t|d�rF|j}t|j�� |jj |j | j | j |||d�S )a) Instantiate a BucketStorageUri from the current BucketStorageUri, by replacing the object name with the object name and other metadata found in the given Key object (including generation). @type key: Key @param key: key for the new StorageUri to represent �clone_replace_keyNFr= r> rj )r r r8 r( r= r> rj ) r r1 r= r> rj r rb �get_provider_namerE rv r8 r( )r rS r= r> rj r r r rz b s( �z"BucketStorageUri.clone_replace_keyc C sn | � d� | �||�}| jp d}| �� jdkrH|p6| j}|�|||�}n|j||| jd�}| �|d| j � |S )zreturns a bucket's acl�get_aclrF rr rt �acl) r rA r ru rv r= r| r> r r )r rC rD r= rE �key_namer} r r r r| s zBucketStorageUri.get_aclc C s4 | � d� | �||�}|�|�}| �|d| j� |S )z%returns a bucket's default object acl� get_def_aclr} )r rA r r r )r rC rD rE r} r r r r � s z BucketStorageUri.get_def_aclc C s4 | � d� | �||�}|�|�}| �|d| j� |S )zreturns a bucket's CORS XML�get_cors�cors)r rA r� r r )r rC rD rE r� r r r r� � s zBucketStorageUri.get_corsc C sB | � d� | �||�}| jdkr.|�||� n|�|�� |� dS )z#sets or updates a bucket's CORS XMLz set_cors r) N)r rA r4 �set_corsZto_xml)r r� rC rD rE r r r r� � s zBucketStorageUri.set_corsc C s | � d� | �||�}|�|�S )N� get_location)r rA r� rV r r r r� � s z BucketStorageUri.get_locationc C s8 | � d� | jdkr"td| j ��| �||�}|�|�S )N�get_storage_classr+ z.get_storage_class() not supported for %s URIs.)r r4 � ValueErrorrA r� rV r r r r� � s � z"BucketStorageUri.get_storage_classc C s> | � d� | jdkr"td| j ��| �||�}|�||� dS )z!Updates a bucket's storage class.�set_storage_classr+ z.set_storage_class() not supported for %s URIs.N)r r4 r� rA r� )r � storage_classrC rD rE r r r r� � s � z"BucketStorageUri.set_storage_classc C s( | � d� | �||�}|�|| j||�S )N�get_subresource)r rA r� r )r � subresourcerC rD r= rE r r r r� � s �z BucketStorageUri.get_subresourcec C s� | � d� | jdkr"td| j ��| jr`|r4td��| �||�}| �|d| j� |�|||� n2| jr�| � ||�}|�||||� nt d| j ��d S )N�add_group_email_grantr+ z2add_group_email_grant() not supported for %s URIs.zDadd_group_email_grant() on key-ful URI cannot specify recursive=TruerS z-add_group_email_grant() on bucket-less URI %s) r r4 r� r rR r r r� r rA r )r � permission� email_address� recursiverC rD rS rE r r r r� � s& � ��z&BucketStorageUri.add_group_email_grantc C sZ | � d� | js.| �||�}|�||||� n(| �||�}| �|d| j� |�||� d S )N�add_email_grantrS )r r rA r� rR r r )r r� r� r� rC rD rE rS r r r r� � s � z BucketStorageUri.add_email_grantc C sZ | � d� | js.| �||�}|�||||� n(| �||�}| �|d| j� |�||� d S )N�add_user_grantrS )r r rA r� rR r r )r r� Zuser_idr� rC rD rE rS r r r r� � s z BucketStorageUri.add_user_grantc C s | � d� | �|�}|�|�S )Nz list_grants )r rA � list_grants�r rD rE r r r r� � s z BucketStorageUri.list_grantsc C s dS )�3Returns True if this URI names a file or directory.Fr r r r r � is_file_uri� s z BucketStorageUri.is_file_uric C s dS )�2Returns True if this URI names a bucket or object.Tr r r r r � is_cloud_uri� s z BucketStorageUri.is_cloud_uric C s t | j �S )z� Returns True if this URI names a directory or bucket. Will return False for bucket subdirs; providing bucket subdir semantics needs to be done by the caller (like gsutil does). �rk r r r r r �names_container s z BucketStorageUri.names_containerc C s t | j�S )z0Returns True if this URI names a file or object.r� r r r r �names_singleton s z BucketStorageUri.names_singletonc C s dS �z+Returns True if this URI names a directory.Fr r r r r �names_directory s z BucketStorageUri.names_directoryc C s t | j �S )�*Returns True if this URI names a provider.)rk r r r r r �names_provider s z BucketStorageUri.names_providerc C s t | j�ot | j �S )�(Returns True if this URI names a bucket.)rk r r r r r r � names_bucket s z BucketStorageUri.names_bucketc C s dS )�&Returns True if this URI names a file.Fr r r r r � names_file s zBucketStorageUri.names_filec C s | � � S )�)Returns True if this URI names an object.�r� r r r r � names_object s z BucketStorageUri.names_objectc C s dS )z8Returns True if this URI represents input/output stream.Fr r r r r � is_stream! s zBucketStorageUri.is_streamrF c C sF | � d� | �� }| jdkr0|�| j||||�S |�| j|||�S d S )Nzcreate_bucket r+ )r r; r4 � create_bucketr )r rD �location�policyr� rQ r r r r� % s �z BucketStorageUri.create_bucketc C s | � d� | �� }|�| j|�S )N� delete_bucket)r r; r� r �r rD rQ r r r r� 1 s z BucketStorageUri.delete_bucketc C s | � � }|�|�S r< )r; �get_all_bucketsr� r r r r� 6 s z BucketStorageUri.get_all_bucketsc C s" | � � }|j}| �|d| j� |S )Nrb )r; rb r r )r rQ rb r r r ru : s z BucketStorageUri.get_providerc C sb | � d� |p| jpd}| �||�}| jrD|j|||| j||d� n|pL| j}|�||||� dS )z Sets or updates a bucket's ACL.�set_aclrF �r> � if_generation�if_metagenerationN)r r rA r> r� r= ) r � acl_or_strr~ rC rD r= r� r� rE r r r r� @ s � zBucketStorageUri.set_aclc C sd | � d� |p| jpd}| �||�}| jrD|j|||| j||d� n |pL| j}|j||||d� dS )z2Sets or updates a bucket's ACL with an XML string.� set_xml_aclrF r� rW N)r r rA r> r� r= ) r � xmlstringr~ rC rD r= r� r� rE r r r r� N s � �z BucketStorageUri.set_xml_aclc C s" | � d� | �||��||� dS )zASets or updates a bucket's default object ACL with an XML string.�set_def_xml_aclN)r rA r� )r r� rC rD r r r r� ] s z BucketStorageUri.set_def_xml_aclc C s" | � d� | �||��||� dS )z.Sets or updates a bucket's default object ACL.� set_def_aclN)r rA r� )r r� rC rD r= r r r r� b s z BucketStorageUri.set_def_aclc C sD | � d� | jd|d� | �||�}| �|d| j� |�||� dS )z>Sets or updates a bucket's acl to a predefined (canned) value.�set_canned_aclrW rS N)r r' rR r r r� �r Zacl_strrC rD r= rS r r r r� h s z BucketStorageUri.set_canned_aclc C s8 | � d� | �||�}| �|d| j� |�|||� dS )zXSets or updates a bucket's default object acl to a predefined (canned) value.zset_def_canned_acl rS N)r rR r r �set_def_canned_aclr� r r r r� q s z#BucketStorageUri.set_def_canned_aclc C s. | � d� | �||�}|�||| j||� d S )N�set_subresource)r rA r� r )r r� �valuerC rD r= rE r r r r� z s �z BucketStorageUri.set_subresourcer[ c C sn | � d� | j|d�} | jdkrH|r0tj�d� | �|||||||�} n| �||||||||�} | �| � | S )N�set_contents_from_stringrH r+ z^Warning: GCS does not support reduced_redundancy; argument ignored by set_contents_from_string)r rU r4 r! r" r# r� rq ) r �srD �replacer] r^ r� rn �reduced_redundancyrS �resultr r r r� � s �� z)BucketStorageUri.set_contents_from_stringc C s� | � d� | j|d�} | jdkrV| j||||||||| | d� } | r�| �d | jd |� n*| jd| d� | j||||||||| d� } | �| � | S )N�set_contents_from_filerH r+ )�size�rewind�res_upload_handler)r� )r� r� )r rU r4 r� ro r> r' rq ) r rY rD r� r] r^ r� rn r� r� r� rS r� r r r r� � s( � ��� z'BucketStorageUri.set_contents_from_filec C s: | � d� | �d|�}|j||||||d�}| �|� |S )N�set_contents_from_streamF)r� r� )r rU r� rq ) r rY rD r� r] r� r� Zdst_keyr� r r r r� � s � z)BucketStorageUri.set_contents_from_stream�STANDARDc C s` | � d� | jd|d�} | r<| j| j|||||||| | d� S | j| j||||||||| d� S dS )zReturns newly created key.�copy_keyF)rC rD ) � new_key_name�src_bucket_name� src_key_name�metadatar� � preserve_acl� encrypt_keyrD � query_args�src_generation) r� r� r� r� �src_version_idr� r� r� rD r� N)r rA r� r ) r r� r� r� r� r� r� r� rD r� r� Z dst_bucketr r r r� � s$ ��zBucketStorageUri.copy_keyc C s* | � d� | �||�}|j|||d� d S )N�enable_loggingrH )r rA r� )r Z target_bucketZ target_prefixrC rD r= rE r r r r� � s z BucketStorageUri.enable_loggingc C s& | � d� | �||�}|j|d� d S )N�disable_loggingrH )r rA r� �r rC rD r= rE r r r r� � s z BucketStorageUri.disable_loggingc C s" | � d� | �||�}|j|d�S )N�get_logging_configrH )r rA r� r� r r r r� � s z#BucketStorageUri.get_logging_configc C s< | � d� | �||�}|s*|s*|�|� n|�|||� d S )N�set_website_config)r rA Z delete_website_configurationZconfigure_website)r Zmain_page_suffixZ error_keyrC rD rE r r r r� � s z#BucketStorageUri.set_website_configc C s | � d� | �||�}|�|�S )N�get_website_config)r rA Zget_website_configurationrV r r r r� � s z#BucketStorageUri.get_website_configc C s | � d� | �d|�}|�|�S )N�get_versioning_configF)r rA Zget_versioning_statusr� r r r r� � s z&BucketStorageUri.get_versioning_configc C s" | � d� | �d|�}|�||�S )N�configure_versioningF)r rA r� )r �enabledrD rE r r r r� � s z%BucketStorageUri.configure_versioningc C s | � d�j||||d�S )NFrH )rR Zset_remote_metadata)r Z metadata_plusZmetadata_minusr� rD r r r � set_metadata� s �z BucketStorageUri.set_metadatac C sT | � d� g }|D ] }|�|�� � |j|d _q| �� j|||d�| _| �� | S )N�compose���)� content_typerD )r �appendrU r> r� rm )r � componentsr� rD Zcomponent_keysZsurir r r r� � s �zBucketStorageUri.composec C s4 | � d� | �||�}|�|�}| �|d| j� |S )z+Returns a bucket's lifecycle configuration.�get_lifecycle_configZ lifecycle)r rA r� r r )r rC rD rE �lifecycle_configr r r r� s z%BucketStorageUri.get_lifecycle_configc C s& | � d� | �||�}|�||� dS )z3Sets or updates a bucket's lifecycle configuration.�configure_lifecycleN)r rA r� )r r� rC rD rE r r r r� s z$BucketStorageUri.configure_lifecyclec C s8 | � d� | jdkr"td| j ��| �d|�}|�|�S )N�get_billing_configr+ z/get_billing_config() not supported for %s URIs.F)r r4 r� rA r� r� r r r r� s � z#BucketStorageUri.get_billing_configc C s@ | � d� | jdkr"td| j ��| �||�}|j||d� dS )z1Sets or updates a bucket's billing configuration.�configure_billingr+ z.configure_billing() not supported for %s URIs.)�requester_paysrD N)r r4 r� rA r� )r r� rC rD rE r r r r� s � z"BucketStorageUri.configure_billingc C s: | � d� | jdkr"td| j ��| �||�}|j|d�S )z0Returns a GCS bucket's encryption configuration.�get_encryption_configr+ z2get_encryption_config() not supported for %s URIs.rH )r r4 r� rA r� rV r r r r� ' s � z&BucketStorageUri.get_encryption_configc C s( | � d� | �||�}|j||d� dS )z-Sets a GCS bucket's encryption configuration.�set_encryption_config)�default_kms_key_namerD N)r rA r� )r r� rC rD rE r r r r� 2 s �z&BucketStorageUri.set_encryption_configc C s6 | j std| j ��| �|�}|j| j |d�}t|�S )z8Returns True if the object exists or False if it doesn'tz exists on object-less URI (%s)rH )r r r rA rR rk )r rD rE rS r r r �exists: s zBucketStorageUri.exists)NNr NTNNF)FNN)FNNN)FNN)FN)FN)FN)FN)FN)FN)FNN)FFN)FFN)FFN)N)NrF NN)N)N)rF FNNNN)rF FNNNN)FN)FNN)FNN)FNN)FNN)NTNr[ NNF) NTNr[ NNNFN)NTNNF)NNr� FFNNN)NFNN)FNN)FNN)NNFN)FN)N)N)N)NN)FN)FN)N)FFN)FN)NFN)N)Drd re rf rg �delim�setZ capabilitiesr rm rq ro rR r@ rw rz r| r r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� ru r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r � s� � 1 � � � � � � � � � � � � � � � � � � � � � � r c @ s� e Zd ZdZejZd dd�Zdd� Zdd� Z d d � Z d d � Z d d� Z dd� Z dd� Zdd� Zdd� Zdd� Zdd� Zdd � Zd!d d �Zd S )"�FileStorageUriz� StorageUri subclass that handles files in the local file system. Callers should instantiate this class by calling boto.storage_uri(). See file/README about how we map StorageUri operations onto a file system. Fc C s, d| _ d| _|| _d| | _|| _|| _dS )a� Instantiate a FileStorageUri from a path name. @type object_name: string @param object_name: object name @type debug: boolean @param debug: whether to enable debugging on this StorageUri After instantiation the components are available in the following fields: uri, scheme, bucket_name (always blank for this "anonymous" bucket), object_name. r- rF zfile://N)r4 r r r r8 �stream)r r r8 r� r r r r M s zFileStorageUri.__init__c C s t || j| j�S )z�Instantiate a FileStorageUri from the current FileStorageUri, but replacing the object_name. @type new_name: string @param new_name: new object name )r� r8 r� rx r r r rw a s z!FileStorageUri.clone_replace_namec C s dS )r� Tr r r r r r� j s zFileStorageUri.is_file_uric C s dS )r� Fr r r r r r� n s zFileStorageUri.is_cloud_uric C s | � � S )z5Returns True if this URI names a directory or bucket.)r� r r r r r� r s z FileStorageUri.names_containerc C s | � � S )z<Returns True if this URI names a file (or stream) or object.)r� r r r r r� v s z FileStorageUri.names_singletonc C s | j r dS tj�| j�S r� )r� �os�path�isdirr r r r r r� z s z FileStorageUri.names_directoryc C s dS )r� Fr r r r r r� � s z FileStorageUri.names_providerc C s dS )r� Fr r r r r r� � s zFileStorageUri.names_bucketc C s | � � S )r� r� r r r r r� � s zFileStorageUri.names_filec C s dS )r� Fr r r r r r� � s zFileStorageUri.names_objectc C s t | j�S )zAReturns True if this URI represents input/output stream. )rk r� r r r r r� � s zFileStorageUri.is_streamc C s | � � �� dS )z$Closes the underlying file. N)rR �closer r r r r� � s zFileStorageUri.closeNc C s t j�| j�S )z6Returns True if the file exists or False if it doesn't)r� r� r� r )r Z_headers_not_usedr r r r� � s zFileStorageUri.exists)F)N)rd re rf rg r� �sepr� r rw r� r� r� r� r� r� r� r� r� r� r� r� r r r r r� C s r� ) �botor� r! r Zboto.s3.deletemarkerr �boto.exceptionr r �objectr r r� r r r r �<module> s C e vendored/boto/boto/__pycache__/https_connection.cpython-39.pyc 0000644 00000011420 15025013617 0020500 0 ustar 00 a sFg� � @ sl d Z ddlZddlZddlZddlZddlmZmZ G dd� dej�Z dd� Z dd � Z G d d � d ej �Z dS ) zCExtensions to allow HTTPS requests with SSL certificate validation.� N)�six� http_clientc @ s e Zd ZdZdd� Zdd� ZdS )�InvalidCertificateExceptionz?Raised when a certificate is provided with an invalid hostname.c C s" t j�| � || _|| _|| _dS )z�Constructor. Args: host: The hostname the connection was made to. cert: The SSL certificate (as a dictionary) the host returned. N)r � HTTPException�__init__�host�cert�reason)�selfr r r � r �8/opt/gsutil/gslib/vendored/boto/boto/https_connection.pyr " s z$InvalidCertificateException.__init__c C s d| j | j| jf S )Nz0Host %s returned an invalid certificate (%s): %s)r r r )r r r r �__str__. s �z#InvalidCertificateException.__str__N)�__name__� __module__� __qualname__�__doc__r r r r r r r s r c C s0 d| v rdd� | d D �S dd� | d D �S dS )z�Returns a list of valid host globs for an SSL certificate. Args: cert: A dictionary representing an SSL certificate. Returns: list: A list of valid host globs. �subjectAltNamec S s$ g | ] }|d � � dkr|d �qS )r Zdns� ��lower��.0�xr r r � <listcomp>< � z(GetValidHostsForCert.<locals>.<listcomp>c S s, g | ]$}|d d � � dkr|d d �qS )r Z commonnamer r r r r r r > s ��subjectNr )r r r r �GetValidHostsForCert3 s r c C sV t | �}tj�d||� |D ]4}|�dd��dd�}t�d|f |tj�r dS q dS ) a Validates that a given hostname is valid for an SSL certificate. Args: cert: A dictionary representing an SSL certificate. hostname: The hostname to test. Returns: bool: Whether or not the hostname is valid for this certificate. z@validating server certificate: hostname=%s, certificate hosts=%s�.z\.�*z[^.]*z^%s$TF)r �boto�log�debug�replace�re�search�I)r �hostname�hostsr Zhost_rer r r �ValidateCertificateHostnameB s �r( c @ s2 e Zd ZdZejZeddddfdd�Zdd� ZdS )� CertValidatingHTTPSConnectionzDAn HTTPConnection that connects over SSL and validates certificates.Nc K s> t jr||d< tjj| f||d�|�� || _|| _|| _dS )a* Constructor. Args: host: The hostname. Can be in 'host:port' form. port: The port. Defaults to 443. key_file: A file containing the client's private key cert_file: A file containing the client's certificates ca_certs: A file contianing a set of concatenated certificate authority certs for validating the server against. strict: When true, causes BadStatusLine to be raised if the status line can't be parsed as a valid HTTP/1.0 or 1.1 status line. �strict)r �portN)r �PY2r �HTTPConnectionr �key_file� cert_file�ca_certs)r r r+ r. r/ r0 r* �kwargsr r r r [ s z&CertValidatingHTTPSConnection.__init__c C s` t | d�r"t�| j| jf| j�}nt�| j| jf�}d}| jrN|d| j 7 }n|d7 }tj� |� t t d��rt t dd��rt � t j �}t j|_| jr�|�| j� | jr�|�| j| j� |j|| jd�| _| j| j_| j| j_|j| j_t j | j_| j| j_d | j_n t j|| j| jt j| jd �| _| j�� }| j� d d �d }t ||��s\t ||d | ��d S )z(Connect to a host on a given (SSL) port.�timeoutzwrapping ssl socket; zCA certificate file=%sz using system provided SSL certs� SSLContext�HAS_SNIF)�server_hostnameN)�keyfile�certfile� cert_reqsr0 �:r z/remote hostname "%s" does not match certificate) �hasattr�socket�create_connectionr r+ r2 r0 r r r! �ssl�getattrr3 �PROTOCOL_SSLv23� CERT_REQUIRED� verify_mode�load_verify_locationsr/ �load_cert_chainr. � wrap_socket�sockr6 r7 r8 � ssl_version�ciphers� getpeercert�splitr( r )r rE �msg�contextr r& r r r �connectt sH � ��z%CertValidatingHTTPSConnection.connect) r r r r r � HTTPS_PORT� default_portr rL r r r r r) V s � r) )r r# r; r= r � boto.compatr r r r r r( r- r) r r r r �<module> s vendored/boto/boto/__pycache__/plugin.cpython-39.pyc 0000644 00000003436 15025013617 0016425 0 ustar 00 a sFg\ � @ sN d Z ddlZddlZddlZG dd� de�Zd dd�Zdd� Zd a d d � Z dS ) a Implements plugin related api. To define a new plugin just subclass Plugin, like this. class AuthPlugin(Plugin): pass Then start creating subclasses of your new plugin. class MyFancyAuth(AuthPlugin): capability = ['sign', 'vmac'] The actual interface is duck typed. � Nc @ s e Zd ZdZg Zedd� �ZdS )�PluginzBase class for all plugins.c C s |D ]}|| j vr dS qdS )zMReturns true if the requested capability is supported by this plugin FT)� capability)�cls�requested_capability�c� r �./opt/gsutil/gslib/vendored/boto/boto/plugin.py� is_capable1 s zPlugin.is_capableN)�__name__� __module__� __qualname__�__doc__r � classmethodr r r r r r , s r c C s2 |sg }g }| � � D ]}|�|�r|�|� q|S �N)�__subclasses__r �append)r r �result�handlerr r r � get_plugin; s r c C s@ t j�| �\}}t j�|�\}}tj�� �||g�}tj� |�S r ) �os�path�split�splitext� importlib� machinery� PathFinder� find_spec�util�module_from_spec)�filenamer �name�ext�specr r r �_import_moduleE s r# Fc C sN t rd S da | �dd�s d S | �dd�}t�tj�|d��D ] }t|� q<d S )NTr Zplugin_directoryz*.py)�_plugin_loaded� has_option�get�globr r �joinr# )�configZ directory�filer r r � load_pluginsO s r+ )N) r r' r Zos.pathr �objectr r r# r$ r+ r r r r �<module> s vendored/boto/boto/__pycache__/connection.cpython-39.pyc 0000644 00000107613 15025013617 0017270 0 ustar 00 a sFg�� � @ s d Z ddlmZ ddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z ddl mZ ddl Z ddlZ ddlZ ddlZ ddl mZmZ ddlmZmZmZmZmZ ddlmZ dd lm Z dd lm Z dd lm Z dd lm Z dd l m!Z! ddl"m#Z# dZ$z(ddl%Z%ddl m&Z& e'e%d��r:dZ$W n e(�yP Y n0 z ddl)Z)W n e(�yz ddl*Z)Y n0 e+dd� dD ��Z,ddd�Z-ej.�/ej.�0ej.�1e j2j3��d�Z4G dd� de5�Z6G d d � d e5�Z7G d d � d e5�Z8G d d!� d!ej9�Z9G d"d#� d#e5�Z:G d$d%� d%e:�Z;dS )&z" Handles basic connections to AWS � )�datetimeN)�auth)� auth_handler)�config� UserAgent)�six� http_client�urlparse�quote� encodebytes)�AWSConnectionError)�BotoClientError)�BotoServerError)�PleaseRetryException)�S3ResponseError)�Provider)� ResultSetF)�https_connection�SSLErrorTc c s | ]}|t jv V qd S �N)�os�environ)�.0�key� r �2/opt/gsutil/gslib/vendored/boto/boto/connection.py� <genexpr>X � r )Z USER_IS_ADMINZCURRENT_VERSION_IDZGAE_APPLICATION� �P )TFz cacerts.txtc @ sH e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dS )�HostConnectionPoola� A pool of connections for one remote (host,port,is_secure). When connections are added to the pool, they are put into a pending queue. The _mexe method returns connections to the pool before the response body has been read, so they connections aren't ready to send another request yet. They stay in the pending queue until they are ready for another request, at which point they are returned to the pool of ready connections. The pool of ready connections is an ordered list of (connection,time) pairs, where the time is the time the connection was returned from _mexe. After a certain period of time, connections are considered stale, and discarded rather than being reused. This saves having to wait for the connection to time out if AWS has decided to close it on the other end because of inactivity. Thread Safety: This class is used only from ConnectionPool while it's mutex is held. c C s g | _ d S r )�queue��selfr r r �__init__{ s zHostConnectionPool.__init__c C s t | j�S )z� Returns the number of connections in the pool for this host. Some of the connections may still be in use, and may not be ready to be returned by get(). )�lenr! r" r r r �size~ s zHostConnectionPool.sizec C s | j �|t�� f� dS )zZ Adds a connection to the pool, along with the time it was added. N)r! �append�time)r# �connr r r �put� s zHostConnectionPool.putc C sL | � � tt| j��D ]0}| j�d�\}}| �|�r<| S | �|� qdS )z� Returns the next connection in this pool that is ready to be reused. Returns None if there aren't any. r N)�clean�ranger% r! �pop� _conn_readyr* )r# �_r) r r r �get� s zHostConnectionPool.getc C s( t rdS t|dd�}|du p"|�� S dS )aZ There is a nice state diagram at the top of http_client.py. It indicates that once the response headers have been read (which _mexe does before adding the connection to the pool), a response is attached to the connection, and it stays there until it's done reading. This isn't entirely true: even after the client is done reading, the response may be closed, but not removed from the connection yet. This is ugly, reading a private instance variable, but the state we care about isn't available in any public methods. F�_HTTPConnection__responseN)� ON_APP_ENGINE�getattr�isclosed)r# r) �responser r r r. � s z HostConnectionPool._conn_readyc C s0 t | j�dkr,| �| jd �r,| j�d� q dS )z/ Get rid of stale connections. r N)r% r! � _pair_staler- r" r r r r+ � s zHostConnectionPool.cleanc C s |\}}t � � }|tj |k S )z[ Returns true of the (connection,time) pair is too old to be used. )r( �ConnectionPool�STALE_DURATION)r# �pairZ_connZ return_time�nowr r r r6 � s z HostConnectionPool._pair_staleN) �__name__� __module__� __qualname__�__doc__r$ r& r* r0 r. r+ r6 r r r r r a s r c @ sP e Zd ZdZdZdZdd� Zdd� Zdd � Zd d � Z d d � Z dd� Z dd� Z dS )r7 z� A connection pool that expires connections after a fixed period of time. This saves time spent waiting for a connection that AWS has timed out on the other end. This class is thread-safe. g @g N@c C s, i | _ d| _t�� | _t�ddtj�t_d S )Ng �BotoZconnection_stale_duration) � host_to_pool�last_clean_time� threading�Lock�mutexr �getfloatr7 r8 r" r r r r$ � s ��zConnectionPool.__init__c C s t � | j�}i |d< |d= |S )Nr@ rD )�copy�__dict__)r# � pickled_dictr r r � __getstate__� s zConnectionPool.__getstate__c C s | � � d S r )r$ )r# �dctr r r � __setstate__� s zConnectionPool.__setstate__c C s t dd� | j�� D ��S )z@ Returns the number of connections in the pool. c s s | ]}|� � V qd S r )r& )r �poolr r r r r z&ConnectionPool.size.<locals>.<genexpr>)�sumr@ �valuesr" r r r r& � s zConnectionPool.sizec C sd | � � | j�B |||f}|| jvr4W d � dS | j| �� W d � S 1 sV0 Y dS )z� Gets a connection from the pool for the named host. Returns None if there is no connection that can be reused. It's the caller's responsibility to call close() on the connection when it's no longer needed. N)r+ rD r@ r0 )r# �host�port� is_securer r r r �get_http_connection s z"ConnectionPool.get_http_connectionc C sZ | j �@ |||f}|| jvr(t� | j|< | j| �|� W d � n1 sL0 Y dS )zm Adds a connection to the pool of connections that can be reused for the named host. N)rD r@ r r* )r# rO rP rQ r) r r r r �put_http_connection s z"ConnectionPool.put_http_connectionc C s� | j �v t�� }| j| j |k rng }| j�� D ]&\}}|�� |�� dkr.|�|� q.|D ] }| j|= qZ|| _W d � n1 s�0 Y dS )a+ Clean up the stale connections in all of the pools, and then get rid of empty pools. Pools clean themselves every time a connection is fetched; this cleaning takes care of pools that aren't being used any more, so nothing is being gotten from them. r N) rD r( rA �CLEAN_INTERVALr@ �itemsr+ r& r' )r# r: � to_removerO rL r r r r+ s zConnectionPool.cleanN) r; r< r= r> rT r8 r$ rI rK r& rR rS r+ r r r r r7 � s r7 c @ s$ e Zd Zdd� Zdd� Zdd� ZdS )� HTTPRequestc C s| || _ || _|| _|| _|| _|du r*|}|| _|| _|rld|v rl|d dkrl| j dkrl|�� | _| jd= n|| _| | _ dS )ay Represents an HTTP request. :type method: string :param method: The HTTP method name, 'GET', 'POST', 'PUT' etc. :type protocol: string :param protocol: The http protocol used, 'http' or 'https'. :type host: string :param host: Host to which the request is addressed. eg. abc.com :type port: int :param port: port on which the request is being sent. Zero means unset, in which case default port will be chosen. :type path: string :param path: URL path that is being accessed. :type auth_path: string :param path: The part of the URL path used when creating the authentication string. :type params: dict :param params: HTTP url query parameters, with key as name of the param, and value as value of param. :type headers: dict :param headers: HTTP headers, with key as name of the header and value as value of header. :type body: string :param body: Body of the HTTP request. If not present, will be None or empty string (''). N�Transfer-Encoding�chunked�PUT) �method�protocolrO rP �path� auth_path�paramsrF �headers�body) r# r[ r\ rO rP r] r^ r_ r` ra r r r r$ 2 s$ $ �� zHTTPRequest.__init__c C s( d| j | j| j| j| j| j| j| jf S )NzTmethod:(%s) protocol:(%s) host(%s) port(%s) path(%s) params(%s) headers(%s) body(%s))r[ r\ rO rP r] r_ r` ra r" r r r �__str__i s ��zHTTPRequest.__str__c K s� t | dd�sT| jD ]4}| j| }t|tj�rd}t|�d�|�| j|< qt| dd� t| jd< |j j | fi |�� d| jvr�d| jvs�| jd d kr�t t | j ��| jd< d S ) NZ_headers_quotedFz !"#$%&'()*+,/:;<=>?@[\]^`{|}~ �utf-8Tz User-AgentzContent-LengthrX rY )r3 r` � isinstancer � text_typer �encode�setattrr � _auth_handler�add_auth�strr% ra )r# � connection�kwargsr �val�safer r r � authorizeo s �zHTTPRequest.authorizeN)r; r< r= r$ rb ro r r r r rW 0 s 7rW c @ s e Zd Zdd� Zddd�ZdS )� HTTPResponsec O s$ t jj| g|�R i |�� d| _d S )N� )r rp r$ �_cached_response)r# �argsrl r r r r$ � s zHTTPResponse.__init__Nc C s4 |du r"| j s tj�| �| _ | j S tj�| |�S dS )a< Read the response. This method does not have the same behavior as http_client.HTTPResponse.read. Instead, if this method is called with no ``amt`` arg, then the response body will be cached. Subsequent calls to ``read()`` with no args **will return the cached response**. N)rr r rp �read)r# �amtr r r rt � s zHTTPResponse.read)N)r; r< r= r$ rt r r r r rp � s rp c @ sz e Zd ZdMdd�Zdd � Zd d � Zd d � Zdd� Zeee�Z dd� Z dd� Z ee e �Z dd� Z ee �Z dd� Zee�ZeZeZdd� Zee�ZeZeZdd� Zee�ZdNd d �ZdOd d �Zd d!� Zd"d#� Zd$d%� Zd&d'� Zd(d)� ZdPd*d+�Z dQd,d-�Z d.d/� Z d0d1� Z d2d3� Z d4d5� Z!dRd6d7�Z"dSd9d:�Z#d;d<� Z$d=d>� Z%d?d@� Z&dAdB� Z'dCdD� Z(dTdEdF�Z)dGdH� Z*dUdIdJ�Z+dKdL� Z,dS )V�AWSAuthConnectionNTr �/�awsc C sH || _ d| _t�dd�r$t�dd�}|| _t�dd|�| _| jrLtsLt d��t� ddt �}|dkrfd}|| _ |rx|| _ n t| | _ | �|||| � tjtjtjtjf| _g | _tr�| j�tj� tj f| _ | dur�| d | _ | j| d 7 _nd| _ |�rd | _ nd | _ || _ | | _!t"| t#j$��s(d } t�%dd | �| _&d| _'i | _(t)j*d t)j*d fdk�rrt�%ddd�| j(d< t+| dd�}t"| t,��r�| | _-n | | _.t,| j.|||||d�| _-| j-j �r�| j-j | _ | j-j �r�| j-j | _ | j-j'�r�| j-j'| _'t/� | _0| j | j | jf| _1d| _2t3�4|t| j-| �5� �| _6t+| dd�du�r>| j7| _8d| _9dS )a0 :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :keyword str security_token: The security token associated with temporary credentials issued by STS. Optional unless using temporary credentials. If none is specified, the environment variable ``AWS_SECURITY_TOKEN`` is used if defined. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{http_client.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect :type suppress_consec_slashes: bool :param suppress_consec_slashes: If provided, controls whether consecutive slashes will be suppressed in key paths. :type validate_certs: bool :param validate_certs: Controls whether SSL certificates will be validated or not. Defaults to True. :type profile_name: str :param profile_name: Override usual Credentials section in config file to use a named set of keys instead. � r? rQ �https_validate_certificatesz�SSL server certificate validation is enabled in boto configuration, but Python dependencies required to support this feature are not available. Certificate validation is only supported when running under Python 2.6 or later.�ca_certificates_file�systemNr � �https�http�debug)� ry �http_socket_timeout�F �timeout�anonF)r� �AuthServiceName):�suppress_consec_slashes� num_retriesr � has_option� getbooleanrQ �getboolrz �HAVE_HTTPS_CONNECTIONr � get_value�DEFAULT_CA_CERTS_FILEr{ rP �PORTS_BY_SECURITY� handle_proxyr � HTTPException�socket�error�gaierror� BadStatusLine�http_exceptions�http_unretryable_exceptionsr' r �InvalidCertificateException�errnoZEINTRZsocket_exception_values�https_connection_factoryr\ rO r] rd r � integer_types�getintr� � host_header�http_connection_kwargs�sys� version_infor3 r �providerZ_provider_typer7 �_pool� _connectionZ_last_rsr �get_auth_handler�_required_auth_capabilityrh r� �auth_service_name� request_hook)r# rO �aws_access_key_id�aws_secret_access_keyrQ rP �proxy� proxy_port� proxy_user� proxy_passr� r� r] r� �security_tokenr� �validate_certs� profile_name� certs_fileZis_anonymous_connectionr r r r$ � s� 9 � �� �� � � �zAWSAuthConnection.__init__c C s d| j j| jf S )Nz%s:%s)� __class__r; rO r" r r r �__repr__B s zAWSAuthConnection.__repr__c C s g S r r r" r r r r� E s z+AWSAuthConnection._required_auth_capabilityc C s t | jd�S )N� service_name�r3 rh r" r r r �_get_auth_service_nameH s z(AWSAuthConnection._get_auth_service_namec C s || j _d S r )rh r� �r# �valuer r r �_set_auth_service_nameN s z(AWSAuthConnection._set_auth_service_namec C s t | jd�S )N� region_namer� r" r r r �_get_auth_region_nameR s z'AWSAuthConnection._get_auth_region_namec C s || j _d S r )rh r� r� r r r �_set_auth_region_nameU s z'AWSAuthConnection._set_auth_region_namec C s | j | j� S r )rR r� r" r r r rk Y s z AWSAuthConnection.connectionc C s | j jS r )r� � access_keyr" r r r r� ] s z#AWSAuthConnection.aws_access_key_idc C s | j jS r )r� � secret_keyr" r r r r� c s z'AWSAuthConnection.aws_secret_access_keyc C s | j jS r )r� r� r" r r r r� i s z AWSAuthConnection.profile_namec C s� | j s| jt�dd|� S |�d�}|dkrF||d � }|d |� }nd }|d dkr\d}nd}| j�d�}|�|�d�� d d � |D �}dd�|� }|d dkr�|r�|d7 }|r�|| }|S ) Nz^(/*)/z\1�?r ���rw TFc S s g | ] }|r|�qS r r )r �pr r r � <listcomp>� r z.AWSAuthConnection.get_path.<locals>.<listcomp>)r� r] �re�sub�find�split�extend�join)r# r] �posr_ Z need_trailingZ path_elementsr r r �get_pathm s& zAWSAuthConnection.get_pathc C s\ |s | j }|dkr| j}n>tjd d tjd }|dkrJ|dkrJ| j}nd| j|f }|S )Nr r � r} r � �%s:%d)rP rO r� r� )r# rP Zsignature_hostZver_intr r r � server_name� s z AWSAuthConnection.server_namec C s || _ || _|| _|| _dtjv rx| j sxt�d�}|�tjd �}|r�|� d�| _ |� d�| _|� d�| _|� d�| _nX| j s�t � ddd �| _ | js�t � dd d �| _| js�t � dd d �| _| js�t � dd d �| _| js�| j r�t d � | j | _tj�d d��ptj�dd�| _| j d u| _d S )N� http_proxyzX(?:http://)?(?:(?P<user>[\w\-\.]+):(?P<pass>.*)@)?(?P<host>[\w\-\.]+)(?::(?P<port>\d+))?rO rP �user�passr? r� r� r� r� zFhttp_proxy environment variable does not specify a port, using default�no_proxyrq �NO_PROXY)r� r� r� r� r r r� �compile�match�groupr r� �printrP r0 r� � use_proxy)r# r� r� r� r� �patternr� r r r r� � s6 � z AWSAuthConnection.handle_proxyc C s. | j �|||�}|d ur |S | �|||�S d S r )r� rR �new_http_connection)r# rO rP rQ r) r r r rR � s z%AWSAuthConnection.get_http_connectionc C s^ | j s dS | j dkrdS |}|�d�d }| j �d�D ]"}|r6|�|�sR|�|�r6 dS q6dS )NF�*T�:r �,)r� r� �endswith)r# rO �hostonly�namer r r � skip_proxy� s z AWSAuthConnection.skip_proxyc C sh |d u r| � � }tj�|�}| j�� }||d< | jrV|sV| �|�sV| j}t | j �|d< |r�tj � d||� | jr�| �|�s�| � ||r�dp�d�}nH| jr�| �|�}n6| jr�tr�tj|fd| ji|��}ntj|fi |��}n>tj � d| � | j�r| j|fi |��}ntj|fi |��}| j dk�r0|�| j � |�d�d | jk�r^|| jk�r^|||f| _t |_ |S ) NrP z1establishing HTTPS connection: host=%s, kwargs=%sr r �ca_certsz'establishing HTTP connection: kwargs=%sr} r� r ) r� �boto�utils� parse_hostr� rF r� r� r� �intr� �logr� � proxy_sslr� rz r� r � CertValidatingHTTPSConnectionr{ r �HTTPSConnection�HTTPConnection�set_debuglevelr� rO rQ r� rp �response_class)r# rO rP rQ r� rk r r r r� � sh �� ���������� " z%AWSAuthConnection.new_http_connectionc C s | j �||||� d S r )r� rS )r# rO rP rQ rk r r r rS s z%AWSAuthConnection.put_http_connectionc C sz |r|rd||f }nd| j | jf }| j�d�}|d urTt�| jt| j�f|�}nt�| jt| j�f�}t j � d|� |� t �d| �� |� t �dt �� | jr�| jr�| �� �� D ] \}}|� t �d||f �� q�t�ddd �r�|� t �d �� n|� t �d �� tj|| j d �}|�� |jd k�rTt�d t �d| j| j|j|jf ���|� � t� |�}| j �r<t �r<d} | j �r�| d| j 7 } n| d7 } t j � | � | j�dd �} | j�dd �} t!�"� } t!j#| _$d| _%| �r�| �&| | � | �'| j � | j(|| j d�} | �)� }| j �*dd�d }t+�,||��spt+�-||d��n4t.td��rVtj!�/|�} nt�!|d d �} t�0|| �} | |_1|S )Nr� r� z'Proxy connection: CONNECT %s HTTP/1.0 zCONNECT %s HTTP/1.0 zUser-Agent: %s z%s: %s r? Z"send_crlf_after_proxy_auth_headersFz )� debuglevel�� i����z*Error talking to HTTP proxy %s:%s: %s (%s)z,wrapping ssl socket for proxied connection; zCA certificate file=%sz using system provided SSL certs�key_file� cert_fileT)�server_hostnamer� r zhostname mismatch�ssl)2rO rP r� r0 r� �create_connectionr� r� r� r� r� r� �sendallr � ensure_binaryr r� r� �get_proxy_auth_headerrU r r� r rp �begin�statusr� �reason�closer� rz r� r{ r� �create_default_context� CERT_REQUIRED� verify_mode�check_hostname�load_cert_chain�load_verify_locations� wrap_socket� getpeercertr� r �ValidateCertificateHostnamer� �hasattr� SSLSocketZ FakeSocket�sock)r# rO rP r� r �k�v�resp�h�msgr� r� �contextZsslSock�cert�hostnamer r r r� s� �� ���� � zAWSAuthConnection.proxy_sslc C s | j d |p| �� | }|S )N�://)r\ r� )r# r] rO r r r �prefix_proxy_to_path] s z&AWSAuthConnection.prefix_proxy_to_pathc C s t | jd | j �}dd| iS )Nr� zProxy-AuthorizationzBasic %s)r r� r� )r# r r r r r� a s z'AWSAuthConnection.get_proxy_auth_headerc C sZ | j s d S | js| jr:| jr.d| j| jf }q>d| j }nd}d|| jt| jpR| j�f S )Nz%s:%s@z%s@rq zhttp://%s%s:%s)r� r� r� r� rj r� rP )r# Z login_infor r r �get_proxy_url_with_authf s z)AWSAuthConnection.get_proxy_url_with_authc C sJ z| j �| j|�|jd< W n* tyD | j�dd�d |jd< Y n0 d S )N�Hostr� r} r )rh r� rO r` �AttributeErrorr� )r# �requestr r r �set_host_headert s � z!AWSAuthConnection.set_host_headerc C s || _ d S r )r� )r# �hookr r r �set_request_hook{ s z"AWSAuthConnection.set_request_hookc C s� t j�d|j � t j�d|j � t j�d|j � t j�d|j � t j�d|j � t j�d|j � t j�d|j � d}d}d}|du r�t � d d | j �}n|}d } | � |j|j| j�} t|jt�s�t|jd �r�|j�d �|_| |k�r�tt�� d| tt j �d dd���} �z�t j�d| jj � |j| d� d| �� v�rnt| dd��sn|j�d��sn| � |� t j�d|j � t � � |_ t |��r�|| |j|j|j|j�}n | �!|j|j|j|j� | �"� }t j�d|�#� � |�$d�} |jdk�r t|dd��r d |_%t |��rR||| | �} | �rR| \}} } |�rDt j�|� t&�'| � W q�|j(d v �r�d |j( }|d | 7 }t j�|� |�)� }t|t��r�|�*d �}�n |j(d k �s�|j(d k�s�| �s|�$d!�}|d"k�r�| �+� n| �,|j|j| j| � | j-du�r| j-�.||� |W S t/| �\}|_|_}}}|�rD| jd#| 7 _d$|jv �rf|j�0d$d%�\|_|_d&| d' }||j|j 7 }t j�|� | � |j|j|d(k�} d}W q�W n� t1�y } z:t j�d)| � | �2|j|j| j�} |j3}|}W Y d}~n�d}~0 | j4�y� } zh| j5D ](}t||��rt j�d*|j6j7 � � �qt j�d+|j6j7 � | �2|j|j| j�} |}W Y d}~n d}~0 0 t&�'| � | d%7 } q�| j-du�r�| j-j.||d,d-� |�r�t8|j(|j9|��n|�r�|�n d.}t:|��dS )/a> mexe - Multi-execute inside a loop, retrying multiple times to handle transient Internet errors by simply trying again. Also handles redirects. This code was inspired by the S3Utils classes posted to the boto-users Google group by Larry Bates. Thanks! z Method: %szPath: %szData: %sz Headers: %szHost: %szPort: %sz Params: %sNr? r� r rf rc r� �max_retry_delay�< z Token: %s)rk �s3r� Fr zFinal headers: %szResponse headers: %s�location�HEADrY )i� i� i� i� zReceived %d response. zRetrying in %3.1f secondsi, � rk r� r� r� r} zRedirecting: %sr r~ z!encountered a retry exception: %sz0encountered unretryable %s exception, re-raisingz&encountered %s exception, reconnectingT)r� z-Please report this exception as a Boto Issue!);r� r� r� r[ r] ra r` rO rP r_ r r� r� rR rQ rd �bytesr rf �min�random�floatr0 r� r� ro r� r3 r r r: � start_time�callabler � getresponse� getheaders� getheaderrY r( �sleepr� rt �decoder� rS r� �handle_request_datar r� r r� r5 r� r� r� r; r r� r )r# r �sender�override_num_retries� retry_handlerr5 ra �exr� �irk � next_sleepr r� r Zconn_header_value�schemer_ �query�fragment�eZ unretryabler r r �_mexe~ s� �� � � � � � � � � � ��� � zAWSAuthConnection._mexerq c C s� | � |�}|d ur | � |�}|d u r*i }n|�� }|d u r@i }n|�� }| jrftj�d|�sf| j|d< |pn| j}| jr�| �|�s�|s�|}| � ||�}| j r�| j r�| j s�|� | �� � t|| j|| j|||||� S )NrO )r� rF r� r� r� �find_matching_headersrO r� r� r r� r� rQ �updater� rW r\ rP )r# r[ r] r^ r_ r` �datarO r r r �build_base_http_request s* �z)AWSAuthConnection.build_base_http_requestc C s |� d�}|dkrd S |d S )Nz.s3.r� r} )�rfind�r# �endpoint�ixr r r � _find_s3_host. s z AWSAuthConnection._find_s3_hostc C s | � |�}|r||d � S d S r �r> r; r r r � _get_s3_host8 s z AWSAuthConnection._get_s3_hostc C s" | � |�}|r |d |� | S d S r r? )r# r< �new_hostr= r r r �_change_s3_host= s z!AWSAuthConnection._change_s3_hostc C s< |r|s dS d| }| � ||�}|r(|S tj�d| � dS )zyReturn a new bucket endpoint that uses correct_region. Return None if host substitution is not possible. Nzs3.%s.amazonaws.comz Could not change s3 host for %s)rB r� r� r� )r# r< Zcorrect_regionrA � new_endpointr r r �_fix_s3_endpoint_regionB s z)AWSAuthConnection._fix_s3_endpoint_regionc C s t |�r.|d�}|r.tj�d� | �|j|�S |jrPtj�d� | �|j|j�S |jdkr�d}t� ||j �}|r�|� d�dkr�|� d�}tj�d� | �|j|�S n|j r�tj�d � |j S tj�d � | � d d d i d d|j�}| �|d d �}|�d�}|�rtj�d� | �|j|�S d S )z�Attempt to return a new s3 endpoint using the correct region to access a bucket. Return None if a retry is not possible.zx-amz-bucket-regionz)Got correct region from response headers.z1Got correct region from parsed xml in err.region.Z"IllegalLocationConstraintExceptionziThe (.*?) location constraint is incompatible for the region specific endpoint this request was sent to\.r} � unspecifiedz&Got correct region from response body.z(Got correct endpoint from response body.z4Sending a bucket HEAD request to get correct region.r rw Nrq z.Got correct region from a bucket head request.)r$ r� r� r� rD rO �region� error_coder� �searchra r� r< r9 r5 r' ) r# r �err� get_headerrF Z region_regexr� �reqZbucket_head_responser r r �&_get_correct_s3_endpoint_from_responseN s8 � � z8AWSAuthConnection._get_correct_s3_endpoint_from_responsec C s� | � |||�}|sd S d|j }|d| 7 }||_| �|�}|r�|| jkr�|d7 }|d| j 7 }|d| 7 }|d7 }|d7 }|| _tj�|� |S )Nz This request was sent to %s, z%when it should have been sent to %s. z1This error may have arisen because your S3 host, z)currently %s, is configured incorrectly. z+Please change your configuration to use %s z(to avoid multiple unnecessary redirects zand signing attempts.)rL rO r@ r� r� r� )r# r rI rJ rC r rA r r r �_change_s3_host_from_errorv s( � z,AWSAuthConnection._change_s3_host_from_errorc C sP |r<|� � }|r|�d�}t|j|j|�}| j|||jd�S |rL| �||�S d S )Nrc )rJ )rt r) r r� r� rM r' )r# � http_requestr5 rI ra r r r �_get_request_for_s3_retry� s ��z+AWSAuthConnection._get_request_for_s3_retryc C s� | du r i } | � |||| |||�} d\} } z| j| ||| d�} W n( tyh } z|} W Y d}~n d}~0 0 | pp| j}| j�d�r�|dv r�| �| | | �}|r�| j|||| d�S | r�| S | r�| �dS )z^Make a request to the server. Include logic for retrying on s3 region errors. N)NN)r- z amazonaws.com)i- r )r9 r5 r r� rO r� rO )r# r[ r] r` r8 rO r^ r+ r, r_ r- rN r5 rI r4 r� Z retry_requestr r r � make_request� s6 � � � �z AWSAuthConnection.make_requestc C s t j�d� d| _dS )z�(Optional) Close any open HTTP connections. This is non-destructive, and making a new request will open a connection again.z closing all HTTP connectionsN)r� r� r� r� r" r r r r� � s zAWSAuthConnection.close)NNTNNNNNr Nrw rx NTTN)rw )N)NN)N)NNN)NNrq N)N)Nrq NNNNNN)-r; r< r= r$ r� r� r� r� �propertyr� r� r� Zauth_region_namerk r� �gs_access_key_idr� r� �gs_secret_access_keyr� r� r� r� r� rR r� r� rS r� r r� r r r r5 r9 r> r@ rB rD rL rM rO rP r� r r r r rv � sz � ! $9 N � � ( � ,rv c sn e Zd ZdZeZd� fdd� Zd d � Zd d � Zddd�Z dd� Z dd� Z d dd�Z d dd�Z d dd�Z� ZS ) �AWSQueryConnectionrq NTr rw rx c s4 t t| �j| ||||||||| | | | |||d� d S )N)r� r� r� r� )�superrT r$ )r# r� r� rQ rP r� r� r� r� rO r� r� r] r� r� r� r� �r� r r r$ � s �zAWSQueryConnection.__init__c C s g S r r r" r r r r� � s z,AWSQueryConnection._required_auth_capabilityc C s t j�|�S r )r� r� �get_utf8_valuer� r r r rW � s z!AWSQueryConnection.get_utf8_value�GETc C sB | � ||d |i d| j�}|r&||jd< | jr8| j|jd< | �|�S )Nrq �Action�Version)r9 rO r_ � APIVersionr5 )r# �actionr_ r] �verbrN r r r rP � s � z AWSQueryConnection.make_requestc C sF t |tj�r|g}tdt|�d �D ] }||d |d||f < q$d S )Nr} z%s.%d)rd r � string_typesr, r% )r# r_ rU �labelr/ r r r �build_list_params� s z$AWSQueryConnection.build_list_paramsc C sL t |d�D ]<\}}d||f }t||�D ] \}} d||f } | || < q(q dS )a Serialize a list of structures. For example:: items = [('foo', 'bar', 'baz'), ('foo2', 'bar2', 'baz2')] label = 'ParamName.member' names = ('One', 'Two', 'Three') self.build_complex_list_params(params, items, label, names) would result in the params dict being updated with these params:: ParamName.member.1.One = foo ParamName.member.1.Two = bar ParamName.member.1.Three = baz ParamName.member.2.One = foo2 ParamName.member.2.Two = bar2 ParamName.member.2.Three = baz2 :type params: dict :param params: The params dict. The complex list params will be added to this dict. :type items: list of tuples :param items: The list to serialize. :type label: string :param label: The prefix to apply to the parameter. :type names: tuple of strings :param names: The names associated with each tuple element. r} z%s.%sN)� enumerate�zip) r# r_ rU r_ �namesr/ �itemZcurrent_prefixr r� Zfull_keyr r r �build_complex_list_params s " z,AWSQueryConnection.build_complex_list_paramsc C s� |s| }| � ||||�}|�� }tj�|� |sTtj�d| � | �|j|j|��n�|jdkr�t |�} tj � | |�} t |t j�r�|�d�}tj�|| � | S tj�d|j|jf � tj�d| � | �|j|j|��d S �N� Null body %sr� rc �%s %s�%s)rP rt r� r� r� r� � ResponseErrorr� r� r �handler� XmlHandlerrd r re rf �xml�sax� parseString) r# r\ r_ �markersr] �parentr] r5 ra �rsr r r r �get_list, s$ zAWSQueryConnection.get_listc C s� |s| }| � ||||�}|�� }tj�|� |sTtj�d| � | �|j|j|��n�|jdkr�||�} tj � | |�} t |t j �r�|�d�}tj�|| � | S tj�d|j|jf � tj�d| � | �|j|j|��d S rf )rP rt r� r� r� r� rj r� r� rk rl rd r re rf rm rn ro ) r# r\ r_ �clsr] rq r] r5 ra �objr r r r � get_objectB s$ z AWSQueryConnection.get_objectc C s� |s| }| � ||||�}|�� }tj�|� |sTtj�d| � | �|j|j|��nl|jdkr�t � }tj � ||�} t j �|| � |jS tj�d|j|jf � tj�d| � | �|j|j|��d S )Nrg r� rh ri )rP rt r� r� r� r� rj r� r� r rk rl rm rn ro ) r# r\ r_ r] rq r] r5 ra rr r r r r � get_statusX s z AWSQueryConnection.get_status)NNTNNNNNNr Nrw NTNrx )Nrw rX )rw NrX )rw NrX )rw NrX )r; r< r= r[ r rj r$ r� rW rP r` re rs rv rw � __classcell__r r rV r rT � s( � * � � rT )<r> r r� r r! r� r� r� r( �xml.saxrm rF r� r r � boto.utilsZ boto.handlerZ boto.cacertsr r � boto.compatr r r r r �boto.exceptionr r r r r � boto.providerr �boto.resultsetr r� r� r r � ImportErrorrB Zdummy_threading�allr2 r� r] r� �dirname�abspath�cacerts�__file__r� �objectr r7 rW rp rv rT r r r r �<module>* sl �"jeT : vendored/boto/boto/__pycache__/compat.cpython-39.pyc 0000644 00000002150 15025013617 0016402 0 ustar 00 a sFgl � @ s8 d dl Z z d dlZW n ey. d dlZY n0 zd dlmZ W n ey^ d dlmZ Y n0 ze j�d� e jjZW n e efy� dd� ZY n0 d dl m Z d dl m Z d dlmZmZ d dlZd dlZd d lmZ d d lmZmZmZmZmZ d d lm Z d d l m Z e Z e!Z"d d l#m$Z$m%Z%m&Z& e Z'eZ(dS )� N)� encodebytes)� encodestring�~c C s | S )N� )�xr r �./opt/gsutil/gslib/vendored/boto/boto/compat.py�<lambda>/ � r )�six)�BytesIO�StringIO)�Queue)�parse_qs�quote�unquote�urlparse�urlsplit)� unquote_plus)�urlopen)� ConfigParser� NoOptionError�NoSectionError))�osZ simplejson�json� ImportError�base64r r �path� expanduser�AttributeErrorZ boto.vendoredr � http.client�client� http_client�ior r �urllib�_thread�queuer � urllib.parser r r r r r Zurllib.requestr � Exception� StandardError�intZ long_type� configparserr r r Z unquote_strZ parse_qs_safer r r r �<module> s6 vendored/boto/boto/__pycache__/handler.cpython-39.pyc 0000644 00000003555 15025013617 0016546 0 ustar 00 a sFgN � @ s< d dl Zd dlmZ G dd� dejj�ZG dd� de�ZdS )� N)�StringIOc @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S ) � XmlHandlerc C s || _ d|fg| _d| _d S )N�root� )� connection�nodes� current_text��selfZ root_noder � r �//opt/gsutil/gslib/vendored/boto/boto/handler.py�__init__ s zXmlHandler.__init__c C s< d| _ | jd d �||| j�}|d ur8| j�||f� d S )Nr ���� )r r � startElementr �append)r �name�attrsZnew_noder r r r " s zXmlHandler.startElementc C sl | j d d �|| j| j� | j d d |krbt| j d d d�rX| j d d �| j� | j �� d| _d S )Nr r r �endNoder )r � endElementr r �hasattrr �pop)r r r r r r ( s zXmlHandler.endElementc C s | j |7 _ d S �N)r �r �contentr r r � characters0 s zXmlHandler.charactersN)�__name__� __module__� __qualname__r r r r r r r r r s r c @ s e Zd Zdd� Zdd� ZdS )�XmlHandlerWrapperc C s> t ||�| _tj�� | _| j�| j� | j�tjjjd� d S )Nr ) r �handler�xml�sax� make_parser�parser�setContentHandler� setFeature�feature_external_gesr r r r r 5 s zXmlHandlerWrapper.__init__c C s | j �t|��S r )r$ �parser r r r r � parseString; s z XmlHandlerWrapper.parseStringN)r r r r r) r r r r r 4 s r ) �xml.saxr! � boto.compatr r" �ContentHandlerr �objectr r r r r �<module> s vendored/boto/boto/__pycache__/__init__.cpython-39.pyc 0000644 00000101533 15025013617 0016663 0 ustar 00 a sFg�� � @ s| d dl mZmZ d dlmZmZ d dlZd dlZd dl Z d dl Z d dl Z d dl Z d dl Z d dlZ d dlmZ d dlmZ dZeZej�dd� dee �� e �� e �� f Ze� Ze �d �Z e �d �Z e �d �Z e �d �Z e j �!e j �"e#�d �Z$dd� Z%G dd� de j&�Z'e �(d�a)e �(d�Z*t)�+e'� � e*�+e'� � e%� e j,dfdd�Z-e j.dfdd�Z/d�dd�Z0d�dd�Z1d�d d �Z2d�d d �Z3d�d d!�Z4d�d"d#�Z5d�d$d%�Z6d�d&d'�Z7d�d(d)�Z8d�d*d+�Z9d�d,d-�Z:d�d.d/�Z;d�d0d1�Z<d�d2d3�Z=d�d4d5�Z>d�d6d7�Z?d�d8d9�Z@d�d:d;�ZAd�d<d=�ZBd�dAdB�ZCd�dCdD�ZDd�dEdF�ZEd�dHdI�ZFd�dJdK�ZGd�dLdM�ZHd�dNdO�ZId�dPdQ�ZJd�dRdS�ZKd�dTdU�ZLd�dVdW�ZMd�dXdY�ZNd�dZd[�ZOd�d\d]�ZPd�d^d_�ZQd�d`da�ZRd�dbdc�ZSd�ddde�ZTd�dfdg�ZUd�dhdi�ZVd�djdk�ZWd�dldm�ZXd�dndo�ZYd�dpdq�ZZd�drds�Z[d�dtdu�Z\d�dvdw�Z]d�dxdy�Z^d�dzd{�Z_d�d|d}�Z`d�d~d�Zad�d d�ed�d@fd�d��Zbd�d�� Zcejd�ee� dS )�� )�Config�BotoConfigLocations)�BucketStorageUri�FileStorageUriN)�urlparse)�InvalidUriErrorz2.49.0� zBoto/%s Python/%s %s/%sz.^[a-zA-Z0-9][a-zA-Z0-9\._-]{1,253}[a-zA-Z0-9]$z[-_a-z0-9]{64}z3(?P<versionless_uri_str>.+)#(?P<generation>[0-9]+)$z/(?P<versionless_uri_str>.+)#(?P<version_id>.+)$zendpoints.jsonc C s4 t D ]*} ztj�tj�| �� W q Y q0 qd S �N)r �logging�configZ fileConfig�os�path� expanduser)�file� r �0/opt/gsutil/gslib/vendored/boto/boto/__init__.py� init_loggingA s r c @ s e Zd Zdd� ZdS )� NullHandlerc C s d S r r )�self�recordr r r �emitJ s zNullHandler.emitN)�__name__� __module__� __qualname__r r r r r r I s r �botoz boto.perfc C sV |sd}t �| �}|�|� t �|�}|�|� t �|�}|�|� |�|� |ad S �Nz0%(asctime)s %(name)s [%(levelname)s]:%(message)s)r � getLogger�setLevel� FileHandler� Formatter� setFormatter� addHandler�log)�name�filepath�level� format_string�logger�fh� formatterr r r �set_file_loggerV s r* c C sT |sd}t �| �}|�|� t �� }|�|� t �|�}|�|� |�|� |ad S r )r r r � StreamHandlerr r r! r" )r# r% r& r' r( r) r r r �set_stream_loggerd s r, c K s ddl m} || |fi |��S )a& :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.sqs.connection.SQSConnection` :return: A connection to Amazon's SQS r )� SQSConnection)Zboto.sqs.connectionr- )�aws_access_key_id�aws_secret_access_key�kwargsr- r r r � connect_sqsr s r1 c K s ddl m} || |fi |��S )a# :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.s3.connection.S3Connection` :return: A connection to Amazon's S3 r �� S3Connection)�boto.s3.connectionr3 )r. r/ r0 r3 r r r � connect_s3� s r5 c K s ddl m} || |fi |��S )aV @type gs_access_key_id: string @param gs_access_key_id: Your Google Cloud Storage Access Key ID @type gs_secret_access_key: string @param gs_secret_access_key: Your Google Cloud Storage Secret Access Key @rtype: L{GSConnection<boto.gs.connection.GSConnection>} @return: A connection to Google's Storage service r )� GSConnection)Zboto.gs.connectionr6 )Zgs_access_key_idZgs_secret_access_keyr0 r6 r r r � connect_gs� s r7 c K s ddl m} || |fi |��S )a& :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.ec2.connection.EC2Connection` :return: A connection to Amazon's EC2 r �� EC2Connection)Zboto.ec2.connectionr9 )r. r/ r0 r9 r r r � connect_ec2� s r: c K s ddl m} || |fi |��S )a2 :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.ec2.elb.ELBConnection` :return: A connection to Amazon's Load Balancing Service r )� ELBConnection)Z boto.ec2.elbr; )r. r/ r0 r; r r r � connect_elb� s r< c K s ddl m} || |fi |��S )a� :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.ec2.autoscale.AutoScaleConnection` :return: A connection to Amazon's Auto Scaling Service :type use_block_device_types bool :param use_block_device_types: Specifies whether to return described Launch Configs with block device mappings containing block device types, or a list of old style block device mappings (deprecated). This defaults to false for compatability with the old incorrect style. r )�AutoScaleConnection)Zboto.ec2.autoscaler= )r. r/ r0 r= r r r �connect_autoscale� s �r> c K s ddl m} || |fi |��S )a@ :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.ec2.cloudwatch.CloudWatchConnection` :return: A connection to Amazon's EC2 Monitoring service r )�CloudWatchConnection)Zboto.ec2.cloudwatchr? )r. r/ r0 r? r r r �connect_cloudwatch� s �r@ c K s ddl m} || |fi |��S )a& :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.sdb.connection.SDBConnection` :return: A connection to Amazon's SDB r )� SDBConnection)Zboto.sdb.connectionrA )r. r/ r0 rA r r r � connect_sdb� s rB c K s ddl m} || |fi |��S )� :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.fps.connection.FPSConnection` :return: A connection to FPS r )� FPSConnection)Zboto.fps.connectionrD )r. r/ r0 rD r r r � connect_fps� s rE c K s ddl m} || |fi |��S )a# :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.mturk.connection.MTurkConnection` :return: A connection to MTurk r )�MTurkConnection)Zboto.mturk.connectionrF )r. r/ r0 rF r r r � connect_mturk s rG c K s ddl m} || |fi |��S )rC r )�CloudFrontConnection)Zboto.cloudfrontrH )r. r/ r0 rH r r r �connect_cloudfront s �rI c K s ddl m} || |fi |��S )a :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.vpc.VPCConnection` :return: A connection to VPC r )� VPCConnection)Zboto.vpcrJ )r. r/ r0 rJ r r r � connect_vpc# s rK c K s ddl m} || |fi |��S )a :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.rds.RDSConnection` :return: A connection to RDS r �� RDSConnection)Zboto.rdsrM �r. r/ r0 rM r r r � connect_rds2 s rO c K s ddl m} |f | |d�|��S )a :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.rds2.layer1.RDSConnection` :return: A connection to RDS r rL �r. r/ )Zboto.rds2.layer1rM rN r r r � connect_rds2A s ��rQ c K s ddl m} || |fi |��S )a :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.emr.EmrConnection` :return: A connection to Elastic mapreduce r )� EmrConnection)Zboto.emrrR )r. r/ r0 rR r r r � connect_emrT s rS c K s ddl m} || |fi |��S )a :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.sns.SNSConnection` :return: A connection to Amazon's SNS r )� SNSConnection)Zboto.snsrT )r. r/ r0 rT r r r � connect_snsc s rU c K s ddl m} || |fi |��S )a :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.iam.IAMConnection` :return: A connection to Amazon's IAM r )� IAMConnection)Zboto.iamrV )r. r/ r0 rV r r r � connect_iamr s rW c K s ddl m} || |fi |��S )a/ :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.dns.Route53Connection` :return: A connection to Amazon's Route53 DNS Service r )�Route53Connection)Z boto.route53rX )r. r/ r0 rX r r r �connect_route53� s �rY c K s ddl m} || |fi |��S )aD :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.cloudformation.CloudFormationConnection` :return: A connection to Amazon's CloudFormation Service r )�CloudFormationConnection)Zboto.cloudformationrZ )r. r/ r0 rZ r r r �connect_cloudformation� s �r[ �E" �/services/EucalyptusFc K sv ddl m} ddlm} |s*t�ddd�}|s<t�ddd�}| sNt�dd d�} |d | d �} |||f| |||d �|��S ) a� Connect to a Eucalyptus service. :type host: string :param host: the host name or ip address of the Eucalyptus server :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.ec2.connection.EC2Connection` :return: A connection to Eucalyptus server r r8 �� RegionInfo� Credentials�euca_access_key_idN�euca_secret_access_key�BotoZeucalyptus_hostZ eucalyptus�r# Zendpoint)�region�portr � is_secure)Zboto.ec2r9 �boto.ec2.regioninfor_ r �get) �hostr. r/ rf r rg r0 r9 r_ Zregr r r � connect_euca� s* �� ��rk c K s ddl m} || |fi |��S )a+ :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.glacier.layer2.Layer2` :return: A connection to Amazon's Glacier Service r ��Layer2)Zboto.glacier.layer2rm �r. r/ r0 rm r r r �connect_glacier� s �ro c K sz ddl m} t| �}|j|d< |j|d< |j|d< d|vrH|jdk|d< ||j|jd�|d < ||d < ||d < tf i |��S ) a� Connect to an EC2 Api endpoint. Additional arguments are passed through to connect_ec2. :type url: string :param url: A url for the ec2 api endpoint to connect to :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.ec2.connection.EC2Connection` :return: A connection to Eucalyptus server r r^ rf rj r rg �httpsrd re r. r/ )rh r_ r rf �hostnamer �schemer: )�urlr. r/ r0 r_ Zpurlr r r �connect_ec2_endpoint� s � rt �/services/Walrusc K sn ddl m} ddl m} |s*t�ddd�}|s<t�ddd�}| sNt�dd d�} |||f| |||� |d �|��S ) a� Connect to a Walrus service. :type host: string :param host: the host name or ip address of the Walrus server :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.s3.connection.S3Connection` :return: A connection to Walrus r r2 ��OrdinaryCallingFormatr` ra Nrb rc Z walrus_host)rj rf r �calling_formatrg �r4 r3 rw r ri ) rj r. r/ rf r rg r0 r3 rw r r r �connect_walrus� s* ����rz c K s ddl m} || |fi |��S )a :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.ses.SESConnection` :return: A connection to Amazon's SES r )� SESConnection)Zboto.sesr{ )r. r/ r0 r{ r r r � connect_ses' s r| c K s ddl m} || |fi |��S )a :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.sts.STSConnection` :return: A connection to Amazon's STS r )� STSConnection)Zboto.stsr} )r. r/ r0 r} r r r � connect_sts6 s r~ c K sP ddl m} ddl m} t�dd| �}t�dd|�}|||fd|� |d�|��S ) ah Connect to the Internet Archive via their S3-like API. :type ia_access_key_id: string :param ia_access_key_id: Your IA Access Key ID. This will also look in your boto config file for an entry in the Credentials section called "ia_access_key_id" :type ia_secret_access_key: string :param ia_secret_access_key: Your IA Secret Access Key. This will also look in your boto config file for an entry in the Credentials section called "ia_secret_access_key" :rtype: :class:`boto.s3.connection.S3Connection` :return: A connection to the Internet Archive r r2 rv r` �ia_access_key_id�ia_secret_access_keyzs3.us.archive.org)rj rx rg ry )r r� rg r0 r3 rw Z access_keyZ secret_keyr r r � connect_iaE s ����r� c K s ddl m} || |fi |��S )a6 :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.dynamodb.layer2.Layer2` :return: A connection to the Layer2 interface for DynamoDB. r rl )Zboto.dynamodb.layer2rm rn r r r �connect_dynamodbe s r� c K s ddl m} || |fi |��S )a, :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.swf.layer1.Layer1` :return: A connection to the Layer1 interface for SWF. r ��Layer1)Zboto.swf.layer1r� �r. r/ r0 r� r r r � connect_swfv s r� c K s ddl m} || |fi |��S )a3 :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.cloudsearch.layer2.Layer2` :return: A connection to Amazon's CloudSearch service r rl )Zboto.cloudsearch.layer2rm rn r r r �connect_cloudsearch� s �r� c K s" ddl m} || |fd|i|��S )a� :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :type sign_request: bool :param sign_request: whether or not to sign search and upload requests :rtype: :class:`boto.cloudsearch2.layer2.Layer2` :return: A connection to Amazon's CloudSearch2 service r rl � sign_request)Zboto.cloudsearch2.layer2rm )r. r/ r� r0 rm r r r �connect_cloudsearch2� s ��r� c K s ddl m} || |fi |��S )aU :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.cloudsearchdomain.layer1.CloudSearchDomainConnection` :return: A connection to Amazon's CloudSearch Domain service r )�CloudSearchDomainConnection)Z boto.cloudsearchdomain.layer1r� )r. r/ r0 r� r r r �connect_cloudsearchdomain� s ��r� c K s ddl m} || |fi |��S )a7 :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.beanstalk.layer1.Layer1` :return: A connection to Amazon's Elastic Beanstalk service r r� )Zboto.beanstalk.layer1r� r� r r r �connect_beanstalk� s r� c K s ddl m} |f | |d�|��S )aG :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.ets.layer1.ElasticTranscoderConnection` :return: A connection to Amazon's Elastic Transcoder service r )�ElasticTranscoderConnectionrP )Z boto.elastictranscoder.layer1r� )r. r/ r0 r� r r r �connect_elastictranscoder� s ��r� c K s ddl m} |f | |d�|��S )Nr )�OpsWorksConnectionrP )Zboto.opsworks.layer1r� )r. r/ r0 r� r r r �connect_opsworks� s ��r� c K s ddl m} |f | |d�|��S )a9 :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.redshift.layer1.RedshiftConnection` :return: A connection to Amazon's Redshift service r )�RedshiftConnectionrP )Zboto.redshift.layer1r� )r. r/ r0 r� r r r �connect_redshift� s ��r� c K s ddl m} |f | |d�|��S )a6 :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.support.layer1.SupportConnection` :return: A connection to Amazon's Support service r )�SupportConnectionrP )Zboto.support.layer1r� )r. r/ r0 r� r r r �connect_support s ��r� c K s ddl m} |f | |d�|��S )a] Connect to AWS CloudTrail :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.cloudtrail.layer1.CloudtrailConnection` :return: A connection to the AWS Cloudtrail service r )�CloudTrailConnectionrP )Zboto.cloudtrail.layer1r� )r. r/ r0 r� r r r �connect_cloudtrail s ��r� c K s ddl m} |f | |d�|��S )ai Connect to AWS DirectConnect :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key :rtype: :class:`boto.directconnect.layer1.DirectConnectConnection` :return: A connection to the AWS DirectConnect service r )�DirectConnectConnectionrP )Zboto.directconnect.layer1r� )r. r/ r0 r� r r r �connect_directconnect3 s ��r� c K s ddl m} |f | |d�|��S )aV Connect to Amazon Kinesis :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key rtype: :class:`boto.kinesis.layer1.KinesisConnection` :return: A connection to the Amazon Kinesis service r )�KinesisConnectionrP )Zboto.kinesis.layer1r� )r. r/ r0 r� r r r �connect_kinesisI s ��r� c K s ddl m} |f | |d�|��S )am Connect to Amazon CloudWatch Logs :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key rtype: :class:`boto.kinesis.layer1.CloudWatchLogsConnection` :return: A connection to the Amazon CloudWatch Logs service r )�CloudWatchLogsConnectionrP )Zboto.logs.layer1r� )r. r/ r0 r� r r r � connect_logs_ s ��r� c K s ddl m} |f | |d�|��S )aw Connect to Amazon Route 53 Domains :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key rtype: :class:`boto.route53.domains.layer1.Route53DomainsConnection` :return: A connection to the Amazon Route 53 Domains service r )�Route53DomainsConnectionrP )Zboto.route53.domains.layer1r� )r. r/ r0 r� r r r �connect_route53domainsv s ��r� c K s ddl m} |f | |d�|��S )ay Connect to Amazon Cognito Identity :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key rtype: :class:`boto.cognito.identity.layer1.CognitoIdentityConnection` :return: A connection to the Amazon Cognito Identity service r )�CognitoIdentityConnectionrP )Z boto.cognito.identity.layer1r� )r. r/ r0 r� r r r �connect_cognito_identity� s ��r� c K s ddl m} |f | |d�|��S )ai Connect to Amazon Cognito Sync :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key rtype: :class:`boto.cognito.sync.layer1.CognitoSyncConnection` :return: A connection to the Amazon Cognito Sync service r )�CognitoSyncConnectionrP )Zboto.cognito.sync.layer1r� )r. r/ r0 r� r r r �connect_cognito_sync� s ��r� c K s ddl m} |f | |d�|��S )a^ Connect to AWS Key Management Service :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key rtype: :class:`boto.kms.layer1.KMSConnection` :return: A connection to the AWS Key Management Service r )� KMSConnectionrP )Zboto.kms.layer1r� )r. r/ r0 r� r r r � connect_kms� s ��r� c K s ddl m} |f | |d�|��S )aR Connect to AWS Lambda :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key rtype: :class:`boto.awslambda.layer1.AWSLambdaConnection` :return: A connection to the AWS Lambda service r )�AWSLambdaConnectionrP )Zboto.awslambda.layer1r� )r. r/ r0 r� r r r �connect_awslambda� s ��r� c K s ddl m} |f | |d�|��S )a^ Connect to AWS CodeDeploy :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key rtype: :class:`boto.cognito.sync.layer1.CodeDeployConnection` :return: A connection to the AWS CodeDeploy service r )�CodeDeployConnectionrP )Zboto.codedeploy.layer1r� )r. r/ r0 r� r r r �connect_codedeploy� s ��r� c K s ddl m} |f | |d�|��S )aP Connect to AWS Config :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key rtype: :class:`boto.kms.layer1.ConfigServiceConnection` :return: A connection to the AWS Config service r )�ConfigServiceConnectionrP )Zboto.configservice.layer1r� )r. r/ r0 r� r r r �connect_configservice s ��r� c K s ddl m} |f | |d�|��S )aT Connect to AWS CloudHSM :type aws_access_key_id: string :param aws_access_key_id: Your AWS Access Key ID :type aws_secret_access_key: string :param aws_secret_access_key: Your AWS Secret Access Key rtype: :class:`boto.cloudhsm.layer1.CloudHSMConnection` :return: A connection to the AWS CloudHSM service r )�CloudHSMConnectionrP )Zboto.cloudhsm.layer1r� )r. r/ r0 r� r r r �connect_cloudhsm s ��r� c K s ddl m} |f | |d�|��S )z� Connect to Amazon EC2 Container Service rtype: :class:`boto.ec2containerservice.layer1.EC2ContainerServiceConnection` :return: A connection to the Amazon EC2 Container Service r )� EC2ContainerServiceConnectionrP )Z boto.ec2containerservice.layer1r� )r. r/ r0 r� r r r �connect_ec2containerservice. s ��r� c K s ddl m} |f | |d�|��S )z� Connect to Amazon Machine Learning service rtype: :class:`boto.machinelearning.layer1.MachineLearningConnection` :return: A connection to the Amazon Machine Learning service r )�MachineLearningConnectionrP )Zboto.machinelearning.layer1r� )r. r/ r0 r� r r r �connect_machinelearning> s ��r� r Tc C s� d}d}| � d�} | dkr(|�� } | } n | d| � �� } | | d d� } | dvr\td| ��| dkr�d } | d krtd } t| || �S | �d d �} | d }d}|r�|r�t�|�r�t�|�r�td| ��| dk�rt �| �}|�rP|� � }|d }|�d d �} t |d �}nL| dk�rDt �| �}|�rP|� � }|d }|�d d �} |d }n td| ��t | �d k�rf| d }|| |||||||d�S dS )aR Instantiate a StorageUri from a URI string. :type uri_str: string :param uri_str: URI naming bucket + optional object. :type default_scheme: string :param default_scheme: default scheme for scheme-less URIs. :type debug: int :param debug: debug level to pass in to boto connection (range 0..2). :type validate: bool :param validate: whether to check for bucket name validity. :type bucket_storage_uri_class: BucketStorageUri interface. :param bucket_storage_uri_class: Allows mocking for unit tests. :param suppress_consec_slashes: If provided, controls whether consecutive slashes will be suppressed in key paths. :type is_latest: bool :param is_latest: whether this versioned object represents the current version. We allow validate to be disabled to allow caller to implement bucket-level wildcarding (outside the boto library; see gsutil). :rtype: :class:`boto.StorageUri` subclass :return: StorageUri subclass for given URI. ``uri_str`` must be one of the following formats: * gs://bucket/name * gs://bucket/name#ver * s3://bucket/name * gs://bucket * s3://bucket * filename (which could be a Unix path like /a/b/c or a Windows path like C:\c) The last example uses the default scheme ('file', unless overridden). Nz://���r � )r �s3�gszUnrecognized scheme "%s"r F�-T�/� r z Invalid bucket name in URI "%s"r� �versionless_uri_str� generationr� � version_id)�suppress_consec_slashesr� r� � is_latest)�find�lowerr r �split�BUCKET_NAME_RE�match�TOO_LONG_DNS_NAME_COMP�search� GENERATION_RE� groupdict�int� VERSION_RE�len)�uri_strZdefault_scheme�debug�validateZbucket_storage_uri_classr� r� r� r� �end_scheme_idxrr r � is_stream� path_parts� bucket_name� object_namer� Zmdr� r r r � storage_uriN s\ ) �� �r� c C sN t | tjjj�s$tdtt| �� ��| jj j � � }d|| jj | j f }t |�S )z�Returns a StorageUri for the given key. :type key: :class:`boto.s3.key.Key` or subclass :param key: URI naming bucket + optional object. z7Requested key (%s) is not a subclass of boto.s3.key.Keyz %s://%s/%s)� isinstancer r� �keyZKeyr �str�type�bucket� connection�providerZget_provider_namer# r� )r� Z prov_namer� r r r �storage_uri_for_key� s �r� )NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NNNr\ r] F)NN)NN)NNNr\ ru F)NN)NN)NNF)NN)NN)NN)NNF)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)NN)fZboto.pyami.configr r Zboto.storage_urir r Z boto.pluginr �datetimer �platform�re�sysr Zlogging.configZ boto.compatr Zboto.exceptionr � __version__ZVersion�strptimeZpython_version�system�release� UserAgentr �compiler� r� r� r� r �join�dirname�__file__ZENDPOINTS_PATHr �Handlerr r r" Zperflogr! �INFOr* �DEBUGr, r1 r5 r7 r: r< r> r@ rB rE rG rI rK rO rQ rS rU rW rY r[ rk ro rt rz r| r~ r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� ZpluginZ load_pluginsr r r r �<module> s* � � ' � $ � ( � � � � � � � � � � � � � � � � � � � � � � � � � � e vendored/boto/boto/__pycache__/auth.cpython-39.pyc 0000644 00000072201 15025013617 0016064 0 ustar 00 a sFg~� � @ s d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl Z ddl Z ddl Z ddlmZmZmZmZmZ ddlmZ ddlmZ ddlmZ z ddlmZ dd lmZ W n ey� ddlZdZY n0 g d �Z g d �Z G d d � d e �Z G dd� dee �Z G dd� dee �Z!G dd� dee �Z"G dd� dee �Z#G dd� dee �Z$G dd� dee �Z%G dd� de%e�Z&G d d � d e�Z'G d d � d e �Z(G d d!� d!e(e�Z)G d"d#� d#e(e�Z*G d$d%� d%e(e�Z+G d&d'� d'e+e�Z,d0d(d)�Z-d*d+� Z.d,d-� Z/d.d/� Z0dS )1z/ Handles authentication required to AWS and GS � N)� formatdate)�urllib� encodebytes� parse_qs_safe�urlparse�six)� AuthHandler)�BotoClientError)�get_utf8able_str)�sha1)�sha256)z-ap-northeast-1z.ap-northeast-1z-ap-southeast-1z.ap-southeast-1z-ap-southeast-2z.ap-southeast-2z -eu-west-1z .eu-west-1z -external-1z .external-1z -sa-east-1z .sa-east-1z -us-east-1z .us-east-1z-us-gov-west-1z.us-gov-west-1z -us-west-1z .us-west-1z -us-west-2z .us-west-2) z.cn-z .eu-centralz -eu-centralz.ap-northeast-2z-ap-northeast-2z .ap-south-1z -ap-south-1z .us-east-2z -us-east-2z -ca-centralz .ca-centralz .eu-west-2z -eu-west-2c @ sJ e Zd ZdZddd�Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dS )�HmacKeysz Key based Auth handler helper.Fc C sD |j d u s|jd u r0|s$tj�� �n d | _d | _|| _| �|� d S �N) � access_key� secret_key�boto� auth_handler�NotReadyToAuthenticate�_hmac� _hmac_256�host�update_provider)�selfr �config�provider�anon� r �,/opt/gsutil/gslib/vendored/boto/boto/auth.py�__init__f s zHmacKeys.__init__c C sR || _ | j jrNtj| j j�d�td�| _trHtj| j j�d�td�| _nd | _d S �N�utf-8)� digestmod) � _providerr �hmac�new�encode�shar r r �r r r r r r p s �� zHmacKeys.update_providerc C s | j r dS dS d S )NZ HmacSHA256ZHmacSHA1)r )r r r r � algorithm{ s zHmacKeys.algorithmc C s( | j r t}nt}tj| jj�d�|d�S r )r r r& r# r$ r" r r% )r r! r r r � _get_hmac� s �zHmacKeys._get_hmacc C s. | � � }|�|�d�� t|�� ��d��� S �Nr )r) �updater% r �digest�decode�strip)r �string_to_signZnew_hmacr r r � sign_string� s zHmacKeys.sign_stringc C s t � | j�}|d= |d= |S )Nr r )�copy�__dict__)r Z pickled_dictr r r � __getstate__� s zHmacKeys.__getstate__c C s || _ | �| j� d S r )r2 r r" )r �dctr r r � __setstate__� s zHmacKeys.__setstate__N)F) �__name__� __module__� __qualname__�__doc__r r r( r) r0 r3 r5 r r r r r c s r c @ s& e Zd ZdZdgZdd� Zdd� ZdS )�AnonAuthHandlerz( Implements Anonymous requests. r c C s( t �| |||� tj| |||dd� d S )NT)r �r r r �r r r r r r r r � s zAnonAuthHandler.__init__c K s d S r r )r � http_request�kwargsr r r �add_auth� s zAnonAuthHandler.add_authN�r6 r7 r8 r9 � capabilityr r? r r r r r: � s r: c s8 e Zd ZdZddgZdd� Z� fdd�Zdd � Z� ZS ) �HmacAuthV1Handlerz: Implements the HMAC request signing used by S3 and GS.zhmac-v1�s3c C s* t �| |||� t�| |||� d | _d S r �r r r r r< r r r r � s zHmacAuthV1Handler.__init__c s t t| ��|� d | _d S r )�superrB r r r' �� __class__r r r � s z!HmacAuthV1Handler.update_providerc K s� |j }|j}|j}d|vr(tdd�|d< | jjrD| jj}| jj||< tj� |||d | j�}tj � d| � | � |�}| jj } d| | jj|f } tj � d| � | |d< d S )N�DateT��usegmt�StringToSign: %s�%s %s:%s� Signature: %s� Authorization)�headers�method� auth_pathr r" �security_token�security_token_headerr �utilsZcanonical_string�log�debugr0 � auth_headerr ) r r= r> rO rP rQ �keyr/ �b64_hmac�auth_hdr�authr r r r? � s$ � zHmacAuthV1Handler.add_auth� r6 r7 r8 r9 rA r r r? � __classcell__r r rF r rB � s rB c s8 e Zd ZdZddgZdd� Z� fdd�Zdd � Z� ZS ) �HmacAuthV2HandlerzJ Implements the simplified HMAC authorization used by CloudFront. zhmac-v2Z cloudfrontc C s* t �| |||� t�| |||� d | _d S r rD r< r r r r � s zHmacAuthV2Handler.__init__c s t t| ��|� d | _d S r )rE r^ r r r' rF r r r � s z!HmacAuthV2Handler.update_providerc K sh |j }d|vr tdd�|d< | jjr8| jj}| jj||< | �|d �}| jj}d|| jj|f |d< d S )NrH TrI rL rN )rO r r" rR rS r0 rW r )r r= r> rO rX rY rZ r r r r? � s ��zHmacAuthV2Handler.add_authr\ r r rF r r^ � s r^ c @ s( e Zd ZdZg d�Zdd� Zdd� ZdS )�HmacAuthV3Handlerz@Implements the new Version 3 HMAC authorization used by Route53.)zhmac-v3Zroute53�sesc C s$ t �| |||� t�| |||� d S r r; r< r r r r � s zHmacAuthV3Handler.__init__c K sr |j }d|vr tdd�|d< | jjr8| jj}| jj||< | �|d �}d| jj }|d| �� |f 7 }||d< d S )NrH TrI z AWS3-HTTPS AWSAccessKeyId=%s,zAlgorithm=%s,Signature=%s�X-Amzn-Authorization)rO r r" rR rS r0 r r( )r r= r> rO rX rY �sr r r r? � s zHmacAuthV3Handler.add_authNr@ r r r r r_ � s r_ c @ s> e Zd ZdZdgZdd� Zdd� Zdd� Zd d � Zd d � Z d S )�HmacAuthV3HTTPHandlerzK Implements the new Version 3 HMAC authorization used by DynamoDB. z hmac-v3-httpc C s$ t �| |||� t�| |||� d S r r; r< r r r r s z HmacAuthV3HTTPHandler.__init__c C s< d| j i}|j�� D ]"\}}|�� }|�d�r|||< q|S )�k Select the headers from the request that need to be included in the StringToSign. �Host�x-amz)r rO �items�lower� startswith)r r= �headers_to_sign�name�value�lnamer r r rj s z%HmacAuthV3HTTPHandler.headers_to_signc s t � fdd�� D ��}d�|�S )� Return the headers that need to be included in the StringToSign in their canonical form by converting all header keys to lower case, sorting them in alphabetical order and then joining them into a string, separated by newlines. c s( g | ] }d |� � �� � | �� f �qS )�%s:%s�rh r. ��.0�n�rj r r � <listcomp> s � �z;HmacAuthV3HTTPHandler.canonical_headers.<locals>.<listcomp>� ��sorted�join�r rj �lr rt r �canonical_headers s �z'HmacAuthV3HTTPHandler.canonical_headersc C s8 | � |�}| �|�}d�|j|jd|d|jg�}||fS )�� Return the canonical StringToSign as well as a dict containing the original version of all headers that were included in the StringToSign. rv � )rj r| ry rP rQ �body)r r= rj r| r/ r r r r/ # s �z$HmacAuthV3HTTPHandler.string_to_signc K s� d|j v r|j d= tdd�|j d< | jjr8| jj|j d< | �|�\}}tj�d| � t|� d��� � }| � |�}d| jj }|d | � � 7 }|d d �|� 7 }|d | 7 }||j d< d S )z� Add AWS3 authentication to a request. :type req: :class`boto.connection.HTTPRequest` :param req: The HTTPRequest object. ra TrI � X-Amz-Date�X-Amz-Security-TokenrK r zAWS3 AWSAccessKeyId=%s,z Algorithm=%s,zSignedHeaders=%s,�;� Signature=%sN)rO r r" rR r/ r rU rV r r% r, r0 r r( ry )r �reqr> r/ rj � hash_valuerY rb r r r r? 3 s z HmacAuthV3HTTPHandler.add_authN) r6 r7 r8 r9 rA r rj r| r/ r? r r r r rc s rc c @ s� e Zd ZdZdgZd+dd�Zd,dd�Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� ZdS )-�HmacAuthV4Handlerz: Implements the new Version 4 HMAC authorization. �hmac-v4Nc C s0 t �| |||� t�| |||� || _|| _d S r )r r r � service_name� region_name)r r r r r� r� r r r r S s zHmacAuthV4Handler.__init__Fc C sN t |t�s|�d�}|r2t�||�d�t��� }nt�||�d�t��� }|S r* )� isinstance�bytesr% r# r$ r � hexdigestr, )r rX �msg�hex�sigr r r �_sign] s zHmacAuthV4Handler._signc C sr | � | j|�}|j�d�r$|jd }d|i}|j�� D ]6\}}|�� }|�d�r6t|t�rd|� d�}|||< q6|S )rd re rf r ) � host_headerr rO �getrg rh ri r� r� r- �r r= Zhost_header_valuerj rk rl rm r r r rj g s z!HmacAuthV4Handler.headers_to_signc C s8 |j }|jdk}|dkr |r(|dkr,|r,|S d||f S �N�https�P i� ro )�port�protocol�r r r= r� �securer r r r� x s z HmacAuthV4Handler.host_headerc C sj t |j�� �}g }|D ]H}t|j| �}|�tjj|d�d�d�d tjj|d�d�d� � qd� |�S )Nr~ �ascii��safe�=�-_~�&) rx �params�keysr �appendr �parse�quoter% ry )r r= Zparameter_names�pairsZpnameZpvalr r r � query_string s ��z HmacAuthV4Handler.query_stringc C sd |j dkrdS g }t|j�D ]<}t|j| �}|�dtjj|dd�tjj|dd�f � q d�|�S )N�POSTr~ �%s=%s�-_.~r� r� ) rP rx r� r r� r r� r� ry �r r= r{ �paramrl r r r �canonical_query_string� s � z(HmacAuthV4Handler.canonical_query_stringc C s� g }i }|D ]H}|� � �� }t|| �}d|v r:|�� }nd�|�� �� �}|||< q t|�D ]}|�d||| f � q^d�|�S )rn �"� ro rv )rh r. �strry �splitrx r� ) r rj � canonicalZnormalized_headers�headerZc_nameZ raw_valueZc_valuerX r r r r| � s z#HmacAuthV4Handler.canonical_headersc C s dd� |D �}t |�}d�|�S )Nc S s g | ]}d |� � �� �qS �z%srp rq r r r ru � � z4HmacAuthV4Handler.signed_headers.<locals>.<listcomp>r� rw rz r r r �signed_headers� s z HmacAuthV4Handler.signed_headersc C sF |j }t�|��dd�}tj�|�}t|�dkrB|�d�rB|d7 }|S )N�\�/� ) rQ � posixpath�normpath�replacer r� r� �len�endswith)r r= �pathZ normalized�encodedr r r � canonical_uri� s z HmacAuthV4Handler.canonical_uric C sN |j }t|d�r.t|d�r.tjj|td�d S t|t�sB|�d�}t|�� � S )N�seek�read)Zhash_algorithmr r ) r �hasattrr rT � compute_hashr r� r� r% r� )r r= r r r r �payload� s zHmacAuthV4Handler.payloadc C st |j �� g}|�| �|�� |�| �|�� | �|�}|�| �|�d � |�| �|�� |�| �|�� d� |�S )Nrv ) rP �upperr� r� r� rj r| r� r� ry )r r= �crrj r r r �canonical_request� s z#HmacAuthV4Handler.canonical_requestc C sB | j jg}|�|j� |�|j� |�|j� |�d� d�|�S )N� aws4_requestr� )r" r r� � timestampr� r� ry )r r= �scoper r r r� � s zHmacAuthV4Handler.scopec C s |� d�S )N�.)r� �r r r r r �split_host_parts� s z"HmacAuthV4Handler.split_host_partsc C sb | � |�}| jd ur | j}nBt|�dkrV|d dkr:d}q^t|�dkrLd}q^|d }n|d }|S )Nr� zus-govz us-gov-west-1� � us-east-1r )r� r� r� )r r �partsr� r r r �determine_region_name� s z'HmacAuthV4Handler.determine_region_namec C s( | � |�}| jd ur | j}n|d }|S )Nr )r� r� )r r r� r� r r r �determine_service_name� s z(HmacAuthV4Handler.determine_service_namec C st g }|j d dd� |_|�|j� | �|j�}| �|j�}||_||_|�|j� |�|j� |�d� d�|�S )Nr� r � r� r� ) rO r� r� r� r r� r� r� ry )r r= r� r� r� r r r �credential_scope� s z"HmacAuthV4Handler.credential_scopec C sH dg}|� |jd � |� | �|�� |� t|�d���� � d�|�S )r} �AWS4-HMAC-SHA256r� r rv )r� rO r� r r% r� ry )r r= r� �stsr r r r/ s z HmacAuthV4Handler.string_to_signc C sX | j j}| �d| �d�|j�}| �||j�}| �||j�}| �|d�}| j||dd�S )NZAWS4r r� T)r� )r" r r� r% r� r� r� )r r= r/ rX Zk_dateZk_regionZ k_serviceZ k_signingr r r � signature s � zHmacAuthV4Handler.signaturec K sr d|j v r|j d= tjjtjjd�jdd�}|�d�|j d< | jjrR| jj|j d< | � |�}|}d|v rv| � |d �}|r�|j d kr�||_ d |j d < t t |j ��|j d < n&|j�d �d |_|r�|jd | |_| �|�}tj�d| � | �||�}tj�d| � | �||�}tj�d| � | �|�} d| �|� g} | �d| �| � � | �d| � d�| �|j d< dS )z� Add AWS4 authentication to a request. :type req: :class`boto.connection.HTTPRequest` :param req: The HTTPRequest object. ra ��tzN��tzinfo�%Y%m%dT%H%M%SZr� r� � unmangled_reqr� �0application/x-www-form-urlencoded; charset=UTF-8� Content-Type�Content-Length�?r zCanonicalRequest: %srK rM z AWS4-HMAC-SHA256 Credential=%szSignedHeaders=%sr� �,rN )rO �datetime�now�timezone�utcr� �strftimer" rR r� rP r r� r� r� r� r� r rU rV r/ r� rj r� r� r� ry ) r r� r> r� �qsZ qs_to_postr� r/ r� rj r{ r r r r? s8 zHmacAuthV4Handler.add_auth)NN)F)r6 r7 r8 r9 rA r r� rj r� r� r� r| r� r� r� r� r� r� r� r� r� r/ r� r? r r r r r� L s, � r� c s� e Zd ZdZdgZ� fdd�Zdd� Zdd� Zd d � Zd d � Z d d� Z dd� Z dd� Z dd� Z � fdd�Z� fdd�Zd dd�Z� ZS ) �S3HmacAuthV4HandlerzN Implements a variant of Version 4 HMAC authorization specific to S3. � hmac-v4-s3c s. t t| �j|i |�� | jr*| �| j�| _d S r )rE r� r r� �clean_region_name)r �argsr> rF r r r [ s z S3HmacAuthV4Handler.__init__c C s |� d�r|dd � S |S )N�s3-r� )ri )r r� r r r r� a s z%S3HmacAuthV4Handler.clean_region_namec C s: t j�|j�}t�|j�}t j�|�}t jj|dd�}|S )Nz/~r� )r r� r r� r � ensure_str�unquoter� )r r= r� Zpath_strZunquotedr� r r r r� g s z!S3HmacAuthV4Handler.canonical_uric C sV g }t |j�D ]<}t|j| �}|�dtjj|dd�tjj|dd�f � qd�|�S )Nr� r� r� r� )rx r� r r� r r� r� ry r� r r r r� t s � z*S3HmacAuthV4Handler.canonical_query_stringc C s< |j }|jdk}|dkr |r(|dkr.|r.|jS d|j|f S r� )r� r� r r� r r r r� s z S3HmacAuthV4Handler.host_headerc C sF | � | j|�}d|i}|j�� D ] \}}|�� }|dvr |||< q |S )rd re )� authorization)r� r rO rg rh r� r r r rj � s z#S3HmacAuthV4Handler.headers_to_signc C s� | � |�}| jd ur | j}n�t|�dkrD| �|d �}|dkr�d}nZtt|��D ]L\}}|�� }|dkr�|| }|dkr~d} q�qP|�d�rP| �|�} q�qP|S )Nr� r rC r� Z amazonawsr� )r� r� r� r� � enumerate�reversedrh ri )r r r� r� �offset�partr r r r� � s$ z)S3HmacAuthV4Handler.determine_region_namec C s dS )NrC r r� r r r r� � s z*S3HmacAuthV4Handler.determine_service_namec C s� t � |�}tj�|j�}|j|_|jdu r2i |_n|j� � }||_|j}t|dd�}|� � D ].\}}t |t t f�r\t |�dkr\|d ||< q\|j�|� |S )z| Returns a copy of the request object with fixed ``auth_path/params`` attributes from the original. NT)�keep_blank_valuesr� r )r1 r r� r rQ r� r� �queryr rg r� �list�tupler� r+ ) r r� Z modified_reqZ parsed_pathZ copy_paramsZraw_qsZ existing_qsrX rl r r r �mangle_path_and_params� s$ � z*S3HmacAuthV4Handler.mangle_path_and_paramsc s& |j �d�r|j d S tt| ��|�S )N�x-amz-content-sha256)rO r� rE r� r� )r r= rF r r r� � s zS3HmacAuthV4Handler.payloadc s^ d|j vr8d|j v r(|j �d�|j d< n| �|�|j d< | �|�}tt| �j|fd|i|��S )Nr� �_sha256r� )rO �popr� r� rE r� r? )r r� r> Z updated_reqrF r r r? � s ��z S3HmacAuthV4Handler.add_authNc C s$ |du r(t j jt jjd�jdd��d�}| �|j�}| �|j�}dd| j j |dd� ||f ||dd �}| j j r|| j j |d < | � |�}t d d � |D ��}d �|�|d< |j�|� | �|�} d�| �d�dd� �d } ||jd< | �|| �} | �|| �} | |jd< d|j|j|jtj�|j�f S )z� Presign a request using SigV4 query params. Takes in an HTTP request and an expiration time in seconds and returns a URL. http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html Nr� r� r� r� z%s/%s/%s/%s/aws4_requestr� r )zX-Amz-AlgorithmzX-Amz-Credentialr� z X-Amz-Expires�X-Amz-SignedHeadersr� c S s g | ]}d |� � �� �qS r� rp rq r r r ru " r� z/S3HmacAuthV4Handler.presign.<locals>.<listcomp>r� r� rv ���z UNSIGNED-PAYLOADr� zX-Amz-Signaturez %s://%s%s?%s)r� r� r� r� r� r� r� r r� r" r rR rj rx ry r� r+ r� r� rO r/ r� r� r� r r� � urlencode) r r� �expiresZiso_date�region�servicer� rj r{ r� r� r� r r r �presign s< �� �zS3HmacAuthV4Handler.presign)N)r6 r7 r8 r9 rA r r� r� r� r� rj r� r� r� r� r? r r] r r rF r r� U s -* r� c @ s. e Zd ZdZdgZdd� Zdd� Zdd� Zd S ) �STSAnonHandlerz� Provides pure query construction (no actual signing). Used for making anonymous STS request for operations like ``assume_role_with_web_identity``. zsts-anonc C s t j�|�S r )r r� r� )r rl r r r � _escape_valueC s z STSAnonHandler._escape_valuec C s\ t |�� �}|jdd� d� g }|D ],}t|| �}|�|d | �t|�� � q$d�|�S )Nc S s | � � S r �rh ��xr r r �<lambda>L r� z4STSAnonHandler._build_query_string.<locals>.<lambda>�rX r� r� )r� r� �sortr r� r ry )r r� r� r� rX �valr r r �_build_query_stringJ s z"STSAnonHandler._build_query_stringc K s4 |j }| �|j�}tj�d| � d|d< ||_d S )Nzquery_string in body: %sz!application/x-www-form-urlencodedr� )rO r r� r rU rV r )r r= r> rO r� r r r r? S s �zSTSAnonHandler.add_authN)r6 r7 r8 r9 rA r r r? r r r r r 9 s r c @ s e Zd ZdZdd� ZdS )�QuerySignatureHelperzy Helper for Query signature based Auth handler. Concrete sub class need to implement _calc_sigature method. c K s� |j }|j}| jj|d< | j|d< tj�� |d< | �|j|j |j |j �\}}tj � d||f � |j dkr�d|d< |d tj�|� |_tt|j��|j d < n8d |_|j�d �d |_|jd | d tj�|� |_d S ) N�AWSAccessKeyId�SignatureVersion� Timestamp� query_string: %s Signature: %sr� r� r� � &Signature=r� r~ r� r )rO r� r" r r r rT �get_ts�_calc_signaturerP rQ r rU rV r r� � quote_plusr r� r� r� r� )r r= r> rO r� r� r� r r r r? f s* � � �z QuerySignatureHelper.add_authN)r6 r7 r8 r9 r? r r r r r _ s r c @ s" e Zd ZdZdZdgZdd� ZdS )�QuerySignatureV0AuthHandlerz Provides Signature V0 Signingr zsign-v0c G s� t j�d� | �� }|d |d }|�|�d�� |�� }|jdd� d� g }|D ]*}t|| �}|� |d t j � |� � qTd � |�} | t�|�� �fS ) Nzusing _calc_signature_0ZActionr r c S s t | �� |�� �S r )�cmprh )r �yr r r r � r� z=QuerySignatureV0AuthHandler._calc_signature.<locals>.<lambda>)r r� r� )r rU rV r) r+ r% r� r r r� r r� r� ry �base64� b64encoder, ) r r� r� r# rb r� r� rX r r� r r r r � s z+QuerySignatureV0AuthHandler._calc_signatureN�r6 r7 r8 r9 r rA r r r r r r } s r c @ s, e Zd ZdZdZddgZdd� Zdd� Zd S ) �QuerySignatureV1AuthHandlerz5 Provides Query Signature V1 Authentication. r� zsign-v1Zmturkc O s: t j| g|�R i |�� tj| g|�R i |�� d | _d S r )r r r r )r r� �kwr r r r � s z$QuerySignatureV1AuthHandler.__init__c G s� t j�d� | �� }t|�� �}|jdd� d� g }|D ]J}|�|�d�� t || ��d�}|�|� |� |d t j � |� � q8d�|�}|t�|�� �fS )Nzusing _calc_signature_1c S s | � � S r r r r r r r � r� z=QuerySignatureV1AuthHandler._calc_signature.<locals>.<lambda>r r r� r� )r rU rV r) r� r� r r+ r% r r� r r� r� ry r r r, ) r r� r� r# r� r� rX r r� r r r r � s z+QuerySignatureV1AuthHandler._calc_signatureN)r6 r7 r8 r9 r rA r r r r r r r � s r c @ s$ e Zd ZdZdZg d�Zdd� ZdS )�QuerySignatureV2AuthHandlerz+Provides Query Signature V2 Authentication.� ) zsign-v2�ec2r# ZemrZfps�ecs�sdbZiamZrdsZsnsZsqsZcloudformationc C s t j�d� d||�� |f }| �� }| �� |d< | jjrF| jj|d< t|� � �}g }|D ]@} t || �� d�} |� t jj| dd�d t jj| d d� � qZd �|�} t j�d | � || 7 }t j�d | � |�|� d�� t�|�� �} t j�d t| � � t j�d| � | | fS )Nzusing _calc_signature_2z %s %s %s ZSignatureMethodZ SecurityTokenr r~ r� r� r� r� zquery string: %szstring_to_sign: %sz len(b64)=%dzbase64 encoded digest: %s)r rU rV rh r) r( r" rR rx r� r r% r� r r� r� ry r+ r r r, r� ) r r� Zverbr� Z server_namer/ r# r� r� rX r r� �b64r r r r � s, � z+QuerySignatureV2AuthHandler._calc_signatureNr r r r r r! � s r! c @ s e Zd ZdZdgZdd� ZdS )�POSTPathQSV2AuthHandlerz� Query Signature V2 Authentication relocating signed query into the path and allowing POST requests with Content-Types. Zmwsc K s� | j j|jd< | j|jd< tj�� |jd< | �|j|j|j |j �\}}tj � d||f � |jdkr�t t|j��|jd< |j�dd�|jd< nd |_|j�d �d |_|jd | d tj�|� |_d S ) Nr r r r r� r� r� z text/plainr~ r� r r )r" r r� r r rT r r rP rQ r rU rV r� r� r rO r� r� r� r r� r )r r� r> r� r� r r r r? � s& � � � �z POSTPathQSV2AuthHandler.add_authN)r6 r7 r8 r9 rA r? r r r r r' � s r' c C s� g }t j�t|�}|D ]2}z|�|| ||�� W q t jjyF Y q0 q|s||}dd� |D �}t j�dt |�t |�f ��|d S )a� Finds an AuthHandler that is ready to authenticate. Lists through all the registered AuthHandlers to find one that is willing to handle for the requested capabilities, config and provider. :type host: string :param host: The name of the host :type config: :param config: :type provider: :param provider: Returns: An implementation of AuthHandler. Raises: boto.exception.NoAuthHandlerFound c S s g | ] }|j �qS r )r6 )rr �handlerr r r ru r� z$get_auth_handler.<locals>.<listcomp>zYNo handler was ready to authenticate. %d handlers were checked. %s Check your credentialsr ) r �plugin� get_pluginr r� r r � exception�NoAuthHandlerFoundr� r� ) r r r �requested_capabilityZready_handlersZ auth_handlersr( Zchecked_handlers�namesr r r �get_auth_handler� s ��r/ c s � fdd�}|S )Nc sj t j�dd�rdgS tj�ddd�r*dgS t| d�rbt| jdd�rbtD ]}|| jj v rFdg S qF� | �S ) NZ EC2_USE_SIGV4Fr� r# � use-sigv4r �endpointr~ ) �os�environr� r r r� �getattrr � SIGV4_DETECTr1 )r �test��funcr r �_wrapper s z(detect_potential_sigv4.<locals>._wrapperr �r8 r9 r r7 r �detect_potential_sigv4 s r; c C s: | d u st | t�r| S | �� dkr&dS | �� dkr6dS d S )N�trueT�falseF)r� �boolrh )r r r r �convert_to_bool0 s r? c s � fdd�}|S )Nc s� t j�d�}tj�dd�}||fD ]*}t|�}|d ur"|r@dgn� | � S q"| j}| j�d�rl| j�d�rtd| }t|�j }|� d�s�|� d�s�� | �S t | d �r�| j r�� | �S dgS ) NZ S3_USE_SIGV4rC r0 r� zhttp://zhttps://z amazonaws.comzamazonaws.com.cnr ) r2 r3 r� r r r? r ri r �netlocr� r� r )r Zenv_use_sigv4_flagZcfg_use_sigv4_flag�flagr r@ r7 r r r9 ; s& � �z*detect_potential_s3sigv4.<locals>._wrapperr r: r r7 r �detect_potential_s3sigv4: s rB )N)1r9 r r Zboto.auth_handler�boto.exception� boto.pluginZ boto.utilsr1 r� � email.utilsr r# r2 r� � boto.compatr r r r r r r r �hashlibr r& r � ImportErrorZS3_AUTH_DETECTr5 �objectr r: rB r^ r_ rc r� r� r r r r r! r' r/ r; r? rB r r r r �<module> sZ 6# K e& 2 vendored/boto/boto/__pycache__/provider.cpython-39.pyc 0000644 00000027374 15025013617 0016770 0 ustar 00 a sFg�X � @ s* d Z ddlZddlmZ ddlmZ ddlmZ ddlZddlmZ ddlmZ ddl m Z dd l m Z dd l mZ dd l mZ dd lmZ dd lmZ d ZdZdZdZdZdZdZdZdZ dZ dZ dZ dZ dZ!dZ"d Z#d Z$d Z%d Z&d Z'd!Z(d"Z)d#Z*d$Z+e,� Z-G d%d&� d&e.�Z/G d'd(� d(e,�Z0d)d*� Z1dS )+zC This class encapsulates the provider-specific header differences. � N)�six)�datetime)�timezone)�config)� expanduser)�Config)� InvalidInstanceMetadataError)�ACL)�CannedACLStrings)�Policy� header_prefix�metadata_prefixzx-amz-zx-goog-z acl-headerz auth-headerzcopy-source-headerz copy-source-version-id-headerzcopy-source-range-headerzdelete-marker-headerz date-headerzmetadata-directive-headerzresumable-upload-headerzsecurity-token-header� storage-classz mfa-headerz server-side-encryption-headerzversion-id-headerzrestore-header�StorageCopyError�StorageCreateError�StorageDataError�StoragePermissionsError�StorageResponseErrorc @ s e Zd ZdS )�ProfileNotFoundErrorN)�__name__� __module__� __qualname__� r r �0/opt/gsutil/gslib/vendored/boto/boto/provider.pyr K s r c # @ s: e Zd Zddd�Zeed�Zeed�Z ddd�Z ddd�Z ddd�Z e eeed eed ed eed eed eed eed eed eed edeed eed eed e ed e ed e ed ie e ee d ee d edee d ee d edee d ee d ee d ee d ee d edee d e e d e de did�Z e!e"j#j$e%e"j#j&e'e"j#j(e)e"j#j*e+e"j#j,ie!e"j#j-e%e"j#j.e'e"j#j/e)e"j#j0e+e"j#j1id�Z2d:dd�Z3d d � Z4d d � Z5e6e4e5�Z7d d!� Z8d"d#� Z9e6e8e9�Z:d$d%� Z;d&d'� Z<e6e;e<�Z=d(d)� Z>d;d*d+�Z?d,d-� Z@d.d/� ZAd0d1� ZBd2d3� ZCd4d5� ZDd6d7� ZEd8d9� ZFdS )<�Provider)�aws_access_key_id�aws_secret_access_keyZaws_security_tokenZ aws_profile)�gs_access_key_id�gs_secret_access_keyNN)�aws�google�s3�gsFTzmeta-�aclZAWSz copy-sourcezcopy-source-version-idzcopy-source-range�datez delete-markerzmetadata-directiveNzsecurity-tokenzserver-side-encryptionz version-idr Zmfa�restoreZGOOG1Z resumablec C s: d | _ d | _d | _|| _|| _|| _|| _|| _|| _| j | j | _ | j | j | _ d | _ tj�td�d| d�}tdd�| _tj�|�r�| j�|� | �||||� | �� | �� d| j| j }t�d|�r�t� d|�| _ d| j| j } t�d| ��r t� d| �| _d | j| j } t�d| ��r6t� d| �| _d S ) N�~�.� credentialsF)�do_loadz%s_host� Credentialsz%s_portz%s_host_header) �host�port� host_header� access_key� secret_key�security_token� profile_name�anon�name� AclClassMap� acl_class� CannedAclsMap� canned_acls�_credential_expiry_time�os�path�joinr r �shared_credentials�isfile�load_from_path�get_credentials�configure_headers�configure_errors� HostKeyMapr � has_option�get�getint) �selfr3 r. r/ r0 r1 r2 Z shared_pathZ host_opt_nameZ port_opt_nameZhost_header_opt_namer r r �__init__� s8 zProvider.__init__c C s | � � r| �� | jS �N)�_credentials_need_refresh�#_populate_keys_from_metadata_server� _access_key�rF r r r �get_access_key� s zProvider.get_access_keyc C s || _ d S rH )rK �rF �valuer r r �set_access_key� s zProvider.set_access_keyc C s | � � r| �� | jS rH )rI rJ � _secret_keyrL r r r �get_secret_key� s zProvider.get_secret_keyc C s || _ d S rH )rQ rN r r r �set_secret_key� s zProvider.set_secret_keyc C s | � � r| �� | jS rH )rI rJ �_security_tokenrL r r r �get_security_token� s zProvider.get_security_tokenc C s || _ d S rH )rT rN r r r �set_security_token� s zProvider.set_security_tokenc C sv | j r dS | jd u rdS | jtjtjd�jd d� }|j|j|j d d d d }|dk rnt j � d� d S dS d S ) NF)�tz)�tzinfo� i i@B i, z!Credentials need to be refreshed.T) r2 r8 r �nowr �utc�replace� microseconds�seconds�days�boto�log�debug)rF �deltaZ seconds_leftr r r rI � s ��� z"Provider._credentials_need_refreshc C s> | j r d S | j| j \}}}}|d u rJ|d urJ|�� tjv rJtj|�� }| j} |d url|| _tj � d� n�|�� tjv r�tj|�� | _tj � d� n�|d u�r| � ||�r�| � ||�| _tj � d| � nBt � d| |��rt � d| |�| _tj � d| � n td| ��nR| � d|��r<| � d|�| _tj � d� n(t � d |��rdt � d |�| _tj � d � |d u�r�|| _tj � d � �nf|�� tjv �r�tj|�� | _tj � d � �n6|d u�r0| � ||��r�| � ||�| _tj � d | � nBt � d| |��r"t � d| |�| _tj � d| � n td| ��n�| � d|��rZ| � d|�| _tj � d� n�t � d |��r�t � d |�| _tj � d� nft � d d��r�t � d d�} z dd l} W n" t�y� tj �d� � Y n0 | �| | j�| _tj � d� |d u�r|| _tj � d� n�|d u�r�|d u �r�|d u �r�|�� tjv �rTtj|�� | _tj � d� n�| � |�p`d|��r�| � |�ptd|�| _tj � d� nt|d u�r�t � d| |��r�tj � d� t � d| |�| _tj � d| � n(t � d |��r�t � d |�| _tj � d� | jd u �s| jd u �r,| j| j �r,| �� | �| j�| _d S )Nz$Using access key provided by client.z/Using access key found in environment variable.z@Using access key found in shared credential file for profile %s.z profile %sz2Using access key found in config file: profile %s.zProfile "%s" not found!�defaultz1Using access key found in shared credential file.r* z&Using access key found in config file.z$Using secret key provided by client.z/Using secret key found in environment variable.z@Using secret key found in shared credential file for profile %s.z2Using secret key found in config file: profile %s.z1Using secret key found in shared credential file.z&Using secret key found in config file.�keyringr zZThe keyring module could not be imported. For keyring support, install the keyring module.z"Using secret key found in keyring.z(Using security token provided by client.z3Using security token found in environment variable.z5Using security token found in shared credential file.zconfig has optionz6Using security token found in config file: profile %s.z*Using security token found in config file.)r2 � CredentialMapr3 �upperr9 �environr< r. r` ra rb rC rD r r r/ re � ImportError�errorZ get_passwordr0 rK rQ �MetadataServiceSupportrJ �_convert_key_to_str) rF r. r/ r0 r1 Zaccess_key_nameZsecret_key_nameZsecurity_token_nameZprofile_name_nameZsharedZ keyring_namere r r r r? s� � � � � ��� � ��� � �� � � � ��� �zProvider.get_credentialsc C s� t j�d� ddlm} t�ddd�}t�ddd�}|||d d �}|r�| �|�}|d | _ |d | _ |d | _ |d }t � |d �| _t j�d| jt �� |� d S )Nz,Retrieving credentials from metadata server.r )�get_instance_metadata�BotoZmetadata_service_timeoutg �?Z metadata_service_num_attempts� z#meta-data/iam/security-credentials/)�timeout� num_retries�data� � z%Y-%m-%dT%H:%M:%SZz.Retrieved credentials will expire in %s at: %s)r` ra rb � boto.utilsrm r �getfloatrE � _get_credentials_from_metadatarK rQ rT r �strptimer8 rZ )rF rm rp Zattempts�metadata�creds� expires_atr r r rJ � s* � � �z,Provider._populate_keys_from_metadata_serverc C s� t |�� �d }t|t�s<|dkr(d}nd| }td| ��z*|d }| �|d �}|d }|d }W n0 ty� } ztd | ��W Y d }~n d }~0 0 ||||fS ) Nr � zan empty stringztype: %sz7Expected a dict type of credentials instead received %sZ AccessKeyIdZSecretAccessKey�TokenZ Expirationz;Credentials from instance metadata missing required key: %s)�list�values� isinstance�dictr rl �KeyError) rF ry rz �msgr. r/ r0 r{ �er r r rw � s( � ��z'Provider._get_credentials_from_metadatac C s t |tj�rt|�S |S rH )r� r � text_type�str)rF �keyr r r rl � s z Provider._convert_key_to_strc C s� | j | j }|t | _|t | _|t | _|t | _ |t | _ |t | _ |t | _|t | _|t | _|t | _|t | _|t | _|t | _|t | _ |t | _ |t | _!|t" | _#d S rH )$� HeaderInfoMapr3 �METADATA_PREFIX_KEYr �HEADER_PREFIX_KEYr �ACL_HEADER_KEYZ acl_header�AUTH_HEADER_KEY� auth_header�COPY_SOURCE_HEADER_KEYZcopy_source_header�!COPY_SOURCE_VERSION_ID_HEADER_KEYZcopy_source_version_id� COPY_SOURCE_RANGE_HEADER_KEYZcopy_source_range_header�DATE_HEADER_KEY� date_header�DELETE_MARKER_HEADER_KEYZ delete_marker� METADATA_DIRECTIVE_HEADER_KEYZmetadata_directive_header�SECURITY_TOKEN_HEADER_KEY�security_token_header�RESUMABLE_UPLOAD_HEADER_KEYZresumable_upload_header�SERVER_SIDE_ENCRYPTION_KEYZ server_side_encryption_header�STORAGE_CLASS_HEADER_KEYZstorage_class_header�VERSION_ID_HEADER_KEY� version_id�MFA_HEADER_KEYZ mfa_header�RESTORE_HEADER_KEYZrestore_header)rF Zheader_info_mapr r r r@ � s0 �� � � zProvider.configure_headersc C sB | j | j }|t | _|t | _|t | _|t | _ |t | _ d S rH ) �ErrorMapr3 �STORAGE_COPY_ERRORZstorage_copy_error�STORAGE_CREATE_ERRORZstorage_create_error�STORAGE_DATA_ERROR�storage_data_error�STORAGE_PERMISSIONS_ERRORZstorage_permissions_error�STORAGE_RESPONSE_ERROR�storage_response_error)rF Z error_mapr r r rA � s zProvider.configure_errorsc C s | j | j S rH )rB r3 rL r r r �get_provider_name� s zProvider.get_provider_namec C s | j | j S rH )�ChunkedTransferSupportr3 rL r r r �supports_chunked_transfer� s z"Provider.supports_chunked_transfer)NNNNF)NNNN)Gr r r rf r r r4 �CannedS3ACLStrings�CannedGSACLStringsr6 rB r� rk r� �AWS_HEADER_PREFIXr� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� �GOOG_HEADER_PREFIXr� r� r` � exception� S3CopyErrorr� � S3CreateErrorr� � S3DataErrorr� �S3PermissionsErrorr� �S3ResponseError� GSCopyError� GSCreateError� GSDataError�GSPermissionsError�GSResponseErrorr� rG rM rP �propertyr. rR rS r/ rU rV r0 rI r? rJ rw rl r@ rA r� r� r r r r r O s� ������ ���������3��� � $ � v r c C s t d�S )Nr )r r r r r � get_default� s r� )2�__doc__r9 � boto.compatr r r r` r r �boto.pyami.configr �boto.exceptionr Z boto.gs.aclr r r� Z boto.s3.aclr� r r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� �objectZNO_CREDENTIALS_PROVIDED� ValueErrorr r r� r r r r �<module> sX vendored/boto/boto/__pycache__/exception.cpython-39.pyc 0000644 00000046302 15025013617 0017124 0 ustar 00 a sFg�H � @ s� d Z ddlZddlZddlZddlmZ ddlmZmZ ddl m Z G dd� de�Z G dd � d e�Z G d d � d e �Z G d d � d e �ZG dd� de �ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�ZG dd� de�ZG d d � d e�ZG d d � d e�ZG d d!� d!e�ZG d"d#� d#e �ZG d$d%� d%e�ZG d&d'� d'e�Z G d(d)� d)e�Z G d*d+� d+e�Z G d,d-� d-e�Z G d.d/� d/e �Z G d0d1� d1e �Z!G d2d3� d3e�Z"G d4d5� d5e�Z#G d6d7� d7e�Z$G d8d9� d9e �Z%G d:d;� d;e �Z&G d<d=� d=e&�Z'G d>d?� d?e&�Z(G d@dA� dAe)�Z*G dBdC� dCe)�Z+G dDdE� dEe)�Z,G dFdG� dGe)�Z-G dHdI� dIe)�Z.G dJdK� dKe)�Z/G dLdM� dMe�Z0G dNdO� dOe)�Z1G dPdQ� dQe)�Z2G dRdS� dSe)�Z3G dTdU� dUe)�Z4G dVdW� dWe)�Z5dS )XzI Exception classes - Subclassing allows you to check for specific errors � N)�handler)�json� StandardError)� ResultSetc s0 e Zd ZdZ� fdd�Zdd� Zdd� Z� ZS )�BotoClientErrorz9 General Boto Client error (error accessing AWS) c s" t t| �j|g|�R � || _d S �N)�superr �__init__�reason)�selfr �args�� __class__� �1/opt/gsutil/gslib/vendored/boto/boto/exception.pyr ) s zBotoClientError.__init__c C s d| j S �NzBotoClientError: %s�r �r r r r �__repr__- s zBotoClientError.__repr__c C s d| j S r r r r r r �__str__0 s zBotoClientError.__str__��__name__� __module__� __qualname__�__doc__r r r � __classcell__r r r r r % s r c @ s e Zd ZdS )�SDBPersistenceErrorN�r r r r r r r r 4 s r c @ s e Zd ZdZdS )�StoragePermissionsErrorzP Permissions error when accessing a bucket or key on a storage service. N�r r r r r r r r r 8 s r c @ s e Zd ZdZdS )�S3PermissionsErrorzA Permissions error when accessing a bucket or key on S3. Nr r r r r r ? s r c @ s e Zd ZdZdS )�GSPermissionsErrorzA Permissions error when accessing a bucket or key on GS. Nr r r r r r! F s r! c sZ e Zd Zd� fdd� Zdd� Z� fdd�Zdd � Zd d � Zd d � Zdd� Z dd� Z � Z S )�BotoServerErrorNc s� t t| �j|||g|�R � || _|| _|p.d| _d | _d | _d | _d| _ d | _ t | jt �r�z| j� d�| _W n ty� tj�d� Y n0 | j�r�t| jd�r�| j�dd �| _d| jv r�| j�di �}|�dd �| _|�dd �| _ n| j�d d �| _ n�z t�| | �}|�| j� W n� ttjjf�y� zft� | j�}d|v �rP|d | _d|v �r�d|d v �rv|d d | _d|d v �r�|d d | _ W n& tt f�y� | j| _ d | _Y n0 Y n0 d S ) N� zutf-8z!Unable to decode body from bytes!�items� RequestId�Error�Code�Message�message) r r" r �statusr �body� request_id� error_codeZ_error_messager) � box_usage� isinstance�bytes�decode�UnicodeDecodeError�boto�log�debug�hasattr�getr ZXmlHandlerWrapperZ parseString� TypeError�xmlZsaxZSAXParseExceptionr �loads� ValueError)r r* r r+ r �error�h�parsedr r r r N sL zBotoServerError.__init__c C s$ |dkr| j S |dkr | jS t�d S )N� error_message�code)r) r- �AttributeError)r �namer r r � __getattr__� s zBotoServerError.__getattr__c s& |dkr|| _ ntt| ��||� d S )Nr? )r) r r" � __setattr__)r rB �valuer r r rD � s zBotoServerError.__setattr__c C s d| j j| j| j| jf S �Nz %s: %s %s %s�r r r* r r+ r r r r r � s �zBotoServerError.__repr__c C s d| j j| j| j| jf S rF rG r r r r r � s �zBotoServerError.__str__c C s d S r r �r rB �attrs� connectionr r r � startElement� s z BotoServerError.startElementc C sB |dv r|| _ n.|dkr || _n |dkr0|| _n|dkr>|| _d S )N)r% � RequestIDr' r( ZBoxUsage�r, r- r) r. �r rB rE rJ r r r � endElement� s zBotoServerError.endElementc C s d | _ d | _d | _d | _d S r rM r r r r �_cleanupParsedProperties� s z(BotoServerError._cleanupParsedProperties)N) r r r r rC rD r r rK rO rP r r r r r r" M s ; r" c @ s& e Zd Zddd�Zdd� Zdd� ZdS ) � ConsoleOutputNc C s" || _ d | _d | _d | _d | _d S r )�parent� instance_id� timestampZcomment�output)r rR r r r r � s zConsoleOutput.__init__c C s d S r r rH r r r rK � s zConsoleOutput.startElementc C s6 |dkr|| _ n"|dkr&t�|�| _n t| ||� d S )NZ instanceIdrU )rS �base64� b64decoderU �setattrrN r r r rO � s zConsoleOutput.endElement)N�r r r r rK rO r r r r rQ � s rQ c s. e Zd ZdZd� fdd� Z� fdd�Z� ZS )�StorageCreateErrorz> Error creating a bucket or key on a storage service. Nc s d | _ tt| ��|||� d S r )�bucketr rZ r �r r* r r+ r r r r � s zStorageCreateError.__init__c s( |dkr|| _ ntt| ��|||�S d S )NZ BucketName)r[ r rZ rO rN r r r rO � s z StorageCreateError.endElement)N)r r r r r rO r r r r r rZ � s rZ c @ s e Zd ZdZdS )� S3CreateErrorz/ Error creating a bucket or key on S3. Nr r r r r r] � s r] c @ s e Zd ZdZdS )� GSCreateErrorz/ Error creating a bucket or key on GS. Nr r r r r r^ � s r^ c @ s e Zd ZdZdS )�StorageCopyErrorz3 Error copying a key on a storage service. Nr r r r r r_ � s r_ c @ s e Zd ZdZdS )� S3CopyErrorz$ Error copying a key on S3. Nr r r r r r` � s r` c @ s e Zd ZdZdS )� GSCopyErrorz$ Error copying a key on GS. Nr r r r r ra � s ra c sF e Zd ZdZd � fdd� Z� fdd�Z� fdd�Z� fd d �Z� ZS ) �SQSErrorz0 General Error on Simple Queue Service. Nc s$ d | _ d | _tt| ��|||� d S r )�detail�typer rb r r\ r r r r � s zSQSError.__init__c s t t| ��|||�S r )r rb rK rH r r r rK s zSQSError.startElementc s8 |dkr|| _ n$|dkr || _ntt| ��|||�S d S )NZDetail�Type)rc rd r rb rO rN r r r rO s zSQSError.endElementc s( t t| ��� dD ]}t| |d � qd S )N)rc rd )r rb rP rX �r �pr r r rP s z!SQSError._cleanupParsedProperties)N� r r r r r rK rO rP r r r r r rb � s rb c s0 e Zd ZdZ� fdd�Zdd� Zdd� Z� ZS )�SQSDecodeErrorz- Error when decoding an SQS message. c s t t| ��||� || _d S r )r ri r r) )r r r) r r r r s zSQSDecodeError.__init__c C s d| j S �NzSQSDecodeError: %sr r r r r r s zSQSDecodeError.__repr__c C s d| j S rj r r r r r r s zSQSDecodeError.__str__r r r r r ri s ri c sF e Zd ZdZd � fdd� Z� fdd�Z� fdd�Z� fd d �Z� ZS ) �StorageResponseErrorz3 Error in response from a storage service. Nc s d | _ tt| ��|||� d S r )�resourcer rk r r\ r r r r & s z StorageResponseError.__init__c s t t| ��|||�S r �r rk rK rH r r r rK * s �z!StorageResponseError.startElementc s( |dkr|| _ ntt| ��|||�S d S )NZResource)rl r rk rO rN r r r rO . s �z StorageResponseError.endElementc s( t t| ��� dD ]}t| |d � qd S )Nrl �r rk rP rX rf r r r rP 5 s z-StorageResponseError._cleanupParsedProperties)Nrh r r r r rk " s rk c sF e Zd ZdZd � fdd� Z� fdd�Z� fdd�Z� fd d �Z� ZS ) �S3ResponseErrorz$ Error in response from S3. Nc s$ d | _ d | _tt| ��|||� d S r )�region�endpointr rk r r\ r r r r ? s zS3ResponseError.__init__c s t t| ��|||�S r rm rH r r r rK D s �z S3ResponseError.startElementc sH |dkr|| _ n4|dkr || _ n$|dkr0|| _ntt| ��|||�S d S )NZRegionZLocationConstraintZEndpoint)rp rq r rk rO rN r r r rO H s �zS3ResponseError.endElementc s( t t| ��� dD ]}t| |d � qd S )N)rp rq rn rf r r r rP S s z(S3ResponseError._cleanupParsedProperties)Nrh r r r r ro ; s ro c @ s e Zd ZdZdS )�GSResponseErrorz$ Error in response from GS. Nr r r r r rr Y s rr c s> e Zd ZdZd � fdd� Zdd� Zdd� Z� fd d �Z� ZS ) �EC2ResponseErrorz% Error in response from EC2. Nc sR d | _ g | _tt| ��|||� dd� | jD �| _ t| j �rN| j d \| _| _d S )Nc S s g | ]}|j |jf�qS r �r- r? )�.0�er r r � <listcomp>h s z-EC2ResponseError.__init__.<locals>.<listcomp>r )�errors�_errorResultSetr rs r �lenr- r? r\ r r r r d s � zEC2ResponseError.__init__c C s& |dkr t dtfg�| _| jS d S d S )NZErrorsr& )r � _EC2Errorry rH r r r rK m s z EC2ResponseError.startElementc C s |dkr|| _ nd S d S )NrL )r, rN r r r rO t s zEC2ResponseError.endElementc s. t t| ��� g | _dD ]}t| |d � qd S )Nrx )r rs rP ry rX rf r r r rP z s z)EC2ResponseError._cleanupParsedProperties)Nrh r r r r rs ` s rs c @ s e Zd ZdZddd�ZdS )�JSONResponseErrora� This exception expects the fully parsed and decoded JSON response body to be passed as the body parameter. :ivar status: The HTTP status code. :ivar reason: The HTTP reason message. :ivar body: The Python dict that represents the decoded JSON response body. :ivar error_message: The full description of the AWS error encountered. :ivar error_code: A short string that identifies the AWS error (e.g. ConditionalCheckFailedException) Nc G sT || _ || _|| _| jrP| j�dd �| _| j�dd �| _| jrP| j�d�d | _d S )Nr) Z__type�#���)r* r r+ r7 r? r- �split)r r* r r+ r r r r r � s zJSONResponseError.__init__)N)r r r r r r r r r r| � s r| c @ s e Zd ZdS )�DynamoDBResponseErrorNr r r r r r� � s r� c @ s e Zd ZdS )�SWFResponseErrorNr r r r r r� � s r� c @ s e Zd ZdZdS )�EmrResponseErrorz$ Error in response from EMR Nr r r r r r� � s r� c @ s& e Zd Zddd�Zdd� Zdd� ZdS ) r{ Nc C s || _ d | _d | _d S r )rJ r- r? )r rJ r r r r � s z_EC2Error.__init__c C s d S r r rH r r r rK � s z_EC2Error.startElementc C s( |dkr|| _ n|dkr || _nd S d S )Nr' r( rt rN r r r rO � s z_EC2Error.endElement)NrY r r r r r{ � s r{ c @ s e Zd ZdZdS )�SDBResponseErrorz& Error in responses from SDB. Nr r r r r r� � s r� c @ s e Zd ZdZdS )�AWSConnectionErrorz: General error connecting to Amazon Web Services. Nr r r r r r� � s r� c @ s e Zd ZdZdS )�StorageDataErrorz6 Error receiving data from a storage service. Nr r r r r r� � s r� c @ s e Zd ZdZdS )� S3DataErrorz' Error receiving data from S3. Nr r r r r r� � s r� c @ s e Zd ZdZdS )� GSDataErrorz' Error receiving data from GS. Nr r r r r r� � s r� c s e Zd ZdZ� fdd�Z� ZS )�InvalidUriErrorz%Exception raised when URI is invalid.c s t t| ��|� || _d S r )r r� r r) �r r) r r r r � s zInvalidUriError.__init__�r r r r r r r r r r r� � s r� c s e Zd ZdZ� fdd�Z� ZS )�InvalidAclErrorz)Exception raised when ACL XML is invalid.c s t t| ��|� || _d S r )r r� r r) r� r r r r � s zInvalidAclError.__init__r� r r r r r� � s r� c s e Zd ZdZ� fdd�Z� ZS )�InvalidCorsErrorz*Exception raised when CORS XML is invalid.c s t t| ��|� || _d S r )r r� r r) r� r r r r � s zInvalidCorsError.__init__r� r r r r r� � s r� c s e Zd ZdZ� fdd�Z� ZS )� InvalidEncryptionConfigErrorzBException raised when GCS encryption configuration XML is invalid.c s t t| ��|� || _d S r )r r� r r) r� r r r r � s z%InvalidEncryptionConfigError.__init__r� r r r r r� � s r� c s e Zd ZdZ� fdd�Z� ZS )�InvalidLifecycleConfigErrorzAException raised when GCS lifecycle configuration XML is invalid.c s t t| ��|� || _d S r )r r� r r) r� r r r r s z$InvalidLifecycleConfigError.__init__r� r r r r r� � s r� c @ s e Zd ZdZdS )�NoAuthHandlerFoundzAIs raised when no auth handlers were found ready to authenticate.Nr r r r r r� s r� c @ s e Zd ZdZdZdZdZdS )� ResumableTransferDisposition� START_OVER�WAIT_BEFORE_RETRY�ABORT_CUR_PROCESS�ABORTN)r r r r� r� r� r� r r r r r� s r� c s( e Zd ZdZ� fdd�Zdd� Z� ZS )�ResumableUploadExceptionz� Exception raised for various resumable upload problems. self.disposition is of type ResumableTransferDisposition. c s" t t| ��||� || _|| _d S r )r r� r r) � disposition�r r) r� r r r r * s z!ResumableUploadException.__init__c C s d| j | jf S )Nz"ResumableUploadException("%s", %s)�r) r� r r r r r / s �z!ResumableUploadException.__repr__�r r r r r r r r r r r r� # s r� c s( e Zd ZdZ� fdd�Zdd� Z� ZS )�ResumableDownloadExceptionz� Exception raised for various resumable download problems. self.disposition is of type ResumableTransferDisposition. c s" t t| ��||� || _|| _d S r )r r� r r) r� r� r r r r ; s z#ResumableDownloadException.__init__c C s d| j | jf S )Nz$ResumableDownloadException("%s", %s)r� r r r r r @ s �z#ResumableDownloadException.__repr__r� r r r r r� 4 s r� c s e Zd ZdZ� fdd�Z� ZS )�TooManyRecordsExceptionzd Exception raised when a search of Route53 records returns more records than requested. c s t t| ��|� || _d S r )r r� r r) r� r r r r K s z TooManyRecordsException.__init__r� r r r r r� E s r� c @ s" e Zd ZdZddd�Zdd� ZdS )�PleaseRetryExceptionz0 Indicates a request should be retried. Nc C s || _ || _d S r �r) �response)r r) r� r r r r T s z PleaseRetryException.__init__c C s d| j | jf S )Nz PleaseRetryException("%s", %s)r� r r r r r X s �z PleaseRetryException.__repr__)N)r r r r r r r r r r r� P s r� c s e Zd ZdZ� fdd�Z� ZS )� InvalidInstanceMetadataErrorz�You can set the 'metadata_service_num_attempts' in your boto config file to increase the number of times boto will attempt to retrieve credentials from the instance metadata service.c s" |d | j }tt| ��|� d S )N� )�MSGr r� r )r �msgZ final_msgr r r r f s z%InvalidInstanceMetadataError.__init__)r r r r� r r r r r r r� _ s �r� )6r rV Zxml.saxr9 r3 r � boto.compatr r Zboto.resultsetr r r r r r! r" �objectrQ rZ r] r^ r_ r` ra rb ri rk ro rr rs r| r� r� r� r{ r� r� r� r� r� � Exceptionr� r� r� r� r� r� r� r� r� r� r� r� r r r r �<module> s^ f ! vendored/boto/boto/__pycache__/auth_handler.cpython-39.pyc 0000644 00000002777 15025013617 0017574 0 ustar 00 a sFg � @ s4 d Z ddlmZ G dd� de�ZG dd� de�ZdS )zA Defines an interface which all Auth handlers need to implement. � )�Pluginc @ s e Zd ZdS )�NotReadyToAuthenticateN)�__name__� __module__� __qualname__� r r �4/opt/gsutil/gslib/vendored/boto/boto/auth_handler.pyr s r c @ s e Zd Zg Zdd� Zdd� ZdS )� AuthHandlerc C s dS )a� Constructs the handlers. :type host: string :param host: The host to which the request is being sent. :type config: boto.pyami.Config :param config: Boto configuration. :type provider: boto.provider.Provider :param provider: Provider details. Raises: NotReadyToAuthenticate: if this handler is not willing to authenticate for the given provider and config. Nr )�self�host�config�providerr r r �__init__% s zAuthHandler.__init__c C s dS )z�Invoked to add authentication details to request. :type http_request: boto.connection.HTTPRequest :param http_request: HTTP request that needs to be authenticated. Nr )r � http_requestr r r �add_auth6 s zAuthHandler.add_authN)r r r � capabilityr r r r r r r ! s r N)�__doc__� boto.pluginr � Exceptionr r r r r r �<module> s vendored/boto/boto/__pycache__/resultset.cpython-39.pyc 0000644 00000011173 15025013617 0017156 0 ustar 00 a sFg� � @ s0 d dl mZ G dd� de�ZG dd� de�ZdS )� )�Userc @ s4 e Zd ZdZd dd�Zdd� Zd dd �Zd d � ZdS )� ResultSeta& The ResultSet is used to pass results back from the Amazon services to the client. It is light wrapper around Python's :py:class:`list` class, with some additional methods for parsing XML results from AWS. Because I don't really want any dependencies on external libraries, I'm using the standard SAX parser that comes with Python. The good news is that it's quite fast and efficient but it makes some things rather difficult. You can pass in, as the marker_elem parameter, a list of tuples. Each tuple contains a string as the first element which represents the XML element that the resultset needs to be on the lookout for and a Python class as the second element of the tuple. Each time the specified element is found in the XML, a new instance of the class will be created and popped onto the stack. :ivar str next_token: A hash used to assist in paging through very long result sets. In most cases, passing this value to certain methods will give you another 'page' of results. Nc C sh t �| � t|t �r || _ng | _d | _d | _d | _d | _d | _d | _ d | _ d | _ d| _ d | _ d| _d S )NFT)�list�__init__� isinstance�markers�marker� key_marker� next_marker�next_key_marker�next_upload_id_marker�next_version_id_marker�next_generation_marker�version_id_marker� is_truncated� next_token�status��selfZ marker_elem� r �1/opt/gsutil/gslib/vendored/boto/boto/resultset.pyr . s zResultSet.__init__c C sP | j D ].}||d kr|d |�}| �|� | S q|dkrLt� | _| jS d S )Nr � ZOwner)r �appendr �owner)r �name�attrs� connection�t�objr r r � startElement@ s zResultSet.startElement�truec C s ||kr dS dS d S �NTFr �r �valueZ true_valuer r r � to_booleanN s zResultSet.to_booleanc C s� |dkr| � |�| _�n�|dkr*|| _�n�|dkr<|| _�n�|dkrN|| _�n�|dkr`|| _�nn|dkrr|| _�n\|dkr�|| _�nJ|dkr�|| _�n8|d kr�|| _ �n&|d kr�|| _ �n|d kr�|| _ �n|d kr�t |�| _ n�|d kr�t |�| _n�|dk�r|| _n�|dk�r | � |�| _n�|dk�r8| � |d�| _n�|dk�rN| �|� n�|dk�r`|| _nn|dk�rx|| _|| _nV|dk�r�z| jt|�7 _W n Y n0 n&|dk�r�| � |d�| _n t| ||� d S )NZ IsTruncatedZMarkerZ KeyMarkerZ NextMarkerZ NextKeyMarkerZVersionIdMarkerZNextVersionIdMarkerZNextGenerationMarkerZUploadIdMarkerZNextUploadIdMarkerZBucketZ MaxUploadsZMaxItemsZPrefix�return� StatusCode�SuccessZItemNameZ NextToken� nextToken�BoxUsage�IsValid�True)r$ r r r r r r r r Zupload_id_markerr �bucket�intZ max_uploadsZ max_items�prefixr r r r( � box_usage�float�setattr�r r r# r r r r � endElementT s^ zResultSet.endElement)N)r )�__name__� __module__� __qualname__�__doc__r r r$ r3 r r r r r s r c @ s@ e Zd Zddd�Zdd� Zdd� Zdd � Zdd d �Zd d� ZdS )� BooleanResultNc C s d| _ d | _d | _d S )NT)r � request_idr/ r r r r r � s zBooleanResult.__init__c C s | j r dS dS d S )Nr+ �False�r �r r r r �__repr__� s zBooleanResult.__repr__c C s | j S �Nr; r< r r r � __nonzero__� s zBooleanResult.__nonzero__c C s d S r> r )r r r r r r r r � s zBooleanResult.startElementr c C s ||kr dS dS d S r! r r" r r r r$ � s zBooleanResult.to_booleanc C s� |dkr| � |�| _nl|dkr.| � |d�| _nT|dkrF| � |d�| _n<|dkrV|| _n,|dkrf|| _n |dkrv|| _n t| ||� d S ) Nr% r& r' r* r+ � RequestIdZ requestIdr) )r$ r r9 r1 r2 r r r r3 � s zBooleanResult.endElement)N)r ) r4 r5 r6 r r= r? r r$ r3 r r r r r8 � s r8 N)Z boto.s3.userr r r �objectr8 r r r r �<module> s p vendored/boto/boto/__pycache__/jsonresponse.cpython-39.pyc 0000644 00000010405 15025013617 0017651 0 ustar 00 a sFg� � @ sL d dl Zd dlmZ G dd� dejj�ZG dd� de�ZG dd� de �Z dS ) � N)�utilsc @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) � XmlHandlerc C s || _ d|fg| _d| _d S )N�root� )� connection�nodes� current_text)�self� root_noder � r �4/opt/gsutil/gslib/vendored/boto/boto/jsonresponse.py�__init__ s zXmlHandler.__init__c C sT d| _ | jd d �||| j�}|d urPt|t�r@| j�|� n| j�||f� d S )Nr ���� )r r � startElementr � isinstance�tuple�append)r �name�attrs�tr r r r " s zXmlHandler.startElementc C sB | j d d �|| j| j� | j d d |kr8| j �� d| _d S )Nr r r r )r � endElementr r �pop�r r r r r r + s zXmlHandler.endElementc C s | j |7 _ d S �N)r )r �contentr r r � characters1 s zXmlHandler.charactersc C s& t |t�s|�d�}tj�|| � d S )Nzutf-8)r �bytes�encode�xml�sax� parseString)r �sr r r �parse4 s zXmlHandler.parseN)�__name__� __module__� __qualname__r r r r r# r r r r r s r c @ s6 e Zd Zddd�Zdd� Zd d � Zd d � Zd d� ZdS )�ElementN��Set��member�itemFc C sT t �| � || _|| _t�|�| _t�|�| _|d u r>g | _n|| _|| _ || _ d S r ) �dictr r � element_namer �mklist� list_marker� item_marker�stack�pythonize_name�parent)r r r. r2 r4 r0 r1 r3 r r r r < s zElement.__init__c C sX || v r| | S | D ]:}| | }t |t�rzt||�W S tyL Y q0 qt�d S r )r r' �getattr�AttributeError)r �key�k�er r r � __getattr__L s zElement.__getattr__c C s | j rt� |�}|S r �r3 r r r r r �get_nameX s zElement.get_namec C s� | j �|� | jD ]<}|�|�rt| j|| j| j| j�}|| | �|�< | S qt | j �dkr�| j d }t | j|| j | | j| j| j�}|| | �|�< ||fS d S d S )Nr r ) r2 r r0 �endswith� ListElementr r1 r3 r< �lenr' )r r r r �lm�lr. r9 r r r r ] s" � �zElement.startElementc C s^ t | j�dkr| j�� |�� }|rZt| jt�rB|| j| �|�< nt| jt�rZ| j� |� d S �Nr ) r? r2 r �stripr r4 r' r< r> r )r r �valuer r r r r o s zElement.endElement)NNNNr( r* F)r$ r% r& r r: r< r r r r r r r' : s � r' c @ s: e Zd Zdddgddfdd�Zdd� Zd d � Zd d � ZdS ) r> Nr) r* Fc C s, t �| � || _|| _|| _|| _|| _d S r )�listr r r. r0 r1 r3 )r r r. r0 r1 r3 r r r r | s zListElement.__init__c C s | j rt� |�}|S r r; r r r r r< � s zListElement.get_namec C s� | j D ]@}|�|�rt| j|| j | j| j�}t| | �|�|� | S q|| jv r|t| j|| | j | j| jd�}| � |� |S d S d S )N)r4 r0 r1 r3 ) r0 r= r> r r1 r3 �setattrr< r' r )r r r r r@ rA r9 r r r r � s" � � zListElement.startElementc C sp || j krZt| �dkrlg }| D ]$}t|t�r t|�dkr |�|� q |D ]}| �|� qHnt| | �|�|� d S rB )r. r? r r' r �removerF r< )r r rD r �emptyr9 r r r r � s zListElement.endElement)r$ r% r& r r< r r r r r r r> z s � r> ) �xml.saxr �botor r �ContentHandlerr r- r' rE r> r r r r �<module> s @ vendored/boto/boto/__pycache__/endpoints.cpython-39.pyc 0000644 00000020130 15025013617 0017120 0 ustar 00 a sFg& � @ sN d dl m m mZ G dd� dej�ZG dd� de�ZG dd� de�ZdS )� Nc s� e Zd ZdZdddddd�Zd � fd d � Zd � fd d� Zdd� Zd � fdd� Zdd� Z d dd�Z dd� Z dd� Z dd � Z � ZS )!�_CompatEndpointResolverzyEndpoint resolver which handles boto2 compatibility concerns. This is NOT intended for external use whatsoever. �lambdaZ monitoring�emailZecs�config)Z awslambdaZ cloudwatchZsesZec2containerserviceZ configserviceNc s@ t t| ��|� |du r | j}|| _tdd� |�� D ��| _dS )�. :type endpoint_data: dict :param endpoint_data: Regions and endpoints data in the same format as is used by botocore / boto3. :type service_rename_map: dict :param service_rename_map: A mapping of boto2 service name to endpoint prefix. Nc s s | ]\}}||fV qd S �N� )�.0�k�vr r �1/opt/gsutil/gslib/vendored/boto/boto/endpoints.py� <genexpr>2 s z3_CompatEndpointResolver.__init__.<locals>.<genexpr>)�superr �__init__�_DEFAULT_SERVICE_RENAMES�_endpoint_prefix_map�dict�items�_service_name_map��selfZ endpoint_dataZservice_rename_map�� __class__r r r " s �z _CompatEndpointResolver.__init__�awsFc s | � |�}tt| ��|||�S r )�_endpoint_prefixr r �get_available_endpoints)r � service_name�partition_nameZallow_non_regional�endpoint_prefixr r r r 5 s �z/_CompatEndpointResolver.get_available_endpointsc C sd t � }| �|�}| �� D ]B}| �||�rH| �|�}|�|d �� � qq|�| �||�� qt|�S )z6Retrieve every region across partitions for a service.�regions) �setr Zget_available_partitions�_is_global_service�_get_partition_data�update�keysr �list)r r r r r � partitionr r r �get_all_available_regions; s ��z1_CompatEndpointResolver.get_all_available_regionsc s | � |�}tt| ��||�S r )r r r �construct_endpoint)r r � region_namer r r r r( P s �z*_CompatEndpointResolver.construct_endpointc s: t � }� jd D ]}|�|d �� � q� fdd�|D �S )zAGet a list of all the available services in the endpoints file(s)� partitions�servicesc s g | ]}� � |��qS r )� _service_name)r �s�r r r � <listcomp>\ � zB_CompatEndpointResolver.get_available_services.<locals>.<listcomp>)r �_endpoint_datar# r$ )r r+ r& r r. r �get_available_servicesU s z._CompatEndpointResolver.get_available_servicesc C s, | � |�}| �|�}|d �|i �}d|v S )z�Determines whether a service uses a global endpoint. In theory a service can be 'global' in one partition but regional in another. In practice, each service is all global or all regional. r+ ZpartitionEndpoint)r r" �get)r r r r r& �servicer r r r! ^ s z*_CompatEndpointResolver._is_global_servicec C s4 | j d D ]}|d |kr | S q td| ��dS )a� Get partition information for a particular partition. This should NOT be used to get service endpoint data because it only loads from the new endpoint format. It should only be used for partition metadata and partition specific service metadata. :type partition_name: str :param partition_name: The name of the partition to search for. :returns: Partition info from the new endpoints format. :rtype: dict or None r* r& z%Could not find partition data for: %sN)r1 � ValueError)r r r& r r r r" i s �z+_CompatEndpointResolver._get_partition_datac C s | j �||�S )z4Given a boto2 service name, get the endpoint prefix.)r r3 �r r r r r r | s z(_CompatEndpointResolver._endpoint_prefixc C s | j �||�S )z5Given an endpoint prefix, get the boto2 service name.)r r3 )r r r r r r, � s z%_CompatEndpointResolver._service_name)N)r F)N)r )�__name__� __module__� __qualname__�__doc__r r r r' r( r2 r! r" r r, � __classcell__r r r r r s$ � � r c @ s2 e Zd ZdZd dd�Zdd� Zdd� Zd d � ZdS ) �BotoEndpointResolverz^Resolves endpoint hostnames for AWS services. This is NOT intended for external use. Nc C s t ||�| _dS )r N)r � _resolverr r r r r � s �z BotoEndpointResolver.__init__c C s* | j �||�}|du rdS |�d|d �S )aD Resolve the hostname for a service in a particular region. :type service_name: str :param service_name: The service to look up. :type region_name: str :param region_name: The region to find the endpoint for. :return: The hostname for the given service in the given region. NZ sslCommonName�hostname)r= r( r3 )r r r) �endpointr r r �resolve_hostname� s z%BotoEndpointResolver.resolve_hostnamec C s | j �|�S )z�Get all the regions a service is available in. :type service_name: str :param service_name: The service to look up. :rtype: list of str :return: A list of all the regions the given service is available in. )r= r' r6 r r r r' � s z.BotoEndpointResolver.get_all_available_regionsc C s | j �� S )z�Get all the services supported by the endpoint data. :rtype: list of str :return: A list of all the services explicitly contained within the endpoint data provided during instantiation. )r= r2 r. r r r r2 � s z+BotoEndpointResolver.get_available_services)N)r7 r8 r9 r: r r@ r' r2 r r r r r< � s r< c @ s2 e Zd ZdZdd� Zd dd�Zdd� Zd d � ZdS ) �StaticEndpointBuilderz:Builds a static mapping of endpoints in the legacy format.c C s || _ dS )ze :type resolver: BotoEndpointResolver :param resolver: An endpoint resolver. N)r= )r �resolverr r r r � s z StaticEndpointBuilder.__init__Nc C sD |du r| j �� }i }|D ]}| �|�}|r|||< q| �|� |S )a� Build a set of static endpoints in the legacy boto2 format. :param service_names: The names of the services to build. They must use the names that boto2 uses, not boto3, e.g "ec2containerservice" and not "ecs". If no service names are provided, all available services will be built. :return: A dict consisting of:: {"service": {"region": "full.host.name"}} N)r= r2 � _build_endpoints_for_service�_handle_special_cases)r Z service_names�static_endpoints�nameZendpoints_for_servicer r r �build_static_endpoints� s z,StaticEndpointBuilder.build_static_endpointsc C s0 i }| j �|�}|D ]}| j �||�||< q|S r )r= r' r@ )r r � endpointsr r) r r r rC � s � z2StaticEndpointBuilder._build_endpoints_for_servicec C s d|v r|d }||d< d S )NZ cloudsearchZcloudsearchdomainr )r rE Zcloudsearch_endpointsr r r rD � s z+StaticEndpointBuilder._handle_special_cases)N)r7 r8 r9 r: r rG rC rD r r r r rA � s rA ) Z boto.vendored.regions.regions�vendoredr Z_regionsZEndpointResolverr �objectr< rA r r r r �<module> s u8 vendored/boto/boto/__pycache__/utils.cpython-39.pyc 0000644 00000100260 15025013617 0016260 0 ustar 00 a sFg�� � @ s� d Z ddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlmZmZmZm Z ddl m Z ddl m Z m!Z! e!Z"ddlm#Z# zddl$m%Z% W n e&�y e'Z%Y n0 g d�Z(e �)d �Z*e �)d �Z+e �)d �Z,d d � Z-dedd�Z.dfdd�Z/dgdd�Z0dhdd�Z1didd�Z2G dd� de3�Z4d d � Z5djd"d#�Z6dkd$d%�Z7dld&d'�Z8d(Z9d)Z:d*Z;e�<� Z=e d+d,� �Z>dmd-d.�Z?d/d0� Z@dnd1d2�ZAd3d4� ZBdod5d6�ZCG d7d8� d8eD�ZEG d9d:� d:ejFjG�ZHG d;d<� d<e3�ZIG d=d>� d>eD�ZJdpd?d@�ZKdAdB� ZLdCdD� ZMdEdF� ZNdqdJdK�ZOdLdM� ZPdrdOdP�ZQdNde fdQdR�ZRdSdT� ZSdUdV� ZTG dWdX� dXeD�ZUdYdZ� ZVd[d\� ZWdsd^d_�ZXd`dB� ZLdadb� ZYdcdd� ZZdS )tz7 Some handy utility functions used by several classes. � N)�six�StringIO�urllib� encodebytes)�contextmanager)�md5�sha512)�json)�JSONDecodeError) �acl�corsZdefaultObjectAcl�location�loggingZ partNumber�policyZrequestPayment�torrentZ versioningZ versionIdZversionsZwebsiteZuploadsZuploadIdzresponse-content-typezresponse-content-languagezresponse-expireszresponse-cache-controlz response-content-dispositionzresponse-content-encoding�delete� lifecycleZtagging�restoreZ storageClassZ websiteConfig�composeZbillingZ userProjectZencryptionConfigz(.)([A-Z][a-z]+)z([a-z])([0-9]+)z([a-z0-9])([A-Z])c C s, t | �dkr| S | d tj�| d �fS d S )N� r )�lenr �parse�unquote)�nv� r �-/opt/gsutil/gslib/vendored/boto/boto/utils.py� unquote_vn s r c C s~ |st j�� }i }|D ]@}|�� }|| dur|dv sB|�|j�rt|| ��� ||< qd|vrhd|d< d|vrxd|d< |j|v r�d|d< |r�t|�|d< t |� � �}d| } |D ]6}|| } |�|j�r�| d|| f 7 } q�| d| 7 } q�|� d �} | | d 7 } t | �d k�rz| d � d �} d d� | D �} dd� | D �} t | �d k�rz| j dd� d� dd� | D �} | d 7 } | d �| �7 } | S )zE Generates the aws canonical string for the given parameters N)� content-md5� content-type�dater � r r z%s z%s:%s �?r r �&c S s g | ]}|� d d��qS )�=r )�split��.0�ar r r � <listcomp>� � z$canonical_string.<locals>.<listcomp>c S s g | ]}|d t v rt|��qS )r )�qsa_of_interestr r% r r r r( � r) c S s | d S �Nr r )�xr r r �<lambda>� r) z"canonical_string.<locals>.<lambda>)�keyc S s g | ]}d � |��qS )r# )�joinr% r r r r( � r) )�boto�provider� get_default�lower� startswithZ header_prefix�str�stripZ date_header�sorted�keysr$ r �sortr/ ) �method�path�headers�expiresr1 Zinteresting_headersr. ZlkZsorted_header_keys�buf�val�tZqsar r r �canonical_stringu sL � � rA c C s` |st j�� }|j}| �� }|�� D ]6}|�� t jjj j v rJ|| ||< q$|| ||| < q$|S �N) r0 r1 r2 �metadata_prefix�copyr8 r3 �s3r. �KeyZbase_user_settable_fields)r< �metadatar1 rC Z final_headers�kr r r � merge_meta� s rI c C s� |st j�� }|j}i }| �� D ]h}|�� �|�r tj� | | �}t |t �rnz|� d�}W n t yl Y n0 |||t|�d � < | |= q |S �N�utf-8)r0 r1 r2 rC r8 r3 r4 r r r � isinstance�bytes�decode�UnicodeDecodeErrorr )r< r1 rC rG �hkeyr? r r r �get_aws_metadata� s rQ T� c C s: t d|�D �] }zdtj�i �}tj�|�}tj�| �}|j||d�}|�� } t| t j �slt | d�rl| � d�} | W S tj jy� } z0| �� } | dkr�|s�W Y d} ~ dS W Y d} ~ n8d} ~ 0 ty� } ztj�d� W Y d} ~ n d} ~ 0 0 |d |kr tj�d � t�td | tj�d d d��� q tj� d� dS )a Retry a url. This is specifically used for accessing the metadata service on an instance. Since this address should never be proxied (for security reasons), we create a ProxyHandler with a NULL dictionary to override any proxy settings in the environment. r )�timeoutrN rK i� Nr z&Caught exception reading instance datar zSleeping before retrying� �Boto�max_retry_delay�< z'Unable to read instance data, giving up)�ranger �request� ProxyHandler� build_opener�Request�open�readrL r � string_types�hasattrrN �error� HTTPError�getcode� Exceptionr0 �log� exception�debug�time�sleep�min�config�get) �url� retry_on_404� num_retriesrS �iZ proxy_handler�opener�req�r�result�e�coder r r � retry_url� s2 � $" � rw c C s t | ||�S rB )�LazyLoadMetadata)rm ro rS r r r �_get_instance_metadata� s ry c sh e Zd Zddd�Zdd� Z� fdd�Zddd �Z� fd d �Z� fd d �Z� fdd�Z � fdd�Z � Z S )rx Nc C s� || _ || _i | _g | _|| _tjj| j | j| jd�}|r�|�d�}|D ]v}|� d�rp|dd� }| j� |� nF|� d�}|dkr�||d d � }|d|� d } n| }} | | j|< d | |< qHd S ) N�ro rS � �/r ���r# r z /openssh-key) �_url� _num_retries�_leaves�_dicts�_timeoutr0 �utilsrw r$ �endswith�append�find) �selfrm ro rS �data�fields�fieldr. �p�resourcer r r �__init__� s& zLazyLoadMetadata.__init__c C s | D ] }| | qd S rB r �r� r. r r r � _materialize s z LazyLoadMetadata._materializec s& || vrt t| ��|�S t t| ��|�}|d ur4|S || jv �r�| j| }d }td| j�D �]`}zttjj| j t j j |dd� | j| j d�}|r�|d dkr�t�|�}W �q�n$|�d�}|dkr�|�d�}W �q�W n� t�y& } z8tj�d|jj|f � tj�d| � |}W Y d }~nLd }~0 t�yh } z*tj�d d |jj � |}� W Y d }~n d }~0 0 tj�d d |d � |d | jkrZtt�� d| tj� ddd��}t � |� qZtj�d� tj�d|jj|f � |�|| |< n&|| j v �rt| j | d | j�| |< t t| ��|�S )Nr z/:)�saferz �{r{ z encountered '%s' exception: %sz corrupted JSON data found: %szencountered unretryablez '%s' exception, re-raisingz"Caught exception reading meta dataz for the '%s' tryr rT rU rV rW z#Unable to read meta data, giving upr| ) �superrx � __getitem__r� rX r r0 r� rw r~ r r �quoter� r �loadsr� r$ r re rg � __class__�__name__rd ra rj �randomrk rl rh ri r� ) r� r. r? r� Zlast_exceptionrp r� ru Z next_sleep�r� r r r� s� �� ����� �� �� �z LazyLoadMetadata.__getitem__c C s& z | | W S t y | Y S 0 d S rB )�KeyError)r� r. �defaultr r r rl V s zLazyLoadMetadata.getc s | � � tt| ��� S rB )r� r� rx �values�r� r� r r r� \ s zLazyLoadMetadata.valuesc s | � � tt| ��� S rB )r� r� rx �itemsr� r� r r r� ` s zLazyLoadMetadata.itemsc s | � � tt| ��� S rB )r� r� rx �__str__r� r� r r r� d s zLazyLoadMetadata.__str__c s | � � tt| ��� S rB )r� r� rx �__repr__r� r� r r r� h s zLazyLoadMetadata.__repr__)N)N) r� � __module__� __qualname__r� r� r� rl r� r� r� r� � __classcell__r r r� r rx � s C rx c C s d| ||f S )ax Builds an EC2 metadata URL for fetching information about an instance. Example: >>> _build_instance_metadata_url('http://169.254.169.254', 'latest', 'meta-data/') http://169.254.169.254/latest/meta-data/ :type url: string :param url: URL to metadata service, e.g. 'http://169.254.169.254' :type version: string :param version: Version of the metadata to get, e.g. 'latest' :type path: string :param path: Path of the metadata to get, e.g. 'meta-data/'. If a trailing slash is required it must be passed in with the path. :return: The full metadata URL z%s/%s/%sr )rm �versionr; r r r � _build_instance_metadata_urlm s r� �latest�http://169.254.169.254� meta-data/� c C sH z t || |�}t|||d�W S tjjyB tj�d|� Y dS 0 dS )a� Returns the instance metadata as a nested Python dictionary. Simple values (e.g. local_hostname, hostname, etc.) will be stored as string values. Values such as ancestor-ami-ids will be stored in the dict as a list of string values. More complex fields such as public-keys and will be stored as nested dicts. If the timeout is specified, the connection to the specified url will time out after the specified number of seconds. rz zBException caught when trying to retrieve instance metadata for: %sN)r� ry r ra �URLErrorr0 re rf )r� rm r� rS ro Z metadata_urlr r r �get_instance_metadata� s �r� c C s� i }t || d�}zdt|||d�}|�d�}|D ]@}t|d | d ||d�} | d dkrbt�| �} |r.| ||< q.|W S tjjy� Y dS 0 dS )zF Returns the instance identity as a nested Python dictionary. zdynamic/instance-identity/rz r{ r| r r� N)r� rw r$ r r� r ra r� ) r� rm rS ro ZiidZbase_urlr� r� r� r? r r r �get_instance_identity� s � r� c C sb t || d�}t|d||d�}|r^|r^|�|�}i }|D ]&}|�d�} | d �� || d �� < q6|S )Nz user-dataF)rn ro rS r# r r )r� rw r$ r6 ) r� �seprm rS ro Zud_urlZ user_data�lZnvpairr@ r r r �get_instance_userdata� s r� z%Y-%m-%dT%H:%M:%SZz%Y-%m-%dT%H:%M:%S.%fZz%a, %d %b %Y %H:%M:%S %Zc c sh t �P t�tj�}z"t�tj| �V W t�tj|� nt�tj|� 0 W d � n1 sZ0 Y dS )zE A context manager to set the locale in a threadsafe manner. N)� LOCALE_LOCK�locale� setlocale�LC_ALL)�nameZsavedr r r r� � s r� c C s | s t �� } t �t| �S rB )rh �gmtime�strftime�ISO8601)�tsr r r �get_ts� s r� c C s� t d��� | �� } z"tj�| t�}|W W d � S ty� z&tj�| t�}|W Y W d � S ty� tj�| t�}| Y Y W d � S 0 Y n0 W d � n1 s�0 Y d S )N�C)r� r6 �datetime�strptimer� � ValueError� ISO8601_MS�RFC1123)r� �dtr r r �parse_ts� s r� c C sv |rd| |f } | � d�}d }zD|dd � D ]0}|r@t||�}q,ttd�|dd� ��|�}q,|W S Y d S 0 d S )Nz%s.%s�.r r r} )r$ �getattr� __import__r/ )� module_name� class_name�modules�c�mr r r � find_class� s r� c C s, d}|d7 }t j�|| |||f �}|�� S )z= Update your Dynamic DNS record with DNSMadeEasy.com z,https://www.dnsmadeeasy.com/servlet/updateipz$?username=%s&password=%s&id=%s&ip=%s)r rY �urlopenr^ )�username�passwordZdme_id� ip_addressZdme_url�sr r r � update_dme� s r� c C s t j�d| � |du r t�� }z�| �d�rv| td�d� �dd�\}}t j||d�}|� |�}|� |�}|� |� n`|r�|r�t j �� } | �d| ||� t j �| �} t j �| �} t j �| � t j �| �} |�| �� � |�d� W n" � t j�d| � d}Y n0 |S ) z� Fetch a file based on the URI provided. If you do not pass in a file pointer a tempfile.NamedTemporaryFile, or None if the file could not be retrieved is returned. The URI can be either an HTTP url, or "s3://bucket_name/key_name" z Fetching %sNzs3://r| r )�aws_access_key_id�aws_secret_access_keyr zProblem Retrieving file: %s)r0 re �info�tempfile�NamedTemporaryFiler4 r r$ � connect_s3� get_bucket�get_key�get_contents_to_filer rY � HTTPPasswordMgrWithDefaultRealm� add_password�HTTPBasicAuthHandlerr[ �install_openerr� �writer^ �seekrf ) �uri�filer� r� � bucket_name�key_namer� �bucketr. ZpassmanZ authhandlerrq r� r r r � fetch_file s4 � r� c @ sT e Zd Zddd�Zddd�Zdd � Zd d � Zeeedd �Zd d� Z ee edd�Z dS )� ShellCommandTFNc C s0 d| _ || _t� | _|| _|| _| j|d� d S )Nr )�cwd)� exit_code�commandr �log_fp�wait� fail_fast�run)r� r� r� r� r� r r r r� ' s zShellCommand.__init__c C s� t j�d| j � tj| jdtjtjtj|d�| _| jr�| j� � d u r|t � d� | j� � }| j �|d � | j �|d � q8t j�| j �� � | jj| _| jr�| jdkr�td| j d | j ��| jS d S )Nz running:%sT)�shell�stdin�stdout�stderrr� r r zCommand z failed with status )r0 re r� r� � subprocess�Popen�PIPE�processr� �pollrh ri � communicater� r� �getvalue� returncoder� r� rd )r� r� r@ r r r r� / s, � ��zShellCommand.runc C s t �d S rB )�AttributeError�r� �valuer r r � setReadOnlyE s zShellCommand.setReadOnlyc C s | j S rB )r� r� r r r � getStatusH s zShellCommand.getStatusz The exit code for the commandc C s | j �� S rB )r� r� r� r r r � getOutputN s zShellCommand.getOutputz*The STDIN and STDERR output of the command)TFN)N) r� r� r� r� r� r� r� �property�statusr� �outputr r r r r� % s ��r� c s( e Zd ZdZ� fdd�Zdd� Z� ZS )�AuthSMTPHandlera� This class extends the SMTPHandler in the standard Python logging module to accept a username and password on the constructor and to then use those credentials to authenticate with the SMTP server. To use this, you could add something like this in your boto config file: [handler_hand07] class=boto.utils.AuthSMTPHandler level=WARN formatter=form07 args=('localhost', 'username', 'password', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject') c s& t t| ��||||� || _|| _dS )z� Initialize the handler. We have extended the constructor to accept a username/password for SMTP authentication. N)r� r r� r� r� )r� �mailhostr� r� �fromaddr�toaddrs�subjectr� r r r� c s �zAuthSMTPHandler.__init__c C s� z~| j }|stj}t�| j|�}|�| j| j� | �|�}d| j d� | j �| � |�t j�� |f }|�| j | j |� |�� W n, ttfy� � Y n | �|� Y n0 dS )z� Emit a record. Format the record and send it to the specified addressees. It would be really nice if I could add authorization to this class without having to resort to cut and paste inheritance but, no. z-From: %s To: %s Subject: %s Date: %s %s�,N)�mailport�smtplib� SMTP_PORT�SMTPr �loginr� r� �formatr r/ r � getSubject�emailr� � formatdate�sendmail�quit�KeyboardInterrupt� SystemExit� handleError)r� �record�port�smtp�msgr r r �emitp s& � zAuthSMTPHandler.emit)r� r� r� �__doc__r� r r� r r r� r r U s r c @ sp e Zd ZdZG dd� de�Zdd� Zdd� Zdd � Zd d � Z d d � Z dd� Z dd� Z dd� Z dd� Zdd� ZdS )�LRUCacheaq A dictionary-like object that stores only a certain number of items, and discards its least recently used item when full. >>> cache = LRUCache(3) >>> cache['A'] = 0 >>> cache['B'] = 1 >>> cache['C'] = 2 >>> len(cache) 3 >>> cache['A'] 0 Adding new items to the cache does not increase its size. Instead, the least recently used item is dropped: >>> cache['D'] = 3 >>> len(cache) 3 >>> 'B' in cache False Iterating over the cache returns the keys, starting with the most recently used: >>> for key in cache: ... print key D A C This code is based on the LRUCache class from Genshi which is based on `Myghty <http://www.myghty.org>`_'s LRUCache from ``myghtyutils.util``, written by Mike Bayer and released under the MIT license (Genshi uses the BSD License). c @ s e Zd Zdd� Zdd� ZdS )zLRUCache._Itemc C s d | _ | _|| _|| _d S rB )�previous�nextr. r� )r� r. r� r r r r� � s zLRUCache._Item.__init__c C s t | j�S rB )�reprr� r� r r r r� � s zLRUCache._Item.__repr__N)r� r� r� r� r� r r r r �_Item� s r! c C s t � | _|| _d | _d | _d S rB )�dict�_dict�capacity�head�tail)r� r$ r r r r� � s zLRUCache.__init__c C s || j v S rB )r# r� r r r � __contains__� s zLRUCache.__contains__c c s | j }|r|jV |j}qd S rB )r% r. r )r� �curr r r �__iter__� s zLRUCache.__iter__c C s t | j�S rB )r r# r� r r r �__len__� s zLRUCache.__len__c C s | j | }| �|� |jS rB )r# � _update_itemr� )r� r. �itemr r r r� � s zLRUCache.__getitem__c C sR | j �|�}|d u r6| �||�}|| j |< | �|� n||_| �|� | �� d S rB )r# rl r! � _insert_itemr� r+ � _manage_size)r� r. r� r, r r r � __setitem__� s zLRUCache.__setitem__c C s t | j�S rB )r r# r� r r r r� � s zLRUCache.__repr__c C s: d |_ | j|_| jd ur"|| j_ n|| _|| _| �� d S rB )r r% r r& r. )r� r, r r r r- � s zLRUCache._insert_itemc C sN t | j�| jkrJ| j| jj= | j| jkr<| jj| _d | j_q d | _| _q d S rB )r r# r$ r& r. r% r r r� r r r r. � s zLRUCache._manage_sizec C sV | j |krd S |j}|j|_|jd ur0||j_n|| _d |_| j |_| | j _| _ d S rB )r% r r r& )r� r, r r r r r+ � s zLRUCache._update_itemN)r� r� r� r �objectr! r� r' r) r* r� r/ r� r- r. r+ r r r r r � s % r c @ s> e Zd ZdZeZd dd�Zdd� Zdd� Zd d � Z d d � Z dS )�Passwordzp Password object that stores itself as hashed. Hash defaults to SHA512 if available, MD5 otherwise. Nc C s || _ |r|| _dS )zh Load the string from an initial value, this should be the raw hashed password. N)r5 �hashfunc)r� r5 r2 r r r r� s zPassword.__init__c C s( t |t�s|�d�}| �|��� | _d S rJ )rL rM �encoder2 � hexdigestr5 r� r r r �set s z Password.setc C s t | j �S rB �r5 r� r r r r� s zPassword.__str__c C s< |d u r dS t |t�s |�d�}t| �|��� �t| j�kS )NFrK )rL rM r3 r5 r2 r4 )r� �otherr r r �__eq__ s zPassword.__eq__c C s | j rt| j �S dS d S r+ )r5 r r� r r r r* " s zPassword.__len__)NN) r� r� r� r �_hashfnr2 r� r5 r� r8 r* r r r r r1 s r1 c C s� |pg }|r"dt j�dd�| f } |s6t j�ddd �}|�rĐzlt j�ddd�}tjj�� }||d< ||d < ||d < tjjd d �|d < | |d< |r�|� tjj � |�� |r�tjj � dd�}|�|� tj�|� |� |� |D ]}|� |� q�t j�ddd�} t j�dd��r*t�| tt j�dd���} n t�| �} t j�dd��r\| �� | �� | �� t j�ddd�} t j�ddd�} | �r�| �| | � | �|||�� � | �� W n t j� d� Y n0 d S )Nz[%s] %s�Instancez instance-idZ NotificationZsmtp_toZ smtp_fromr0 �FromzReply-To�ToT)� localtime�Date�Subject�text�html� smtp_host� localhostZ smtp_portZsmtp_tls� smtp_userr � smtp_passz notify failed) r0 rk � get_valuer �mime� multipart� MIMEMultipartr� r �attachr@ �MIMEText�base�MIMEBase� set_payload�encoders� encode_base64r r �int�getbool�ehlo�starttlsr r � as_stringr re rf ) r �bodyZ html_bodyZ to_string� attachmentsZappend_instance_id� from_stringr �partrB �serverrD rE r r r �notify) s^ �� � � r[ c C sH t | t�r| �d� | S t | tj�s.t�| �} t | tj�rD| �d�} | S rJ �rL rM rN r r_ � text_typer3 �r� r r r �get_utf8_valuea s r_ c C s( t | t�s$t | t�r t| �} n| g} | S rB )rL �list�tupler^ r r r �mklisto s rb c C s( t �d| �}t�d|�}t�d|��� S )a~ Convert camel case to a "pythonic" name. Examples:: pythonize_name('CamelCase') -> 'camel_case' pythonize_name('already_pythonized') -> 'already_pythonized' pythonize_name('HTTPRequest') -> 'http_request' pythonize_name('HTTPStatus200Ok') -> 'http_status_200_ok' pythonize_name('UPPER') -> 'upper' pythonize_name('') -> '' z\1_\2)�_first_cap_regex�sub�_number_cap_regex�_end_cap_regexr3 )r� �s1�s2r r r �pythonize_namex s ri F� text/plain�:c C s� t jj�� }| D ]~\}}t||�}|�dd�\}} |dkrNt jjj|| d�} n&t jj� || �} | � |� t j � | � | j dd|d� |�| � q|�� } |r�t� } tjd| d �} z| �| � W | �� n | �� 0 | �� } | S ) a� Description: :param content: A list of tuples of name-content pairs. This is used instead of a dict to ensure that scripts run in order :type list of tuples: :param compress: Use gzip to compress the scripts, defaults to no compression :type bool: :param deftype: The type that should be assumed if nothing else can be figured out :type str: :param delimiter: mime delimiter :type str: :return: Final mime multipart :rtype: str: r| r r@ )�_subtypezContent-Disposition� attachment)�filename�wb)�mode�fileobj)r rG rH rI �guess_mime_typer$ r@ rK rL rM rN rO rP � add_headerrJ rU r �gzip�GzipFiler� �closer� )�content�compress�deftype� delimiter�wrapperr� ZconZ definite_type�maintype�subtypeZmime_conZrcontentr> �gzr r r �write_mime_multipart� s( r c C s> ddddddd�}|}|� � D ]\}}| �|�r |} q:q |S )z�Description: Guess the mime type of a block of text :param content: content we're finding the type of :type str: :param deftype: Default mime type :type str: :rtype: <type>: :return: <description> ztext/x-include-urlztext/x-shellscriptztext/cloud-configztext/upstart-jobztext/part-handlerztext/cloud-boothook)z#includez#!z #cloud-configz #upstart-jobz #part-handlerz#cloud-boothook)r� r4 )rw ry Zstarts_with_mappingsZrtypeZ possible_typeZmimetyper r r rr � s � rr � c C s t | ||td�S )a� Compute MD5 hash on passed file and return results in a tuple of values. :type fp: file :param fp: File pointer to the file to MD5 hash. The file pointer will be reset to its current location before the method returns. :type buf_size: integer :param buf_size: Number of bytes per read request. :type size: int :param size: (optional) The Maximum number of bytes to read from the file pointer (fp). This is useful when uploading a file in multiple parts where the file is being split inplace into different parts. Less bytes may be available. :rtype: tuple :return: A tuple containing the hex digest version of the MD5 hash as the first element, the base64 encoded version of the plain digest as the second element and the data size as the third element. )�hash_algorithm)� compute_hashr )�fp�buf_size�sizer r r � compute_md5� s r� c C s� |� }| � � }|r&||k r&| �|�}n | �|�}|r�t|t�sH|�d�}|�|� |rl|t|�8 }|dkrlq�|r�||k r�| �|�}q0| �|�}q0|�� }t|� � �� d�}|d dkr�|dd� }| � � | } | � |� ||| fS )NrK r r} r{ ) �tellr^ rL rM r3 �updater r4 r �digestrN r� ) r� r� r� r� Zhash_obj�sposr� Z hex_digestZ base64_digest� data_sizer r r r� � s. r� c s � fdd�|D �S )z� Takes a specific header name and a dict of headers {"name": "value"}. Returns a list of matching header names, case-insensitive. c s g | ]}|� � � � � kr|�qS r )r3 �r& �h�r� r r r( r) z)find_matching_headers.<locals>.<listcomp>r )r� r< r r� r �find_matching_headers s r� c s" t | � �}d�� fdd�|D ��S )z� Takes a specific header name and a dict of headers {"name": "value"}. Returns a string of all header values, comma-separated, that match the input header name, case-insensitive. r c 3 s& | ] }� | d urt � | �V qd S rB r6 r� �r< r r � <genexpr> s �z(merge_headers_by_name.<locals>.<genexpr>)r� r/ )r� r< Zmatching_headersr r� r �merge_headers_by_name s r� c @ s e Zd ZdZddd�ZdS )� RequestHookz� This can be extended and supplied to the connection object to gain access to request and response object after the request completes. One use for this would be to implement some specific request logging. Fc C s d S rB r )r� rY �responsera r r r �handle_request_data' s z RequestHook.handle_request_dataN)F)r� r� r� r r� r r r r r� ! s r� c C s: | rt | t�sdS | �d�r dS t| �d��dkr6dS dS )zQ Detect (naively) if the hostname is an IPV6 host. Return a boolean. F�[Trk rT )rL r5 r4 r r$ ��hostnamer r r � host_is_ipv6+ s r� c C s: | � � } t| �r&| �dd�d � d�S | �dd�d S dS )z� Given a hostname that may have a port name, ensure that the port is trimmed returning only the host, including hostnames that are IPV6 and may include brackets. z]:r r z[]rk N)r6 r� r$ r� r r r � parse_hostB s r� �strictc C s� t | tjtjf�st| �} tjrTt | tj�r8| �d|�S t | tj�r~| �d� | S n*t | tj�rd| S t | tj�r~| �d�} | S tdt | � ��dS )a� Returns a UTF8-encodable string in PY3, UTF8 bytes in PY2. This method is similar to six's `ensure_str()`, except it also makes sure that any bytes passed in can be decoded using the utf-8 codec (and raises a UnicodeDecodeError if not). If the object isn't a string, this method will attempt to coerce it to a string with `str()`. Objects without `__str__` property or `__repr__` property will raise an exception. rK znot expecting type "%s"N) rL r r] � binary_typer5 �PY2r3 rN � TypeError�type)r� �errorsr r r �get_utf8able_strQ s r� c C sH t | t�r| �d� | S t | tj�s.t�| �} t | tj�rD| �d�} | S rJ r\ r^ r r r r_ o s c O s` dd� }dd� }|f i |��\}}}t �|�}t �|�}|| � }|�|�}||7 }t||� dS )a A Python 2/3 compatible analogue to the print function. This function writes text to a file descriptor as the builtin print function would, favoring utf-8 encoding. Arguments and return values are the same as documented in the Python 2 print function. c [ s` t �dddtjfg�}| �� D ]8\}}||vrNd}t|�|d�|�� ����q |||< q |� � S )a& Validates keyword arguments that would be used in Print Valid keyword arguments, mirroring print(), are 'sep', 'end', and 'file'. These must be of types string, string, and file / file interface respectively. Returns the above kwargs of the above types. )r� � )�endr{ r� z9{} is not a valid keyword argument. Please use one of: {}r� ) � collections� OrderedDict�sysr� r� r� r r/ r8 r� )�kwargs�expected_keywordsr. r� � error_msgr r r � _get_args� s" �� �� z print_to_fd.<locals>._get_argsc W sT g }| D ]F}t |tjtjf�s&t|�}t |tj�r>|�|� q|�t�|�� q|S )zAGets a `bytes` string for each item in list of printable objects.)rL r r� r] r5 r� � ensure_binary)�objects� byte_objectsr, r r r �_get_byte_strings� s z&print_to_fd.<locals>._get_byte_stringsN)r r� r/ � write_to_fd)r� r� r� r� r� r� r� r� r r r � print_to_fd} s r� c C s� t jr| �|� dS t|t�rtt| d�r2d| jv s>t| tj�rJ| �|� q�t| d�rb| j �|� q�| �t � |�� n&d| jv r�| �t � |�� n | �|� dS )zGWrite given data to given file descriptor, doing any conversions neededNrp �b�buffer) r r� r� rL rM r` rp �io�BytesIOr� � ensure_textr� )�fdr� r r r r� � s � r� )NN)N)N)TrR N)N)r� r� r� Nr� )r� r� Nr� )r� Nr� Nr� )N)N)NNN)NNNNT)Frj rk )r� N)r� )[r r� rh �logging.handlersr r0 � boto.providerr� r� r� r r� �rer� Zemail.mime.multipartr Zemail.mime.baseZemail.mime.text� email.utils�email.encodersrt � threadingr� r� � boto.compatr r r r � contextlibr �hashlibr r r9 r Zboto.compat.jsonr � ImportErrorr� r* �compilerc re rf r rA rI rQ rw ry r" rx r� r� r� r� r� r� r� �Lockr� r� r� r� r� r� r� r0 r� �handlers� SMTPHandlerr r r1 r[ r_ rb ri r rr r� r� r� r� r� r� r� r� r� r� r r r r �<module>& s� � : # v � � � #07w& � 8 - < vendored/boto/boto/https_connection.py 0000644 00000013761 15025013617 0014223 0 ustar 00 # Copyright 2007,2011 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # This file is derived from # http://googleappengine.googlecode.com/svn-history/r136/trunk/python/google/appengine/tools/https_wrapper.py """Extensions to allow HTTPS requests with SSL certificate validation.""" import re import socket import ssl import boto from boto.compat import six, http_client class InvalidCertificateException(http_client.HTTPException): """Raised when a certificate is provided with an invalid hostname.""" def __init__(self, host, cert, reason): """Constructor. Args: host: The hostname the connection was made to. cert: The SSL certificate (as a dictionary) the host returned. """ http_client.HTTPException.__init__(self) self.host = host self.cert = cert self.reason = reason def __str__(self): return ('Host %s returned an invalid certificate (%s): %s' % (self.host, self.reason, self.cert)) def GetValidHostsForCert(cert): """Returns a list of valid host globs for an SSL certificate. Args: cert: A dictionary representing an SSL certificate. Returns: list: A list of valid host globs. """ if 'subjectAltName' in cert: return [x[1] for x in cert['subjectAltName'] if x[0].lower() == 'dns'] else: return [x[0][1] for x in cert['subject'] if x[0][0].lower() == 'commonname'] def ValidateCertificateHostname(cert, hostname): """Validates that a given hostname is valid for an SSL certificate. Args: cert: A dictionary representing an SSL certificate. hostname: The hostname to test. Returns: bool: Whether or not the hostname is valid for this certificate. """ hosts = GetValidHostsForCert(cert) boto.log.debug( "validating server certificate: hostname=%s, certificate hosts=%s", hostname, hosts) for host in hosts: host_re = host.replace(r'.', r'\.').replace(r'*', r'[^.]*') if re.search('^%s$' % (host_re,), hostname, re.I): return True return False class CertValidatingHTTPSConnection(http_client.HTTPConnection): """An HTTPConnection that connects over SSL and validates certificates.""" default_port = http_client.HTTPS_PORT def __init__(self, host, port=default_port, key_file=None, cert_file=None, ca_certs=None, strict=None, **kwargs): """Constructor. Args: host: The hostname. Can be in 'host:port' form. port: The port. Defaults to 443. key_file: A file containing the client's private key cert_file: A file containing the client's certificates ca_certs: A file contianing a set of concatenated certificate authority certs for validating the server against. strict: When true, causes BadStatusLine to be raised if the status line can't be parsed as a valid HTTP/1.0 or 1.1 status line. """ if six.PY2: # Python 3.2 and newer have deprecated and removed the strict # parameter. Since the params are supported as keyword arguments # we conditionally add it here. kwargs['strict'] = strict http_client.HTTPConnection.__init__(self, host=host, port=port, **kwargs) self.key_file = key_file self.cert_file = cert_file self.ca_certs = ca_certs def connect(self): "Connect to a host on a given (SSL) port." if hasattr(self, "timeout"): sock = socket.create_connection((self.host, self.port), self.timeout) else: sock = socket.create_connection((self.host, self.port)) msg = "wrapping ssl socket; " if self.ca_certs: msg += "CA certificate file=%s" % self.ca_certs else: msg += "using system provided SSL certs" boto.log.debug(msg) if hasattr(ssl, 'SSLContext') and getattr(ssl, 'HAS_SNI', False): # Use SSLContext so we can specify server_hostname for SNI # (Required for connections to storage.googleapis.com) context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = ssl.CERT_REQUIRED if self.ca_certs: context.load_verify_locations(self.ca_certs) if self.cert_file: context.load_cert_chain(self.cert_file, self.key_file) self.sock = context.wrap_socket(sock, server_hostname=self.host) # Add attributes only set in SSLSocket constructor without context: self.sock.keyfile = self.key_file self.sock.certfile = self.cert_file self.sock.cert_reqs = context.verify_mode self.sock.ssl_version = ssl.PROTOCOL_SSLv23 self.sock.ca_certs = self.ca_certs self.sock.ciphers = None else: self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_certs) cert = self.sock.getpeercert() hostname = self.host.split(':', 0)[0] if not ValidateCertificateHostname(cert, hostname): raise InvalidCertificateException(hostname, cert, 'remote hostname "%s" does not match ' 'certificate' % hostname) vendored/boto/boto/cognito/identity/layer1.py 0000644 00000060175 15025013617 0015333 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # import boto from boto.compat import json from boto.connection import AWSQueryConnection from boto.regioninfo import RegionInfo from boto.exception import JSONResponseError from boto.cognito.identity import exceptions class CognitoIdentityConnection(AWSQueryConnection): """ Amazon Cognito Amazon Cognito is a web service that delivers scoped temporary credentials to mobile devices and other untrusted environments. Amazon Cognito uniquely identifies a device and supplies the user with a consistent identity over the lifetime of an application. Using Amazon Cognito, you can enable authentication with one or more third-party identity providers (Facebook, Google, or Login with Amazon), and you can also choose to support unauthenticated access from your app. Cognito delivers a unique identifier for each user and acts as an OpenID token provider trusted by AWS Security Token Service (STS) to access temporary, limited- privilege AWS credentials. To provide end-user credentials, first make an unsigned call to GetId. If the end user is authenticated with one of the supported identity providers, set the `Logins` map with the identity provider token. `GetId` returns a unique identifier for the user. Next, make an unsigned call to GetOpenIdToken, which returns the OpenID token necessary to call STS and retrieve AWS credentials. This call expects the same `Logins` map as the `GetId` call, as well as the `IdentityID` originally returned by `GetId`. The token returned by `GetOpenIdToken` can be passed to the STS operation `AssumeRoleWithWebIdentity`_ to retrieve AWS credentials. """ APIVersion = "2014-06-30" DefaultRegionName = "us-east-1" DefaultRegionEndpoint = "cognito-identity.us-east-1.amazonaws.com" ServiceName = "CognitoIdentity" TargetPrefix = "AWSCognitoIdentityService" ResponseError = JSONResponseError _faults = { "LimitExceededException": exceptions.LimitExceededException, "ResourceConflictException": exceptions.ResourceConflictException, "DeveloperUserAlreadyRegisteredException": exceptions.DeveloperUserAlreadyRegisteredException, "TooManyRequestsException": exceptions.TooManyRequestsException, "InvalidParameterException": exceptions.InvalidParameterException, "ResourceNotFoundException": exceptions.ResourceNotFoundException, "InternalErrorException": exceptions.InternalErrorException, "NotAuthorizedException": exceptions.NotAuthorizedException, } def __init__(self, **kwargs): region = kwargs.pop('region', None) if not region: region = RegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) if 'host' not in kwargs or kwargs['host'] is None: kwargs['host'] = region.endpoint super(CognitoIdentityConnection, self).__init__(**kwargs) self.region = region def _required_auth_capability(self): return ['hmac-v4'] def create_identity_pool(self, identity_pool_name, allow_unauthenticated_identities, supported_login_providers=None, developer_provider_name=None, open_id_connect_provider_ar_ns=None): """ Creates a new identity pool. The identity pool is a store of user identity information that is specific to your AWS account. The limit on identity pools is 60 per account. :type identity_pool_name: string :param identity_pool_name: A string that you provide. :type allow_unauthenticated_identities: boolean :param allow_unauthenticated_identities: TRUE if the identity pool supports unauthenticated logins. :type supported_login_providers: map :param supported_login_providers: Optional key:value pairs mapping provider names to provider app IDs. :type developer_provider_name: string :param developer_provider_name: The "domain" by which Cognito will refer to your users. This name acts as a placeholder that allows your backend and the Cognito service to communicate about the developer provider. For the `DeveloperProviderName`, you can use letters as well as period ( `.`), underscore ( `_`), and dash ( `-`). Once you have set a developer provider name, you cannot change it. Please take care in setting this parameter. :type open_id_connect_provider_ar_ns: list :param open_id_connect_provider_ar_ns: """ params = { 'IdentityPoolName': identity_pool_name, 'AllowUnauthenticatedIdentities': allow_unauthenticated_identities, } if supported_login_providers is not None: params['SupportedLoginProviders'] = supported_login_providers if developer_provider_name is not None: params['DeveloperProviderName'] = developer_provider_name if open_id_connect_provider_ar_ns is not None: params['OpenIdConnectProviderARNs'] = open_id_connect_provider_ar_ns return self.make_request(action='CreateIdentityPool', body=json.dumps(params)) def delete_identity_pool(self, identity_pool_id): """ Deletes a user pool. Once a pool is deleted, users will not be able to authenticate with the pool. :type identity_pool_id: string :param identity_pool_id: An identity pool ID in the format REGION:GUID. """ params = {'IdentityPoolId': identity_pool_id, } return self.make_request(action='DeleteIdentityPool', body=json.dumps(params)) def describe_identity_pool(self, identity_pool_id): """ Gets details about a particular identity pool, including the pool name, ID description, creation date, and current number of users. :type identity_pool_id: string :param identity_pool_id: An identity pool ID in the format REGION:GUID. """ params = {'IdentityPoolId': identity_pool_id, } return self.make_request(action='DescribeIdentityPool', body=json.dumps(params)) def get_id(self, account_id, identity_pool_id, logins=None): """ Generates (or retrieves) a Cognito ID. Supplying multiple logins will create an implicit linked account. :type account_id: string :param account_id: A standard AWS account ID (9+ digits). :type identity_pool_id: string :param identity_pool_id: An identity pool ID in the format REGION:GUID. :type logins: map :param logins: A set of optional name-value pairs that map provider names to provider tokens. The available provider names for `Logins` are as follows: + Facebook: `graph.facebook.com` + Google: `accounts.google.com` + Amazon: `www.amazon.com` """ params = { 'AccountId': account_id, 'IdentityPoolId': identity_pool_id, } if logins is not None: params['Logins'] = logins return self.make_request(action='GetId', body=json.dumps(params)) def get_open_id_token(self, identity_id, logins=None): """ Gets an OpenID token, using a known Cognito ID. This known Cognito ID is returned by GetId. You can optionally add additional logins for the identity. Supplying multiple logins creates an implicit link. The OpenId token is valid for 15 minutes. :type identity_id: string :param identity_id: A unique identifier in the format REGION:GUID. :type logins: map :param logins: A set of optional name-value pairs that map provider names to provider tokens. """ params = {'IdentityId': identity_id, } if logins is not None: params['Logins'] = logins return self.make_request(action='GetOpenIdToken', body=json.dumps(params)) def get_open_id_token_for_developer_identity(self, identity_pool_id, logins, identity_id=None, token_duration=None): """ Registers (or retrieves) a Cognito `IdentityId` and an OpenID Connect token for a user authenticated by your backend authentication process. Supplying multiple logins will create an implicit linked account. You can only specify one developer provider as part of the `Logins` map, which is linked to the identity pool. The developer provider is the "domain" by which Cognito will refer to your users. You can use `GetOpenIdTokenForDeveloperIdentity` to create a new identity and to link new logins (that is, user credentials issued by a public provider or developer provider) to an existing identity. When you want to create a new identity, the `IdentityId` should be null. When you want to associate a new login with an existing authenticated/unauthenticated identity, you can do so by providing the existing `IdentityId`. This API will create the identity in the specified `IdentityPoolId`. :type identity_pool_id: string :param identity_pool_id: An identity pool ID in the format REGION:GUID. :type identity_id: string :param identity_id: A unique identifier in the format REGION:GUID. :type logins: map :param logins: A set of optional name-value pairs that map provider names to provider tokens. Each name-value pair represents a user from a public provider or developer provider. If the user is from a developer provider, the name-value pair will follow the syntax `"developer_provider_name": "developer_user_identifier"`. The developer provider is the "domain" by which Cognito will refer to your users; you provided this domain while creating/updating the identity pool. The developer user identifier is an identifier from your backend that uniquely identifies a user. When you create an identity pool, you can specify the supported logins. :type token_duration: long :param token_duration: The expiration time of the token, in seconds. You can specify a custom expiration time for the token so that you can cache it. If you don't provide an expiration time, the token is valid for 15 minutes. You can exchange the token with Amazon STS for temporary AWS credentials, which are valid for a maximum of one hour. The maximum token duration you can set is 24 hours. You should take care in setting the expiration time for a token, as there are significant security implications: an attacker could use a leaked token to access your AWS resources for the token's duration. """ params = { 'IdentityPoolId': identity_pool_id, 'Logins': logins, } if identity_id is not None: params['IdentityId'] = identity_id if token_duration is not None: params['TokenDuration'] = token_duration return self.make_request(action='GetOpenIdTokenForDeveloperIdentity', body=json.dumps(params)) def list_identities(self, identity_pool_id, max_results, next_token=None): """ Lists the identities in a pool. :type identity_pool_id: string :param identity_pool_id: An identity pool ID in the format REGION:GUID. :type max_results: integer :param max_results: The maximum number of identities to return. :type next_token: string :param next_token: A pagination token. """ params = { 'IdentityPoolId': identity_pool_id, 'MaxResults': max_results, } if next_token is not None: params['NextToken'] = next_token return self.make_request(action='ListIdentities', body=json.dumps(params)) def list_identity_pools(self, max_results, next_token=None): """ Lists all of the Cognito identity pools registered for your account. :type max_results: integer :param max_results: The maximum number of identities to return. :type next_token: string :param next_token: A pagination token. """ params = {'MaxResults': max_results, } if next_token is not None: params['NextToken'] = next_token return self.make_request(action='ListIdentityPools', body=json.dumps(params)) def lookup_developer_identity(self, identity_pool_id, identity_id=None, developer_user_identifier=None, max_results=None, next_token=None): """ Retrieves the `IdentityID` associated with a `DeveloperUserIdentifier` or the list of `DeveloperUserIdentifier`s associated with an `IdentityId` for an existing identity. Either `IdentityID` or `DeveloperUserIdentifier` must not be null. If you supply only one of these values, the other value will be searched in the database and returned as a part of the response. If you supply both, `DeveloperUserIdentifier` will be matched against `IdentityID`. If the values are verified against the database, the response returns both values and is the same as the request. Otherwise a `ResourceConflictException` is thrown. :type identity_pool_id: string :param identity_pool_id: An identity pool ID in the format REGION:GUID. :type identity_id: string :param identity_id: A unique identifier in the format REGION:GUID. :type developer_user_identifier: string :param developer_user_identifier: A unique ID used by your backend authentication process to identify a user. Typically, a developer identity provider would issue many developer user identifiers, in keeping with the number of users. :type max_results: integer :param max_results: The maximum number of identities to return. :type next_token: string :param next_token: A pagination token. The first call you make will have `NextToken` set to null. After that the service will return `NextToken` values as needed. For example, let's say you make a request with `MaxResults` set to 10, and there are 20 matches in the database. The service will return a pagination token as a part of the response. This token can be used to call the API again and get results starting from the 11th match. """ params = {'IdentityPoolId': identity_pool_id, } if identity_id is not None: params['IdentityId'] = identity_id if developer_user_identifier is not None: params['DeveloperUserIdentifier'] = developer_user_identifier if max_results is not None: params['MaxResults'] = max_results if next_token is not None: params['NextToken'] = next_token return self.make_request(action='LookupDeveloperIdentity', body=json.dumps(params)) def merge_developer_identities(self, source_user_identifier, destination_user_identifier, developer_provider_name, identity_pool_id): """ Merges two users having different `IdentityId`s, existing in the same identity pool, and identified by the same developer provider. You can use this action to request that discrete users be merged and identified as a single user in the Cognito environment. Cognito associates the given source user ( `SourceUserIdentifier`) with the `IdentityId` of the `DestinationUserIdentifier`. Only developer-authenticated users can be merged. If the users to be merged are associated with the same public provider, but as two different users, an exception will be thrown. :type source_user_identifier: string :param source_user_identifier: User identifier for the source user. The value should be a `DeveloperUserIdentifier`. :type destination_user_identifier: string :param destination_user_identifier: User identifier for the destination user. The value should be a `DeveloperUserIdentifier`. :type developer_provider_name: string :param developer_provider_name: The "domain" by which Cognito will refer to your users. This is a (pseudo) domain name that you provide while creating an identity pool. This name acts as a placeholder that allows your backend and the Cognito service to communicate about the developer provider. For the `DeveloperProviderName`, you can use letters as well as period (.), underscore (_), and dash (-). :type identity_pool_id: string :param identity_pool_id: An identity pool ID in the format REGION:GUID. """ params = { 'SourceUserIdentifier': source_user_identifier, 'DestinationUserIdentifier': destination_user_identifier, 'DeveloperProviderName': developer_provider_name, 'IdentityPoolId': identity_pool_id, } return self.make_request(action='MergeDeveloperIdentities', body=json.dumps(params)) def unlink_developer_identity(self, identity_id, identity_pool_id, developer_provider_name, developer_user_identifier): """ Unlinks a `DeveloperUserIdentifier` from an existing identity. Unlinked developer users will be considered new identities next time they are seen. If, for a given Cognito identity, you remove all federated identities as well as the developer user identifier, the Cognito identity becomes inaccessible. :type identity_id: string :param identity_id: A unique identifier in the format REGION:GUID. :type identity_pool_id: string :param identity_pool_id: An identity pool ID in the format REGION:GUID. :type developer_provider_name: string :param developer_provider_name: The "domain" by which Cognito will refer to your users. :type developer_user_identifier: string :param developer_user_identifier: A unique ID used by your backend authentication process to identify a user. """ params = { 'IdentityId': identity_id, 'IdentityPoolId': identity_pool_id, 'DeveloperProviderName': developer_provider_name, 'DeveloperUserIdentifier': developer_user_identifier, } return self.make_request(action='UnlinkDeveloperIdentity', body=json.dumps(params)) def unlink_identity(self, identity_id, logins, logins_to_remove): """ Unlinks a federated identity from an existing account. Unlinked logins will be considered new identities next time they are seen. Removing the last linked login will make this identity inaccessible. :type identity_id: string :param identity_id: A unique identifier in the format REGION:GUID. :type logins: map :param logins: A set of optional name-value pairs that map provider names to provider tokens. :type logins_to_remove: list :param logins_to_remove: Provider names to unlink from this identity. """ params = { 'IdentityId': identity_id, 'Logins': logins, 'LoginsToRemove': logins_to_remove, } return self.make_request(action='UnlinkIdentity', body=json.dumps(params)) def update_identity_pool(self, identity_pool_id, identity_pool_name, allow_unauthenticated_identities, supported_login_providers=None, developer_provider_name=None, open_id_connect_provider_ar_ns=None): """ Updates a user pool. :type identity_pool_id: string :param identity_pool_id: An identity pool ID in the format REGION:GUID. :type identity_pool_name: string :param identity_pool_name: A string that you provide. :type allow_unauthenticated_identities: boolean :param allow_unauthenticated_identities: TRUE if the identity pool supports unauthenticated logins. :type supported_login_providers: map :param supported_login_providers: Optional key:value pairs mapping provider names to provider app IDs. :type developer_provider_name: string :param developer_provider_name: The "domain" by which Cognito will refer to your users. :type open_id_connect_provider_ar_ns: list :param open_id_connect_provider_ar_ns: """ params = { 'IdentityPoolId': identity_pool_id, 'IdentityPoolName': identity_pool_name, 'AllowUnauthenticatedIdentities': allow_unauthenticated_identities, } if supported_login_providers is not None: params['SupportedLoginProviders'] = supported_login_providers if developer_provider_name is not None: params['DeveloperProviderName'] = developer_provider_name if open_id_connect_provider_ar_ns is not None: params['OpenIdConnectProviderARNs'] = open_id_connect_provider_ar_ns return self.make_request(action='UpdateIdentityPool', body=json.dumps(params)) def make_request(self, action, body): headers = { 'X-Amz-Target': '%s.%s' % (self.TargetPrefix, action), 'Host': self.region.endpoint, 'Content-Type': 'application/x-amz-json-1.1', 'Content-Length': str(len(body)), } http_request = self.build_base_http_request( method='POST', path='/', auth_path='/', params={}, headers=headers, data=body) response = self._mexe(http_request, sender=None, override_num_retries=10) response_body = response.read().decode('utf-8') boto.log.debug(response_body) if response.status == 200: if response_body: return json.loads(response_body) else: json_body = json.loads(response_body) fault_name = json_body.get('__type', None) exception_class = self._faults.get(fault_name, self.ResponseError) raise exception_class(response.status, response.reason, body=json_body) vendored/boto/boto/cognito/identity/exceptions.py 0000644 00000002145 15025013617 0016310 0 ustar 00 # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.exception import BotoServerError class LimitExceededException(BotoServerError): pass class ResourceConflictException(BotoServerError): pass class DeveloperUserAlreadyRegisteredException(BotoServerError): pass class TooManyRequestsException(BotoServerError): pass class InvalidParameterException(BotoServerError): pass class ResourceNotFoundException(BotoServerError): pass class InternalErrorException(BotoServerError): pass class NotAuthorizedException(BotoServerError): pass vendored/boto/boto/cognito/identity/__init__.py 0000644 00000003434 15025013617 0015670 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. # All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.regioninfo import RegionInfo, get_regions from boto.regioninfo import connect def regions(): """ Get all available regions for the Amazon Cognito Identity service. :rtype: list :return: A list of :class:`boto.regioninfo.RegionInfo` """ from boto.cognito.identity.layer1 import CognitoIdentityConnection return get_regions('cognito-identity', connection_cls=CognitoIdentityConnection) def connect_to_region(region_name, **kw_params): from boto.cognito.identity.layer1 import CognitoIdentityConnection return connect('cognito-identity', region_name, connection_cls=CognitoIdentityConnection, **kw_params) vendored/boto/boto/cognito/__init__.py 0000644 00000002143 15025013617 0014033 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # vendored/boto/boto/cognito/sync/layer1.py 0000644 00000051202 15025013617 0014445 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.compat import json from boto.exception import JSONResponseError from boto.connection import AWSAuthConnection from boto.regioninfo import RegionInfo from boto.cognito.sync import exceptions class CognitoSyncConnection(AWSAuthConnection): """ Amazon Cognito Sync Amazon Cognito Sync provides an AWS service and client library that enable cross-device syncing of application-related user data. High-level client libraries are available for both iOS and Android. You can use these libraries to persist data locally so that it's available even if the device is offline. Developer credentials don't need to be stored on the mobile device to access the service. You can use Amazon Cognito to obtain a normalized user ID and credentials. User data is persisted in a dataset that can store up to 1 MB of key-value pairs, and you can have up to 20 datasets per user identity. With Amazon Cognito Sync, the data stored for each identity is accessible only to credentials assigned to that identity. In order to use the Cognito Sync service, you need to make API calls using credentials retrieved with `Amazon Cognito Identity service`_. """ APIVersion = "2014-06-30" DefaultRegionName = "us-east-1" DefaultRegionEndpoint = "cognito-sync.us-east-1.amazonaws.com" ResponseError = JSONResponseError _faults = { "LimitExceededException": exceptions.LimitExceededException, "ResourceConflictException": exceptions.ResourceConflictException, "InvalidConfigurationException": exceptions.InvalidConfigurationException, "TooManyRequestsException": exceptions.TooManyRequestsException, "InvalidParameterException": exceptions.InvalidParameterException, "ResourceNotFoundException": exceptions.ResourceNotFoundException, "InternalErrorException": exceptions.InternalErrorException, "NotAuthorizedException": exceptions.NotAuthorizedException, } def __init__(self, **kwargs): region = kwargs.get('region') if not region: region = RegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) else: del kwargs['region'] kwargs['host'] = region.endpoint super(CognitoSyncConnection, self).__init__(**kwargs) self.region = region def _required_auth_capability(self): return ['hmac-v4'] def delete_dataset(self, identity_pool_id, identity_id, dataset_name): """ Deletes the specific dataset. The dataset will be deleted permanently, and the action can't be undone. Datasets that this dataset was merged with will no longer report the merge. Any consequent operation on this dataset will result in a ResourceNotFoundException. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type identity_id: string :param identity_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type dataset_name: string :param dataset_name: A string of up to 128 characters. Allowed characters are a-z, A-Z, 0-9, '_' (underscore), '-' (dash), and '.' (dot). """ uri = '/identitypools/{0}/identities/{1}/datasets/{2}'.format( identity_pool_id, identity_id, dataset_name) return self.make_request('DELETE', uri, expected_status=200) def describe_dataset(self, identity_pool_id, identity_id, dataset_name): """ Gets metadata about a dataset by identity and dataset name. The credentials used to make this API call need to have access to the identity data. With Amazon Cognito Sync, each identity has access only to its own data. You should use Amazon Cognito Identity service to retrieve the credentials necessary to make this API call. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type identity_id: string :param identity_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type dataset_name: string :param dataset_name: A string of up to 128 characters. Allowed characters are a-z, A-Z, 0-9, '_' (underscore), '-' (dash), and '.' (dot). """ uri = '/identitypools/{0}/identities/{1}/datasets/{2}'.format( identity_pool_id, identity_id, dataset_name) return self.make_request('GET', uri, expected_status=200) def describe_identity_pool_usage(self, identity_pool_id): """ Gets usage details (for example, data storage) about a particular identity pool. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. """ uri = '/identitypools/{0}'.format(identity_pool_id) return self.make_request('GET', uri, expected_status=200) def describe_identity_usage(self, identity_pool_id, identity_id): """ Gets usage information for an identity, including number of datasets and data usage. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type identity_id: string :param identity_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. """ uri = '/identitypools/{0}/identities/{1}'.format( identity_pool_id, identity_id) return self.make_request('GET', uri, expected_status=200) def get_identity_pool_configuration(self, identity_pool_id): """ Gets the configuration settings of an identity pool. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. This is the ID of the pool for which to return a configuration. """ uri = '/identitypools/{0}/configuration'.format(identity_pool_id) return self.make_request('GET', uri, expected_status=200) def list_datasets(self, identity_pool_id, identity_id, next_token=None, max_results=None): """ Lists datasets for an identity. The credentials used to make this API call need to have access to the identity data. With Amazon Cognito Sync, each identity has access only to its own data. You should use Amazon Cognito Identity service to retrieve the credentials necessary to make this API call. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type identity_id: string :param identity_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type next_token: string :param next_token: A pagination token for obtaining the next page of results. :type max_results: integer :param max_results: The maximum number of results to be returned. """ uri = '/identitypools/{0}/identities/{1}/datasets'.format( identity_pool_id, identity_id) params = {} headers = {} query_params = {} if next_token is not None: query_params['nextToken'] = next_token if max_results is not None: query_params['maxResults'] = max_results return self.make_request('GET', uri, expected_status=200, data=json.dumps(params), headers=headers, params=query_params) def list_identity_pool_usage(self, next_token=None, max_results=None): """ Gets a list of identity pools registered with Cognito. :type next_token: string :param next_token: A pagination token for obtaining the next page of results. :type max_results: integer :param max_results: The maximum number of results to be returned. """ uri = '/identitypools' params = {} headers = {} query_params = {} if next_token is not None: query_params['nextToken'] = next_token if max_results is not None: query_params['maxResults'] = max_results return self.make_request('GET', uri, expected_status=200, data=json.dumps(params), headers=headers, params=query_params) def list_records(self, identity_pool_id, identity_id, dataset_name, last_sync_count=None, next_token=None, max_results=None, sync_session_token=None): """ Gets paginated records, optionally changed after a particular sync count for a dataset and identity. The credentials used to make this API call need to have access to the identity data. With Amazon Cognito Sync, each identity has access only to its own data. You should use Amazon Cognito Identity service to retrieve the credentials necessary to make this API call. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type identity_id: string :param identity_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type dataset_name: string :param dataset_name: A string of up to 128 characters. Allowed characters are a-z, A-Z, 0-9, '_' (underscore), '-' (dash), and '.' (dot). :type last_sync_count: long :param last_sync_count: The last server sync count for this record. :type next_token: string :param next_token: A pagination token for obtaining the next page of results. :type max_results: integer :param max_results: The maximum number of results to be returned. :type sync_session_token: string :param sync_session_token: A token containing a session ID, identity ID, and expiration. """ uri = '/identitypools/{0}/identities/{1}/datasets/{2}/records'.format( identity_pool_id, identity_id, dataset_name) params = {} headers = {} query_params = {} if last_sync_count is not None: query_params['lastSyncCount'] = last_sync_count if next_token is not None: query_params['nextToken'] = next_token if max_results is not None: query_params['maxResults'] = max_results if sync_session_token is not None: query_params['syncSessionToken'] = sync_session_token return self.make_request('GET', uri, expected_status=200, data=json.dumps(params), headers=headers, params=query_params) def register_device(self, identity_pool_id, identity_id, platform, token): """ Registers a device to receive push sync notifications. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. Here, the ID of the pool that the identity belongs to. :type identity_id: string :param identity_id: The unique ID for this identity. :type platform: string :param platform: The SNS platform type (e.g. GCM, SDM, APNS, APNS_SANDBOX). :type token: string :param token: The push token. """ uri = '/identitypools/{0}/identity/{1}/device'.format( identity_pool_id, identity_id) params = {'Platform': platform, 'Token': token, } headers = {} query_params = {} return self.make_request('POST', uri, expected_status=200, data=json.dumps(params), headers=headers, params=query_params) def set_identity_pool_configuration(self, identity_pool_id, push_sync=None): """ Sets the necessary configuration for push sync. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. This is the ID of the pool to modify. :type push_sync: dict :param push_sync: Configuration options to be applied to the identity pool. """ uri = '/identitypools/{0}/configuration'.format(identity_pool_id) params = {} headers = {} query_params = {} if push_sync is not None: params['PushSync'] = push_sync return self.make_request('POST', uri, expected_status=200, data=json.dumps(params), headers=headers, params=query_params) def subscribe_to_dataset(self, identity_pool_id, identity_id, dataset_name, device_id): """ Subscribes to receive notifications when a dataset is modified by another device. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. The ID of the pool to which the identity belongs. :type identity_id: string :param identity_id: Unique ID for this identity. :type dataset_name: string :param dataset_name: The name of the dataset to subcribe to. :type device_id: string :param device_id: The unique ID generated for this device by Cognito. """ uri = '/identitypools/{0}/identities/{1}/datasets/{2}/subscriptions/{3}'.format( identity_pool_id, identity_id, dataset_name, device_id) return self.make_request('POST', uri, expected_status=200) def unsubscribe_from_dataset(self, identity_pool_id, identity_id, dataset_name, device_id): """ Unsubscribe from receiving notifications when a dataset is modified by another device. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. The ID of the pool to which this identity belongs. :type identity_id: string :param identity_id: Unique ID for this identity. :type dataset_name: string :param dataset_name: The name of the dataset from which to unsubcribe. :type device_id: string :param device_id: The unique ID generated for this device by Cognito. """ uri = '/identitypools/{0}/identities/{1}/datasets/{2}/subscriptions/{3}'.format( identity_pool_id, identity_id, dataset_name, device_id) return self.make_request('DELETE', uri, expected_status=200) def update_records(self, identity_pool_id, identity_id, dataset_name, sync_session_token, device_id=None, record_patches=None, client_context=None): """ Posts updates to records and add and delete records for a dataset and user. The credentials used to make this API call need to have access to the identity data. With Amazon Cognito Sync, each identity has access only to its own data. You should use Amazon Cognito Identity service to retrieve the credentials necessary to make this API call. :type identity_pool_id: string :param identity_pool_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type identity_id: string :param identity_id: A name-spaced GUID (for example, us- east-1:23EC4050-6AEA-7089-A2DD-08002EXAMPLE) created by Amazon Cognito. GUID generation is unique within a region. :type dataset_name: string :param dataset_name: A string of up to 128 characters. Allowed characters are a-z, A-Z, 0-9, '_' (underscore), '-' (dash), and '.' (dot). :type device_id: string :param device_id: The unique ID generated for this device by Cognito. :type record_patches: list :param record_patches: A list of patch operations. :type sync_session_token: string :param sync_session_token: The SyncSessionToken returned by a previous call to ListRecords for this dataset and identity. :type client_context: string :param client_context: Intended to supply a device ID that will populate the `lastModifiedBy` field referenced in other methods. The `ClientContext` field is not yet implemented. """ uri = '/identitypools/{0}/identities/{1}/datasets/{2}'.format( identity_pool_id, identity_id, dataset_name) params = {'SyncSessionToken': sync_session_token, } headers = {} query_params = {} if device_id is not None: params['DeviceId'] = device_id if record_patches is not None: params['RecordPatches'] = record_patches if client_context is not None: headers['x-amz-Client-Context'] = client_context if client_context is not None: headers['x-amz-Client-Context'] = client_context return self.make_request('POST', uri, expected_status=200, data=json.dumps(params), headers=headers, params=query_params) def make_request(self, verb, resource, headers=None, data='', expected_status=None, params=None): if headers is None: headers = {} response = AWSAuthConnection.make_request( self, verb, resource, headers=headers, data=data, params=params) body = json.loads(response.read().decode('utf-8')) if response.status == expected_status: return body else: error_type = response.getheader('x-amzn-ErrorType').split(':')[0] error_class = self._faults.get(error_type, self.ResponseError) raise error_class(response.status, response.reason, body) vendored/boto/boto/cognito/sync/exceptions.py 0000644 00000003160 15025013617 0015431 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.exception import BotoServerError class LimitExceededException(BotoServerError): pass class ResourceConflictException(BotoServerError): pass class InvalidConfigurationException(BotoServerError): pass class TooManyRequestsException(BotoServerError): pass class InvalidParameterException(BotoServerError): pass class ResourceNotFoundException(BotoServerError): pass class InternalErrorException(BotoServerError): pass class NotAuthorizedException(BotoServerError): pass vendored/boto/boto/cognito/sync/__init__.py 0000644 00000003341 15025013617 0015010 0 ustar 00 # Copyright (c) 2014 Amazon.com, Inc. or its affiliates. # All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.regioninfo import RegionInfo, get_regions from boto.regioninfo import connect def regions(): """ Get all available regions for the Amazon Cognito Sync service. :rtype: list :return: A list of :class:`boto.regioninfo.RegionInfo` """ from boto.cognito.sync.layer1 import CognitoSyncConnection return get_regions('cognito-sync', connection_cls=CognitoSyncConnection) def connect_to_region(region_name, **kw_params): from boto.cognito.sync.layer1 import CognitoSyncConnection return connect('cognito-sync', region_name, connection_cls=CognitoSyncConnection, **kw_params) vendored/boto/boto/route53/record.py 0000644 00000034541 15025013617 0013425 0 ustar 00 # Copyright (c) 2010 Chris Moyer http://coredumped.org/ # Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. # All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. RECORD_TYPES = ['A', 'AAAA', 'TXT', 'CNAME', 'MX', 'PTR', 'SRV', 'SPF'] from boto.resultset import ResultSet class ResourceRecordSets(ResultSet): """ A list of resource records. :ivar hosted_zone_id: The ID of the hosted zone. :ivar comment: A comment that will be stored with the change. :ivar changes: A list of changes. """ ChangeResourceRecordSetsBody = """<?xml version="1.0" encoding="UTF-8"?> <ChangeResourceRecordSetsRequest xmlns="https://route53.amazonaws.com/doc/2013-04-01/"> <ChangeBatch> <Comment>%(comment)s</Comment> <Changes>%(changes)s</Changes> </ChangeBatch> </ChangeResourceRecordSetsRequest>""" ChangeXML = """<Change> <Action>%(action)s</Action> %(record)s </Change>""" def __init__(self, connection=None, hosted_zone_id=None, comment=None): self.connection = connection self.hosted_zone_id = hosted_zone_id self.comment = comment self.changes = [] self.next_record_name = None self.next_record_type = None self.next_record_identifier = None super(ResourceRecordSets, self).__init__([('ResourceRecordSet', Record)]) def __repr__(self): if self.changes: record_list = ','.join([c.__repr__() for c in self.changes]) else: record_list = ','.join([record.__repr__() for record in self]) return '<ResourceRecordSets:%s [%s]' % (self.hosted_zone_id, record_list) def add_change(self, action, name, type, ttl=600, alias_hosted_zone_id=None, alias_dns_name=None, identifier=None, weight=None, region=None, alias_evaluate_target_health=None, health_check=None, failover=None): """ Add a change request to the set. :type action: str :param action: The action to perform ('CREATE'|'DELETE'|'UPSERT') :type name: str :param name: The name of the domain you want to perform the action on. :type type: str :param type: The DNS record type. Valid values are: * A * AAAA * CNAME * MX * NS * PTR * SOA * SPF * SRV * TXT :type ttl: int :param ttl: The resource record cache time to live (TTL), in seconds. :type alias_hosted_zone_id: str :param alias_dns_name: *Alias resource record sets only* The value of the hosted zone ID, CanonicalHostedZoneNameId, for the LoadBalancer. :type alias_dns_name: str :param alias_hosted_zone_id: *Alias resource record sets only* Information about the domain to which you are redirecting traffic. :type identifier: str :param identifier: *Weighted and latency-based resource record sets only* An identifier that differentiates among multiple resource record sets that have the same combination of DNS name and type. :type weight: int :param weight: *Weighted resource record sets only* Among resource record sets that have the same combination of DNS name and type, a value that determines what portion of traffic for the current resource record set is routed to the associated location :type region: str :param region: *Latency-based resource record sets only* Among resource record sets that have the same combination of DNS name and type, a value that determines which region this should be associated with for the latency-based routing :type alias_evaluate_target_health: bool :param alias_evaluate_target_health: *Required for alias resource record sets* Indicates whether this Resource Record Set should respect the health status of any health checks associated with the ALIAS target record which it is linked to. :type health_check: str :param health_check: Health check to associate with this record :type failover: str :param failover: *Failover resource record sets only* Whether this is the primary or secondary resource record set. """ change = Record(name, type, ttl, alias_hosted_zone_id=alias_hosted_zone_id, alias_dns_name=alias_dns_name, identifier=identifier, weight=weight, region=region, alias_evaluate_target_health=alias_evaluate_target_health, health_check=health_check, failover=failover) self.changes.append([action, change]) return change def add_change_record(self, action, change): """Add an existing record to a change set with the specified action""" self.changes.append([action, change]) return def to_xml(self): """Convert this ResourceRecordSet into XML to be saved via the ChangeResourceRecordSetsRequest""" changesXML = "" for change in self.changes: changeParams = {"action": change[0], "record": change[1].to_xml()} changesXML += self.ChangeXML % changeParams params = {"comment": self.comment, "changes": changesXML} return self.ChangeResourceRecordSetsBody % params def commit(self): """Commit this change""" if not self.connection: import boto self.connection = boto.connect_route53() return self.connection.change_rrsets(self.hosted_zone_id, self.to_xml()) def endElement(self, name, value, connection): """Overwritten to also add the NextRecordName, NextRecordType and NextRecordIdentifier to the base object""" if name == 'NextRecordName': self.next_record_name = value elif name == 'NextRecordType': self.next_record_type = value elif name == 'NextRecordIdentifier': self.next_record_identifier = value else: return super(ResourceRecordSets, self).endElement(name, value, connection) def __iter__(self): """Override the next function to support paging""" results = super(ResourceRecordSets, self).__iter__() truncated = self.is_truncated while results: for obj in results: yield obj if self.is_truncated: self.is_truncated = False results = self.connection.get_all_rrsets(self.hosted_zone_id, name=self.next_record_name, type=self.next_record_type, identifier=self.next_record_identifier) else: results = None self.is_truncated = truncated class Record(object): """An individual ResourceRecordSet""" HealthCheckBody = """<HealthCheckId>%s</HealthCheckId>""" XMLBody = """<ResourceRecordSet> <Name>%(name)s</Name> <Type>%(type)s</Type> %(weight)s %(body)s %(health_check)s </ResourceRecordSet>""" WRRBody = """ <SetIdentifier>%(identifier)s</SetIdentifier> <Weight>%(weight)s</Weight> """ RRRBody = """ <SetIdentifier>%(identifier)s</SetIdentifier> <Region>%(region)s</Region> """ FailoverBody = """ <SetIdentifier>%(identifier)s</SetIdentifier> <Failover>%(failover)s</Failover> """ ResourceRecordsBody = """ <TTL>%(ttl)s</TTL> <ResourceRecords> %(records)s </ResourceRecords>""" ResourceRecordBody = """<ResourceRecord> <Value>%s</Value> </ResourceRecord>""" AliasBody = """<AliasTarget> <HostedZoneId>%(hosted_zone_id)s</HostedZoneId> <DNSName>%(dns_name)s</DNSName> %(eval_target_health)s </AliasTarget>""" EvaluateTargetHealth = """<EvaluateTargetHealth>%s</EvaluateTargetHealth>""" def __init__(self, name=None, type=None, ttl=600, resource_records=None, alias_hosted_zone_id=None, alias_dns_name=None, identifier=None, weight=None, region=None, alias_evaluate_target_health=None, health_check=None, failover=None): self.name = name self.type = type self.ttl = ttl if resource_records is None: resource_records = [] self.resource_records = resource_records self.alias_hosted_zone_id = alias_hosted_zone_id self.alias_dns_name = alias_dns_name self.identifier = identifier self.weight = weight self.region = region self.alias_evaluate_target_health = alias_evaluate_target_health self.health_check = health_check self.failover = failover def __repr__(self): return '<Record:%s:%s:%s>' % (self.name, self.type, self.to_print()) def add_value(self, value): """Add a resource record value""" self.resource_records.append(value) def set_alias(self, alias_hosted_zone_id, alias_dns_name, alias_evaluate_target_health=False): """Make this an alias resource record set""" self.alias_hosted_zone_id = alias_hosted_zone_id self.alias_dns_name = alias_dns_name self.alias_evaluate_target_health = alias_evaluate_target_health def to_xml(self): """Spit this resource record set out as XML""" if self.alias_hosted_zone_id is not None and self.alias_dns_name is not None: # Use alias if self.alias_evaluate_target_health is not None: eval_target_health = self.EvaluateTargetHealth % ('true' if self.alias_evaluate_target_health else 'false') else: eval_target_health = "" body = self.AliasBody % {"hosted_zone_id": self.alias_hosted_zone_id, "dns_name": self.alias_dns_name, "eval_target_health": eval_target_health} else: # Use resource record(s) records = "" for r in self.resource_records: records += self.ResourceRecordBody % r body = self.ResourceRecordsBody % { "ttl": self.ttl, "records": records, } weight = "" if self.identifier is not None and self.weight is not None: weight = self.WRRBody % {"identifier": self.identifier, "weight": self.weight} elif self.identifier is not None and self.region is not None: weight = self.RRRBody % {"identifier": self.identifier, "region": self.region} elif self.identifier is not None and self.failover is not None: weight = self.FailoverBody % {"identifier": self.identifier, "failover": self.failover} health_check = "" if self.health_check is not None: health_check = self.HealthCheckBody % (self.health_check) params = { "name": self.name, "type": self.type, "weight": weight, "body": body, "health_check": health_check } return self.XMLBody % params def to_print(self): rr = "" if self.alias_hosted_zone_id is not None and self.alias_dns_name is not None: # Show alias rr = 'ALIAS ' + self.alias_hosted_zone_id + ' ' + self.alias_dns_name if self.alias_evaluate_target_health is not None: rr += ' (EvalTarget %s)' % self.alias_evaluate_target_health else: # Show resource record(s) rr = ",".join(self.resource_records) if self.identifier is not None and self.weight is not None: rr += ' (WRR id=%s, w=%s)' % (self.identifier, self.weight) elif self.identifier is not None and self.region is not None: rr += ' (LBR id=%s, region=%s)' % (self.identifier, self.region) elif self.identifier is not None and self.failover is not None: rr += ' (FAILOVER id=%s, failover=%s)' % (self.identifier, self.failover) return rr def endElement(self, name, value, connection): if name == 'Name': self.name = value elif name == 'Type': self.type = value elif name == 'TTL': self.ttl = value elif name == 'Value': self.resource_records.append(value) elif name == 'HostedZoneId': self.alias_hosted_zone_id = value elif name == 'DNSName': self.alias_dns_name = value elif name == 'SetIdentifier': self.identifier = value elif name == 'EvaluateTargetHealth': self.alias_evaluate_target_health = value.lower() == 'true' elif name == 'Weight': self.weight = value elif name == 'Region': self.region = value elif name == 'Failover': self.failover = value elif name == 'HealthCheckId'